copnix/modules/nixos/boot/patches/0001-prjc-cachy-lfbmq.patch

From c4dc19c67ce41b5f43b349ea1f4ced963cb4f6e4 Mon Sep 17 00:00:00 2001
From: Piotr Gorski <[email protected]>
Date: Tue, 30 Jun 2026 12:14:48 +0200
Subject: [PATCH] prjc-cachy-lfbmq

Signed-off-by: Piotr Gorski <[email protected]>
---
 Documentation/admin-guide/sysctl/kernel.rst |    9 +
 Documentation/scheduler/sched-BMQ.txt       |  110 +
 fs/f2fs/checkpoint.c                        |    2 +-
 fs/proc/base.c                              |    2 +-
 include/linux/rseq_entry.h                  |    3 +
 include/linux/sched.h                       |   79 +
 include/linux/sched/deadline.h              |   20 +
 include/linux/sched/nohz.h                  |    2 +-
 include/linux/sched/prio.h                  |   22 +
 include/linux/sched/rt.h                    |    2 +
 include/linux/sched/topology.h              |    3 +-
 init/Kconfig                                |   30 +-
 init/init_task.c                            |   21 +
 kernel/Kconfig.preempt                      |    4 +-
 kernel/cgroup/cpuset.c                      |   14 +-
 kernel/delayacct.c                          |    2 +-
 kernel/exit.c                               |    6 +-
 kernel/locking/rtmutex.c                    |   16 +-
 kernel/locking/ww_mutex.h                   |    2 +
 kernel/sched/Makefile                       |    5 +
 kernel/sched/alt_core.c                     | 7354 +++++++++++++++++++
 kernel/sched/alt_core.h                     |  290 +
 kernel/sched/alt_debug.c                    |   32 +
 kernel/sched/alt_sched.h                    | 1001 +++
 kernel/sched/alt_topology.c                 |  389 +
 kernel/sched/bmq.h                          |   84 +
 kernel/sched/build_policy.c                 |    8 +-
 kernel/sched/build_utility.c                |    6 +-
 kernel/sched/cpufreq_schedutil.c            |   12 +-
 kernel/sched/cputime.c                      |   12 +-
 kernel/sched/debug.c                        |   20 +
 kernel/sched/idle.c                         |   11 +-
 kernel/sched/loadavg.c                      |   14 +-
 kernel/sched/pds.h                          |  139 +
 kernel/sched/pelt.c                         |    6 +-
 kernel/sched/pelt.h                         |    6 +-
 kernel/sched/sched.h                        |    9 +
 kernel/sched/stats.c                        |    4 +
 kernel/sched/stats.h                        |    2 +
 kernel/sched/syscalls.c                     |  156 +-
 kernel/sched/topology.c                     |   42 +
 kernel/sysctl.c                             |   15 +
 kernel/time/posix-cpu-timers.c              |   10 +-
 kernel/trace/trace_osnoise.c                |    4 +
 kernel/trace/trace_selftest.c               |    5 +
 kernel/workqueue.c                          |   15 +
 46 files changed, 9957 insertions(+), 43 deletions(-)
 create mode 100644 Documentation/scheduler/sched-BMQ.txt
 create mode 100644 kernel/sched/alt_core.c
 create mode 100644 kernel/sched/alt_core.h
 create mode 100644 kernel/sched/alt_debug.c
 create mode 100644 kernel/sched/alt_sched.h
 create mode 100644 kernel/sched/alt_topology.c
 create mode 100644 kernel/sched/bmq.h
 create mode 100644 kernel/sched/pds.h

diff --git a/Documentation/admin-guide/sysctl/kernel.rst b/Documentation/admin-guide/sysctl/kernel.rst
index c6994e55d..cf28e5f29 100644
--- a/Documentation/admin-guide/sysctl/kernel.rst
+++ b/Documentation/admin-guide/sysctl/kernel.rst
@@ -1734,3 +1734,12 @@ is 10 seconds.
 
 The softlockup threshold is (``2 * watchdog_thresh``). Setting this
 tunable to zero will disable lockup detection altogether.
+
+yield_type:
+===========
+
+BMQ/PDS CPU scheduler only. This determines what type of yield calls
+to sched_yield() will be performed.
+
+  0 - No yield.
+  1 - Requeue task. (default)
diff --git a/Documentation/scheduler/sched-BMQ.txt b/Documentation/scheduler/sched-BMQ.txt
new file mode 100644
index 000000000..05c84eec0
--- /dev/null
+++ b/Documentation/scheduler/sched-BMQ.txt
@@ -0,0 +1,110 @@
+                         BitMap queue CPU Scheduler
+                         --------------------------
+
+CONTENT
+========
+
+ Background
+ Design
+   Overview
+   Task policy
+   Priority management
+   BitMap Queue
+   CPU Assignment and Migration
+
+
+Background
+==========
+
+BitMap Queue CPU scheduler, referred to as BMQ from here on, is an evolution
+of previous Priority and Deadline based Skiplist multiple queue scheduler(PDS),
+and inspired by Zircon scheduler. The goal of it is to keep the scheduler code
+simple, while efficiency and scalable for interactive tasks, such as desktop,
+movie playback and gaming etc.
+
+Design
+======
+
+Overview
+--------
+
+BMQ use per CPU run queue design, each CPU(logical) has it's own run queue,
+each CPU is responsible for scheduling the tasks that are putting into it's
+run queue.
+
+The run queue is a set of priority queues. Note that these queues are fifo
+queue for non-rt tasks or priority queue for rt tasks in data structure. See
+BitMap Queue below for details. BMQ is optimized for non-rt tasks in the fact
+that most applications are non-rt tasks. No matter the queue is fifo or
+priority, In each queue is an ordered list of runnable tasks awaiting execution
+and the data structures are the same. When it is time for a new task to run,
+the scheduler simply looks the lowest numbered queueue that contains a task,
+and runs the first task from the head of that queue. And per CPU idle task is
+also in the run queue, so the scheduler can always find a task to run on from
+its run queue.
+
+Each task will assigned the same timeslice(default 4ms) when it is picked to
+start running. Task will be reinserted at the end of the appropriate priority
+queue when it uses its whole timeslice. When the scheduler selects a new task
+from the priority queue it sets the CPU's preemption timer for the remainder of
+the previous timeslice. When that timer fires the scheduler will stop execution
+on that task, select another task and start over again.
+
+If a task blocks waiting for a shared resource then it's taken out of its
+priority queue and is placed in a wait queue for the shared resource. When it
+is unblocked it will be reinserted in the appropriate priority queue of an
+eligible CPU.
+
+Task policy
+-----------
+
+BMQ supports DEADLINE, FIFO, RR, NORMAL, BATCH and IDLE task policy like the
+mainline CFS scheduler. But BMQ is heavy optimized for non-rt task, that's
+NORMAL/BATCH/IDLE policy tasks. Below is the implementation detail of each
+policy.
+
+DEADLINE
+	It is squashed as priority 0 FIFO task.
+
+FIFO/RR
+	All RT tasks share one single priority queue in BMQ run queue designed. The
+complexity of insert operation is O(n). BMQ is not designed for system runs
+with major rt policy tasks.
+
+NORMAL/BATCH/IDLE
+	BATCH and IDLE tasks are treated as the same policy. They compete CPU with
+NORMAL policy tasks, but they just don't boost. To control the priority of
+NORMAL/BATCH/IDLE tasks, simply use nice level.
+
+ISO
+	ISO policy is not supported in BMQ. Please use nice level -20 NORMAL policy
+task instead.
+
+Priority management
+-------------------
+
+RT tasks have priority from 0-99. For non-rt tasks, there are three different
+factors used to determine the effective priority of a task. The effective
+priority being what is used to determine which queue it will be in.
+
+The first factor is simply the task’s static priority. Which is assigned from
+task's nice level, within [-20, 19] in userland's point of view and [0, 39]
+internally.
+
+The second factor is the priority boost. This is a value bounded between
+[-MAX_PRIORITY_ADJ, MAX_PRIORITY_ADJ] used to offset the base priority, it is
+modified by the following cases:
+
+*When a thread has used up its entire timeslice, always deboost its boost by
+increasing by one.
+*When a thread gives up cpu control(voluntary or non-voluntary) to reschedule,
+and its switch-in time(time after last switch and run) below the thredhold
+based on its priority boost, will boost its boost by decreasing by one buti is
+capped at 0 (won’t go negative).
+
+The intent in this system is to ensure that interactive threads are serviced
+quickly. These are usually the threads that interact directly with the user
+and cause user-perceivable latency. These threads usually do little work and
+spend most of their time blocked awaiting another user event. So they get the
+priority boost from unblocking while background threads that do most of the
+processing receive the priority penalty for using their entire timeslice.
diff --git a/fs/f2fs/checkpoint.c b/fs/f2fs/checkpoint.c
index c00a6b6eb..0fa976ca0 100644
--- a/fs/f2fs/checkpoint.c
+++ b/fs/f2fs/checkpoint.c
@@ -28,7 +28,7 @@ static inline void get_lock_elapsed_time(struct f2fs_time_stat *ts)
 {
 	ts->total_time = ktime_get();
 #ifdef CONFIG_64BIT
-	ts->running_time = current->se.sum_exec_runtime;
+	ts->running_time = tsk_seruntime(current);
 #endif
 #if defined(CONFIG_SCHED_INFO) && defined(CONFIG_SCHEDSTATS)
 	ts->runnable_time = current->sched_info.run_delay;
diff --git a/fs/proc/base.c b/fs/proc/base.c
index d9acfa89c..61a7726c5 100644
--- a/fs/proc/base.c
+++ b/fs/proc/base.c
@@ -515,7 +515,7 @@ static int proc_pid_schedstat(struct seq_file *m, struct pid_namespace *ns,
 		seq_puts(m, "0 0 0\n");
 	else
 		seq_printf(m, "%llu %llu %lu\n",
-		   (unsigned long long)task->se.sum_exec_runtime,
+		   (unsigned long long)tsk_seruntime(task),
 		   (unsigned long long)task->sched_info.run_delay,
 		   task->sched_info.pcount);
 
diff --git a/include/linux/rseq_entry.h b/include/linux/rseq_entry.h
index ed9da6e41..4ad444d63 100644
--- a/include/linux/rseq_entry.h
+++ b/include/linux/rseq_entry.h
@@ -729,7 +729,10 @@ static __always_inline void rseq_syscall_exit_to_user_mode(void)
 
 	/* Needed to remove the store for the !lockdep case */
 	if (IS_ENABLED(CONFIG_LOCKDEP)) {
+#ifndef CONFIG_SCHED_ALT
+/* WA for sched/alt: [Sync] 9a723ed7facf sched/mmcid: Provide new scheduler CID mechanism */
 		WARN_ON_ONCE(ev->sched_switch);
+#endif /* !CONFIG_SCHED_ALT */
 		ev->events = 0;
 	}
 }
diff --git a/include/linux/sched.h b/include/linux/sched.h
index f9c0b4724..cddd46432 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -847,7 +847,9 @@ struct task_struct {
 #endif
 
 	int				on_cpu;
+
 	struct __call_single_node	wake_entry;
+#ifndef CONFIG_SCHED_ALT
 	unsigned int			wakee_flips;
 	unsigned long			wakee_flip_decay_ts;
 	struct task_struct		*last_wakee;
@@ -860,6 +862,7 @@ struct task_struct {
 	 * used CPU that may be idle.
 	 */
 	int				recent_used_cpu;
+#endif /* !CONFIG_SCHED_ALT */
 	int				wake_cpu;
 	int				on_rq;
 
@@ -868,6 +871,20 @@ struct task_struct {
 	int				normal_prio;
 	unsigned int			rt_priority;
 
+#ifdef CONFIG_SCHED_ALT
+	u64				last_ran;
+	s64				time_slice;
+	struct llist_node		pq_node;
+	int				__sched_prio;
+#ifdef CONFIG_SCHED_BMQ
+	int				boost_prio;
+#endif /* CONFIG_SCHED_BMQ */
+#ifdef CONFIG_SCHED_PDS
+	u64				deadline;
+#endif /* CONFIG_SCHED_PDS */
+	/* sched_clock time spent running */
+	u64				sched_time;
+#else /* !CONFIG_SCHED_ALT */
 	struct sched_entity		se;
 	struct sched_rt_entity		rt;
 	struct sched_dl_entity		dl;
@@ -882,6 +899,7 @@ struct task_struct {
 	unsigned long			core_cookie;
 	unsigned int			core_occupation;
 #endif
+#endif /* !CONFIG_SCHED_ALT */
 
 #ifdef CONFIG_CGROUP_SCHED
 	struct task_group		*sched_task_group;
@@ -923,9 +941,13 @@ struct task_struct {
 	const cpumask_t			*cpus_ptr;
 	cpumask_t			*user_cpus_ptr;
 	cpumask_t			cpus_mask;
+#ifndef CONFIG_SCHED_ALT
 	void				*migration_pending;
+#endif
 	unsigned short			migration_disabled;
+#ifndef CONFIG_SCHED_ALT
 	unsigned short			migration_flags;
+#endif
 
 #ifdef CONFIG_PREEMPT_RCU
 	int				rcu_read_lock_nesting;
@@ -956,8 +978,10 @@ struct task_struct {
 	struct sched_info		sched_info;
 
 	struct list_head		tasks;
+#ifndef CONFIG_SCHED_ALT
 	struct plist_node		pushable_tasks;
 	struct rb_node			pushable_dl_tasks;
+#endif
 
 	struct mm_struct		*mm;
 	struct mm_struct		*active_mm;
@@ -1670,6 +1694,15 @@ static inline bool sched_proxy_exec(void)
 }
 #endif
 
+#ifdef CONFIG_SCHED_ALT
+#define tsk_seruntime(t)		((t)->sched_time)
+/* replace the uncertian rt_timeout with 0UL */
+#define tsk_rttimeout(t)		(0UL)
+#else /* !CONFIG_SCHED_ALT: */
+#define tsk_seruntime(t)	((t)->se.sum_exec_runtime)
+#define tsk_rttimeout(t)	((t)->rt.timeout)
+#endif /* !CONFIG_SCHED_ALT */
+
 #define TASK_REPORT_IDLE	(TASK_REPORT + 1)
 #define TASK_REPORT_MAX		(TASK_REPORT_IDLE << 1)
 
@@ -2302,7 +2335,9 @@ static inline unsigned int task_cpu(const struct task_struct *p)
 	return READ_ONCE(task_thread_info(p)->cpu);
 }
 
+#ifndef CONFIG_SCHED_ALT
 extern void set_task_cpu(struct task_struct *p, unsigned int cpu);
+#endif
 
 #else
 
@@ -2319,7 +2354,11 @@ static inline void set_task_cpu(struct task_struct *p, unsigned int cpu)
 
 static inline bool task_is_runnable(struct task_struct *p)
 {
+#ifdef CONFIG_SCHED_ALT
+	return p->on_rq;
+#else
 	return p->on_rq && !p->se.sched_delayed;
+#endif /* !CONFIG_SCHED_ALT */
 }
 
 extern bool sched_task_on_rq(struct task_struct *p);
@@ -2440,6 +2479,25 @@ DECLARE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
 #endif
 #define this_rq_pinned() (*(unsigned int *)((void *)this_rq_raw() + RQ_nr_pinned))
 
+static inline void
+__do_set_cpus_ptr(struct task_struct *p, const struct cpumask *new_mask)
+{
+	/*
+	 * This here violates the locking rules for affinity, since we're only
+	 * supposed to change these variables while holding both rq->lock and
+	 * p->pi_lock.
+	 *
+	 * HOWEVER, it magically works, because ttwu() is the only code that
+	 * accesses these variables under p->pi_lock and only does so after
+	 * smp_cond_load_acquire(&p->on_cpu, !VAL), and we're in __schedule()
+	 * before finish_task().
+	 *
+	 * XXX do further audits, this smells like something putrid.
+	 */
+	WARN_ON_ONCE(!p->on_cpu);
+	p->cpus_ptr = new_mask;
+}
+
 static inline void __migrate_enable(void)
 {
 	struct task_struct *p = current;
@@ -2463,8 +2521,17 @@ static inline void __migrate_enable(void)
 	 * __set_cpus_allowed_ptr(SCA_MIGRATE_ENABLE) doesn't schedule().
 	 */
 	guard(preempt)();
+#ifdef CONFIG_SCHED_ALT
+	/*
+	 * Assumption: current should be running on allowed cpu
+	 */
+	WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(), &p->cpus_mask));
+	if (p->cpus_ptr != &p->cpus_mask)
+		__do_set_cpus_ptr(p, &p->cpus_mask);
+#else
 	if (unlikely(p->cpus_ptr != &p->cpus_mask))
 		___migrate_enable();
+#endif
 	/*
 	 * Mustn't clear migration_disabled() until cpus_ptr points back at the
 	 * regular cpus_mask, otherwise things that race (eg.
@@ -2491,8 +2558,20 @@ static inline void __migrate_disable(void)
 	}
 
 	guard(preempt)();
+#ifdef CONFIG_SCHED_ALT
+	int cpu = smp_processor_id();
+	if (cpumask_test_cpu(cpu, &p->cpus_mask)) {
+#endif
 	this_rq_pinned()++;
 	p->migration_disabled = 1;
+#ifdef CONFIG_SCHED_ALT
+		/*
+		 * Violates locking rules! see comment in __do_set_cpus_ptr().
+		 */
+		if (p->cpus_ptr == &p->cpus_mask)
+				__do_set_cpus_ptr(p, cpumask_of(cpu));
+	}
+#endif
 }
 #else /* !COMPILE_OFFSETS */
 static inline void __migrate_disable(void) { }
diff --git a/include/linux/sched/deadline.h b/include/linux/sched/deadline.h
index 273538200..12b41502a 100644
--- a/include/linux/sched/deadline.h
+++ b/include/linux/sched/deadline.h
@@ -2,6 +2,25 @@
 #ifndef _LINUX_SCHED_DEADLINE_H
 #define _LINUX_SCHED_DEADLINE_H
 
+#ifdef CONFIG_SCHED_ALT
+
+static inline int dl_task(struct task_struct *p)
+{
+	return 0;
+}
+
+#ifdef CONFIG_SCHED_BMQ
+#define __tsk_deadline(p)	(0UL)
+#endif
+
+#ifdef CONFIG_SCHED_PDS
+#define __tsk_deadline(p)	((((u64) ((p)->prio))<<56) | (p)->deadline)
+#endif
+
+#else
+
+#define __tsk_deadline(p)	((p)->dl.deadline)
+
 /*
  * SCHED_DEADLINE tasks has negative priorities, reflecting
  * the fact that any of them has higher prio than RT and
@@ -72,5 +91,6 @@ static inline bool dl_is_implicit(struct sched_dl_entity *dl_se)
 {
 	return dl_se->dl_deadline == dl_se->dl_period;
 }
+#endif /* !CONFIG_SCHED_ALT */
 
 #endif /* _LINUX_SCHED_DEADLINE_H */
diff --git a/include/linux/sched/nohz.h b/include/linux/sched/nohz.h
index 0db7f6793..4352cd9c5 100644
--- a/include/linux/sched/nohz.h
+++ b/include/linux/sched/nohz.h
@@ -13,7 +13,7 @@ extern int get_nohz_timer_target(void);
 static inline void nohz_balance_enter_idle(int cpu) { }
 #endif
 
-#ifdef CONFIG_NO_HZ_COMMON
+#if defined(CONFIG_NO_HZ_COMMON) && !defined(CONFIG_SCHED_ALT)
 void calc_load_nohz_start(void);
 void calc_load_nohz_remote(struct rq *rq);
 void calc_load_nohz_stop(void);
diff --git a/include/linux/sched/prio.h b/include/linux/sched/prio.h
index 6ab43b4f7..ef1cff556 100644
--- a/include/linux/sched/prio.h
+++ b/include/linux/sched/prio.h
@@ -19,6 +19,28 @@
 #define MAX_PRIO		(MAX_RT_PRIO + NICE_WIDTH)
 #define DEFAULT_PRIO		(MAX_RT_PRIO + NICE_WIDTH / 2)
 
+#ifdef CONFIG_SCHED_ALT
+
+/* Undefine MAX_PRIO and DEFAULT_PRIO */
+#undef MAX_PRIO
+#undef DEFAULT_PRIO
+
+/* +/- priority levels from the base priority */
+#ifdef CONFIG_SCHED_BMQ
+#define MAX_PRIORITY_ADJ	(12)
+#endif
+
+#ifdef CONFIG_SCHED_PDS
+#define MAX_PRIORITY_ADJ	(0)
+#endif
+
+#define MIN_NORMAL_PRIO		(128)
+#define NORMAL_PRIO_NUM		(64)
+#define MAX_PRIO		(MIN_NORMAL_PRIO + NORMAL_PRIO_NUM)
+#define DEFAULT_PRIO		(MAX_PRIO - MAX_PRIORITY_ADJ - NICE_WIDTH / 2)
+
+#endif /* CONFIG_SCHED_ALT */
+
 /*
  * Convert user-nice values [ -20 ... 0 ... 19 ]
  * to static priority [ MAX_RT_PRIO..MAX_PRIO-1 ],
diff --git a/include/linux/sched/rt.h b/include/linux/sched/rt.h
index 4e3338103..6dfef878f 100644
--- a/include/linux/sched/rt.h
+++ b/include/linux/sched/rt.h
@@ -45,8 +45,10 @@ static inline bool rt_or_dl_task_policy(struct task_struct *tsk)
 
 	if (policy == SCHED_FIFO || policy == SCHED_RR)
 		return true;
+#ifndef CONFIG_SCHED_ALT
 	if (policy == SCHED_DEADLINE)
 		return true;
+#endif
 	return false;
 }
 
diff --git a/include/linux/sched/topology.h b/include/linux/sched/topology.h
index 35c9c37a0..89c67556b 100644
--- a/include/linux/sched/topology.h
+++ b/include/linux/sched/topology.h
@@ -248,7 +248,8 @@ extern void sched_update_asym_prefer_cpu(int cpu, int old_prio, int new_prio);
 #define SDTL_INIT(maskfn, flagsfn, dname) ((struct sched_domain_topology_level) \
 	    { .mask = maskfn, .sd_flags = flagsfn, .name = #dname })
 
-#if defined(CONFIG_ENERGY_MODEL) && defined(CONFIG_CPU_FREQ_GOV_SCHEDUTIL)
+#if defined(CONFIG_ENERGY_MODEL) && defined(CONFIG_CPU_FREQ_GOV_SCHEDUTIL) && \
+	!defined(CONFIG_SCHED_ALT)
 extern void rebuild_sched_domains_energy(void);
 #else
 static inline void rebuild_sched_domains_energy(void)
diff --git a/init/Kconfig b/init/Kconfig
index c8f3cccd8..bb37f11dd 100644
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -723,6 +723,7 @@ config TASK_IO_ACCOUNTING
 
 config PSI
 	bool "Pressure stall information tracking"
+	depends on !SCHED_ALT
 	select KERNFS
 	help
 	  Collect metrics that indicate how overcommitted the CPU, memory,
@@ -946,6 +947,29 @@ config SCHED_PROXY_EXEC
 	  This option enables proxy execution, a mechanism for mutex-owning
 	  tasks to inherit the scheduling context of higher priority waiters.
 
+menuconfig SCHED_ALT
+	bool "Alternative CPU Schedulers"
+	default y
+	help
+	  This feature enable alternative CPU scheduler"
+
+if SCHED_ALT
+
+choice
+	prompt "Alternative CPU Scheduler"
+	default SCHED_BMQ
+
+config SCHED_BMQ
+	bool "BMQ CPU scheduler"
+	help
+	  The BitMap Queue CPU scheduler for excellent interactivity and
+	  responsiveness on the desktop and solid scalability on normal
+	  hardware and commodity servers.
+
+endchoice
+
+endif
+
 endmenu
 
 #
@@ -1016,6 +1040,7 @@ config NUMA_BALANCING
 	depends on ARCH_SUPPORTS_NUMA_BALANCING
 	depends on !ARCH_WANT_NUMA_VARIABLE_LOCALITY
 	depends on SMP && NUMA_MIGRATION && !PREEMPT_RT
+	depends on !SCHED_ALT
 	help
 	  This option adds support for automatic NUMA aware memory/task placement.
 	  The mechanism is quite primitive and is based on migrating memory when
@@ -1302,7 +1327,7 @@ config CPUSETS
 
 config CPUSETS_V1
 	bool "Legacy cgroup v1 cpusets controller"
-	depends on CPUSETS
+	depends on (CPUSETS && !SCHED_ALT)
 	default n
 	help
 	  Legacy cgroup v1 cpusets controller which has been deprecated by
@@ -1472,6 +1497,7 @@ config CHECKPOINT_RESTORE
 
 config SCHED_AUTOGROUP
 	bool "Automatic process group scheduling"
+	depends on !SCHED_ALT
 	select CGROUPS
 	select CGROUP_SCHED
 	select FAIR_GROUP_SCHED
@@ -1484,7 +1510,7 @@ config SCHED_AUTOGROUP
 
 config SCHED_POC_SELECTOR
 	bool "Piece-Of-Cake Fast Idle CPU Selector"
-	depends on SMP
+	depends on SMP && !SCHED_ALT
 	default y
 	help
 	  Idle CPU selector using cached bitmasks inspired by the scx_cake BPF
diff --git a/init/init_task.c b/init/init_task.c
index b5f48ebdc..760b5626a 100644
--- a/init/init_task.c
+++ b/init/init_task.c
@@ -102,9 +102,16 @@ struct task_struct init_task __aligned(L1_CACHE_BYTES) = {
 	.stack		= init_stack,
 	.usage		= REFCOUNT_INIT(2),
 	.flags		= PF_KTHREAD,
+#ifdef CONFIG_SCHED_ALT
+	.on_cpu		= 1,
+	.prio		= DEFAULT_PRIO,
+	.static_prio	= DEFAULT_PRIO,
+	.normal_prio	= DEFAULT_PRIO,
+#else
 	.prio		= MAX_PRIO - 20,
 	.static_prio	= MAX_PRIO - 20,
 	.normal_prio	= MAX_PRIO - 20,
+#endif
 	.policy		= SCHED_NORMAL,
 	.cpus_ptr	= &init_task.cpus_mask,
 	.user_cpus_ptr	= NULL,
@@ -116,6 +123,17 @@ struct task_struct init_task __aligned(L1_CACHE_BYTES) = {
 	.restart_block	= {
 		.fn = do_no_restart_syscall,
 	},
+#ifdef CONFIG_SCHED_ALT
+	.pq_node	= { NULL },
+	.__sched_prio	= -1,
+#ifdef CONFIG_SCHED_BMQ
+	.boost_prio	= 0,
+#endif
+#ifdef CONFIG_SCHED_PDS
+	.deadline	= 0,
+#endif
+	.time_slice	= HZ,
+#else
 	.se		= {
 		.group_node 	= LIST_HEAD_INIT(init_task.se.group_node),
 	},
@@ -123,10 +141,13 @@ struct task_struct init_task __aligned(L1_CACHE_BYTES) = {
 		.run_list	= LIST_HEAD_INIT(init_task.rt.run_list),
 		.time_slice	= RR_TIMESLICE,
 	},
+#endif
 	.tasks		= LIST_HEAD_INIT(init_task.tasks),
+#ifndef CONFIG_SCHED_ALT
 #ifdef CONFIG_SMP
 	.pushable_tasks	= PLIST_NODE_INIT(init_task.pushable_tasks, MAX_PRIO),
 #endif
+#endif
 #ifdef CONFIG_CGROUP_SCHED
 	.sched_task_group = &root_task_group,
 #endif
diff --git a/kernel/Kconfig.preempt b/kernel/Kconfig.preempt
index 2471a502c..67aaddc46 100644
--- a/kernel/Kconfig.preempt
+++ b/kernel/Kconfig.preempt
@@ -150,7 +150,7 @@ config PREEMPT_DYNAMIC
 
 config SCHED_CORE
 	bool "Core Scheduling for SMT"
-	depends on SCHED_SMT
+	depends on SCHED_SMT && !SCHED_ALT
 	help
 	  This option permits Core Scheduling, a means of coordinated task
 	  selection across SMT siblings. When enabled -- see
@@ -168,7 +168,7 @@ config SCHED_CORE
 
 config SCHED_CLASS_EXT
 	bool "Extensible Scheduling Class"
-	depends on BPF_SYSCALL && BPF_JIT && DEBUG_INFO_BTF
+	depends on BPF_SYSCALL && BPF_JIT && DEBUG_INFO_BTF && !SCHED_ALT
 	select STACKTRACE if STACKTRACE_SUPPORT
 	help
 	  This option enables a new scheduler class sched_ext (SCX), which
diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c
index c9e14fda3..02a6a1557 100644
--- a/kernel/cgroup/cpuset.c
+++ b/kernel/cgroup/cpuset.c
@@ -767,7 +767,7 @@ static int validate_change(struct cpuset *cur, struct cpuset *trial)
 	return ret;
 }
 
-#ifdef CONFIG_SMP
+#if defined(CONFIG_SMP) && !defined(CONFIG_SCHED_ALT)
 
 /*
  * generate_sched_domains()
@@ -1011,7 +1011,7 @@ void rebuild_sched_domains_locked(void)
 	/* Have scheduler rebuild the domains */
 	partition_sched_domains(ndoms, doms, attr);
 }
-#else /* !CONFIG_SMP */
+#else /* !CONFIG_SMP || CONFIG_SCHED_ALT */
 void rebuild_sched_domains_locked(void)
 {
 }
@@ -2995,7 +2995,7 @@ static int cpuset_can_attach(struct cgroup_taskset *tset)
 	struct cpuset *cs, *oldcs;
 	struct task_struct *task;
 	bool setsched_check;
-	int cpu, ret;
+	int ret;
 
 	/* used later by cpuset_attach() */
 	cpuset_attach_old_cs = task_cs(cgroup_taskset_first(tset, &css));
@@ -3039,6 +3039,7 @@ static int cpuset_can_attach(struct cgroup_taskset *tset)
 				goto out_unlock;
 		}
 
+#ifndef CONFIG_SCHED_ALT
 		if (dl_task(task)) {
 			/*
 			 * Count all migrating DL tasks for cpuset task accounting.
@@ -3049,12 +3050,14 @@ static int cpuset_can_attach(struct cgroup_taskset *tset)
 			if (dl_task_needs_bw_move(task, cs->effective_cpus))
 				cs->sum_migrate_dl_bw += task->dl.dl_bw;
 		}
+#endif
 	}
 
+#ifndef CONFIG_SCHED_ALT
 	if (!cs->sum_migrate_dl_bw)
 		goto out_success;
 
-	cpu = cpumask_any_and(cpu_active_mask, cs->effective_cpus);
+	int cpu = cpumask_any_and(cpu_active_mask, cs->effective_cpus);
 	if (unlikely(cpu >= nr_cpu_ids)) {
 		ret = -EINVAL;
 		goto out_unlock;
@@ -3067,6 +3070,7 @@ static int cpuset_can_attach(struct cgroup_taskset *tset)
 	cs->dl_bw_cpu = cpu;
 
 out_success:
+#endif
 	/*
 	 * Mark attach is in progress.  This makes validate_change() fail
 	 * changes which zero cpus/mems_allowed.
@@ -3091,11 +3095,13 @@ static void cpuset_cancel_attach(struct cgroup_taskset *tset)
 	mutex_lock(&cpuset_mutex);
 	dec_attach_in_progress_locked(cs);
 
+#ifndef CONFIG_SCHED_ALT
 	if (cs->dl_bw_cpu >= 0)
 		dl_bw_free(cs->dl_bw_cpu, cs->sum_migrate_dl_bw);
 
 	if (cs->nr_migrate_dl_tasks)
 		reset_migrate_dl_data(cs);
+#endif
 
 	mutex_unlock(&cpuset_mutex);
 }
diff --git a/kernel/delayacct.c b/kernel/delayacct.c
index 2e55c493c..5c2262a24 100644
--- a/kernel/delayacct.c
+++ b/kernel/delayacct.c
@@ -170,7 +170,7 @@ int delayacct_add_tsk(struct taskstats *d, struct task_struct *tsk)
 	 */
 	t1 = tsk->sched_info.pcount;
 	t2 = tsk->sched_info.run_delay;
-	t3 = tsk->se.sum_exec_runtime;
+	t3 = tsk_seruntime(tsk);
 
 	d->cpu_count += t1;
 
diff --git a/kernel/exit.c b/kernel/exit.c
index f50d73c27..a62d6f4d2 100644
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -207,7 +207,7 @@ static void __exit_signal(struct release_task_post *post, struct task_struct *ts
 	sig->inblock += task_io_get_inblock(tsk);
 	sig->oublock += task_io_get_oublock(tsk);
 	task_io_accounting_add(&sig->ioac, &tsk->ioac);
-	sig->sum_sched_runtime += tsk->se.sum_exec_runtime;
+	sig->sum_sched_runtime += tsk_seruntime(tsk);
 	sig->nr_threads--;
 	__unhash_process(post, tsk, group_dead);
 	write_sequnlock(&sig->stats_lock);
@@ -290,8 +290,8 @@ void release_task(struct task_struct *p)
 	/* @thread_pid can't go away until free_pids() below */
 	proc_flush_pid(thread_pid);
 	exit_cred_namespaces(p);
-	add_device_randomness(&p->se.sum_exec_runtime,
-			      sizeof(p->se.sum_exec_runtime));
+	add_device_randomness((const void*) &tsk_seruntime(p),
+			      sizeof(unsigned long long));
 	free_pids(post.pids);
 	release_thread(p);
 	/*
diff --git a/kernel/locking/rtmutex.c b/kernel/locking/rtmutex.c
index daeeeef97..b7bb3cdec 100644
--- a/kernel/locking/rtmutex.c
+++ b/kernel/locking/rtmutex.c
@@ -372,7 +372,7 @@ waiter_update_prio(struct rt_mutex_waiter *waiter, struct task_struct *task)
 	lockdep_assert(RB_EMPTY_NODE(&waiter->tree.entry));
 
 	waiter->tree.prio = __waiter_prio(task);
-	waiter->tree.deadline = task->dl.deadline;
+	waiter->tree.deadline = __tsk_deadline(task);
 }
 
 /*
@@ -393,16 +393,20 @@ waiter_clone_prio(struct rt_mutex_waiter *waiter, struct task_struct *task)
  * Only use with rt_waiter_node_{less,equal}()
  */
 #define task_to_waiter_node(p)	\
-	&(struct rt_waiter_node){ .prio = __waiter_prio(p), .deadline = (p)->dl.deadline }
+	&(struct rt_waiter_node){ .prio = __waiter_prio(p), .deadline = __tsk_deadline(p) }
 #define task_to_waiter(p)	\
 	&(struct rt_mutex_waiter){ .tree = *task_to_waiter_node(p) }
 
 static __always_inline int rt_waiter_node_less(struct rt_waiter_node *left,
 					       struct rt_waiter_node *right)
 {
+#ifdef CONFIG_SCHED_PDS
+	return (left->deadline < right->deadline);
+#else
 	if (left->prio < right->prio)
 		return 1;
 
+#ifndef CONFIG_SCHED_BMQ
 	/*
 	 * If both waiters have dl_prio(), we check the deadlines of the
 	 * associated tasks.
@@ -411,16 +415,22 @@ static __always_inline int rt_waiter_node_less(struct rt_waiter_node *left,
 	 */
 	if (dl_prio(left->prio))
 		return dl_time_before(left->deadline, right->deadline);
+#endif
 
 	return 0;
+#endif
 }
 
 static __always_inline int rt_waiter_node_equal(struct rt_waiter_node *left,
 						 struct rt_waiter_node *right)
 {
+#ifdef CONFIG_SCHED_PDS
+	return (left->deadline == right->deadline);
+#else
 	if (left->prio != right->prio)
 		return 0;
 
+#ifndef CONFIG_SCHED_BMQ
 	/*
 	 * If both waiters have dl_prio(), we check the deadlines of the
 	 * associated tasks.
@@ -429,8 +439,10 @@ static __always_inline int rt_waiter_node_equal(struct rt_waiter_node *left,
 	 */
 	if (dl_prio(left->prio))
 		return left->deadline == right->deadline;
+#endif
 
 	return 1;
+#endif
 }
 
 static inline bool rt_mutex_steal(struct rt_mutex_waiter *waiter,
diff --git a/kernel/locking/ww_mutex.h b/kernel/locking/ww_mutex.h
index 6c1245209..330fa4ac2 100644
--- a/kernel/locking/ww_mutex.h
+++ b/kernel/locking/ww_mutex.h
@@ -282,6 +282,7 @@ __ww_ctx_less(struct ww_acquire_ctx *a, struct ww_acquire_ctx *b)
 
 		/* equal static prio */
 
+#ifndef	CONFIG_SCHED_ALT
 		if (dl_prio(a_prio)) {
 			if (dl_time_before(b->task->dl.deadline,
 					   a->task->dl.deadline))
@@ -291,6 +292,7 @@ __ww_ctx_less(struct ww_acquire_ctx *a, struct ww_acquire_ctx *b)
 					   b->task->dl.deadline))
 				return false;
 		}
+#endif
 
 		/* equal prio */
 	}
diff --git a/kernel/sched/Makefile b/kernel/sched/Makefile
index b1f1a3670..b7f5f9931 100644
--- a/kernel/sched/Makefile
+++ b/kernel/sched/Makefile
@@ -36,7 +36,12 @@ endif
 # These compilation units have roughly the same size and complexity - so their
 # build parallelizes well and finishes roughly at once:
 #
+ifdef CONFIG_SCHED_ALT
+obj-y += alt_core.o
+obj-$(CONFIG_SCHED_DEBUG) += alt_debug.o
+else
 obj-y += core.o
 obj-y += fair.o
+endif
 obj-y += build_policy.o
 obj-y += build_utility.o
diff --git a/kernel/sched/alt_core.c b/kernel/sched/alt_core.c
new file mode 100644
index 000000000..48f1654bc
--- /dev/null
+++ b/kernel/sched/alt_core.c
@@ -0,0 +1,7354 @@
+/*
+ *  kernel/sched/alt_core.c
+ *
+ *  Core alternative kernel scheduler code and related syscalls
+ *
+ *  Copyright (C) 1991-2002  Linus Torvalds
+ *
+ *  2009-08-13	Brainfuck deadline scheduling policy by Con Kolivas deletes
+ *		a whole lot of those previous things.
+ *  2017-09-06	Priority and Deadline based Skip list multiple queue kernel
+ *		scheduler by Alfred Chen.
+ *  2019-02-20	BMQ(BitMap Queue) kernel scheduler by Alfred Chen.
+ */
+#define INSTANTIATE_EXPORTED_MIGRATE_DISABLE
+#include <linux/sched.h>
+#include <linux/sched/clock.h>
+#include <linux/sched/cputime.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/hotplug.h>
+#include <linux/sched/init.h>
+#include <linux/sched/isolation.h>
+#include <linux/sched/loadavg.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/nohz.h>
+#include <linux/sched/wake_q.h>
+
+#include <linux/blkdev.h>
+#include <linux/context_tracking.h>
+#include <linux/cpuset.h>
+#include <linux/delayacct.h>
+#include <linux/init_task.h>
+#include <linux/kcov.h>
+#include <linux/kprobes.h>
+#include <linux/nmi.h>
+#include <linux/rseq.h>
+#include <linux/scs.h>
+
+#include <uapi/linux/sched/types.h>
+
+#include <asm/irq_regs.h>
+#include <asm/switch_to.h>
+
+#define CREATE_TRACE_POINTS
+#include <trace/events/sched.h>
+#include <trace/events/ipi.h>
+#undef CREATE_TRACE_POINTS
+
+#include "sched.h"
+#include "smp.h"
+
+#include "pelt.h"
+
+#include "../../io_uring/io-wq.h"
+#include "../smpboot.h"
+
+/* Use CONFIG_PARAVIRT as this will avoid more #ifdef in arch code. */
+#ifdef CONFIG_PARAVIRT
+struct static_key paravirt_steal_rq_enabled;
+#endif
+
+EXPORT_TRACEPOINT_SYMBOL_GPL(ipi_send_cpu);
+EXPORT_TRACEPOINT_SYMBOL_GPL(ipi_send_cpumask);
+
+/*
+ * Export tracepoints that act as a bare tracehook (ie: have no trace event
+ * associated with them) to allow external modules to probe them.
+ */
+EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_irq_tp);
+
+#define sched_feat(x)	(1)
+/*
+ * Print a warning if need_resched is set for the given duration (if
+ * LATENCY_WARN is enabled).
+ *
+ * If sysctl_resched_latency_warn_once is set, only one warning will be shown
+ * per boot.
+ */
+__read_mostly int sysctl_resched_latency_warn_ms = 100;
+__read_mostly int sysctl_resched_latency_warn_once = 1;
+
+#define ALT_SCHED_VERSION "v7.1-r0-lf"
+
+#define STOP_PRIO		(MAX_RT_PRIO - 1)
+
+/*
+ * Time slice
+ * (default: 4 msec, units: nanoseconds)
+ */
+unsigned int sysctl_sched_base_slice __read_mostly	= (4 << 20);
+
+#include "alt_core.h"
+
+/* Reschedule if less than this many μs left */
+#define RESCHED_NS		(100 << 10)
+
+/**
+ * sched_yield_type - Type of sched_yield() will be performed.
+ * 0: No yield.
+ * 1: Requeue task. (default)
+ */
+int sched_yield_type __read_mostly = 1;
+
+#ifdef CONFIG_SCHED_SMT
+DEFINE_STATIC_KEY_FALSE(sched_smt_present);
+EXPORT_SYMBOL_GPL(sched_smt_present);
+
+cpumask_t sched_smt_mask ____cacheline_aligned_in_smp;
+#endif
+
+DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
+
+struct sched_run_queue grq ____cacheline_aligned_in_smp;
+
+DEFINE_PER_CPU_SHARED_ALIGNED(struct llist_head, preempt_list) = { NULL };
+
+#ifndef prepare_arch_switch
+# define prepare_arch_switch(next)	do { } while (0)
+#endif
+#ifndef finish_arch_post_lock_switch
+# define finish_arch_post_lock_switch()	do { } while (0)
+#endif
+
+static int cpu_prio[NR_CPUS] ____cacheline_aligned_in_smp;
+
+cpumask_t cpu_sched_prio_mask[SCHED_LEVELS]			____cacheline_aligned_in_smp;
+DECLARE_BITMAP(cpu_sched_prio_bitmap, SCHED_LEVELS)		____cacheline_aligned_in_smp;
+
+#define sched_idle_mask	(cpu_sched_prio_mask + SCHED_LEVELS - 1 - IDLE_TASK_SCHED_PRIO)
+
+#include "alt_topology.c"
+
+static __always_inline void update_sched_cpu_prio(const int cpu, const int prio)
+{
+	int last_prio = READ_ONCE(cpu_prio[cpu]);
+
+	if (prio == last_prio)
+		return;
+
+	WRITE_ONCE(cpu_prio[cpu], prio);
+
+	if (IDLE_TASK_SCHED_PRIO == last_prio) {
+		sched_clear_idle_mask(cpu);
+	} else {
+		int idx = SCHED_LEVELS - 1 - last_prio;
+		cpumask_clear_cpu(cpu, cpu_sched_prio_mask + idx);
+		if (cpumask_empty(cpu_sched_prio_mask + idx))
+			clear_bit(idx, cpu_sched_prio_bitmap);
+	}
+
+	if (IDLE_TASK_SCHED_PRIO == prio) {
+		sched_set_idle_mask(cpu);
+	} else {
+		int idx = SCHED_LEVELS - 1 - prio;
+		cpumask_set_cpu(cpu, cpu_sched_prio_mask + idx);
+		set_bit(idx, cpu_sched_prio_bitmap);
+	}
+}
+
+/* task function */
+static inline const struct cpumask *task_user_cpus(struct task_struct *p)
+{
+	if (!p->user_cpus_ptr)
+		return cpu_possible_mask; /* &init_task.cpus_mask */
+	return p->user_cpus_ptr;
+}
+
+/* sched_run_queue related functions */
+static inline void sched_run_queue_init(struct sched_run_queue *q)
+{
+	int i;
+
+	for(i = 0; i < SCHED_QUEUE_BITS; i++) {
+		raw_spin_lock_init(&q->_lock[i]);
+		init_llist_head(&q->_head[i]);
+	}
+	bitmap_zero(q->bitmap, SCHED_QUEUE_BITS);
+	atomic_set(&q->nr_queued, 0);
+	atomic_set(&q->nr_uninterruptible, 0);
+	atomic_set(&q->nr_iowait, 0);
+}
+
+/* need a wrapper since we may need to trace from modules */
+EXPORT_TRACEPOINT_SYMBOL(sched_set_state_tp);
+
+/* Call via the helper macro trace_set_current_state. */
+void __trace_set_current_state(int state_value)
+{
+	trace_sched_set_state_tp(current, state_value);
+}
+EXPORT_SYMBOL(__trace_set_current_state);
+
+/*
+ * Serialization rules:
+ *
+ * Lock order:
+ *
+ *   p->pi_lock
+ *     rq->lock
+ *       hrtimer_cpu_base->lock (hrtimer_start() for bandwidth controls)
+ *
+ *  rq1->lock
+ *    rq2->lock  where: rq1 < rq2
+ *
+ * Regular state:
+ *
+ * Normal scheduling state is serialized by rq->lock. __schedule() takes the
+ * local CPU's rq->lock, it optionally removes the task from the runqueue and
+ * always looks at the local rq data structures to find the most eligible task
+ * to run next.
+ *
+ * Task enqueue is also under rq->lock, possibly taken from another CPU.
+ * Wakeups from another LLC domain might use an IPI to transfer the enqueue to
+ * the local CPU to avoid bouncing the runqueue state around [ see
+ * ttwu_queue_wakelist() ]
+ *
+ * Task wakeup, specifically wakeups that involve migration, are horribly
+ * complicated to avoid having to take two rq->locks.
+ *
+ * Special state:
+ *
+ * System-calls and anything external will use task_rq_lock() which acquires
+ * both p->pi_lock and rq->lock. As a consequence the state they change is
+ * stable while holding either lock:
+ *
+ *  - sched_setaffinity()/
+ *    set_cpus_allowed_ptr():	p->cpus_ptr, p->nr_cpus_allowed
+ *  - set_user_nice():		p->se.load, p->*prio
+ *  - __sched_setscheduler():	p->sched_class, p->policy, p->*prio,
+ *				p->se.load, p->rt_priority,
+ *				p->dl.dl_{runtime, deadline, period, flags, bw, density}
+ *  - sched_setnuma():		p->numa_preferred_nid
+ *  - sched_move_task():        p->sched_task_group
+ *  - uclamp_update_active()	p->uclamp*
+ *
+ * p->state <- TASK_*:
+ *
+ *   is changed locklessly using set_current_state(), __set_current_state() or
+ *   set_special_state(), see their respective comments, or by
+ *   try_to_wake_up(). This latter uses p->pi_lock to serialize against
+ *   concurrent self.
+ *
+ * p->on_rq <- { 0, 1 = TASK_ON_RQ_QUEUED, 2 = TASK_ON_RQ_MIGRATING }:
+ *
+ *   is set by activate_task() and cleared by deactivate_task()/block_task(),
+ *   under rq->lock. Non-zero indicates the task is runnable, the special
+ *   ON_RQ_MIGRATING state is used for migration without holding both
+ *   rq->locks. It indicates task_cpu() is not stable, see task_rq_lock().
+ *
+ *   Additionally it is possible to be ->on_rq but still be considered not
+ *   runnable when p->se.sched_delayed is true. These tasks are on the runqueue
+ *   but will be dequeued as soon as they get picked again. See the
+ *   task_is_runnable() helper.
+ *
+ * p->on_cpu <- { 0, 1 }:
+ *
+ *   is set by prepare_task() and cleared by finish_task() such that it will be
+ *   set before p is scheduled-in and cleared after p is scheduled-out, both
+ *   under rq->lock. Non-zero indicates the task is running on its CPU.
+ *
+ *   [ The astute reader will observe that it is possible for two tasks on one
+ *     CPU to have ->on_cpu = 1 at the same time. ]
+ *
+ * task_cpu(p): is changed by set_task_cpu(), the rules are:
+ *
+ *  - Don't call set_task_cpu() on a blocked task:
+ *
+ *    We don't care what CPU we're not running on, this simplifies hotplug,
+ *    the CPU assignment of blocked tasks isn't required to be valid.
+ *
+ *  - for try_to_wake_up(), called under p->pi_lock:
+ *
+ *    This allows try_to_wake_up() to only take one rq->lock, see its comment.
+ *
+ *  - for migration called under rq->lock:
+ *    [ see task_on_rq_migrating() in task_rq_lock() ]
+ *
+ *    o move_queued_task()
+ *    o detach_task()
+ *
+ *  - for migration called under double_rq_lock():
+ *
+ *    o __migrate_swap_task()
+ *    o push_rt_task() / pull_rt_task()
+ *    o push_dl_task() / pull_dl_task()
+ *    o dl_task_offline_migration()
+ *
+ */
+
+/*
+ * Context: p->pi_lock
+ */
+
+/*
+ * __task_rq_lock - lock the rq @p resides on.
+ */
+static inline struct rq *__task_rq_lock(struct task_struct *p, struct rq_flags *rf)
+{
+	lockdep_assert_held(&p->pi_lock);
+
+	for (;;) {
+		if (TASK_ON_RQ_WAKING == p->on_rq) {
+			struct rq *rq = cpu_rq(p->wake_cpu);
+
+			raw_spin_lock(&rq->lock);
+			if (likely(TASK_ON_RQ_WAKING == p->on_rq && rq == task_rq(p))) {
+				rf->lock = &rq->lock;
+				rf->queued = false;
+				return rq;
+			}
+			raw_spin_unlock(&rq->lock);
+		} else if (task_on_rq_queued(p)) {
+			if (p->on_cpu) {
+				struct rq *rq = task_rq(p);
+
+				raw_spin_lock(&rq->lock);
+				if (likely(p->on_cpu && rq == task_rq(p))) {
+					rf->lock = &rq->lock;
+					return rq;
+				}
+				raw_spin_unlock(&rq->lock);
+			} else {
+				rf->lock = NULL;
+				return NULL;
+			}
+		} else if (task_on_rq_migrating(p)) {
+			do {
+				cpu_relax();
+			} while (unlikely(task_on_rq_migrating(p)));
+		} else {
+			rf->lock = NULL;
+			return NULL;
+		}
+	}
+}
+
+static inline void __task_rq_unlock(struct rq_flags *rf)
+{
+	if (NULL != rf->lock)
+		raw_spin_unlock(rf->lock);
+}
+
+/*
+ * task_rq_lock - lock p->pi_lock and lock the rq @p resides on.
+ */
+struct rq *_task_rq_lock(struct task_struct *p, struct rq_flags *rf)
+{
+	struct rq *rq;
+
+	for (;;) {
+		raw_spin_lock_irqsave(&p->pi_lock, rf->flags);
+		rq = task_rq(p);
+		raw_spin_lock(&rq->lock);
+		/*
+		 *	move_queued_task()		task_rq_lock()
+		 *
+		 *	ACQUIRE (rq->lock)
+		 *	[S] ->on_rq = MIGRATING		[L] rq = task_rq()
+		 *	WMB (__set_task_cpu())		ACQUIRE (rq->lock);
+		 *	[S] ->cpu = new_cpu		[L] task_rq()
+		 *					[L] ->on_rq
+		 *	RELEASE (rq->lock)
+		 *
+		 * If we observe the old CPU in task_rq_lock(), the acquire of
+		 * the old rq->lock will fully serialize against the stores.
+		 *
+		 * If we observe the new CPU in task_rq_lock(), the address
+		 * dependency headed by '[L] rq = task_rq()' and the acquire
+		 * will pair with the WMB to ensure we then also see migrating.
+		 */
+		if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) {
+			return rq;
+		}
+		raw_spin_unlock(&rq->lock);
+		raw_spin_unlock_irqrestore(&p->pi_lock, rf->flags);
+
+		while (unlikely(task_on_rq_migrating(p)))
+			cpu_relax();
+	}
+}
+
+static inline void rq_lock_irqsave(struct rq *rq, struct rq_flags *rf)
+	__acquires(rq->lock)
+{
+	raw_spin_lock_irqsave(&rq->lock, rf->flags);
+}
+
+static inline void rq_unlock_irqrestore(struct rq *rq, struct rq_flags *rf)
+	__releases(rq->lock)
+{
+	raw_spin_unlock_irqrestore(&rq->lock, rf->flags);
+}
+
+DEFINE_LOCK_GUARD_1(rq_lock_irqsave, struct rq,
+		    rq_lock_irqsave(_T->lock, &_T->rf),
+		    rq_unlock_irqrestore(_T->lock, &_T->rf),
+		    struct rq_flags rf)
+
+/*
+ * RQ-clock updating methods:
+ */
+
+static void update_rq_clock_task(struct rq *rq, s64 delta)
+{
+/*
+ * In theory, the compile should just see 0 here, and optimize out the call
+ * to sched_rt_avg_update. But I don't trust it...
+ */
+	s64 __maybe_unused steal = 0, irq_delta = 0;
+
+#ifdef CONFIG_IRQ_TIME_ACCOUNTING
+	if (irqtime_enabled()) {
+		irq_delta = irq_time_read(cpu_of(rq)) - rq->prev_irq_time;
+
+		/*
+		 * Since irq_time is only updated on {soft,}irq_exit, we might run into
+		 * this case when a previous update_rq_clock() happened inside a
+		 * {soft,}IRQ region.
+		 *
+		 * When this happens, we stop ->clock_task and only update the
+		 * prev_irq_time stamp to account for the part that fit, so that a next
+		 * update will consume the rest. This ensures ->clock_task is
+		 * monotonic.
+		 *
+		 * It does however cause some slight miss-attribution of {soft,}IRQ
+		 * time, a more accurate solution would be to update the irq_time using
+		 * the current rq->clock timestamp, except that would require using
+		 * atomic ops.
+		 */
+		if (irq_delta > delta)
+			irq_delta = delta;
+
+		rq->prev_irq_time += irq_delta;
+		delta -= irq_delta;
+		delayacct_irq(rq->curr, irq_delta);
+	}
+#endif
+#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
+	if (static_key_false((&paravirt_steal_rq_enabled))) {
+		u64 prev_steal;
+
+		steal = prev_steal = paravirt_steal_clock(cpu_of(rq));
+		steal -= rq->prev_steal_time_rq;
+
+		if (unlikely(steal > delta))
+			steal = delta;
+
+		rq->prev_steal_time_rq = prev_steal;
+		delta -= steal;
+	}
+#endif
+
+	rq->clock_task += delta;
+
+#ifdef CONFIG_HAVE_SCHED_AVG_IRQ
+	if ((irq_delta + steal))
+		update_irq_load_avg(rq, irq_delta + steal);
+#endif
+}
+
+static inline void update_rq_clock(struct rq *rq)
+{
+	s64 delta = sched_clock_cpu(cpu_of(rq)) - rq->clock;
+
+	if (unlikely(delta <= 0))
+		return;
+	rq->clock += delta;
+	sched_update_rq_clock(rq);
+	update_rq_clock_task(rq, delta);
+}
+
+/*
+ * RQ Load update routine
+ */
+#define RQ_LOAD_HISTORY_BITS		(sizeof(s32) * 8ULL)
+#define RQ_UTIL_SHIFT			(8)
+#define RQ_LOAD_HISTORY_TO_UTIL(l)	(((l) >> (RQ_LOAD_HISTORY_BITS - 1 - RQ_UTIL_SHIFT)) & 0xff)
+
+#define LOAD_BLOCK(t)		((t) >> 17)
+#define LOAD_HALF_BLOCK(t)	((t) >> 16)
+#define BLOCK_MASK(t)		((t) & ((0x01 << 18) - 1))
+#define LOAD_BLOCK_BIT(b)	(1UL << (RQ_LOAD_HISTORY_BITS - 1 - (b)))
+#define CURRENT_LOAD_BIT	LOAD_BLOCK_BIT(0)
+
+static inline void rq_load_update(struct rq *rq)
+{
+	u64 time = rq->clock;
+	u64 delta = min(LOAD_BLOCK(time) - LOAD_BLOCK(rq->load_stamp), RQ_LOAD_HISTORY_BITS - 1);
+	u64 prev = !!(rq->load_history & CURRENT_LOAD_BIT);
+	u64 curr = (rq->curr != rq->idle);
+
+	if (delta) {
+		rq->load_history = rq->load_history >> delta;
+
+		if (delta < RQ_UTIL_SHIFT) {
+			rq->load_block += (~BLOCK_MASK(rq->load_stamp)) * prev;
+			if (!!LOAD_HALF_BLOCK(rq->load_block) ^ curr)
+				rq->load_history ^= LOAD_BLOCK_BIT(delta);
+		}
+
+		rq->load_block = BLOCK_MASK(time) * prev;
+	} else {
+		rq->load_block += (time - rq->load_stamp) * prev;
+	}
+	if (prev ^ curr)
+		rq->load_history ^= CURRENT_LOAD_BIT;
+	rq->load_stamp = time;
+}
+
+unsigned long rq_load_util(struct rq *rq, unsigned long max)
+{
+	return RQ_LOAD_HISTORY_TO_UTIL(rq->load_history) * (max >> RQ_UTIL_SHIFT);
+}
+
+unsigned long sched_cpu_util(int cpu)
+{
+	return rq_load_util(cpu_rq(cpu), arch_scale_cpu_capacity(cpu));
+}
+
+#ifdef CONFIG_CPU_FREQ
+/**
+ * cpufreq_update_util - Take a note about CPU utilization changes.
+ * @rq: Runqueue to carry out the update for.
+ * @flags: Update reason flags.
+ *
+ * This function is called by the scheduler on the CPU whose utilization is
+ * being updated.
+ *
+ * It can only be called from RCU-sched read-side critical sections.
+ *
+ * The way cpufreq is currently arranged requires it to evaluate the CPU
+ * performance state (frequency/voltage) on a regular basis to prevent it from
+ * being stuck in a completely inadequate performance level for too long.
+ * That is not guaranteed to happen if the updates are only triggered from CFS
+ * and DL, though, because they may not be coming in if only RT tasks are
+ * active all the time (or there are RT tasks only).
+ *
+ * As a workaround for that issue, this function is called periodically by the
+ * RT sched class to trigger extra cpufreq updates to prevent it from stalling,
+ * but that really is a band-aid.  Going forward it should be replaced with
+ * solutions targeted more specifically at RT tasks.
+ */
+static inline void cpufreq_update_util(struct rq *rq, unsigned int flags)
+{
+	struct update_util_data *data;
+
+	rq_load_update(rq);
+	data = rcu_dereference_sched(*per_cpu_ptr(&cpufreq_update_util_data, cpu_of(rq)));
+	if (data)
+		data->func(data, rq_clock(rq), flags);
+}
+#else /* !CONFIG_CPU_FREQ: */
+static inline void cpufreq_update_util(struct rq *rq, unsigned int flags)
+{
+	rq_load_update(rq);
+}
+#endif /* !CONFIG_CPU_FREQ */
+
+#ifdef CONFIG_NO_HZ_FULL
+/*
+ * Tick may be needed by tasks in the runqueue depending on their policy and
+ * requirements. If tick is needed, lets send the target an IPI to kick it out
+ * of nohz mode if necessary.
+ */
+static inline void sched_update_tick_dependency(struct rq *rq)
+{
+	int cpu = cpu_of(rq);
+
+	if (!tick_nohz_full_cpu(cpu))
+		return;
+
+	if ((IDLE_TASK_SCHED_PRIO != READ_ONCE(cpu_prio[cpu])) &&
+	    (0 == srq_nr_queued(cpu)) && is_preempt_list_empty(cpu))
+		tick_nohz_dep_clear_cpu(cpu, TICK_DEP_BIT_SCHED);
+	else
+		tick_nohz_dep_set_cpu(cpu, TICK_DEP_BIT_SCHED);
+}
+#else /* !CONFIG_NO_HZ_FULL: */
+static inline void sched_update_tick_dependency(struct rq *rq) { }
+#endif /* !CONFIG_NO_HZ_FULL */
+
+bool sched_task_on_rq(struct task_struct *p)
+{
+	return task_on_rq_queued(p);
+}
+
+unsigned long get_wchan(struct task_struct *p)
+{
+	unsigned long ip = 0;
+	unsigned int state;
+
+	if (!p || p == current)
+		return 0;
+
+	/* Only get wchan if task is blocked and we can keep it that way. */
+	raw_spin_lock_irq(&p->pi_lock);
+	state = READ_ONCE(p->__state);
+	smp_rmb(); /* see try_to_wake_up() */
+	if (state != TASK_RUNNING && state != TASK_WAKING && !p->on_rq)
+		ip = __get_wchan(p);
+	raw_spin_unlock_irq(&p->pi_lock);
+
+	return ip;
+}
+
+/*
+ * try_cmpxchg based fetch_or() macro so it works for different integer types:
+ */
+#define fetch_or(ptr, mask)						\
+	({								\
+		typeof(ptr) _ptr = (ptr);				\
+		typeof(mask) _mask = (mask);				\
+		typeof(*_ptr) _val = *_ptr;				\
+									\
+		do {							\
+		} while (!try_cmpxchg(_ptr, &_val, _val | _mask));	\
+	_val;								\
+})
+
+#ifdef TIF_POLLING_NRFLAG
+/*
+ * Atomically set TIF_NEED_RESCHED and test for TIF_POLLING_NRFLAG,
+ * this avoids any races wrt polling state changes and thereby avoids
+ * spurious IPIs.
+ */
+static inline bool set_nr_and_not_polling(struct thread_info *ti, int tif)
+{
+	return !(fetch_or(&ti->flags, 1 << tif) & _TIF_POLLING_NRFLAG);
+}
+
+/*
+ * Atomically set TIF_NEED_RESCHED if TIF_POLLING_NRFLAG is set.
+ *
+ * If this returns true, then the idle task promises to call
+ * sched_ttwu_pending() and reschedule soon.
+ */
+static bool set_nr_if_polling(struct task_struct *p)
+{
+	struct thread_info *ti = task_thread_info(p);
+	typeof(ti->flags) val = READ_ONCE(ti->flags);
+
+	do {
+		if (!(val & _TIF_POLLING_NRFLAG))
+			return false;
+		if (val & _TIF_NEED_RESCHED)
+			return true;
+	} while (!try_cmpxchg(&ti->flags, &val, val | _TIF_NEED_RESCHED));
+
+	return true;
+}
+
+#else /* !TIF_POLLING_NRFLAG: */
+static inline bool set_nr_and_not_polling(struct thread_info *ti, int tif)
+{
+	set_ti_thread_flag(ti, tif);
+	return true;
+}
+
+static inline bool set_nr_if_polling(struct task_struct *p)
+{
+	return false;
+}
+#endif /* !TIF_POLLING_NRFLAG */
+
+static bool __wake_q_add(struct wake_q_head *head, struct task_struct *task)
+{
+	struct wake_q_node *node = &task->wake_q;
+
+	/*
+	 * Atomically grab the task, if ->wake_q is !nil already it means
+	 * it's already queued (either by us or someone else) and will get the
+	 * wakeup due to that.
+	 *
+	 * In order to ensure that a pending wakeup will observe our pending
+	 * state, even in the failed case, an explicit smp_mb() must be used.
+	 */
+	smp_mb__before_atomic();
+	if (unlikely(cmpxchg_relaxed(&node->next, NULL, WAKE_Q_TAIL)))
+		return false;
+
+	/*
+	 * The head is context local, there can be no concurrency.
+	 */
+	*head->lastp = node;
+	head->lastp = &node->next;
+	return true;
+}
+
+/**
+ * wake_q_add() - queue a wakeup for 'later' waking.
+ * @head: the wake_q_head to add @task to
+ * @task: the task to queue for 'later' wakeup
+ *
+ * Queue a task for later wakeup, most likely by the wake_up_q() call in the
+ * same context, _HOWEVER_ this is not guaranteed, the wakeup can come
+ * instantly.
+ *
+ * This function must be used as-if it were wake_up_process(); IOW the task
+ * must be ready to be woken at this location.
+ */
+void wake_q_add(struct wake_q_head *head, struct task_struct *task)
+{
+	if (__wake_q_add(head, task))
+		get_task_struct(task);
+}
+
+/**
+ * wake_q_add_safe() - safely queue a wakeup for 'later' waking.
+ * @head: the wake_q_head to add @task to
+ * @task: the task to queue for 'later' wakeup
+ *
+ * Queue a task for later wakeup, most likely by the wake_up_q() call in the
+ * same context, _HOWEVER_ this is not guaranteed, the wakeup can come
+ * instantly.
+ *
+ * This function must be used as-if it were wake_up_process(); IOW the task
+ * must be ready to be woken at this location.
+ *
+ * This function is essentially a task-safe equivalent to wake_q_add(). Callers
+ * that already hold reference to @task can call the 'safe' version and trust
+ * wake_q to do the right thing depending whether or not the @task is already
+ * queued for wakeup.
+ */
+void wake_q_add_safe(struct wake_q_head *head, struct task_struct *task)
+{
+	if (!__wake_q_add(head, task))
+		put_task_struct(task);
+}
+
+void wake_up_q(struct wake_q_head *head)
+{
+	struct wake_q_node *node = head->first;
+
+	while (node != WAKE_Q_TAIL) {
+		struct task_struct *task;
+
+		task = container_of(node, struct task_struct, wake_q);
+		node = node->next;
+		/* pairs with cmpxchg_relaxed() in __wake_q_add() */
+		WRITE_ONCE(task->wake_q.next, NULL);
+		/* Task can safely be re-inserted now. */
+
+		/*
+		 * wake_up_process() executes a full barrier, which pairs with
+		 * the queueing in wake_q_add() so as not to miss wakeups.
+		 */
+		wake_up_process(task);
+		put_task_struct(task);
+	}
+}
+
+/*
+ * resched_curr - mark rq's current task 'to be rescheduled now'.
+ *
+ * On UP this means the setting of the need_resched flag, on SMP it
+ * might also involve a cross-CPU call to trigger the scheduler on
+ * the target CPU.
+ */
+static inline void __resched_curr(struct rq *rq, int tif)
+{
+	struct task_struct *curr = rq->curr;
+	struct thread_info *cti = task_thread_info(curr);
+	int cpu;
+
+	lockdep_assert_held(&rq->lock);
+
+	/*
+	 * Always immediately preempt the idle task; no point in delaying doing
+	 * actual work.
+	 */
+	if (is_idle_task(curr) && tif == TIF_NEED_RESCHED_LAZY)
+		tif = TIF_NEED_RESCHED;
+
+	if (cti->flags & ((1 << tif) | _TIF_NEED_RESCHED))
+		return;
+
+	cpu = cpu_of(rq);
+
+	trace_sched_set_need_resched_tp(curr, cpu, tif);
+	if (cpu == smp_processor_id()) {
+		set_ti_thread_flag(cti, tif);
+		if (tif == TIF_NEED_RESCHED)
+			set_preempt_need_resched();
+		return;
+	}
+
+	if (set_nr_and_not_polling(cti, tif)) {
+		if (tif == TIF_NEED_RESCHED)
+			smp_send_reschedule(cpu);
+	} else {
+		trace_sched_wake_idle_without_ipi(cpu);
+	}
+}
+
+void __trace_set_need_resched(struct task_struct *curr, int tif)
+{
+	trace_sched_set_need_resched_tp(curr, smp_processor_id(), tif);
+}
+EXPORT_SYMBOL_GPL(__trace_set_need_resched);
+
+static inline void resched_curr(struct rq *rq)
+{
+	__resched_curr(rq, TIF_NEED_RESCHED);
+}
+
+#ifdef CONFIG_PREEMPT_DYNAMIC
+static DEFINE_STATIC_KEY_FALSE(sk_dynamic_preempt_lazy);
+static __always_inline bool dynamic_preempt_lazy(void)
+{
+	return static_branch_unlikely(&sk_dynamic_preempt_lazy);
+}
+#else /* !CONFIG_PREEMPT_DYNAMIC: */
+static __always_inline bool dynamic_preempt_lazy(void)
+{
+	return IS_ENABLED(CONFIG_PREEMPT_LAZY);
+}
+#endif /* !CONFIG_PREEMPT_DYNAMIC */
+
+static __always_inline int get_lazy_tif_bit(void)
+{
+	if (dynamic_preempt_lazy())
+		return TIF_NEED_RESCHED_LAZY;
+
+	return TIF_NEED_RESCHED;
+}
+
+static inline void resched_curr_lazy(struct rq *rq)
+{
+	__resched_curr(rq, get_lazy_tif_bit());
+}
+
+void resched_cpu(int cpu)
+{
+	struct rq *rq = cpu_rq(cpu);
+	unsigned long flags;
+
+	raw_spin_lock_irqsave(&rq->lock, flags);
+	if (cpu_online(cpu) || cpu == smp_processor_id())
+		resched_curr(cpu_rq(cpu));
+	raw_spin_unlock_irqrestore(&rq->lock, flags);
+}
+
+#ifdef CONFIG_NO_HZ_COMMON
+/*
+ * This routine will record that the CPU is going idle with tick stopped.
+ * This info will be used in performing idle load balancing in the future.
+ */
+void nohz_balance_enter_idle(int cpu) {}
+
+/*
+ * In the semi idle case, use the nearest busy CPU for migrating timers
+ * from an idle CPU.  This is good for power-savings.
+ *
+ * We don't do similar optimization for completely idle system, as
+ * selecting an idle CPU will add more delays to the timers than intended
+ * (as that CPU's timer base may not be up to date wrt jiffies etc).
+ */
+int get_nohz_timer_target(void)
+{
+	int i, cpu = smp_processor_id(), default_cpu = -1;
+	struct cpumask *mask;
+	const struct cpumask *hk_mask, *end_mask;
+
+	if (housekeeping_cpu(cpu, HK_TYPE_KERNEL_NOISE)) {
+		if (!idle_cpu(cpu))
+			return cpu;
+		default_cpu = cpu;
+	}
+
+	hk_mask = housekeeping_cpumask(HK_TYPE_KERNEL_NOISE);
+	end_mask = per_cpu(cpu_affinity_end_mask, cpu);
+
+	for (mask = per_cpu(cpu_affinity_masks, cpu); mask < end_mask; mask++)
+		for_each_cpu_and(i, mask, hk_mask)
+			if (!idle_cpu(i))
+				return i;
+
+	if (default_cpu == -1)
+		default_cpu = housekeeping_any_cpu(HK_TYPE_KERNEL_NOISE);
+	cpu = default_cpu;
+
+	return cpu;
+}
+
+/*
+ * When add_timer_on() enqueues a timer into the timer wheel of an
+ * idle CPU then this timer might expire before the next timer event
+ * which is scheduled to wake up that CPU. In case of a completely
+ * idle system the next event might even be infinite time into the
+ * future. wake_up_idle_cpu() ensures that the CPU is woken up and
+ * leaves the inner idle loop so the newly added timer is taken into
+ * account when the CPU goes back to idle and evaluates the timer
+ * wheel for the next timer event.
+ */
+static inline void wake_up_idle_cpu(int cpu)
+{
+	struct rq *rq = cpu_rq(cpu);
+
+	if (cpu == smp_processor_id())
+		return;
+
+	/*
+	 * Set TIF_NEED_RESCHED and send an IPI if in the non-polling
+	 * part of the idle loop. This forces an exit from the idle loop
+	 * and a round trip to schedule(). Now this could be optimized
+	 * because a simple new idle loop iteration is enough to
+	 * re-evaluate the next tick. Provided some re-ordering of tick
+	 * nohz functions that would need to follow TIF_NR_POLLING
+	 * clearing:
+	 *
+	 * - On most architectures, a simple fetch_or on ti::flags with a
+	 *   "0" value would be enough to know if an IPI needs to be sent.
+	 *
+	 * - x86 needs to perform a last need_resched() check between
+	 *   monitor and mwait which doesn't take timers into account.
+	 *   There a dedicated TIF_TIMER flag would be required to
+	 *   fetch_or here and be checked along with TIF_NEED_RESCHED
+	 *   before mwait().
+	 *
+	 * However, remote timer enqueue is not such a frequent event
+	 * and testing of the above solutions didn't appear to report
+	 * much benefits.
+	 */
+	if (set_nr_and_not_polling(task_thread_info(rq->idle), TIF_NEED_RESCHED))
+		smp_send_reschedule(cpu);
+	else
+		trace_sched_wake_idle_without_ipi(cpu);
+}
+
+static inline bool wake_up_full_nohz_cpu(int cpu)
+{
+	/*
+	 * We just need the target to call irq_exit() and re-evaluate
+	 * the next tick. The nohz full kick at least implies that.
+	 * If needed we can still optimize that later with an
+	 * empty IRQ.
+	 */
+	if (cpu_is_offline(cpu))
+		return true;  /* Don't try to wake offline CPUs. */
+	if (tick_nohz_full_cpu(cpu)) {
+		if (cpu != smp_processor_id() ||
+		    tick_nohz_tick_stopped())
+			tick_nohz_full_kick_cpu(cpu);
+		return true;
+	}
+
+	return false;
+}
+
+void wake_up_nohz_cpu(int cpu)
+{
+	if (!wake_up_full_nohz_cpu(cpu))
+		wake_up_idle_cpu(cpu);
+}
+
+static void nohz_csd_func(void *info)
+{
+	struct rq *rq = info;
+	int cpu = cpu_of(rq);
+	unsigned int flags;
+
+	/*
+	 * Release the rq::nohz_csd.
+	 */
+	flags = atomic_fetch_andnot(NOHZ_KICK_MASK, nohz_flags(cpu));
+	WARN_ON(!(flags & NOHZ_KICK_MASK));
+
+	rq->idle_balance = idle_cpu(cpu);
+	if (rq->idle_balance) {
+		rq->nohz_idle_balance = flags;
+		__raise_softirq_irqoff(SCHED_SOFTIRQ);
+	}
+}
+
+#endif /* CONFIG_NO_HZ_COMMON */
+
+void yield_task(struct rq *rq)
+{
+	struct task_struct *p;
+
+	if (!sched_yield_type)
+		return;
+
+	p = current;
+	if (!rt_task(p) && p != rq->idle) {
+		do_sched_yield_type_1(p, rq);
+		update_sched_cpu_prio(cpu_of(rq), task_sched_prio(p));
+	}
+}
+
+static __always_inline
+int __task_state_match(struct task_struct *p, unsigned int state)
+{
+	if (READ_ONCE(p->__state) & state)
+		return 1;
+
+	if (READ_ONCE(p->saved_state) & state)
+		return -1;
+
+	return 0;
+}
+
+static __always_inline
+int task_state_match(struct task_struct *p, unsigned int state)
+{
+	/*
+	 * Serialize against current_save_and_set_rtlock_wait_state(),
+	 * current_restore_rtlock_saved_state(), and __refrigerator().
+	 */
+	guard(raw_spinlock_irq)(&p->pi_lock);
+
+	return __task_state_match(p, state);
+}
+
+/*
+ * wait_task_inactive - wait for a thread to unschedule.
+ *
+ * Wait for the thread to block in any of the states set in @match_state.
+ * If it changes, i.e. @p might have woken up, then return zero.  When we
+ * succeed in waiting for @p to be off its CPU, we return a positive number
+ * (its total switch count).  If a second call a short while later returns the
+ * same number, the caller can be sure that @p has remained unscheduled the
+ * whole time.
+ *
+ * The caller must ensure that the task *will* unschedule sometime soon,
+ * else this function might spin for a *long* time. This function can't
+ * be called with interrupts off, or it may introduce deadlock with
+ * smp_call_function() if an IPI is sent by the same process we are
+ * waiting to become inactive.
+ */
+unsigned long wait_task_inactive(struct task_struct *p, unsigned int match_state)
+{
+	int running, queued, match;
+	unsigned long ncsw;
+	struct rq_flags rf;
+
+	for (;;) {
+		/*
+		 * If the task is actively running on another CPU
+		 * still, just relax and busy-wait without holding
+		 * any locks.
+		 *
+		 * NOTE! Since we don't hold any locks, it's not
+		 * even sure that "rq" stays as the right runqueue!
+		 * But we don't care, since this will return false
+		 * if the runqueue has changed and p is actually now
+		 * running somewhere else!
+		 */
+		while (task_on_cpu(p)) {
+			if (!task_state_match(p, match_state))
+				return 0;
+			cpu_relax();
+		}
+
+		/*
+		 * Ok, time to look more closely! We need the rq
+		 * lock now, to be *sure*. If we're wrong, we'll
+		 * just go back and repeat.
+		 */
+		raw_spin_lock_irqsave(&p->pi_lock, rf.flags);
+		__task_rq_lock(p, &rf);
+		trace_sched_wait_task(p);
+		running = task_on_cpu(p);
+		queued = task_on_rq_queued(p);
+		ncsw = 0;
+		if ((match = __task_state_match(p, match_state))) {
+			/*
+			 * When matching on p->saved_state, consider this task
+			 * still queued so it will wait.
+			 */
+			if (match < 0)
+				queued = 1;
+			ncsw = p->nvcsw | LONG_MIN; /* sets MSB */
+		}
+		__task_rq_unlock(&rf);
+		raw_spin_unlock_irqrestore(&p->pi_lock, rf.flags);
+
+		/*
+		 * If it changed from the expected state, bail out now.
+		 */
+		if (unlikely(!ncsw))
+			break;
+
+		/*
+		 * Was it really running after all now that we
+		 * checked with the proper locks actually held?
+		 *
+		 * Oops. Go back and try again..
+		 */
+		if (unlikely(running)) {
+			cpu_relax();
+			continue;
+		}
+
+		/*
+		 * It's not enough that it's not actively running,
+		 * it must be off the runqueue _entirely_, and not
+		 * preempted!
+		 *
+		 * So if it was still runnable (but just not actively
+		 * running right now), it's preempted, and we should
+		 * yield - it could be a while.
+		 */
+		if (unlikely(queued)) {
+			ktime_t to = NSEC_PER_SEC / HZ;
+
+			set_current_state(TASK_UNINTERRUPTIBLE);
+			schedule_hrtimeout(&to, HRTIMER_MODE_REL_HARD);
+			continue;
+		}
+
+		/*
+		 * Ahh, all good. It wasn't running, and it wasn't
+		 * runnable, which means that it will never become
+		 * running in the future either. We're all done!
+		 */
+		break;
+	}
+
+	return ncsw;
+}
+
+#ifdef CONFIG_SCHED_HRTICK
+/*
+ * Use HR-timers to deliver accurate preemption points.
+ */
+
+enum {
+	HRTICK_SCHED_NONE		= 0,
+	HRTICK_SCHED_DEFER		= BIT(1),
+	HRTICK_SCHED_START		= BIT(2),
+	HRTICK_SCHED_REARM_HRTIMER	= BIT(3)
+};
+
+static void __used hrtick_clear(struct rq *rq)
+{
+	if (hrtimer_active(&rq->hrtick_timer))
+		hrtimer_cancel(&rq->hrtick_timer);
+}
+
+/*
+ * High-resolution timer tick.
+ * Runs from hardirq context with interrupts disabled.
+ */
+static enum hrtimer_restart hrtick(struct hrtimer *timer)
+{
+	struct rq *rq = container_of(timer, struct rq, hrtick_timer);
+
+	WARN_ON_ONCE(cpu_of(rq) != smp_processor_id());
+
+	raw_spin_lock(&rq->lock);
+	resched_curr(rq);
+	raw_spin_unlock(&rq->lock);
+
+	return HRTIMER_NORESTART;
+}
+
+/*
+ * Use hrtick when:
+ *  - enabled by features
+ *  - hrtimer is actually high res
+ */
+static inline bool hrtick_enabled(struct rq *rq)
+{
+	return cpu_active(cpu_of(rq)) && hrtimer_highres_enabled();
+}
+
+static inline bool hrtick_needs_rearm(struct hrtimer *timer, ktime_t expires)
+{
+	/*
+	 * Queued is false when the timer is not started or currently
+	 * running the callback. In both cases, restart. If queued check
+	 * whether the expiry time actually changes substantially.
+	 */
+	return !hrtimer_is_queued(timer) ||
+		abs(expires - hrtimer_get_expires(timer)) > 5000;
+}
+
+static void hrtick_cond_restart(struct rq *rq)
+{
+	struct hrtimer *timer = &rq->hrtick_timer;
+	ktime_t time = rq->hrtick_time;
+
+	if (hrtick_needs_rearm(timer, time))
+		hrtimer_start(timer, time, HRTIMER_MODE_ABS_PINNED_HARD);
+}
+
+/*
+ * called from hardirq (IPI) context
+ */
+static void __hrtick_start(void *arg)
+{
+	struct rq *rq = arg;
+
+	raw_spin_lock(&rq->lock);
+	hrtick_cond_restart(rq);
+	raw_spin_unlock(&rq->lock);
+}
+
+/*
+ * Called to set the hrtick timer state.
+ *
+ * called with rq->lock held and IRQs disabled
+ */
+static inline void hrtick_start(struct rq *rq, u64 delay)
+{
+	s64 delta;
+
+	/*
+	 * Don't schedule slices shorter than 10000ns, that just
+	 * doesn't make sense and can cause timer DoS.
+	 */
+	delta = max_t(s64, delay, 10000LL);
+
+	rq->hrtick_time = ktime_add_ns(ktime_get(), delta);
+	if (!hrtick_needs_rearm(&rq->hrtick_timer, rq->hrtick_time))
+		return;
+
+	/*
+	 * If this is in the middle of schedule() only note the delay
+	 * and let hrtick_schedule_exit() deal with it.
+	 */
+	if (rq->hrtick_sched) {
+		rq->hrtick_sched |= HRTICK_SCHED_START;
+		rq->hrtick_delay = delta;
+		return;
+	}
+
+	if (rq == this_rq())
+		hrtimer_start(&rq->hrtick_timer, rq->hrtick_time, HRTIMER_MODE_ABS_PINNED_HARD);
+	else
+		smp_call_function_single_async(cpu_of(rq), &rq->hrtick_csd);
+}
+
+static inline void hrtick_schedule_enter(struct rq *rq)
+{
+	rq->hrtick_sched = HRTICK_SCHED_DEFER;
+	if (hrtimer_test_and_clear_rearm_deferred())
+		rq->hrtick_sched |= HRTICK_SCHED_REARM_HRTIMER;
+}
+
+static inline void hrtick_schedule_exit(struct rq *rq)
+{
+	if (rq->hrtick_sched & HRTICK_SCHED_START) {
+		rq->hrtick_time = ktime_add_ns(ktime_get(), rq->hrtick_delay);
+		hrtick_cond_restart(rq);
+	} else if (idle_rq(rq)) {
+		/*
+		 * No need for using hrtimer_is_active(). The timer is CPU local
+		 * and interrupts are disabled, so the callback cannot be
+		 * running and the queued state is valid.
+		 */
+		if (hrtimer_is_queued(&rq->hrtick_timer))
+			hrtimer_cancel(&rq->hrtick_timer);
+	}
+
+	if (rq->hrtick_sched & HRTICK_SCHED_REARM_HRTIMER)
+		__hrtimer_rearm_deferred();
+
+	rq->hrtick_sched = HRTICK_SCHED_NONE;
+}
+
+static void hrtick_rq_init(struct rq *rq)
+{
+	INIT_CSD(&rq->hrtick_csd, __hrtick_start, rq);
+	rq->hrtick_sched = HRTICK_SCHED_NONE;
+	hrtimer_setup(&rq->hrtick_timer, hrtick, CLOCK_MONOTONIC,
+		      HRTIMER_MODE_REL_HARD | HRTIMER_MODE_LAZY_REARM);
+}
+#else	/* !CONFIG_SCHED_HRTICK: */
+static inline void hrtick_clear(struct rq *rq) { }
+static inline void hrtick_rq_init(struct rq *rq) { }
+static inline void hrtick_schedule_enter(struct rq *rq) { }
+static inline void hrtick_schedule_exit(struct rq *rq) { }
+#endif	/* !CONFIG_SCHED_HRTICK */
+
+/*
+ * activate_task - enqueue a task to the schedule runqueue @srq.
+ */
+static inline void activate_task(struct task_struct *p, struct sched_run_queue *srq)
+{
+	SRQ_ENQUEUE_TASK(srq, p,
+			 {
+				WRITE_ONCE(p->on_rq, TASK_ON_RQ_QUEUED);
+				ASSERT_EXCLUSIVE_WRITER(p->on_rq);
+
+				WRITE_ONCE(p->__state, TASK_RUNNING);
+			});
+	/*
+	 * If in_iowait is set, the code below may not trigger any cpufreq
+	 * utilization updates, so do it here explicitly with the IOWAIT flag
+	 * passed.
+	 */
+	//cpufreq_update_util(rq, SCHED_CPUFREQ_IOWAIT * p->in_iowait);
+}
+
+static void block_task(struct rq *rq, struct task_struct *p)
+{
+	sched_task_deactivate(p, rq);
+
+	if (p->sched_contributes_to_load)
+		atomic_inc(&rq_srq(rq)->nr_uninterruptible);
+
+	if (p->in_iowait) {
+		atomic_inc(&rq_srq(rq)->nr_iowait);
+		delayacct_blkio_start();
+	}
+
+	ASSERT_EXCLUSIVE_WRITER(p->on_rq);
+
+	/*
+	 * The moment this write goes through, ttwu() can swoop in and migrate
+	 * this task, rendering our rq->__lock ineffective.
+	 *
+	 * __schedule()				try_to_wake_up()
+	 *   LOCK rq->__lock			  LOCK p->pi_lock
+	 *   pick_next_task()
+	 *     pick_next_task_fair()
+	 *       pick_next_entity()
+	 *         dequeue_entities()
+	 *           __block_task()
+	 *             RELEASE p->on_rq = 0	  if (p->on_rq && ...)
+	 *					    break;
+	 *
+	 *					  ACQUIRE (after ctrl-dep)
+	 *
+	 *					  cpu = select_task_rq();
+	 *					  set_task_cpu(p, cpu);
+	 *					  ttwu_queue()
+	 *					    ttwu_do_activate()
+	 *					      LOCK rq->__lock
+	 *					      activate_task()
+	 *					        STORE p->on_rq = 1
+	 *   UNLOCK rq->__lock
+	 *
+	 * Callers must ensure to not reference @p after this -- we no longer
+	 * own it.
+	 */
+	smp_store_release(&p->on_rq, 0);
+}
+
+static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu)
+{
+	/*
+	 * After ->cpu is set up to a new value, task_access_lock(p, ...) can be
+	 * successfully executed on another CPU. We must ensure that updates of
+	 * per-task data have been completed by this moment.
+	 */
+	smp_wmb();
+
+	WRITE_ONCE(task_thread_info(p)->cpu, cpu);
+	rseq_sched_set_ids_changed(p);
+}
+
+static inline void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
+{
+	unsigned int state = READ_ONCE(p->__state);
+
+	/*
+	 * We should only call set_task_cpu() on a running task in pick_next_task().
+	 */
+	WARN_ON_ONCE(state != TASK_RUNNING && !task_on_rq_queued(p) &&
+		     TASK_ON_RQ_WAKING != p->on_rq);
+
+	/*
+	 * Clearly, migrating tasks to offline CPUs is a fairly daft thing.
+	 */
+	WARN_ON_ONCE(!cpu_online(new_cpu));
+
+	/*
+	 * We should not see migration disabled task be picked in pick_next_task().
+	 * If cpus_ptr changed on migration disabled task, __migrate_force_enable()
+	 * will be called.
+	 */
+	WARN_ON_ONCE(is_migration_disabled(p));
+	trace_sched_migrate_task(p, new_cpu);
+
+	if (task_cpu(p) != new_cpu)
+		perf_event_task_migrate(p);
+
+	__set_task_cpu(p, new_cpu);
+}
+
+void ___migrate_enable(void)
+{
+	struct task_struct *p = current;
+	__do_set_cpus_ptr(p, &p->cpus_mask);
+}
+EXPORT_SYMBOL_GPL(___migrate_enable);
+
+void migrate_disable(void)
+{
+	__migrate_disable();
+}
+EXPORT_SYMBOL_GPL(migrate_disable);
+
+void migrate_enable(void)
+{
+	__migrate_enable();
+}
+EXPORT_SYMBOL_GPL(migrate_enable);
+
+static void __migrate_force_enable(struct task_struct *p, struct rq *rq)
+{
+	if (likely(p->cpus_ptr != &p->cpus_mask))
+		__do_set_cpus_ptr(p, &p->cpus_mask);
+	p->migration_disabled = 0;
+	/* When p is migrate_disabled, rq->lock should be held */
+	rq->nr_pinned--;
+}
+
+static inline bool rq_has_pinned_tasks(struct rq *rq)
+{
+	return rq->nr_pinned;
+}
+
+/*
+ * Per-CPU kthreads are allowed to run on !active && online CPUs, see
+ * __set_cpus_allowed_ptr() and select_fallback_rq().
+ */
+static inline bool is_cpu_allowed(struct task_struct *p, int cpu)
+{
+	/* When not in the task's cpumask, no point in looking further. */
+	if (!cpumask_test_cpu(cpu, p->cpus_ptr))
+		return false;
+
+	/* migrate_disabled() must be allowed to finish. */
+	if (is_migration_disabled(p))
+		return cpu_online(cpu);
+
+	/* Non kernel threads are not allowed during either online or offline. */
+	if (!(p->flags & PF_KTHREAD))
+		return cpu_active(cpu) && task_cpu_possible(cpu, p);
+
+	/* KTHREAD_IS_PER_CPU is always allowed. */
+	if (kthread_is_per_cpu(p))
+		return cpu_online(cpu);
+
+	/* Regular kernel threads don't get to stay during offline. */
+	if (cpu_dying(cpu))
+		return false;
+
+	/* But are allowed during online. */
+	return cpu_online(cpu);
+}
+
+static inline void mm_update_cpus_allowed(struct mm_struct *mm, const cpumask_t *affmask);
+
+static inline void
+set_cpus_allowed_common(struct task_struct *p, struct affinity_context *ctx)
+{
+	cpumask_copy(&p->cpus_mask, ctx->new_mask);
+	p->nr_cpus_allowed = cpumask_weight(ctx->new_mask);
+	mm_update_cpus_allowed(p->mm, ctx->new_mask);
+
+	/*
+	 * Swap in a new user_cpus_ptr if SCA_USER flag set
+	 */
+	if (ctx->flags & SCA_USER)
+		swap(p->user_cpus_ptr, ctx->user_mask);
+}
+
+static void
+do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx)
+{
+	lockdep_assert_held(&p->pi_lock);
+	set_cpus_allowed_common(p, ctx);
+}
+
+/*
+ * Used for kthread_bind() and select_fallback_rq(), in both cases the user
+ * affinity (if any) should be destroyed too.
+ */
+void set_cpus_allowed_force(struct task_struct *p, const struct cpumask *new_mask)
+{
+	struct affinity_context ac = {
+		.new_mask  = new_mask,
+		.user_mask = NULL,
+		.flags     = SCA_USER,	/* clear the user requested mask */
+	};
+	union cpumask_rcuhead {
+		cpumask_t cpumask;
+		struct rcu_head rcu;
+	};
+
+	do_set_cpus_allowed(p, &ac);
+
+	if (is_migration_disabled(p) && !cpumask_test_cpu(task_cpu(p), &p->cpus_mask))
+		__migrate_force_enable(p, task_rq(p));
+
+	/*
+	 * Because this is called with p->pi_lock held, it is not possible
+	 * to use kfree() here (when PREEMPT_RT=y), therefore punt to using
+	 * kfree_rcu().
+	 */
+	kfree_rcu((union cpumask_rcuhead *)ac.user_mask, rcu);
+}
+
+int dup_user_cpus_ptr(struct task_struct *dst, struct task_struct *src,
+		      int node)
+{
+	cpumask_t *user_mask;
+	unsigned long flags;
+
+	/*
+	 * Always clear dst->user_cpus_ptr first as their user_cpus_ptr's
+	 * may differ by now due to racing.
+	 */
+	dst->user_cpus_ptr = NULL;
+
+	/*
+	 * This check is racy and losing the race is a valid situation.
+	 * It is not worth the extra overhead of taking the pi_lock on
+	 * every fork/clone.
+	 */
+	if (data_race(!src->user_cpus_ptr))
+		return 0;
+
+	user_mask = alloc_user_cpus_ptr(node);
+	if (!user_mask)
+		return -ENOMEM;
+
+	/*
+	 * Use pi_lock to protect content of user_cpus_ptr
+	 *
+	 * Though unlikely, user_cpus_ptr can be reset to NULL by a concurrent
+	 * do_set_cpus_allowed().
+	 */
+	raw_spin_lock_irqsave(&src->pi_lock, flags);
+	if (src->user_cpus_ptr) {
+		swap(dst->user_cpus_ptr, user_mask);
+		cpumask_copy(dst->user_cpus_ptr, src->user_cpus_ptr);
+	}
+	raw_spin_unlock_irqrestore(&src->pi_lock, flags);
+
+	if (unlikely(user_mask))
+		kfree(user_mask);
+
+	return 0;
+}
+
+static inline struct cpumask *clear_user_cpus_ptr(struct task_struct *p)
+{
+	struct cpumask *user_mask = NULL;
+
+	swap(p->user_cpus_ptr, user_mask);
+
+	return user_mask;
+}
+
+void release_user_cpus_ptr(struct task_struct *p)
+{
+	kfree(clear_user_cpus_ptr(p));
+}
+
+/**
+ * task_curr - is this task currently executing on a CPU?
+ * @p: the task in question.
+ *
+ * Return: 1 if the task is currently executing. 0 otherwise.
+ */
+inline int task_curr(const struct task_struct *p)
+{
+	return cpu_curr(task_cpu(p)) == p;
+}
+
+/***
+ * kick_process - kick a running thread to enter/exit the kernel
+ * @p: the to-be-kicked thread
+ *
+ * Cause a process which is running on another CPU to enter
+ * kernel-mode, without any delay. (to get signals handled.)
+ *
+ * NOTE: this function doesn't have to take the runqueue lock,
+ * because all it wants to ensure is that the remote task enters
+ * the kernel. If the IPI races and the task has been migrated
+ * to another CPU then no harm is done and the purpose has been
+ * achieved as well.
+ */
+void kick_process(struct task_struct *p)
+{
+	guard(preempt)();
+	int cpu = task_cpu(p);
+
+	if ((cpu != smp_processor_id()) && task_curr(p))
+		smp_send_reschedule(cpu);
+}
+EXPORT_SYMBOL_GPL(kick_process);
+
+/*
+ * ->cpus_ptr is protected by both rq->lock and p->pi_lock
+ *
+ * A few notes on cpu_active vs cpu_online:
+ *
+ *  - cpu_active must be a subset of cpu_online
+ *
+ *  - on CPU-up we allow per-CPU kthreads on the online && !active CPU,
+ *    see __set_cpus_allowed_ptr(). At this point the newly online
+ *    CPU isn't yet part of the sched domains, and balancing will not
+ *    see it.
+ *
+ *  - on cpu-down we clear cpu_active() to mask the sched domains and
+ *    avoid the load balancer to place new tasks on the to be removed
+ *    CPU. Existing tasks will remain running there and will be taken
+ *    off.
+ *
+ * This means that fallback selection must not select !active CPUs.
+ * And can assume that any active CPU must be online. Conversely
+ * select_task_rq() below may allow selection of !active CPUs in order
+ * to satisfy the above rules.
+ */
+static int select_fallback_rq(int cpu, struct task_struct *p)
+{
+	int nid = cpu_to_node(cpu);
+	const struct cpumask *nodemask = NULL;
+	enum { cpuset, possible, fail } state = cpuset;
+	int dest_cpu;
+
+	/*
+	 * If the node that the CPU is on has been offlined, cpu_to_node()
+	 * will return -1. There is no CPU on the node, and we should
+	 * select the CPU on the other node.
+	 */
+	if (nid != -1) {
+		nodemask = cpumask_of_node(nid);
+
+		/* Look for allowed, online CPU in same node. */
+		for_each_cpu(dest_cpu, nodemask) {
+			if (is_cpu_allowed(p, dest_cpu))
+				return dest_cpu;
+		}
+	}
+
+	for (;;) {
+		/* Any allowed, online CPU? */
+		for_each_cpu(dest_cpu, p->cpus_ptr) {
+			if (!is_cpu_allowed(p, dest_cpu))
+				continue;
+			goto out;
+		}
+
+		/* No more Mr. Nice Guy. */
+		switch (state) {
+		case cpuset:
+			if (cpuset_cpus_allowed_fallback(p)) {
+				state = possible;
+				break;
+			}
+			fallthrough;
+		case possible:
+			set_cpus_allowed_force(p, task_cpu_fallback_mask(p));
+			state = fail;
+			break;
+
+		case fail:
+			BUG();
+			break;
+		}
+	}
+
+out:
+	if (state != cpuset) {
+		/*
+		 * Don't tell them about moving exiting tasks or
+		 * kernel threads (both mm NULL), since they never
+		 * leave kernel.
+		 */
+		if (p->mm && printk_ratelimit()) {
+			printk_deferred("process %d (%s) no longer affine to cpu%d\n",
+					task_pid_nr(p), p->comm, cpu);
+		}
+	}
+
+	return dest_cpu;
+}
+
+static inline void __resched_cpu(const int cpu)
+{
+	struct rq *rq = cpu_rq(cpu);
+
+	raw_spin_lock(&rq->lock);
+	resched_curr(rq);
+	raw_spin_unlock(&rq->lock);
+}
+
+static inline struct rq *
+__wakeup_rq_trylock(const struct task_struct *p, const int sched_prio, const cpumask_t *filter)
+{
+	struct rq *rq;
+	int idx, preempt_idx = SCHED_LEVELS - 1 - sched_prio, cpu = task_cpu(p);
+	const bool filter_test_cpu = cpumask_test_cpu(cpu, filter);
+	cpumask_t *end_mask = per_cpu(cpu_affinity_end_mask, cpu);
+
+	for_each_set_bit (idx, cpu_sched_prio_bitmap, preempt_idx) {
+		cpumask_t *mask;
+		const struct cpumask *prio_mask = cpu_sched_prio_mask + idx;
+
+		if (filter_test_cpu && cpumask_test_cpu(cpu, prio_mask)) {
+			rq = cpu_rq(cpu);
+			if (raw_spin_trylock(&rq->lock))
+				return rq;
+		}
+		for (mask = per_cpu(cpu_affinity_masks, cpu); mask < end_mask; mask++) {
+			int i;
+			for_each_cpu_and (i, filter, mask)
+				if (cpumask_test_cpu(i, prio_mask)) {
+					rq = cpu_rq(i);
+					if (raw_spin_trylock(&rq->lock))
+						return rq;
+				}
+		}
+	}
+
+	return NULL;
+}
+
+static inline struct rq *wakeup_rq_trylock(const struct task_struct *p)
+{
+	const int sched_prio = task_sched_prio(p);
+
+	if (1 == p->nr_cpus_allowed || is_migration_disabled(p)) {
+		/* Fastpath */
+		const int cpu = cpumask_any(p->cpus_ptr);
+
+		if (sched_prio < READ_ONCE(cpu_prio[cpu])) {
+			struct rq *rq = cpu_rq(cpu);
+			if (likely(raw_spin_trylock(&rq->lock)))
+				return rq;
+		}
+
+		return NULL;
+	}
+
+	return __wakeup_rq_trylock(p, sched_prio, p->cpus_ptr);
+}
+
+static __always_inline void preempt_on_rq(struct task_struct *p, struct rq *rq)
+{
+	int cpu = p->wake_cpu = cpu_of(rq);
+
+	llist_add(&p->pq_node, per_cpu_ptr(&preempt_list, cpu));
+
+	resched_curr(rq);
+
+	raw_spin_unlock(&rq->lock);
+}
+
+static __always_inline void wakeup_preempt_on_rq(struct task_struct *p, struct rq *rq)
+{
+	WRITE_ONCE(p->on_rq, TASK_ON_RQ_WAKING);
+	ASSERT_EXCLUSIVE_WRITER(p->on_rq);
+
+	WRITE_ONCE(p->__state, TASK_RUNNING);
+
+	preempt_on_rq(p, rq);
+}
+
+void wakeup_modified_task(struct task_struct *p)
+{
+	struct rq *rq = wakeup_rq_trylock(p);
+
+	if (rq) {
+		WRITE_ONCE(p->on_rq, TASK_ON_RQ_WAKING);
+		ASSERT_EXCLUSIVE_WRITER(p->on_rq);
+
+		preempt_on_rq(p, rq);
+		return;
+	}
+
+	SRQ_ENQUEUE_TASK(&grq, p,
+			 {
+				WRITE_ONCE(p->on_rq, TASK_ON_RQ_QUEUED);
+				ASSERT_EXCLUSIVE_WRITER(p->on_rq);
+			 });
+}
+
+void sched_set_stop_task(int cpu, struct task_struct *stop)
+{
+	static struct lock_class_key stop_pi_lock;
+	struct sched_param stop_param = { .sched_priority = STOP_PRIO };
+	struct sched_param start_param = { .sched_priority = 0 };
+	struct task_struct *old_stop = cpu_rq(cpu)->stop;
+
+	if (stop) {
+		/*
+		 * Make it appear like a SCHED_FIFO task, its something
+		 * userspace knows about and won't get confused about.
+		 *
+		 * Also, it will make PI more or less work without too
+		 * much confusion -- but then, stop work should not
+		 * rely on PI working anyway.
+		 */
+		sched_setscheduler_nocheck(stop, SCHED_FIFO, &stop_param);
+
+		/*
+		 * The PI code calls rt_mutex_setprio() with ->pi_lock held to
+		 * adjust the effective priority of a task. As a result,
+		 * rt_mutex_setprio() can trigger (RT) balancing operations,
+		 * which can then trigger wakeups of the stop thread to push
+		 * around the current task.
+		 *
+		 * The stop task itself will never be part of the PI-chain, it
+		 * never blocks, therefore that ->pi_lock recursion is safe.
+		 * Tell lockdep about this by placing the stop->pi_lock in its
+		 * own class.
+		 */
+		lockdep_set_class(&stop->pi_lock, &stop_pi_lock);
+	}
+
+	cpu_rq(cpu)->stop = stop;
+
+	if (old_stop) {
+		/*
+		 * Reset it back to a normal scheduling policy so that
+		 * it can die in pieces.
+		 */
+		sched_setscheduler_nocheck(old_stop, SCHED_NORMAL, &start_param);
+	}
+}
+
+/*
+ * dummy_cpu_stop - this will be executed by a high-prio stopper thread and bump
+ * current thread off CPU to schedule rq, when switching out, this thread will be
+ * waken up on correct idle CPU, see finish_task() for detail.
+ */
+static int dummy_cpu_stop(void *data)
+{
+	return 0;
+}
+
+static int affine_move_task(struct rq *rq, struct task_struct *p, struct rq_flags *rf)
+{
+	/* Can the task run on the task's current CPU? If so, we're done */
+	if (!cpumask_test_cpu(task_cpu(p), &p->cpus_mask)) {
+		if (is_migration_disabled(p))
+			__migrate_force_enable(p, rq);
+
+		if (task_on_cpu(p)) {
+			/* Need help from cpu stop thread: drop lock and wait. */
+			__task_modify_unlock(p, rf);
+			raw_spin_unlock_irqrestore(&p->pi_lock, rf->flags);
+			stop_one_cpu(cpu_of(rq), dummy_cpu_stop, p);
+			return 0;
+		}
+		/* Nothing we should do for task_on_rq_queued(p) */
+	}
+	__task_modify_unlock(p, rf);
+	raw_spin_unlock_irqrestore(&p->pi_lock, rf->flags);
+	return 0;
+}
+
+static int __set_cpus_allowed_ptr_locked(struct task_struct *p,
+					 struct affinity_context *ctx,
+					 struct rq *rq,
+					 struct rq_flags *rf)
+{
+	const struct cpumask *cpu_allowed_mask = task_cpu_possible_mask(p);
+	bool kthread = p->flags & PF_KTHREAD;
+	int ret = 0;
+
+	/*
+	 * Kernel threads are allowed on online && !active CPUs,
+	 * however, during cpu-hot-unplug, even these might get pushed
+	 * away if not KTHREAD_IS_PER_CPU.
+	 *
+	 * Specifically, migration_disabled() tasks must not fail the
+	 * cpumask_any_and_distribute() pick below, esp. so on
+	 * SCA_MIGRATE_ENABLE, otherwise we'll not call
+	 * set_cpus_allowed_common() and actually reset p->cpus_ptr.
+	 */
+	if (unlikely(cpumask_empty(ctx->new_mask))) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	if (!kthread && !cpumask_subset(ctx->new_mask, cpu_allowed_mask)) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	/*
+	 * Must re-check here, to close a race against __kthread_bind(),
+	 * sched_setaffinity() is not guaranteed to observe the flag.
+	 */
+	if ((ctx->flags & SCA_CHECK) && (p->flags & PF_NO_SETAFFINITY)) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	if (cpumask_equal(&p->cpus_mask, ctx->new_mask)) {
+		if (ctx->flags & SCA_USER)
+			swap(p->user_cpus_ptr, ctx->user_mask);
+		goto out;
+	}
+
+	do_set_cpus_allowed(p, ctx);
+
+	return affine_move_task(rq, p, rf);
+
+out:
+	__task_modify_unlock(p, rf);
+	raw_spin_unlock_irqrestore(&p->pi_lock, rf->flags);
+
+	return ret;
+}
+
+/*
+ * Change a given task's CPU affinity. Migrate the thread to a
+ * is removed from the allowed bitmask.
+ *
+ * NOTE: the caller must have a valid reference to the task, the
+ * task must not exit() & deallocate itself prematurely. The
+ * call is not atomic; no spinlocks may be held.
+ */
+int __set_cpus_allowed_ptr(struct task_struct *p,
+			   struct affinity_context *ctx)
+{
+	struct rq *rq;
+	struct rq_flags rf;
+
+	raw_spin_lock_irqsave(&p->pi_lock, rf.flags);
+	rq = __task_modify_lock(p, &rf);
+	/*
+	 * Masking should be skipped if SCA_USER or any of the SCA_MIGRATE_*
+	 * flags are set.
+	 */
+	if (p->user_cpus_ptr &&
+	    !(ctx->flags & SCA_USER) &&
+	    cpumask_and(rq->scratch_mask, ctx->new_mask, p->user_cpus_ptr))
+		ctx->new_mask = rq->scratch_mask;
+
+
+	return __set_cpus_allowed_ptr_locked(p, ctx, rq, &rf);
+}
+
+int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
+{
+	struct affinity_context ac = {
+		.new_mask  = new_mask,
+		.flags     = 0,
+	};
+
+	return __set_cpus_allowed_ptr(p, &ac);
+}
+EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr);
+
+/*
+ * Change a given task's CPU affinity to the intersection of its current
+ * affinity mask and @subset_mask, writing the resulting mask to @new_mask.
+ * If user_cpus_ptr is defined, use it as the basis for restricting CPU
+ * affinity or use cpu_online_mask instead.
+ *
+ * If the resulting mask is empty, leave the affinity unchanged and return
+ * -EINVAL.
+ */
+static int restrict_cpus_allowed_ptr(struct task_struct *p,
+				     struct cpumask *new_mask,
+				     const struct cpumask *subset_mask)
+{
+	struct affinity_context ac = {
+		.new_mask  = new_mask,
+		.flags     = 0,
+	};
+	struct rq *rq;
+	struct rq_flags rf;
+	int err;
+
+	raw_spin_lock_irqsave(&p->pi_lock, rf.flags);
+	rq = __task_modify_lock(p, &rf);
+
+	if (!cpumask_and(new_mask, task_user_cpus(p), subset_mask)) {
+		err = -EINVAL;
+		goto err_unlock;
+	}
+
+	return __set_cpus_allowed_ptr_locked(p, &ac, rq, &rf);
+
+err_unlock:
+	__task_modify_unlock(p, &rf);
+	raw_spin_unlock_irqrestore(&p->pi_lock, rf.flags);
+	return err;
+}
+
+/*
+ * Restrict the CPU affinity of task @p so that it is a subset of
+ * task_cpu_possible_mask() and point @p->user_cpus_ptr to a copy of the
+ * old affinity mask. If the resulting mask is empty, we warn and walk
+ * up the cpuset hierarchy until we find a suitable mask.
+ */
+void force_compatible_cpus_allowed_ptr(struct task_struct *p)
+{
+	cpumask_var_t new_mask;
+	const struct cpumask *override_mask = task_cpu_possible_mask(p);
+
+	alloc_cpumask_var(&new_mask, GFP_KERNEL);
+
+	/*
+	 * __migrate_task() can fail silently in the face of concurrent
+	 * offlining of the chosen destination CPU, so take the hotplug
+	 * lock to ensure that the migration succeeds.
+	 */
+	cpus_read_lock();
+	if (!cpumask_available(new_mask))
+		goto out_set_mask;
+
+	if (!restrict_cpus_allowed_ptr(p, new_mask, override_mask))
+		goto out_free_mask;
+
+	/*
+	 * We failed to find a valid subset of the affinity mask for the
+	 * task, so override it based on its cpuset hierarchy.
+	 */
+	cpuset_cpus_allowed(p, new_mask);
+	override_mask = new_mask;
+
+out_set_mask:
+	if (printk_ratelimit()) {
+		printk_deferred("Overriding affinity for process %d (%s) to CPUs %*pbl\n",
+				task_pid_nr(p), p->comm,
+				cpumask_pr_args(override_mask));
+	}
+
+	WARN_ON(set_cpus_allowed_ptr(p, override_mask));
+out_free_mask:
+	cpus_read_unlock();
+	free_cpumask_var(new_mask);
+}
+
+/*
+ * Restore the affinity of a task @p which was previously restricted by a
+ * call to force_compatible_cpus_allowed_ptr().
+ *
+ * It is the caller's responsibility to serialise this with any calls to
+ * force_compatible_cpus_allowed_ptr(@p).
+ */
+void relax_compatible_cpus_allowed_ptr(struct task_struct *p)
+{
+	struct affinity_context ac = {
+		.new_mask  = task_user_cpus(p),
+		.flags     = 0,
+	};
+	int ret;
+
+	/*
+	 * Try to restore the old affinity mask with __sched_setaffinity().
+	 * Cpuset masking will be done there too.
+	 */
+	ret = __sched_setaffinity(p, &ac);
+	WARN_ON_ONCE(ret);
+}
+
+static void
+ttwu_stat(struct task_struct *p, int cpu, int wake_flags)
+{
+	struct rq *rq;
+
+	if (!schedstat_enabled())
+		return;
+
+	rq = this_rq();
+
+	if (cpu == rq->cpu) {
+		__schedstat_inc(rq->ttwu_local);
+		__schedstat_inc(p->stats.nr_wakeups_local);
+	} else {
+		/** Alt schedule FW ToDo:
+		 * How to do ttwu_wake_remote
+		 */
+	}
+
+	__schedstat_inc(rq->ttwu_count);
+	__schedstat_inc(p->stats.nr_wakeups);
+}
+
+/*
+ * Mark the task runnable.
+ */
+static inline void ttwu_do_wakeup(struct task_struct *p)
+{
+	WRITE_ONCE(p->__state, TASK_RUNNING);
+	trace_sched_wakeup(p);
+}
+
+static inline void ttwu_do_activate(struct task_struct *p, int wake_flags)
+{
+	struct sched_run_queue *srq = cpu_srq(0);
+	struct rq *rq;
+
+	if (p->sched_contributes_to_load)
+		atomic_dec(&srq->nr_uninterruptible);
+
+	if (p->in_iowait) {
+		delayacct_blkio_end(p);
+		atomic_dec(&srq->nr_iowait);
+	}
+
+	trace_sched_wakeup(p);
+
+	if ((wake_flags & WF_CURRENT_CPU) && cpumask_test_cpu(smp_processor_id(), p->cpus_ptr)) {
+		rq = this_rq();
+		raw_spin_lock(&rq->lock);
+	} else
+		rq = wakeup_rq_trylock(p);
+
+	if (rq) {
+		if (unlikely(!cpumask_test_cpu(cpu_of(rq), cpu_active_mask))) {
+			raw_spin_unlock(&rq->lock);
+			rq = cpu_rq(select_fallback_rq(task_cpu(p), p));
+			raw_spin_lock(&rq->lock);
+		}
+
+		wakeup_preempt_on_rq(p, rq);
+	} else
+		activate_task(p, srq);
+}
+
+/*
+ * Consider @p being inside a wait loop:
+ *
+ *   for (;;) {
+ *      set_current_state(TASK_UNINTERRUPTIBLE);
+ *
+ *      if (CONDITION)
+ *         break;
+ *
+ *      schedule();
+ *   }
+ *   __set_current_state(TASK_RUNNING);
+ *
+ * between set_current_state() and schedule(). In this case @p is still
+ * runnable, so all that needs doing is change p->state back to TASK_RUNNING in
+ * an atomic manner.
+ *
+ * By taking task_rq(p)->lock we serialize against schedule(), if @p->on_rq
+ * then schedule() must still happen and p->state can be changed to
+ * TASK_RUNNING. Otherwise we lost the race, schedule() has happened, and we
+ * need to do a full wakeup with enqueue.
+ *
+ * Returns: %true when the wakeup is done,
+ *          %false otherwise.
+ */
+static int ttwu_runnable(struct task_struct *p, int wake_flags)
+{
+	struct rq *rq;
+	int ret = 0;
+
+	rq = task_rq(p);
+	raw_spin_lock(&rq->lock);
+	if (task_on_rq_queued(p)) {
+		update_rq_clock(rq);
+		if (!task_on_cpu(p)) {
+			/*
+			 * When on_rq && !on_cpu the task is preempted, see if
+			 * it should preempt the task that is current now.
+			 */
+			if (task_sched_prio(p) < READ_ONCE(cpu_prio[cpu_of(rq)])) {
+				resched_curr(rq);
+			}
+		}
+		ttwu_do_wakeup(p);
+		ret = 1;
+	}
+	raw_spin_unlock(&rq->lock);
+
+	return ret;
+}
+
+void sched_ttwu_pending(void *arg)
+{
+	struct llist_node *llist = arg;
+	struct rq *rq = this_rq();
+	struct task_struct *p, *t;
+	struct rq_flags rf;
+
+	if (!llist)
+		return;
+
+	rq_lock_irqsave(rq, &rf);
+	update_rq_clock(rq);
+
+	llist_for_each_entry_safe(p, t, llist, wake_entry.llist) {
+		if (WARN_ON_ONCE(p->on_cpu))
+			smp_cond_load_acquire(&p->on_cpu, !VAL);
+
+		if (WARN_ON_ONCE(task_cpu(p) != cpu_of(rq)))
+			set_task_cpu(p, cpu_of(rq));
+
+		ttwu_do_activate(p, p->sched_remote_wakeup ? WF_MIGRATED : 0);
+	}
+
+	/*
+	 * Must be after enqueueing at least once task such that
+	 * idle_cpu() does not observe a false-negative -- if it does,
+	 * it is possible for select_idle_siblings() to stack a number
+	 * of tasks on this CPU during that window.
+	 *
+	 * It is OK to clear ttwu_pending when another task pending.
+	 * We will receive IPI after local IRQ enabled and then enqueue it.
+	 * Since now nr_running > 0, idle_cpu() will always get correct result.
+	 */
+	WRITE_ONCE(rq->ttwu_pending, 0);
+	rq_unlock_irqrestore(rq, &rf);
+}
+
+/*
+ * Prepare the scene for sending an IPI for a remote smp_call
+ *
+ * Returns true if the caller can proceed with sending the IPI.
+ * Returns false otherwise.
+ */
+bool call_function_single_prep_ipi(int cpu)
+{
+	if (set_nr_if_polling(cpu_rq(cpu)->idle)) {
+		trace_sched_wake_idle_without_ipi(cpu);
+		return false;
+	}
+
+	return true;
+}
+
+/*
+ * Queue a task on the target CPUs wake_list and wake the CPU via IPI if
+ * necessary. The wakee CPU on receipt of the IPI will queue the task
+ * via sched_ttwu_wakeup() for activation so the wakee incurs the cost
+ * of the wakeup instead of the waker.
+ */
+static void __ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags)
+{
+	struct rq *rq = cpu_rq(cpu);
+
+	p->sched_remote_wakeup = !!(wake_flags & WF_MIGRATED);
+
+	WRITE_ONCE(rq->ttwu_pending, 1);
+	__smp_call_single_queue(cpu, &p->wake_entry.llist);
+}
+
+static inline bool ttwu_queue_cond(struct task_struct *p, int cpu)
+{
+	int this_cpu = smp_processor_id();
+
+	/*
+	 * Do not complicate things with the async wake_list while the CPU is
+	 * in hotplug state.
+	 */
+	if (!cpu_active(cpu))
+		return false;
+
+	/* Ensure the task will still be allowed to run on the CPU. */
+	if (!cpumask_test_cpu(cpu, p->cpus_ptr))
+		return false;
+
+	/*
+	 * If the CPU does not share cache, then queue the task on the
+	 * remote rqs wakelist to avoid accessing remote data.
+	 */
+	if (!cpus_share_cache(this_cpu, cpu))
+		return true;
+
+	if (cpu == this_cpu)
+		return false;
+
+	/*
+	 * If the wakee cpu is idle, or the task is descheduling and the
+	 * only running task on the CPU, then use the wakelist to offload
+	 * the task activation to the idle (or soon-to-be-idle) CPU as
+	 * the current CPU is likely busy. nr_running is checked to
+	 * avoid unnecessary task stacking.
+	 *
+	 * Note that we can only get here with (wakee) p->on_rq=0,
+	 * p->on_cpu can be whatever, we've done the dequeue, so
+	 * the wakee has been accounted out of ->nr_running.
+	 */
+	if (cpumask_test_cpu(cpu, sched_idle_mask) && !srq_nr_queued(cpu) &&
+	    is_preempt_list_empty(cpu))
+		return true;
+
+	return false;
+}
+
+static bool ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags)
+{
+	if (__is_defined(ALT_SCHED_TTWU_QUEUE) && ttwu_queue_cond(p, cpu)) {
+		sched_clock_cpu(cpu); /* Sync clocks across CPUs */
+		__ttwu_queue_wakelist(p, cpu, wake_flags);
+		return true;
+	}
+
+	return false;
+}
+
+void wake_up_if_idle(int cpu)
+{
+	struct rq *rq = cpu_rq(cpu);
+
+	guard(rcu)();
+	if (is_idle_task(rcu_dereference(rq->curr))) {
+		guard(raw_spinlock_irqsave)(&rq->lock);
+		if (is_idle_task(rq->curr))
+			resched_curr(rq);
+	}
+}
+
+extern struct static_key_false sched_asym_cpucapacity;
+
+static __always_inline bool sched_asym_cpucap_active(void)
+{
+	return static_branch_unlikely(&sched_asym_cpucapacity);
+}
+
+bool cpus_equal_capacity(int this_cpu, int that_cpu)
+{
+	if (!sched_asym_cpucap_active())
+		return true;
+
+	if (this_cpu == that_cpu)
+		return true;
+
+	return arch_scale_cpu_capacity(this_cpu) == arch_scale_cpu_capacity(that_cpu);
+}
+
+static inline void ttwu_queue(struct task_struct *p, int wake_flags)
+{
+	if (ttwu_queue_wakelist(p, 0, wake_flags))
+		return;
+
+	ttwu_do_activate(p, wake_flags);
+}
+
+/*
+ * Invoked from try_to_wake_up() to check whether the task can be woken up.
+ *
+ * The caller holds p::pi_lock if p != current or has preemption
+ * disabled when p == current.
+ *
+ * The rules of saved_state:
+ *
+ *   The related locking code always holds p::pi_lock when updating
+ *   p::saved_state, which means the code is fully serialized in both cases.
+ *
+ *  For PREEMPT_RT, the lock wait and lock wakeups happen via TASK_RTLOCK_WAIT.
+ *  No other bits set. This allows to distinguish all wakeup scenarios.
+ *
+ *  For FREEZER, the wakeup happens via TASK_FROZEN. No other bits set. This
+ *  allows us to prevent early wakeup of tasks before they can be run on
+ *  asymmetric ISA architectures (eg ARMv9).
+ */
+static __always_inline
+bool ttwu_state_match(struct task_struct *p, unsigned int state, int *success)
+{
+	int match;
+
+	if (IS_ENABLED(CONFIG_DEBUG_PREEMPT)) {
+		WARN_ON_ONCE((state & TASK_RTLOCK_WAIT) &&
+			     state != TASK_RTLOCK_WAIT);
+	}
+
+	*success = !!(match = __task_state_match(p, state));
+
+	/*
+	 * Saved state preserves the task state across blocking on
+	 * an RT lock or TASK_FREEZABLE tasks.  If the state matches,
+	 * set p::saved_state to TASK_RUNNING, but do not wake the task
+	 * because it waits for a lock wakeup or __thaw_task(). Also
+	 * indicate success because from the regular waker's point of
+	 * view this has succeeded.
+	 *
+	 * After acquiring the lock the task will restore p::__state
+	 * from p::saved_state which ensures that the regular
+	 * wakeup is not lost. The restore will also set
+	 * p::saved_state to TASK_RUNNING so any further tests will
+	 * not result in false positives vs. @success
+	 */
+	if (match < 0)
+		p->saved_state = TASK_RUNNING;
+
+	return match > 0;
+}
+
+/*
+ * Notes on Program-Order guarantees on SMP systems.
+ *
+ *  MIGRATION
+ *
+ * The basic program-order guarantee on SMP systems is that when a task [t]
+ * migrates, all its activity on its old CPU [c0] happens-before any subsequent
+ * execution on its new CPU [c1].
+ *
+ * For migration (of runnable tasks) this is provided by the following means:
+ *
+ *  A) UNLOCK of the rq(c0)->lock scheduling out task t
+ *  B) migration for t is required to synchronize *both* rq(c0)->lock and
+ *     rq(c1)->lock (if not at the same time, then in that order).
+ *  C) LOCK of the rq(c1)->lock scheduling in task
+ *
+ * Transitivity guarantees that B happens after A and C after B.
+ * Note: we only require RCpc transitivity.
+ * Note: the CPU doing B need not be c0 or c1
+ *
+ * Example:
+ *
+ *   CPU0            CPU1            CPU2
+ *
+ *   LOCK rq(0)->lock
+ *   sched-out X
+ *   sched-in Y
+ *   UNLOCK rq(0)->lock
+ *
+ *                                   LOCK rq(0)->lock // orders against CPU0
+ *                                   dequeue X
+ *                                   UNLOCK rq(0)->lock
+ *
+ *                                   LOCK rq(1)->lock
+ *                                   enqueue X
+ *                                   UNLOCK rq(1)->lock
+ *
+ *                   LOCK rq(1)->lock // orders against CPU2
+ *                   sched-out Z
+ *                   sched-in X
+ *                   UNLOCK rq(1)->lock
+ *
+ *
+ *  BLOCKING -- aka. SLEEP + WAKEUP
+ *
+ * For blocking we (obviously) need to provide the same guarantee as for
+ * migration. However the means are completely different as there is no lock
+ * chain to provide order. Instead we do:
+ *
+ *   1) smp_store_release(X->on_cpu, 0)   -- finish_task()
+ *   2) smp_cond_load_acquire(!X->on_cpu) -- try_to_wake_up()
+ *
+ * Example:
+ *
+ *   CPU0 (schedule)  CPU1 (try_to_wake_up) CPU2 (schedule)
+ *
+ *   LOCK rq(0)->lock LOCK X->pi_lock
+ *   dequeue X
+ *   sched-out X
+ *   smp_store_release(X->on_cpu, 0);
+ *
+ *                    smp_cond_load_acquire(&X->on_cpu, !VAL);
+ *                    X->state = WAKING
+ *                    set_task_cpu(X,2)
+ *
+ *                    LOCK rq(2)->lock
+ *                    enqueue X
+ *                    X->state = RUNNING
+ *                    UNLOCK rq(2)->lock
+ *
+ *                                          LOCK rq(2)->lock // orders against CPU1
+ *                                          sched-out Z
+ *                                          sched-in X
+ *                                          UNLOCK rq(2)->lock
+ *
+ *                    UNLOCK X->pi_lock
+ *   UNLOCK rq(0)->lock
+ *
+ *
+ * However; for wakeups there is a second guarantee we must provide, namely we
+ * must observe the state that lead to our wakeup. That is, not only must our
+ * task observe its own prior state, it must also observe the stores prior to
+ * its wakeup.
+ *
+ * This means that any means of doing remote wakeups must order the CPU doing
+ * the wakeup against the CPU the task is going to end up running on. This,
+ * however, is already required for the regular Program-Order guarantee above,
+ * since the waking CPU is the one issueing the ACQUIRE (smp_cond_load_acquire).
+ *
+ */
+
+/**
+ * try_to_wake_up - wake up a thread
+ * @p: the thread to be awakened
+ * @state: the mask of task states that can be woken
+ * @wake_flags: wake modifier flags (WF_*)
+ *
+ * Conceptually does:
+ *
+ *   If (@state & @p->state) @p->state = TASK_RUNNING.
+ *
+ * If the task was not queued/runnable, also place it back on a runqueue.
+ *
+ * This function is atomic against schedule() which would dequeue the task.
+ *
+ * It issues a full memory barrier before accessing @p->state, see the comment
+ * with set_current_state().
+ *
+ * Uses p->pi_lock to serialize against concurrent wake-ups.
+ *
+ * Relies on p->pi_lock stabilizing:
+ *  - p->sched_class
+ *  - p->cpus_ptr
+ *  - p->sched_task_group
+ * in order to do migration, see its use of select_task_rq()/set_task_cpu().
+ *
+ * Tries really hard to only take one task_rq(p)->lock for performance.
+ * Takes rq->lock in:
+ *  - ttwu_runnable()    -- old rq, unavoidable, see comment there;
+ *  - ttwu_queue()       -- new rq, for enqueue of the task;
+ *  - psi_ttwu_dequeue() -- much sadness :-( accounting will kill us.
+ *
+ * As a consequence we race really badly with just about everything. See the
+ * many memory barriers and their comments for details.
+ *
+ * Return: %true if @p->state changes (an actual wakeup was done),
+ *	   %false otherwise.
+ */
+int try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
+{
+	guard(preempt)();
+	int success = 0;
+
+	if (p == current) {
+		/*
+		 * We're waking current, this means 'p->on_rq' and 'task_cpu(p)
+		 * == smp_processor_id()'. Together this means we can special
+		 * case the whole 'p->on_rq && ttwu_runnable()' case below
+		 * without taking any locks.
+		 *
+		 * In particular:
+		 *  - we rely on Program-Order guarantees for all the ordering,
+		 *  - we're serialized against set_special_state() by virtue of
+		 *    it disabling IRQs (this allows not taking ->pi_lock).
+		 */
+		if (!ttwu_state_match(p, state, &success))
+			goto out;
+
+		trace_sched_waking(p);
+		ttwu_do_wakeup(p);
+		goto out;
+	}
+
+	/*
+	 * If we are going to wake up a thread waiting for CONDITION we
+	 * need to ensure that CONDITION=1 done by the caller can not be
+	 * reordered with p->state check below. This pairs with smp_store_mb()
+	 * in set_current_state() that the waiting thread does.
+	 */
+	scoped_guard (raw_spinlock_irqsave, &p->pi_lock) {
+		smp_mb__after_spinlock();
+		if (!ttwu_state_match(p, state, &success))
+			break;
+
+		trace_sched_waking(p);
+
+		/*
+		 * Ensure we load p->on_rq _after_ p->state, otherwise it would
+		 * be possible to, falsely, observe p->on_rq == 0 and get stuck
+		 * in smp_cond_load_acquire() below.
+		 *
+		 * sched_ttwu_pending()			try_to_wake_up()
+		 *   STORE p->on_rq = 1			  LOAD p->state
+		 *   UNLOCK rq->lock
+		 *
+		 * __schedule() (switch to task 'p')
+		 *   LOCK rq->lock			  smp_rmb();
+		 *   smp_mb__after_spinlock();
+		 *   UNLOCK rq->lock
+		 *
+		 * [task p]
+		 *   STORE p->state = UNINTERRUPTIBLE	  LOAD p->on_rq
+		 *
+		 * Pairs with the LOCK+smp_mb__after_spinlock() on rq->lock in
+		 * __schedule().  See the comment for smp_mb__after_spinlock().
+		 *
+		 * A similar smp_rmb() lives in __task_needs_rq_lock().
+		 */
+		smp_rmb();
+		if (READ_ONCE(p->on_rq) && ttwu_runnable(p, wake_flags))
+			break;
+
+		/*
+		 * Ensure we load p->on_cpu _after_ p->on_rq, otherwise it would be
+		 * possible to, falsely, observe p->on_cpu == 0.
+		 *
+		 * One must be running (->on_cpu == 1) in order to remove oneself
+		 * from the runqueue.
+		 *
+		 * __schedule() (switch to task 'p')	try_to_wake_up()
+		 *   STORE p->on_cpu = 1		  LOAD p->on_rq
+		 *   UNLOCK rq->lock
+		 *
+		 * __schedule() (put 'p' to sleep)
+		 *   LOCK rq->lock			  smp_rmb();
+		 *   smp_mb__after_spinlock();
+		 *   STORE p->on_rq = 0			  LOAD p->on_cpu
+		 *
+		 * Pairs with the LOCK+smp_mb__after_spinlock() on rq->lock in
+		 * __schedule().  See the comment for smp_mb__after_spinlock().
+		 *
+		 * Form a control-dep-acquire with p->on_rq == 0 above, to ensure
+		 * schedule()'s block_task() has 'happened' and p will no longer
+		 * care about it's own p->state. See the comment in __schedule().
+		 */
+		smp_acquire__after_ctrl_dep();
+
+		/*
+		 * We're doing the wakeup (@success == 1), they did a dequeue (p->on_rq
+		 * == 0), which means we need to do an enqueue, change p->state to
+		 * TASK_WAKING such that we can unlock p->pi_lock before doing the
+		 * enqueue, such as ttwu_queue_wakelist().
+		 */
+		WRITE_ONCE(p->__state, TASK_WAKING);
+
+		/*
+		 * If the owning (remote) CPU is still in the middle of schedule() with
+		 * this task as prev, considering queueing p on the remote CPUs wake_list
+		 * which potentially sends an IPI instead of spinning on p->on_cpu to
+		 * let the waker make forward progress. This is safe because IRQs are
+		 * disabled and the IPI will deliver after on_cpu is cleared.
+		 *
+		 * Ensure we load task_cpu(p) after p->on_cpu:
+		 *
+		 * set_task_cpu(p, cpu);
+		 *   STORE p->cpu = @cpu
+		 * __schedule() (switch to task 'p')
+		 *   LOCK rq->lock
+		 *   smp_mb__after_spin_lock()          smp_cond_load_acquire(&p->on_cpu)
+		 *   STORE p->on_cpu = 1                LOAD p->cpu
+		 *
+		 * to ensure we observe the correct CPU on which the task is currently
+		 * scheduling.
+		 */
+		if (smp_load_acquire(&p->on_cpu) &&
+		    ttwu_queue_wakelist(p, task_cpu(p), wake_flags))
+			break;
+
+		/*
+		 * If the owning (remote) CPU is still in the middle of schedule() with
+		 * this task as prev, wait until it's done referencing the task.
+		 *
+		 * Pairs with the smp_store_release() in finish_task().
+		 *
+		 * This ensures that tasks getting woken will be fully ordered against
+		 * their previous state and preserve Program Order.
+		 */
+		smp_cond_load_acquire(&p->on_cpu, !VAL);
+
+		sched_task_ttwu(p);
+
+		ttwu_queue(p, wake_flags);
+	}
+out:
+	if (success)
+		ttwu_stat(p, task_cpu(p), wake_flags);
+
+	return success;
+}
+
+static bool __task_needs_rq_lock(struct task_struct *p)
+{
+	unsigned int state = READ_ONCE(p->__state);
+
+	/*
+	 * Since pi->lock blocks try_to_wake_up(), we don't need rq->lock when
+	 * the task is blocked. Make sure to check @state since ttwu() can drop
+	 * locks at the end, see ttwu_queue_wakelist().
+	 */
+	if (state == TASK_RUNNING || state == TASK_WAKING)
+		return true;
+
+	/*
+	 * Ensure we load p->on_rq after p->__state, otherwise it would be
+	 * possible to, falsely, observe p->on_rq == 0.
+	 *
+	 * See try_to_wake_up() for a longer comment.
+	 */
+	smp_rmb();
+	if (p->on_rq)
+		return true;
+
+	/*
+	 * Ensure the task has finished __schedule() and will not be referenced
+	 * anymore. Again, see try_to_wake_up() for a longer comment.
+	 */
+	smp_rmb();
+	smp_cond_load_acquire(&p->on_cpu, !VAL);
+
+	return false;
+}
+
+/**
+ * task_call_func - Invoke a function on task in fixed state
+ * @p: Process for which the function is to be invoked, can be @current.
+ * @func: Function to invoke.
+ * @arg: Argument to function.
+ *
+ * Fix the task in it's current state by avoiding wakeups and or rq operations
+ * and call @func(@arg) on it.  This function can use task_is_runnable() and
+ * task_curr() to work out what the state is, if required.  Given that @func
+ * can be invoked with a runqueue lock held, it had better be quite
+ * lightweight.
+ *
+ * Returns:
+ *   Whatever @func returns
+ */
+int task_call_func(struct task_struct *p, task_call_f func, void *arg)
+{
+	struct rq *rq = NULL;
+	struct rq_flags rf;
+	int ret;
+
+	raw_spin_lock_irqsave(&p->pi_lock, rf.flags);
+
+	if (__task_needs_rq_lock(p))
+		rq = __task_rq_lock(p, &rf);
+
+	/*
+	 * At this point the task is pinned; either:
+	 *  - blocked and we're holding off wakeups      (pi->lock)
+	 *  - woken, and we're holding off enqueue       (rq->lock)
+	 *  - queued, and we're holding off schedule     (rq->lock)
+	 *  - running, and we're holding off de-schedule (rq->lock)
+	 *
+	 * The called function (@func) can use: task_curr(), p->on_rq and
+	 * p->__state to differentiate between these states.
+	 */
+	ret = func(p, arg);
+
+	if (rq)
+		__task_rq_unlock(&rf);
+
+	raw_spin_unlock_irqrestore(&p->pi_lock, rf.flags);
+	return ret;
+}
+
+/**
+ * cpu_curr_snapshot - Return a snapshot of the currently running task
+ * @cpu: The CPU on which to snapshot the task.
+ *
+ * Returns the task_struct pointer of the task "currently" running on
+ * the specified CPU.  If the same task is running on that CPU throughout,
+ * the return value will be a pointer to that task's task_struct structure.
+ * If the CPU did any context switches even vaguely concurrently with the
+ * execution of this function, the return value will be a pointer to the
+ * task_struct structure of a randomly chosen task that was running on
+ * that CPU somewhere around the time that this function was executing.
+ *
+ * If the specified CPU was offline, the return value is whatever it
+ * is, perhaps a pointer to the task_struct structure of that CPU's idle
+ * task, but there is no guarantee.  Callers wishing a useful return
+ * value must take some action to ensure that the specified CPU remains
+ * online throughout.
+ *
+ * This function executes full memory barriers before and after fetching
+ * the pointer, which permits the caller to confine this function's fetch
+ * with respect to the caller's accesses to other shared variables.
+ */
+struct task_struct *cpu_curr_snapshot(int cpu)
+{
+	struct task_struct *t;
+
+	smp_mb(); /* Pairing determined by caller's synchronization design. */
+	t = rcu_dereference(cpu_curr(cpu));
+	smp_mb(); /* Pairing determined by caller's synchronization design. */
+	return t;
+}
+
+/**
+ * wake_up_process - Wake up a specific process
+ * @p: The process to be woken up.
+ *
+ * Attempt to wake up the nominated process and move it to the set of runnable
+ * processes.
+ *
+ * Return: 1 if the process was woken up, 0 if it was already running.
+ *
+ * This function executes a full memory barrier before accessing the task state.
+ */
+int wake_up_process(struct task_struct *p)
+{
+	return try_to_wake_up(p, TASK_NORMAL, 0);
+}
+EXPORT_SYMBOL(wake_up_process);
+
+int wake_up_state(struct task_struct *p, unsigned int state)
+{
+	return try_to_wake_up(p, state, 0);
+}
+
+/*
+ * Perform scheduler related setup for a newly forked process p.
+ * p is forked by current.
+ *
+ * __sched_fork() is basic setup which is also used by sched_init() to
+ * initialize the boot CPU's idle task.
+ */
+static inline void __sched_fork(u64 clone_flags, struct task_struct *p)
+{
+	p->on_rq			= 0;
+	p->on_cpu			= 0;
+	p->utime			= 0;
+	p->stime			= 0;
+	p->sched_time			= 0;
+
+#ifdef CONFIG_SCHEDSTATS
+	/* Even if schedstat is disabled, there should not be garbage */
+	memset(&p->stats, 0, sizeof(p->stats));
+#endif
+
+#ifdef CONFIG_PREEMPT_NOTIFIERS
+	INIT_HLIST_HEAD(&p->preempt_notifiers);
+#endif
+
+#ifdef CONFIG_COMPACTION
+	p->capture_control = NULL;
+#endif
+	p->wake_entry.u_flags = CSD_TYPE_TTWU;
+}
+
+/*
+ * fork()/clone()-time setup:
+ */
+int sched_fork(u64 clone_flags, struct task_struct *p)
+{
+	__sched_fork(clone_flags, p);
+	/*
+	 * We mark the process as NEW here. This guarantees that
+	 * nobody will actually run it, and a signal or other external
+	 * event cannot wake it up and insert it on the runqueue either.
+	 */
+	p->__state = TASK_NEW;
+
+	/*
+	 * Make sure we do not leak PI boosting priority to the child.
+	 */
+	p->prio = current->normal_prio;
+
+	/*
+	 * Revert to default priority/policy on fork if requested.
+	 */
+	if (unlikely(p->sched_reset_on_fork)) {
+		if (task_has_rt_policy(p)) {
+			p->policy = SCHED_NORMAL;
+			p->static_prio = NICE_TO_PRIO(0);
+			p->rt_priority = 0;
+		} else if (PRIO_TO_NICE(p->static_prio) < 0)
+			p->static_prio = NICE_TO_PRIO(0);
+
+		p->prio = p->normal_prio = p->static_prio;
+
+		/*
+		 * We don't need the reset flag anymore after the fork. It has
+		 * fulfilled its duty:
+		 */
+		p->sched_reset_on_fork = 0;
+	}
+
+#ifdef CONFIG_SCHED_INFO
+	if (unlikely(sched_info_on()))
+		memset(&p->sched_info, 0, sizeof(p->sched_info));
+#endif
+	init_task_preempt_count(p);
+
+	return 0;
+}
+
+int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs)
+{
+	unsigned long flags;
+	struct rq *rq;
+
+	/*
+	 * Because we're not yet on the pid-hash, p->pi_lock isn't strictly
+	 * required yet, but lockdep gets upset if rules are violated.
+	 */
+	raw_spin_lock_irqsave(&p->pi_lock, flags);
+	/*
+	 * Share the timeslice between parent and child, thus the
+	 * total amount of pending timeslices in the system doesn't change,
+	 * resulting in more scheduling fairness.
+	 */
+	rq = this_rq();
+	raw_spin_lock(&rq->lock);
+
+	rq->curr->time_slice /= 2;
+	p->time_slice = rq->curr->time_slice;
+#ifdef CONFIG_SCHED_HRTICK
+	hrtick_start(rq, rq->curr->time_slice);
+#endif
+
+	if (p->time_slice < RESCHED_NS) {
+		p->time_slice = sysctl_sched_base_slice;
+		resched_curr(rq);
+	}
+	sched_task_fork(p, rq);
+	raw_spin_unlock(&rq->lock);
+
+	/*
+	 * We're setting the CPU for the first time, we don't migrate,
+	 * so use __set_task_cpu().
+	 */
+	__set_task_cpu(p, smp_processor_id());
+	raw_spin_unlock_irqrestore(&p->pi_lock, flags);
+
+	return 0;
+}
+
+void sched_cancel_fork(struct task_struct *p)
+{
+}
+
+static void sched_mm_cid_fork(struct task_struct *t);
+
+void sched_post_fork(struct task_struct *p)
+{
+	sched_mm_cid_fork(p);
+}
+
+#ifdef CONFIG_SCHEDSTATS
+
+DEFINE_STATIC_KEY_FALSE(sched_schedstats);
+
+static void set_schedstats(bool enabled)
+{
+	if (enabled)
+		static_branch_enable(&sched_schedstats);
+	else
+		static_branch_disable(&sched_schedstats);
+}
+
+void force_schedstat_enabled(void)
+{
+	if (!schedstat_enabled()) {
+		pr_info("kernel profiling enabled schedstats, disable via kernel.sched_schedstats.\n");
+		static_branch_enable(&sched_schedstats);
+	}
+}
+
+static int __init setup_schedstats(char *str)
+{
+	int ret = 0;
+	if (!str)
+		goto out;
+
+	if (!strcmp(str, "enable")) {
+		set_schedstats(true);
+		ret = 1;
+	} else if (!strcmp(str, "disable")) {
+		set_schedstats(false);
+		ret = 1;
+	}
+out:
+	if (!ret)
+		pr_warn("Unable to parse schedstats=\n");
+
+	return ret;
+}
+__setup("schedstats=", setup_schedstats);
+
+#ifdef CONFIG_PROC_SYSCTL
+static int sysctl_schedstats(const struct ctl_table *table, int write, void *buffer,
+		size_t *lenp, loff_t *ppos)
+{
+	struct ctl_table t;
+	int err;
+	int state = static_branch_likely(&sched_schedstats);
+
+	if (write && !capable(CAP_SYS_ADMIN))
+		return -EPERM;
+
+	t = *table;
+	t.data = &state;
+	err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos);
+	if (err < 0)
+		return err;
+	if (write)
+		set_schedstats(state);
+	return err;
+}
+#endif /* CONFIG_PROC_SYSCTL */
+#endif /* CONFIG_SCHEDSTATS */
+
+#ifdef CONFIG_SYSCTL
+static const struct ctl_table sched_core_sysctls[] = {
+#ifdef CONFIG_SCHEDSTATS
+	{
+		.procname       = "sched_schedstats",
+		.data           = NULL,
+		.maxlen         = sizeof(unsigned int),
+		.mode           = 0644,
+		.proc_handler   = sysctl_schedstats,
+		.extra1         = SYSCTL_ZERO,
+		.extra2         = SYSCTL_ONE,
+	},
+#endif /* CONFIG_SCHEDSTATS */
+};
+static int __init sched_core_sysctl_init(void)
+{
+	register_sysctl_init("kernel", sched_core_sysctls);
+	return 0;
+}
+late_initcall(sched_core_sysctl_init);
+#endif /* CONFIG_SYSCTL */
+
+/*
+ * wake_up_new_task - wake up a newly created task for the first time.
+ *
+ * This function will do some initial scheduler statistics housekeeping
+ * that must be done for every newly created context, then puts the task
+ * on the runqueue and wakes it.
+ */
+void wake_up_new_task(struct task_struct *p)
+{
+	struct rq *rq;
+	unsigned long flags;
+
+	raw_spin_lock_irqsave(&p->pi_lock, flags);
+
+	trace_sched_wakeup_new(p);
+
+	rq = wakeup_rq_trylock(p);
+
+	if (rq)
+		wakeup_preempt_on_rq(p, rq);
+	else
+		activate_task(p, cpu_srq(0));
+
+	raw_spin_unlock_irqrestore(&p->pi_lock, flags);
+}
+
+#ifdef CONFIG_PREEMPT_NOTIFIERS
+
+static DEFINE_STATIC_KEY_FALSE(preempt_notifier_key);
+
+void preempt_notifier_inc(void)
+{
+	static_branch_inc(&preempt_notifier_key);
+}
+EXPORT_SYMBOL_GPL(preempt_notifier_inc);
+
+void preempt_notifier_dec(void)
+{
+	static_branch_dec(&preempt_notifier_key);
+}
+EXPORT_SYMBOL_GPL(preempt_notifier_dec);
+
+/**
+ * preempt_notifier_register - tell me when current is being preempted & rescheduled
+ * @notifier: notifier struct to register
+ */
+void preempt_notifier_register(struct preempt_notifier *notifier)
+{
+	if (!static_branch_unlikely(&preempt_notifier_key))
+		WARN(1, "registering preempt_notifier while notifiers disabled\n");
+
+	hlist_add_head(&notifier->link, &current->preempt_notifiers);
+}
+EXPORT_SYMBOL_GPL(preempt_notifier_register);
+
+/**
+ * preempt_notifier_unregister - no longer interested in preemption notifications
+ * @notifier: notifier struct to unregister
+ *
+ * This is *not* safe to call from within a preemption notifier.
+ */
+void preempt_notifier_unregister(struct preempt_notifier *notifier)
+{
+	hlist_del(&notifier->link);
+}
+EXPORT_SYMBOL_GPL(preempt_notifier_unregister);
+
+static void __fire_sched_in_preempt_notifiers(struct task_struct *curr)
+{
+	struct preempt_notifier *notifier;
+
+	hlist_for_each_entry(notifier, &curr->preempt_notifiers, link)
+		notifier->ops->sched_in(notifier, raw_smp_processor_id());
+}
+
+static __always_inline void fire_sched_in_preempt_notifiers(struct task_struct *curr)
+{
+	if (static_branch_unlikely(&preempt_notifier_key))
+		__fire_sched_in_preempt_notifiers(curr);
+}
+
+static void
+__fire_sched_out_preempt_notifiers(struct task_struct *curr,
+				   struct task_struct *next)
+{
+	struct preempt_notifier *notifier;
+
+	hlist_for_each_entry(notifier, &curr->preempt_notifiers, link)
+		notifier->ops->sched_out(notifier, next);
+}
+
+static __always_inline void
+fire_sched_out_preempt_notifiers(struct task_struct *curr,
+				 struct task_struct *next)
+{
+	if (static_branch_unlikely(&preempt_notifier_key))
+		__fire_sched_out_preempt_notifiers(curr, next);
+}
+
+#else /* !CONFIG_PREEMPT_NOTIFIERS: */
+
+static inline void fire_sched_in_preempt_notifiers(struct task_struct *curr)
+{
+}
+
+static inline void
+fire_sched_out_preempt_notifiers(struct task_struct *curr,
+				 struct task_struct *next)
+{
+}
+
+#endif /* !CONFIG_PREEMPT_NOTIFIERS */
+
+static inline void prepare_task(struct task_struct *next)
+{
+	/*
+	 * Claim the task as running, we do this before switching to it
+	 * such that any running task will have this set.
+	 *
+	 * See the smp_load_acquire(&p->on_cpu) case in ttwu() and
+	 * its ordering comment.
+	 */
+	WRITE_ONCE(next->on_cpu, 1);
+}
+
+static inline void finish_task(struct task_struct *prev, struct rq *rq)
+{
+	/*
+	 * This must be the very last reference to @prev from this CPU. After
+	 * p->on_cpu is cleared, the task can be moved to a different CPU. We
+	 * must ensure this doesn't happen until the switch is completely
+	 * finished.
+	 *
+	 * In particular, the load of prev->state in finish_task_switch() must
+	 * happen before this.
+	 *
+	 * Pairs with the smp_cond_load_acquire() in try_to_wake_up().
+	 */
+	smp_store_release(&prev->on_cpu, 0);
+
+	/*
+	 * Enqueue prev if not rq->idle and prev is not blocked
+	 */
+	if (prev != rq->idle && !rq->block) {
+		struct rq *trq;
+		struct sched_run_queue *srq = rq_srq(rq);
+
+		SRQ_ENQUEUE_TASK(srq, prev, );
+
+		trq = __wakeup_rq_trylock(prev, IDLE_TASK_SCHED_PRIO - 1, prev->cpus_ptr);
+		if (trq) {
+			resched_curr(trq);
+			raw_spin_unlock(&trq->lock);
+		}
+	}
+}
+
+static void do_balance_callbacks(struct rq *rq, struct balance_callback *head)
+{
+	void (*func)(struct rq *rq);
+	struct balance_callback *next;
+
+	lockdep_assert_held(&rq->lock);
+
+	while (head) {
+		func = (void (*)(struct rq *))head->func;
+		next = head->next;
+		head->next = NULL;
+		head = next;
+
+		func(rq);
+	}
+}
+
+static void balance_push(struct rq *rq);
+
+/*
+ * balance_push_callback is a right abuse of the callback interface and plays
+ * by significantly different rules.
+ *
+ * Where the normal balance_callback's purpose is to be ran in the same context
+ * that queued it (only later, when it's safe to drop rq->lock again),
+ * balance_push_callback is specifically targeted at __schedule().
+ *
+ * This abuse is tolerated because it places all the unlikely/odd cases behind
+ * a single test, namely: rq->balance_callback == NULL.
+ */
+struct balance_callback balance_push_callback = {
+	.next = NULL,
+	.func = balance_push,
+};
+
+static inline struct balance_callback *
+__splice_balance_callbacks(struct rq *rq, bool split)
+{
+	struct balance_callback *head = rq->balance_callback;
+
+	if (likely(!head))
+		return NULL;
+
+	lockdep_assert_rq_held(rq);
+	/*
+	 * Must not take balance_push_callback off the list when
+	 * splice_balance_callbacks() and balance_callbacks() are not
+	 * in the same rq->lock section.
+	 *
+	 * In that case it would be possible for __schedule() to interleave
+	 * and observe the list empty.
+	 */
+	if (split && head == &balance_push_callback)
+		head = NULL;
+	else
+		rq->balance_callback = NULL;
+
+	return head;
+}
+
+struct balance_callback *splice_balance_callbacks(struct rq *rq)
+{
+	return __splice_balance_callbacks(rq, true);
+}
+
+static void __balance_callbacks(struct rq *rq)
+{
+	do_balance_callbacks(rq, __splice_balance_callbacks(rq, false));
+}
+
+void balance_callbacks(struct rq *rq, struct balance_callback *head)
+{
+	unsigned long flags;
+
+	if (unlikely(head)) {
+		raw_spin_lock_irqsave(&rq->lock, flags);
+		do_balance_callbacks(rq, head);
+		raw_spin_unlock_irqrestore(&rq->lock, flags);
+	}
+}
+
+static inline void
+prepare_lock_switch(struct rq *rq, struct task_struct *next)
+{
+	/*
+	 * Since the runqueue lock will be released by the next
+	 * task (which is an invalid locking op but in the case
+	 * of the scheduler it's an obvious special-case), so we
+	 * do an early lockdep release here:
+	 */
+	spin_release(&rq->lock.dep_map, _THIS_IP_);
+#ifdef CONFIG_DEBUG_SPINLOCK
+	/* this is a valid case when another task releases the spinlock */
+	rq->lock.owner = next;
+#endif
+}
+
+static inline void finish_lock_switch(struct rq *rq)
+{
+	/*
+	 * If we are tracking spinlock dependencies then we have to
+	 * fix up the runqueue lock - which gets 'carried over' from
+	 * prev into current:
+	 */
+	spin_acquire(&rq->lock.dep_map, 0, 0, _THIS_IP_);
+	__balance_callbacks(rq);
+	hrtick_schedule_exit(rq);
+	raw_spin_unlock_irq(&rq->lock);
+}
+
+/*
+ * NOP if the arch has not defined these:
+ */
+
+#ifndef prepare_arch_switch
+# define prepare_arch_switch(next)	do { } while (0)
+#endif
+
+#ifndef finish_arch_post_lock_switch
+# define finish_arch_post_lock_switch()	do { } while (0)
+#endif
+
+static inline void kmap_local_sched_out(void)
+{
+#ifdef CONFIG_KMAP_LOCAL
+	if (unlikely(current->kmap_ctrl.idx))
+		__kmap_local_sched_out();
+#endif
+}
+
+static inline void kmap_local_sched_in(void)
+{
+#ifdef CONFIG_KMAP_LOCAL
+	if (unlikely(current->kmap_ctrl.idx))
+		__kmap_local_sched_in();
+#endif
+}
+
+/**
+ * prepare_task_switch - prepare to switch tasks
+ * @rq: the runqueue preparing to switch
+ * @next: the task we are going to switch to.
+ *
+ * This is called with the rq lock held and interrupts off. It must
+ * be paired with a subsequent finish_task_switch after the context
+ * switch.
+ *
+ * prepare_task_switch sets up locking and calls architecture specific
+ * hooks.
+ */
+static inline void
+prepare_task_switch(struct rq *rq, struct task_struct *prev,
+		    struct task_struct *next)
+{
+	kcov_prepare_switch(prev);
+	sched_info_switch(rq, prev, next);
+	perf_event_task_sched_out(prev, next);
+	fire_sched_out_preempt_notifiers(prev, next);
+	kmap_local_sched_out();
+	prepare_task(next);
+	prepare_arch_switch(next);
+}
+
+/**
+ * finish_task_switch - clean up after a task-switch
+ * @rq: runqueue associated with task-switch
+ * @prev: the thread we just switched away from.
+ *
+ * finish_task_switch must be called after the context switch, paired
+ * with a prepare_task_switch call before the context switch.
+ * finish_task_switch will reconcile locking set up by prepare_task_switch,
+ * and do any other architecture-specific cleanup actions.
+ *
+ * Note that we may have delayed dropping an mm in context_switch(). If
+ * so, we finish that here outside of the runqueue lock.  (Doing it
+ * with the lock held can cause deadlocks; see schedule() for
+ * details.)
+ *
+ * The context switch have flipped the stack from under us and restored the
+ * local variables which were saved when this task called schedule() in the
+ * past. 'prev == current' is still correct but we need to recalculate this_rq
+ * because prev may have moved to another CPU.
+ */
+static struct rq *finish_task_switch(struct task_struct *prev)
+	__releases(rq->lock)
+{
+	struct rq *rq = this_rq();
+	struct mm_struct *mm = rq->prev_mm;
+	unsigned int prev_state;
+
+	/*
+	 * The previous task will have left us with a preempt_count of 2
+	 * because it left us after:
+	 *
+	 *	schedule()
+	 *	  preempt_disable();			// 1
+	 *	  __schedule()
+	 *	    raw_spin_lock_irq(&rq->lock)	// 2
+	 *
+	 * Also, see FORK_PREEMPT_COUNT.
+	 */
+	if (WARN_ONCE(preempt_count() != 2*PREEMPT_DISABLE_OFFSET,
+		      "corrupted preempt_count: %s/%d/0x%x\n",
+		      current->comm, current->pid, preempt_count()))
+		preempt_count_set(FORK_PREEMPT_COUNT);
+
+	rq->prev_mm = NULL;
+
+	/*
+	 * A task struct has one reference for the use as "current".
+	 * If a task dies, then it sets TASK_DEAD in tsk->state and calls
+	 * schedule one last time. The schedule call will never return, and
+	 * the scheduled task must drop that reference.
+	 *
+	 * We must observe prev->state before clearing prev->on_cpu (in
+	 * finish_task), otherwise a concurrent wakeup can get prev
+	 * running on another CPU and we could rave with its RUNNING -> DEAD
+	 * transition, resulting in a double drop.
+	 */
+	prev_state = READ_ONCE(prev->__state);
+	vtime_task_switch(prev);
+	perf_event_task_sched_in(prev, current);
+	finish_task(prev, rq);
+	tick_nohz_task_switch();
+	finish_lock_switch(rq);
+	finish_arch_post_lock_switch();
+	kcov_finish_switch(current);
+	/*
+	 * kmap_local_sched_out() is invoked with rq::lock held and
+	 * interrupts disabled. There is no requirement for that, but the
+	 * sched out code does not have an interrupt enabled section.
+	 * Restoring the maps on sched in does not require interrupts being
+	 * disabled either.
+	 */
+	kmap_local_sched_in();
+
+	fire_sched_in_preempt_notifiers(current);
+	/*
+	 * When switching through a kernel thread, the loop in
+	 * membarrier_{private,global}_expedited() may have observed that
+	 * kernel thread and not issued an IPI. It is therefore possible to
+	 * schedule between user->kernel->user threads without passing though
+	 * switch_mm(). Membarrier requires a barrier after storing to
+	 * rq->curr, before returning to userspace, so provide them here:
+	 *
+	 * - a full memory barrier for {PRIVATE,GLOBAL}_EXPEDITED, implicitly
+	 *   provided by mmdrop_lazy_tlb(),
+	 * - a sync_core for SYNC_CORE.
+	 */
+	if (mm) {
+		membarrier_mm_sync_core_before_usermode(mm);
+		mmdrop_lazy_tlb_sched(mm);
+	}
+	if (unlikely(prev_state == TASK_DEAD)) {
+		cgroup_task_dead(prev);
+
+		/* Task is done with its stack. */
+		put_task_stack(prev);
+
+		put_task_struct_rcu_user(prev);
+	}
+
+	return rq;
+}
+
+/**
+ * schedule_tail - first thing a freshly forked thread must call.
+ * @prev: the thread we just switched away from.
+ */
+asmlinkage __visible void schedule_tail(struct task_struct *prev)
+	__releases(rq->lock)
+{
+	/*
+	 * New tasks start with FORK_PREEMPT_COUNT, see there and
+	 * finish_task_switch() for details.
+	 *
+	 * finish_task_switch() will drop rq->lock() and lower preempt_count
+	 * and the preempt_enable() will end up enabling preemption (on
+	 * PREEMPT_COUNT kernels).
+	 */
+
+	finish_task_switch(prev);
+	/*
+	 * This is a special case: the newly created task has just
+	 * switched the context for the first time. It is returning from
+	 * schedule for the first time in this path.
+	 */
+	trace_sched_exit_tp(true);
+	preempt_enable();
+
+	if (current->set_child_tid)
+		put_user(task_pid_vnr(current), current->set_child_tid);
+
+	calculate_sigpending();
+}
+
+/*
+ * context_switch - switch to the new MM and the new thread's register state.
+ */
+static __always_inline struct rq *
+context_switch(struct rq *rq, struct task_struct *prev,
+	       struct task_struct *next)
+{
+	prepare_task_switch(rq, prev, next);
+
+	/*
+	 * For paravirt, this is coupled with an exit in switch_to to
+	 * combine the page table reload and the switch backend into
+	 * one hypercall.
+	 */
+	arch_start_context_switch(prev);
+
+	/*
+	 * kernel -> kernel   lazy + transfer active
+	 *   user -> kernel   lazy + mmgrab_lazy_tlb() active
+	 *
+	 * kernel ->   user   switch + mmdrop_lazy_tlb() active
+	 *   user ->   user   switch
+	 */
+	if (!next->mm) {				// to kernel
+		enter_lazy_tlb(prev->active_mm, next);
+
+		next->active_mm = prev->active_mm;
+		if (prev->mm)				// from user
+			mmgrab_lazy_tlb(prev->active_mm);
+		else
+			prev->active_mm = NULL;
+	} else {					// to user
+		membarrier_switch_mm(rq, prev->active_mm, next->mm);
+		/*
+		 * sys_membarrier() requires an smp_mb() between setting
+		 * rq->curr / membarrier_switch_mm() and returning to userspace.
+		 *
+		 * The below provides this either through switch_mm(), or in
+		 * case 'prev->active_mm == next->mm' through
+		 * finish_task_switch()'s mmdrop().
+		 */
+		switch_mm_irqs_off(prev->active_mm, next->mm, next);
+		lru_gen_use_mm(next->mm);
+
+		if (!prev->mm) {			// from kernel
+			/* will mmdrop_lazy_tlb() in finish_task_switch(). */
+			rq->prev_mm = prev->active_mm;
+			prev->active_mm = NULL;
+		}
+	}
+
+	mm_cid_switch_to(prev, next);
+
+	/*
+	 * Tell rseq that the task was scheduled in. Must be after
+	 * switch_mm_cid() to get the TIF flag set.
+	 */
+	rseq_sched_switch_event(prev);
+
+	prepare_lock_switch(rq, next);
+
+	/* Here we just switch the register state and the stack. */
+	switch_to(prev, next, prev);
+	barrier();
+
+	return finish_task_switch(prev);
+}
+
+/*
+ * nr_running, nr_uninterruptible and nr_context_switches:
+ *
+ * externally visible scheduler statistics: current number of runnable
+ * threads, total number of context switches performed since bootup.
+ */
+unsigned int nr_running(void)
+{
+	return srq_nr_queued(0) + num_active_cpus() - cpumask_weight(sched_idle_mask);
+}
+
+/*
+ * Check if only the current task is running on the CPU.
+ *
+ * Caution: this function does not check that the caller has disabled
+ * preemption, thus the result might have a time-of-check-to-time-of-use
+ * race.  The caller is responsible to use it correctly, for example:
+ *
+ * - from a non-preemptible section (of course)
+ *
+ * - from a thread that is bound to a single CPU
+ *
+ * - in a loop with very short iterations (e.g. a polling loop)
+ */
+bool single_task_running(void)
+{
+	const int cpu = cpu_of(raw_rq());
+
+	return cpumask_test_cpu(cpu, sched_idle_mask) &&
+	       (0 == srq_nr_queued(0)) && is_preempt_list_empty(cpu);
+}
+EXPORT_SYMBOL(single_task_running);
+
+unsigned long long nr_context_switches_cpu(int cpu)
+{
+	return cpu_rq(cpu)->nr_switches;
+}
+
+unsigned long long nr_context_switches(void)
+{
+	int i;
+	unsigned long long sum = 0;
+
+	for_each_possible_cpu(i)
+		sum += cpu_rq(i)->nr_switches;
+
+	return sum;
+}
+
+/*
+ * Consumers of these two interfaces, like for example the cpuidle menu
+ * governor, are using nonsensical data. Preferring shallow idle state selection
+ * for a CPU that has IO-wait which might not even end up running the task when
+ * it does become runnable.
+ */
+
+unsigned int nr_iowait_cpu(int cpu)
+{
+	return (0 == cpu) ? atomic_read(&cpu_srq(0)->nr_iowait) : 0;
+}
+
+/*
+ * IO-wait accounting, and how it's mostly bollocks (on SMP).
+ *
+ * The idea behind IO-wait account is to account the idle time that we could
+ * have spend running if it were not for IO. That is, if we were to improve the
+ * storage performance, we'd have a proportional reduction in IO-wait time.
+ *
+ * This all works nicely on UP, where, when a task blocks on IO, we account
+ * idle time as IO-wait, because if the storage were faster, it could've been
+ * running and we'd not be idle.
+ *
+ * This has been extended to SMP, by doing the same for each CPU. This however
+ * is broken.
+ *
+ * Imagine for instance the case where two tasks block on one CPU, only the one
+ * CPU will have IO-wait accounted, while the other has regular idle. Even
+ * though, if the storage were faster, both could've ran at the same time,
+ * utilising both CPUs.
+ *
+ * This means, that when looking globally, the current IO-wait accounting on
+ * SMP is a lower bound, by reason of under accounting.
+ *
+ * Worse, since the numbers are provided per CPU, they are sometimes
+ * interpreted per CPU, and that is nonsensical. A blocked task isn't strictly
+ * associated with any one particular CPU, it can wake to another CPU than it
+ * blocked on. This means the per CPU IO-wait number is meaningless.
+ *
+ * Task CPU affinities can make all that even more 'interesting'.
+ */
+
+unsigned int nr_iowait(void)
+{
+	return nr_iowait_cpu(0);
+}
+
+/*
+ * sched_exec - execve() is a valuable balancing opportunity, because at
+ * this point the task has the smallest effective memory and cache
+ * footprint.
+ */
+void sched_exec(void)
+{
+}
+
+DEFINE_PER_CPU(struct kernel_stat, kstat);
+DEFINE_PER_CPU(struct kernel_cpustat, kernel_cpustat);
+
+EXPORT_PER_CPU_SYMBOL(kstat);
+EXPORT_PER_CPU_SYMBOL(kernel_cpustat);
+
+static inline void update_curr(struct rq *rq, struct task_struct *p)
+{
+	s64 ns = rq->clock_task - p->last_ran;
+
+	p->sched_time += ns;
+	cgroup_account_cputime(p, ns);
+	account_group_exec_runtime(p, ns);
+
+	p->time_slice -= ns;
+	p->last_ran = rq->clock_task;
+}
+
+/*
+ * Return accounted runtime for the task.
+ * Return separately the current's pending runtime that have not been
+ * accounted yet.
+ */
+unsigned long long task_sched_runtime(struct task_struct *p)
+{
+	struct rq *rq;
+	struct rq_flags rf;
+	u64 ns;
+
+#ifdef CONFIG_64BIT
+	/*
+	 * 64-bit doesn't need locks to atomically read a 64-bit value.
+	 * So we have a optimization chance when the task's delta_exec is 0.
+	 * Reading ->on_cpu is racy, but this is OK.
+	 *
+	 * If we race with it leaving CPU, we'll take a lock. So we're correct.
+	 * If we race with it entering CPU, unaccounted time is 0. This is
+	 * indistinguishable from the read occurring a few cycles earlier.
+	 * If we see ->on_cpu without ->on_rq, the task is leaving, and has
+	 * been accounted, so we're correct here as well.
+	 */
+	if (!p->on_cpu || !task_on_rq_queued(p))
+		return tsk_seruntime(p);
+#endif
+
+	raw_spin_lock_irqsave(&p->pi_lock, rf.flags);
+	rq = task_rq(p);
+	/*
+	 * Must be ->curr _and_ ->on_rq.  If dequeued, we would
+	 * project cycles that may never be accounted to this
+	 * thread, breaking clock_gettime().
+	 */
+	if (p == rq->curr && task_on_rq_queued(p)) {
+		raw_spin_lock(&rq->lock);
+		if (likely(p == rq->curr && task_on_rq_queued(p) && task_rq(p) == rq)) {
+			update_rq_clock(rq);
+			update_curr(rq, p);
+		}
+		raw_spin_unlock(&rq->lock);
+	}
+	ns = tsk_seruntime(p);
+	raw_spin_unlock_irqrestore(&p->pi_lock, rf.flags);
+
+	return ns;
+}
+
+/* This manages tasks that have run out of timeslice during a scheduler_tick */
+static inline void scheduler_task_tick(struct rq *rq)
+{
+	struct task_struct *p = rq->curr;
+
+	if (is_idle_task(p))
+		return;
+
+	update_curr(rq, p);
+	cpufreq_update_util(rq, 0);
+
+	/*
+	 * Tasks have less than RESCHED_NS of time slice left they will be
+	 * rescheduled.
+	 */
+	if (p->time_slice >= RESCHED_NS)
+		return;
+	set_tsk_need_resched(p);
+	set_preempt_need_resched();
+}
+
+static u64 cpu_resched_latency(struct rq *rq)
+{
+	int latency_warn_ms = READ_ONCE(sysctl_resched_latency_warn_ms);
+	u64 resched_latency, now = rq_clock(rq);
+	static bool warned_once;
+
+	if (sysctl_resched_latency_warn_once && warned_once)
+		return 0;
+
+	if (!need_resched() || !latency_warn_ms)
+		return 0;
+
+	if (system_state == SYSTEM_BOOTING)
+		return 0;
+
+	if (!rq->last_seen_need_resched_ns) {
+		rq->last_seen_need_resched_ns = now;
+		rq->ticks_without_resched = 0;
+		return 0;
+	}
+
+	rq->ticks_without_resched++;
+	resched_latency = now - rq->last_seen_need_resched_ns;
+	if (resched_latency <= latency_warn_ms * NSEC_PER_MSEC)
+		return 0;
+
+	warned_once = true;
+
+	return resched_latency;
+}
+
+static int __init setup_resched_latency_warn_ms(char *str)
+{
+	long val;
+
+	if ((kstrtol(str, 0, &val))) {
+		pr_warn("Unable to set resched_latency_warn_ms\n");
+		return 1;
+	}
+
+	sysctl_resched_latency_warn_ms = val;
+	return 1;
+}
+__setup("resched_latency_warn_ms=", setup_resched_latency_warn_ms);
+
+/*
+ * This function gets called by the timer code, with HZ frequency.
+ * We call it with interrupts disabled.
+ */
+void sched_tick(void)
+{
+	int cpu __maybe_unused = smp_processor_id();
+	struct rq *rq = cpu_rq(cpu);
+	struct task_struct *curr = rq->curr;
+	u64 resched_latency;
+
+	if (housekeeping_cpu(cpu, HK_TYPE_KERNEL_NOISE))
+		arch_scale_freq_tick();
+
+	sched_clock_tick();
+
+	raw_spin_lock(&rq->lock);
+	update_rq_clock(rq);
+
+	if (dynamic_preempt_lazy() && tif_test_bit(TIF_NEED_RESCHED_LAZY))
+		resched_curr(rq);
+
+	scheduler_task_tick(rq);
+	if (sched_feat(LATENCY_WARN))
+		resched_latency = cpu_resched_latency(rq);
+
+	raw_spin_unlock(&rq->lock);
+
+	if (sched_feat(LATENCY_WARN) && resched_latency)
+		resched_latency_warn(cpu, resched_latency);
+
+	perf_event_task_tick();
+
+	if (curr->flags & PF_WQ_WORKER)
+		wq_worker_tick(curr);
+}
+
+#ifdef CONFIG_NO_HZ_FULL
+
+struct tick_work {
+	int			cpu;
+	atomic_t		state;
+	struct delayed_work	work;
+};
+/* Values for ->state, see diagram below. */
+#define TICK_SCHED_REMOTE_OFFLINE	0
+#define TICK_SCHED_REMOTE_OFFLINING	1
+#define TICK_SCHED_REMOTE_RUNNING	2
+
+/*
+ * State diagram for ->state:
+ *
+ *
+ *          TICK_SCHED_REMOTE_OFFLINE
+ *                    |   ^
+ *                    |   |
+ *                    |   | sched_tick_remote()
+ *                    |   |
+ *                    |   |
+ *                    +--TICK_SCHED_REMOTE_OFFLINING
+ *                    |   ^
+ *                    |   |
+ * sched_tick_start() |   | sched_tick_stop()
+ *                    |   |
+ *                    V   |
+ *          TICK_SCHED_REMOTE_RUNNING
+ *
+ *
+ * Other transitions get WARN_ON_ONCE(), except that sched_tick_remote()
+ * and sched_tick_start() are happy to leave the state in RUNNING.
+ */
+
+static struct tick_work __percpu *tick_work_cpu;
+
+static void sched_tick_remote(struct work_struct *work)
+{
+	struct delayed_work *dwork = to_delayed_work(work);
+	struct tick_work *twork = container_of(dwork, struct tick_work, work);
+	int cpu = twork->cpu;
+	struct rq *rq = cpu_rq(cpu);
+	int os;
+
+	/*
+	 * Handle the tick only if it appears the remote CPU is running in full
+	 * dynticks mode. The check is racy by nature, but missing a tick or
+	 * having one too much is no big deal because the scheduler tick updates
+	 * statistics and checks timeslices in a time-independent way, regardless
+	 * of when exactly it is running.
+	 */
+	if (tick_nohz_tick_stopped_cpu(cpu)) {
+		guard(raw_spinlock_irqsave)(&rq->lock);
+		struct task_struct *curr = rq->curr;
+
+		if (cpu_online(cpu)) {
+			update_rq_clock(rq);
+
+			if (!is_idle_task(curr)) {
+				/*
+				 * Make sure the next tick runs within a
+				 * reasonable amount of time.
+				 */
+				u64 delta = rq_clock_task(rq) - curr->last_ran;
+				WARN_ON_ONCE(delta > (u64)NSEC_PER_SEC * 30);
+			}
+			scheduler_task_tick(rq);
+
+			calc_load_nohz_remote(rq);
+		}
+	}
+
+	/*
+	 * Run the remote tick once per second (1Hz). This arbitrary
+	 * frequency is large enough to avoid overload but short enough
+	 * to keep scheduler internal stats reasonably up to date.  But
+	 * first update state to reflect hotplug activity if required.
+	 */
+	os = atomic_fetch_add_unless(&twork->state, -1, TICK_SCHED_REMOTE_RUNNING);
+	WARN_ON_ONCE(os == TICK_SCHED_REMOTE_OFFLINE);
+	if (os == TICK_SCHED_REMOTE_RUNNING)
+		queue_delayed_work(system_dfl_wq, dwork, HZ);
+}
+
+static void sched_tick_start(int cpu)
+{
+	int os;
+	struct tick_work *twork;
+
+	if (housekeeping_cpu(cpu, HK_TYPE_KERNEL_NOISE))
+		return;
+
+	WARN_ON_ONCE(!tick_work_cpu);
+
+	twork = per_cpu_ptr(tick_work_cpu, cpu);
+	os = atomic_xchg(&twork->state, TICK_SCHED_REMOTE_RUNNING);
+	WARN_ON_ONCE(os == TICK_SCHED_REMOTE_RUNNING);
+	if (os == TICK_SCHED_REMOTE_OFFLINE) {
+		twork->cpu = cpu;
+		INIT_DELAYED_WORK(&twork->work, sched_tick_remote);
+		queue_delayed_work(system_dfl_wq, &twork->work, HZ);
+	}
+}
+
+#ifdef CONFIG_HOTPLUG_CPU
+static void sched_tick_stop(int cpu)
+{
+	struct tick_work *twork;
+	int os;
+
+	if (housekeeping_cpu(cpu, HK_TYPE_KERNEL_NOISE))
+		return;
+
+	WARN_ON_ONCE(!tick_work_cpu);
+
+	twork = per_cpu_ptr(tick_work_cpu, cpu);
+	/* There cannot be competing actions, but don't rely on stop-machine. */
+	os = atomic_xchg(&twork->state, TICK_SCHED_REMOTE_OFFLINING);
+	WARN_ON_ONCE(os != TICK_SCHED_REMOTE_RUNNING);
+	/* Don't cancel, as this would mess up the state machine. */
+}
+#endif /* CONFIG_HOTPLUG_CPU */
+
+int __init sched_tick_offload_init(void)
+{
+	tick_work_cpu = alloc_percpu(struct tick_work);
+	BUG_ON(!tick_work_cpu);
+	return 0;
+}
+
+#else /* !CONFIG_NO_HZ_FULL: */
+static inline void sched_tick_start(int cpu) { }
+static inline void sched_tick_stop(int cpu) { }
+#endif /* !CONFIG_NO_HZ_FULL */
+
+#if defined(CONFIG_PREEMPTION) && (defined(CONFIG_DEBUG_PREEMPT) || \
+				defined(CONFIG_PREEMPT_TRACER))
+/*
+ * If the value passed in is equal to the current preempt count
+ * then we just disabled preemption. Start timing the latency.
+ */
+static inline void preempt_latency_start(int val)
+{
+	if (preempt_count() == val) {
+		unsigned long ip = get_lock_parent_ip();
+#ifdef CONFIG_DEBUG_PREEMPT
+		current->preempt_disable_ip = ip;
+#endif
+		trace_preempt_off(CALLER_ADDR0, ip);
+	}
+}
+
+void preempt_count_add(int val)
+{
+#ifdef CONFIG_DEBUG_PREEMPT
+	/*
+	 * Underflow?
+	 */
+	if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0)))
+		return;
+#endif
+	__preempt_count_add(val);
+#ifdef CONFIG_DEBUG_PREEMPT
+	/*
+	 * Spinlock count overflowing soon?
+	 */
+	DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >=
+				PREEMPT_MASK - 10);
+#endif
+	preempt_latency_start(val);
+}
+EXPORT_SYMBOL(preempt_count_add);
+NOKPROBE_SYMBOL(preempt_count_add);
+
+/*
+ * If the value passed in equals to the current preempt count
+ * then we just enabled preemption. Stop timing the latency.
+ */
+static inline void preempt_latency_stop(int val)
+{
+	if (preempt_count() == val)
+		trace_preempt_on(CALLER_ADDR0, get_lock_parent_ip());
+}
+
+void preempt_count_sub(int val)
+{
+#ifdef CONFIG_DEBUG_PREEMPT
+	/*
+	 * Underflow?
+	 */
+	if (DEBUG_LOCKS_WARN_ON(val > preempt_count()))
+		return;
+	/*
+	 * Is the spinlock portion underflowing?
+	 */
+	if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) &&
+			!(preempt_count() & PREEMPT_MASK)))
+		return;
+#endif
+
+	preempt_latency_stop(val);
+	__preempt_count_sub(val);
+}
+EXPORT_SYMBOL(preempt_count_sub);
+NOKPROBE_SYMBOL(preempt_count_sub);
+
+#else
+static inline void preempt_latency_start(int val) { }
+static inline void preempt_latency_stop(int val) { }
+#endif
+
+static inline unsigned long get_preempt_disable_ip(struct task_struct *p)
+{
+#ifdef CONFIG_DEBUG_PREEMPT
+	return p->preempt_disable_ip;
+#else
+	return 0;
+#endif
+}
+
+/*
+ * Print scheduling while atomic bug:
+ */
+static noinline void __schedule_bug(struct task_struct *prev)
+{
+	/* Save this before calling printk(), since that will clobber it */
+	unsigned long preempt_disable_ip = get_preempt_disable_ip(current);
+
+	if (oops_in_progress)
+		return;
+
+	printk(KERN_ERR "BUG: scheduling while atomic: %s/%d/0x%08x\n",
+		prev->comm, prev->pid, preempt_count());
+
+	debug_show_held_locks(prev);
+	print_modules();
+	if (irqs_disabled())
+		print_irqtrace_events(prev);
+	if (IS_ENABLED(CONFIG_DEBUG_PREEMPT)) {
+		pr_err("Preemption disabled at:");
+		print_ip_sym(KERN_ERR, preempt_disable_ip);
+	}
+	check_panic_on_warn("scheduling while atomic");
+
+	dump_stack();
+	add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
+}
+
+/*
+ * Various schedule()-time debugging checks and statistics:
+ */
+static inline void schedule_debug(struct task_struct *prev, bool preempt)
+{
+#ifdef CONFIG_SCHED_STACK_END_CHECK
+	if (task_stack_end_corrupted(prev))
+		panic("corrupted stack end detected inside scheduler\n");
+
+	if (task_scs_end_corrupted(prev))
+		panic("corrupted shadow stack detected inside scheduler\n");
+#endif
+
+#ifdef CONFIG_DEBUG_ATOMIC_SLEEP
+	if (!preempt && READ_ONCE(prev->__state) && prev->non_block_count) {
+		printk(KERN_ERR "BUG: scheduling in a non-blocking section: %s/%d/%i\n",
+			prev->comm, prev->pid, prev->non_block_count);
+		dump_stack();
+		add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
+	}
+#endif
+
+	if (unlikely(in_atomic_preempt_off())) {
+		__schedule_bug(prev);
+		preempt_count_set(PREEMPT_DISABLED);
+	}
+	rcu_sleep_check();
+	WARN_ON_ONCE(ct_state() == CT_STATE_USER);
+
+	profile_hit(SCHED_PROFILING, __builtin_return_address(0));
+
+	schedstat_inc(this_rq()->sched_count);
+}
+
+#ifdef ALT_SCHED_DEBUG
+static inline void alt_sched_rq_debug(struct rq *rq)
+{
+	const int cpu = cpu_of(rq);
+
+	printk(KERN_INFO "sched: rq(%02d) nr_pinned=%d, prio=%d\n",
+	       cpu, rq->nr_pinned, READ_ONCE(cpu_prio[cpu]));
+}
+
+static inline void alt_sched_task_debug(struct task_struct *p)
+{
+	printk(KERN_INFO "sched: task %px, sched_prio: %d, cpumask: 0x%04lx, cpu: %d\n",
+	       p, task_sched_prio(p), p->cpus_ptr->bits[0], task_cpu(p));
+}
+
+static void alt_sched_srq_debug(struct sched_run_queue *srq)
+{
+	int idx = find_first_bit(srq->bitmap, SCHED_QUEUE_BITS);
+
+	printk(KERN_INFO "sched: srq debug, srq_nr_queued:%d, bitmap:0x%016lx, firstbit:%d\n",
+	       srq_nr_queued(0), srq->bitmap[0], idx);
+
+	if (SCHED_QUEUE_BITS != idx) {
+		struct llist_node *entry = srq->_head[idx].first;
+
+		if (entry) {
+			while (entry->next) {
+				entry = entry->next;
+			}
+			struct task_struct *p = llist_entry(entry, struct task_struct, pq_node);
+			alt_sched_task_debug(p);
+		}
+	}
+}
+
+static void alt_sched_cpu_prio_debug(void)
+{
+	int i;
+
+	printk(KERN_INFO "sched: cpu_prio[idle] : 0x%08lx\n", cpu_sched_prio_mask[6].bits[0]);
+
+	for_each_set_bit (i, cpu_sched_prio_bitmap, SCHED_LEVELS) {
+		printk(KERN_INFO "sched: cpu_prio[%4d] : 0x%08lx\n",
+		       SCHED_LEVELS - 1 - i,
+		       cpu_sched_prio_mask[i].bits[0]);
+	}
+}
+
+void alt_sched_debug(void)
+{
+	int cpu;
+
+	alt_sched_cpu_prio_debug();
+
+	alt_sched_srq_debug(&grq);
+
+	for_each_cpu(cpu, cpu_active_mask)
+		alt_sched_rq_debug(cpu_rq(cpu));
+
+/*
+	for (int i = 0; i < 8; i++)
+		printk(KERN_INFO "sched: [%d] = %d\n", i, alt_debug[i]);
+*/
+}
+#endif
+
+/*
+ * Timeslices below RESCHED_NS are considered as good as expired as there's no
+ * point rescheduling when there's so little time left.
+ *
+ * Return: 1 on timeslice of @p is expired, 0 otherwise.
+ */
+static __always_inline int check_curr(struct task_struct *p, struct rq *rq)
+{
+	if (unlikely(rq->idle == p))
+		return 0;
+
+	update_curr(rq, p);
+
+	if (p->time_slice < RESCHED_NS) {
+		p->time_slice = sysctl_sched_base_slice;
+		sched_task_renew(p, rq);
+
+		return 1;
+	}
+
+	return 0;
+}
+
+static __always_inline void migrate_preempt_task(struct task_struct *p, const int cpu)
+{
+	/* idle preempt success rate ~52.7% */
+	struct rq *rq = __wakeup_rq_trylock(p, task_sched_prio(p), p->cpus_ptr);
+
+	if (rq)
+		preempt_on_rq(p, rq);
+	else
+		activate_task(p, cpu_srq(cpu));
+}
+
+static __always_inline struct task_struct *pick_preempt_task(const int cpu, int pick_sched_prio)
+{
+	struct llist_node *plist;
+	struct task_struct *p, *n, *preempt = NULL;
+
+	plist = llist_del_all(this_cpu_ptr(&preempt_list));
+	if (NULL == plist)
+		return preempt;
+
+	plist = llist_reverse_order(plist);
+	llist_for_each_entry_safe(p, n, plist, pq_node) {
+		int sched_prio = task_sched_prio(p);
+
+		if (is_cpu_allowed(p, cpu) && sched_prio < pick_sched_prio) {
+			if (NULL != preempt)
+				migrate_preempt_task(preempt, cpu);
+			pick_sched_prio = sched_prio;
+			preempt = p;
+		} else {
+			migrate_preempt_task(p, cpu);
+		}
+	}
+	if (NULL != preempt) {
+		WRITE_ONCE(preempt->on_rq, TASK_ON_RQ_QUEUED);
+		ASSERT_EXCLUSIVE_WRITER(preempt->on_rq);
+
+		if (task_cpu(preempt) != cpu)
+			set_task_cpu(preempt, cpu);
+	}
+
+	return preempt;
+}
+
+static __always_inline void wakeup_srq_task(const int cpu)
+{
+	struct sched_run_queue *srq = cpu_srq(cpu);
+	int idx = find_first_bit(srq->bitmap, SCHED_QUEUE_BITS);
+
+	if (unlikely(SCHED_QUEUE_BITS == idx))
+		return;
+
+	struct llist_node *entry= srq->_head[idx].first;
+	if (unlikely(NULL == entry))
+		return;
+
+	struct task_struct *p = llist_entry(entry, struct task_struct, pq_node);
+	int i;
+
+	/* At this point, p is most likely per-cpu task */
+	for_each_cpu_and (i, p->cpus_ptr, cpu_active_mask) {
+		if (idx < READ_ONCE(cpu_prio[i])) {
+			struct rq *rq = cpu_rq(i);
+			if (likely(raw_spin_trylock(&rq->lock))) {
+				resched_curr(rq);
+				raw_spin_unlock(&rq->lock);
+			}
+		}
+	}
+}
+
+#define PQ_PICK_TASK										\
+{												\
+	struct llist_node *first = llist_del_all(head);						\
+	if (first) {										\
+		first = llist_reverse_order(first);						\
+		struct llist_node **curr = &first;						\
+		while (*curr) {									\
+			struct llist_node *entry = *curr;					\
+			struct task_struct *p = llist_entry(entry, struct task_struct, pq_node);\
+			if (is_cpu_allowed(p, cpu)) {						\
+				*curr = entry->next;						\
+				next = p;							\
+				break;								\
+			}									\
+			curr = &entry->next;							\
+		}										\
+		if (first) {									\
+			/* ~3.3% to process all remain tasks to other cpu, no point to do so*/	\
+			struct llist_node *last = first;					\
+			first = llist_reverse_order(first);					\
+			sched_llist_merge(head, first, last);					\
+		} else if (llist_empty(head)) {							\
+			clear_bit(idx, srq->bitmap);						\
+			smp_rmb();								\
+			if (!llist_empty(head))							\
+				set_bit(idx, srq->bitmap);					\
+		}										\
+		if (next) {									\
+			if (task_cpu(next) != cpu)						\
+				set_task_cpu(next, cpu);					\
+			WRITE_ONCE(next->__sched_prio, -1);					\
+			atomic_dec(&srq->nr_queued);						\
+			raw_spin_unlock(lock);							\
+			goto picked;								\
+		}										\
+	}											\
+}
+
+static inline struct task_struct *pick_next_task(const int cpu, struct rq *rq, int expired)
+{
+	struct sched_run_queue *srq;
+	struct task_struct *next;
+	bool blocked = rq->block;
+	int pick_sched_prio = blocked ? IDLE_TASK_SCHED_PRIO : READ_ONCE(cpu_prio[cpu]) + expired;
+
+	if ((next = pick_preempt_task(cpu, pick_sched_prio)))
+		goto picked;
+
+	/* pick next task from schedule run queue */
+	srq = cpu_srq(cpu);
+	if (!bitmap_empty(srq->bitmap, pick_sched_prio)) {
+		int idx;
+		DECLARE_BITMAP(lock_bitmap, SCHED_QUEUE_BITS) = { 0 };
+
+		/* trylock loop, reduce waiting time of pq lock  */
+		for_each_set_bit(idx, srq->bitmap, pick_sched_prio) {
+			raw_spinlock_t	*lock = &srq->_lock[idx];
+			struct llist_head *head = &srq->_head[idx];
+
+			if (raw_spin_trylock(lock)) {
+				PQ_PICK_TASK;
+				raw_spin_unlock(lock);
+			} else {
+				set_bit(idx, lock_bitmap);
+			}
+		}
+
+		/* lock loop */
+		for_each_set_bit(idx, lock_bitmap, pick_sched_prio) {
+			raw_spinlock_t	*lock = &srq->_lock[idx];
+			struct llist_head *head = &srq->_head[idx];
+
+			raw_spin_lock(lock);
+			PQ_PICK_TASK;
+			raw_spin_unlock(lock);
+		}
+	}
+
+	struct task_struct *idle = rq->idle;
+	next = blocked ? idle : rq->curr;
+	if (next == idle) {
+		if (rq->online) {
+			if (srq_nr_queued(cpu))
+				wakeup_srq_task(cpu);
+			else
+				sched_cpu_topology_balance(cpu, rq);
+		}
+
+		schedstat_inc(rq->sched_goidle);
+		/*printk(KERN_INFO "sched: choose_next_task(%d) idle %px\n", cpu, next);*/
+		return idle;
+	}
+picked:
+#ifdef CONFIG_SCHED_HRTICK
+	hrtick_start(rq, next->time_slice);
+#endif
+	/*printk(KERN_INFO "sched: choose_next_task(%d) next %px\n", cpu, next);*/
+	return next;
+}
+
+/*
+ * Constants for the sched_mode argument of __schedule().
+ *
+ * The mode argument allows RT enabled kernels to differentiate a
+ * preemption from blocking on an 'sleeping' spin/rwlock.
+ */
+ #define SM_IDLE		(-1)
+ #define SM_NONE		0
+ #define SM_PREEMPT		1
+ #define SM_RTLOCK_WAIT		2
+
+/*
+ * Helper function for __schedule()
+ *
+ * If a task does not have signals pending, deactivate it
+ * Otherwise marks the task's __state as RUNNING
+ */
+static bool try_to_block_task(struct rq *rq, struct task_struct *p,
+			      unsigned long *task_state_p)
+{
+	unsigned long task_state = *task_state_p;
+	if (signal_pending_state(task_state, p)) {
+		WRITE_ONCE(p->__state, TASK_RUNNING);
+		*task_state_p = TASK_RUNNING;
+		return false;
+	}
+	p->sched_contributes_to_load =
+		(task_state & TASK_UNINTERRUPTIBLE) &&
+		!(task_state & TASK_NOLOAD) &&
+		!(task_state & TASK_FROZEN);
+
+	/*
+	 * __schedule()			ttwu()
+	 *   prev_state = prev->state;    if (p->on_rq && ...)
+	 *   if (prev_state)		    goto out;
+	 *     p->on_rq = 0;		  smp_acquire__after_ctrl_dep();
+	 *				  p->state = TASK_WAKING
+	 *
+	 * Where __schedule() and ttwu() have matching control dependencies.
+	 *
+	 * After this, schedule() must not care about p->state any more.
+	 */
+	block_task(rq, p);
+	return true;
+}
+
+/*
+ * schedule() is the main scheduler function.
+ *
+ * The main means of driving the scheduler and thus entering this function are:
+ *
+ *   1. Explicit blocking: mutex, semaphore, waitqueue, etc.
+ *
+ *   2. TIF_NEED_RESCHED flag is checked on interrupt and userspace return
+ *      paths. For example, see arch/x86/entry_64.S.
+ *
+ *      To drive preemption between tasks, the scheduler sets the flag in timer
+ *      interrupt handler sched_tick().
+ *
+ *   3. Wakeups don't really cause entry into schedule(). They add a
+ *      task to the run-queue and that's it.
+ *
+ *      Now, if the new task added to the run-queue preempts the current
+ *      task, then the wakeup sets TIF_NEED_RESCHED and schedule() gets
+ *      called on the nearest possible occasion:
+ *
+ *       - If the kernel is preemptible (CONFIG_PREEMPTION=y):
+ *
+ *         - in syscall or exception context, at the next outmost
+ *           preempt_enable(). (this might be as soon as the wake_up()'s
+ *           spin_unlock()!)
+ *
+ *         - in IRQ context, return from interrupt-handler to
+ *           preemptible context
+ *
+ *       - If the kernel is not preemptible (CONFIG_PREEMPTION is not set)
+ *         then at the next:
+ *
+ *          - cond_resched() call
+ *          - explicit schedule() call
+ *          - return from syscall or exception to user-space
+ *          - return from interrupt-handler to user-space
+ *
+ * WARNING: must be called with preemption disabled!
+ */
+static void __sched notrace __schedule(int sched_mode)
+{
+	struct task_struct *prev, *next;
+	/*
+	 * On PREEMPT_RT kernel, SM_RTLOCK_WAIT is noted
+	 * as a preemption by schedule_debug() and RCU.
+	 */
+	bool preempt = sched_mode > SM_NONE;
+	bool is_switch = false;
+	unsigned long *switch_count;
+	unsigned long prev_state;
+	struct rq *rq;
+	int cpu, expired;
+
+	/* Trace preemptions consistently with task switches */
+	trace_sched_entry_tp(preempt);
+
+	cpu = smp_processor_id();
+	rq = cpu_rq(cpu);
+	prev = rq->curr;
+	rq->block = false;
+
+	schedule_debug(prev, preempt);
+
+	klp_sched_try_switch(prev);
+
+	local_irq_disable();
+	rcu_note_context_switch(preempt);
+
+	/*
+	 * Make sure that signal_pending_state()->signal_pending() below
+	 * can't be reordered with __set_current_state(TASK_INTERRUPTIBLE)
+	 * done by the caller to avoid the race with signal_wake_up():
+	 *
+	 * __set_current_state(@state)		signal_wake_up()
+	 * schedule()				  set_tsk_thread_flag(p, TIF_SIGPENDING)
+	 *					  wake_up_state(p, state)
+	 *   LOCK rq->lock			    LOCK p->pi_state
+	 *   smp_mb__after_spinlock()		    smp_mb__after_spinlock()
+	 *     if (signal_pending_state())	    if (p->state & @state)
+	 *
+	 * Also, the membarrier system call requires a full memory barrier
+	 * after coming from user-space, before storing to rq->curr; this
+	 * barrier matches a full barrier in the proximity of the membarrier
+	 * system call exit.
+	 */
+	raw_spin_lock(&rq->lock);
+	smp_mb__after_spinlock();
+
+	hrtick_schedule_enter(rq);
+
+	update_rq_clock(rq);
+
+	switch_count = &prev->nivcsw;
+
+	/* Task state changes only considers SM_PREEMPT as preemption */
+	preempt = sched_mode == SM_PREEMPT;
+
+	/*
+	 * We must load prev->state once (task_struct::state is volatile), such
+	 * that we form a control dependency vs deactivate_task() below.
+	 */
+	prev_state = READ_ONCE(prev->__state);
+	if (sched_mode == SM_IDLE) {
+		if (!srq_nr_queued(cpu) && is_preempt_list_empty(cpu)) {
+			next = prev;
+			goto picked;
+		}
+	} else if (!preempt && prev_state) {
+		rq->block = try_to_block_task(rq, prev, &prev_state);
+		switch_count = &prev->nvcsw;
+	}
+
+	expired = check_curr(rq->curr, rq);
+	next = pick_next_task(cpu, rq, expired);
+picked:
+	clear_tsk_need_resched(prev);
+	clear_preempt_need_resched();
+	rq->last_seen_need_resched_ns = 0;
+
+	is_switch = prev != next;
+	if (likely(is_switch)) {
+		next->last_ran = rq->clock_task;
+
+		/*printk(KERN_INFO "sched: %px -> %px\n", prev, next);*/
+		rq->nr_switches++;
+		/*
+		 * RCU users of rcu_dereference(rq->curr) may not see
+		 * changes to task_struct made by pick_next_task().
+		 */
+		RCU_INIT_POINTER(rq->curr, next);
+		/*
+		 * The membarrier system call requires each architecture
+		 * to have a full memory barrier after updating
+		 * rq->curr, before returning to user-space.
+		 *
+		 * Here are the schemes providing that barrier on the
+		 * various architectures:
+		 * - mm ? switch_mm() : mmdrop() for x86, s390, sparc, PowerPC,
+		 *   RISC-V.  switch_mm() relies on membarrier_arch_switch_mm()
+		 *   on PowerPC and on RISC-V.
+		 * - finish_lock_switch() for weakly-ordered
+		 *   architectures where spin_unlock is a full barrier,
+		 * - switch_to() for arm64 (weakly-ordered, spin_unlock
+		 *   is a RELEASE barrier),
+		 *
+		 * The barrier matches a full barrier in the proximity of
+		 * the membarrier system call entry.
+		 *
+		 * On RISC-V, this barrier pairing is also needed for the
+		 * SYNC_CORE command when switching between processes, cf.
+		 * the inline comments in membarrier_arch_switch_mm().
+		 */
+		++*switch_count;
+
+		trace_sched_switch(preempt, prev, next, prev_state);
+
+		update_sched_cpu_prio(cpu, task_sched_prio(next));
+
+		/* Also unlocks the rq: */
+		rq = context_switch(rq, prev, next);
+
+		cpu = cpu_of(rq);
+	} else {
+		if (expired)
+			update_sched_cpu_prio(cpu, task_sched_prio(next));
+		__balance_callbacks(rq);
+		hrtick_schedule_exit(rq);
+		raw_spin_unlock_irq(&rq->lock);
+	}
+	trace_sched_exit_tp(is_switch);
+}
+
+void __noreturn do_task_dead(void)
+{
+	/* Causes final put_task_struct in finish_task_switch(): */
+	set_special_state(TASK_DEAD);
+
+	/* Tell freezer to ignore us: */
+	current->flags |= PF_NOFREEZE;
+
+	__schedule(SM_NONE);
+	BUG();
+
+	/* Avoid "noreturn function does return" - but don't continue if BUG() is a NOP: */
+	for (;;)
+		cpu_relax();
+}
+
+static inline void sched_submit_work(struct task_struct *tsk)
+{
+	static DEFINE_WAIT_OVERRIDE_MAP(sched_map, LD_WAIT_CONFIG);
+	unsigned int task_flags;
+
+	/*
+	 * Establish LD_WAIT_CONFIG context to ensure none of the code called
+	 * will use a blocking primitive -- which would lead to recursion.
+	 */
+	lock_map_acquire_try(&sched_map);
+
+	task_flags = tsk->flags;
+	/*
+	 * If a worker goes to sleep, notify and ask workqueue whether it
+	 * wants to wake up a task to maintain concurrency.
+	 */
+	if (task_flags & PF_WQ_WORKER)
+		wq_worker_sleeping(tsk);
+	else if (task_flags & PF_IO_WORKER)
+		io_wq_worker_sleeping(tsk);
+
+	/*
+	 * spinlock and rwlock must not flush block requests.  This will
+	 * deadlock if the callback attempts to acquire a lock which is
+	 * already acquired.
+	 */
+	WARN_ON_ONCE(current->__state & TASK_RTLOCK_WAIT);
+
+	/*
+	 * If we are going to sleep and we have plugged IO queued,
+	 * make sure to submit it to avoid deadlocks.
+	 */
+	blk_flush_plug(tsk->plug, true);
+
+	lock_map_release(&sched_map);
+}
+
+static void sched_update_worker(struct task_struct *tsk)
+{
+	if (tsk->flags & (PF_WQ_WORKER | PF_IO_WORKER | PF_BLOCK_TS)) {
+		if (tsk->flags & PF_BLOCK_TS)
+			blk_plug_invalidate_ts(tsk);
+		if (tsk->flags & PF_WQ_WORKER)
+			wq_worker_running(tsk);
+		else if (tsk->flags & PF_IO_WORKER)
+			io_wq_worker_running(tsk);
+	}
+}
+
+static __always_inline void __schedule_loop(int sched_mode)
+{
+	do {
+		preempt_disable();
+		__schedule(sched_mode);
+		sched_preempt_enable_no_resched();
+	} while (need_resched());
+}
+
+asmlinkage __visible void __sched schedule(void)
+{
+	struct task_struct *tsk = current;
+
+#ifdef CONFIG_RT_MUTEXES
+	lockdep_assert(!tsk->sched_rt_mutex);
+#endif
+
+	if (!task_is_running(tsk))
+		sched_submit_work(tsk);
+	__schedule_loop(SM_NONE);
+	sched_update_worker(tsk);
+}
+EXPORT_SYMBOL(schedule);
+
+/*
+ * synchronize_rcu_tasks() makes sure that no task is stuck in preempted
+ * state (have scheduled out non-voluntarily) by making sure that all
+ * tasks have either left the run queue or have gone into user space.
+ * As idle tasks do not do either, they must not ever be preempted
+ * (schedule out non-voluntarily).
+ *
+ * schedule_idle() is similar to schedule_preempt_disable() except that it
+ * never enables preemption because it does not call sched_submit_work().
+ */
+void __sched schedule_idle(void)
+{
+	/*
+	 * As this skips calling sched_submit_work(), which the idle task does
+	 * regardless because that function is a NOP when the task is in a
+	 * TASK_RUNNING state, make sure this isn't used someplace that the
+	 * current task can be in any other state. Note, idle is always in the
+	 * TASK_RUNNING state.
+	 */
+	WARN_ON_ONCE(current->__state);
+	do {
+		__schedule(SM_IDLE);
+	} while (need_resched());
+}
+
+#if defined(CONFIG_CONTEXT_TRACKING_USER) && !defined(CONFIG_HAVE_CONTEXT_TRACKING_USER_OFFSTACK)
+asmlinkage __visible void __sched schedule_user(void)
+{
+	/*
+	 * If we come here after a random call to set_need_resched(),
+	 * or we have been woken up remotely but the IPI has not yet arrived,
+	 * we haven't yet exited the RCU idle mode. Do it here manually until
+	 * we find a better solution.
+	 *
+	 * NB: There are buggy callers of this function.  Ideally we
+	 * should warn if prev_state != CT_STATE_USER, but that will trigger
+	 * too frequently to make sense yet.
+	 */
+	enum ctx_state prev_state = exception_enter();
+	schedule();
+	exception_exit(prev_state);
+}
+#endif
+
+/**
+ * schedule_preempt_disabled - called with preemption disabled
+ *
+ * Returns with preemption disabled. Note: preempt_count must be 1
+ */
+void __sched schedule_preempt_disabled(void)
+{
+	sched_preempt_enable_no_resched();
+	schedule();
+	preempt_disable();
+}
+
+#ifdef CONFIG_PREEMPT_RT
+void __sched notrace schedule_rtlock(void)
+{
+	__schedule_loop(SM_RTLOCK_WAIT);
+}
+NOKPROBE_SYMBOL(schedule_rtlock);
+#endif
+
+static void __sched notrace preempt_schedule_common(void)
+{
+	do {
+		/*
+		 * Because the function tracer can trace preempt_count_sub()
+		 * and it also uses preempt_enable/disable_notrace(), if
+		 * NEED_RESCHED is set, the preempt_enable_notrace() called
+		 * by the function tracer will call this function again and
+		 * cause infinite recursion.
+		 *
+		 * Preemption must be disabled here before the function
+		 * tracer can trace. Break up preempt_disable() into two
+		 * calls. One to disable preemption without fear of being
+		 * traced. The other to still record the preemption latency,
+		 * which can also be traced by the function tracer.
+		 */
+		preempt_disable_notrace();
+		preempt_latency_start(1);
+		__schedule(SM_PREEMPT);
+		preempt_latency_stop(1);
+		preempt_enable_no_resched_notrace();
+
+		/*
+		 * Check again in case we missed a preemption opportunity
+		 * between schedule and now.
+		 */
+	} while (need_resched());
+}
+
+#ifdef CONFIG_PREEMPTION
+/*
+ * This is the entry point to schedule() from in-kernel preemption
+ * off of preempt_enable.
+ */
+asmlinkage __visible void __sched notrace preempt_schedule(void)
+{
+	/*
+	 * If there is a non-zero preempt_count or interrupts are disabled,
+	 * we do not want to preempt the current task. Just return..
+	 */
+	if (likely(!preemptible()))
+		return;
+
+	preempt_schedule_common();
+}
+NOKPROBE_SYMBOL(preempt_schedule);
+EXPORT_SYMBOL(preempt_schedule);
+
+#ifdef CONFIG_PREEMPT_DYNAMIC
+# ifdef CONFIG_HAVE_PREEMPT_DYNAMIC_CALL
+#  ifndef preempt_schedule_dynamic_enabled
+#   define preempt_schedule_dynamic_enabled	preempt_schedule
+#   define preempt_schedule_dynamic_disabled	NULL
+#  endif
+DEFINE_STATIC_CALL(preempt_schedule, preempt_schedule_dynamic_enabled);
+EXPORT_STATIC_CALL_TRAMP(preempt_schedule);
+# elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY)
+static DEFINE_STATIC_KEY_TRUE(sk_dynamic_preempt_schedule);
+void __sched notrace dynamic_preempt_schedule(void)
+{
+	if (!static_branch_unlikely(&sk_dynamic_preempt_schedule))
+		return;
+	preempt_schedule();
+}
+NOKPROBE_SYMBOL(dynamic_preempt_schedule);
+EXPORT_SYMBOL(dynamic_preempt_schedule);
+# endif
+#endif /* CONFIG_PREEMPT_DYNAMIC */
+
+/**
+ * preempt_schedule_notrace - preempt_schedule called by tracing
+ *
+ * The tracing infrastructure uses preempt_enable_notrace to prevent
+ * recursion and tracing preempt enabling caused by the tracing
+ * infrastructure itself. But as tracing can happen in areas coming
+ * from userspace or just about to enter userspace, a preempt enable
+ * can occur before user_exit() is called. This will cause the scheduler
+ * to be called when the system is still in usermode.
+ *
+ * To prevent this, the preempt_enable_notrace will use this function
+ * instead of preempt_schedule() to exit user context if needed before
+ * calling the scheduler.
+ */
+asmlinkage __visible void __sched notrace preempt_schedule_notrace(void)
+{
+	enum ctx_state prev_ctx;
+
+	if (likely(!preemptible()))
+		return;
+
+	do {
+		/*
+		 * Because the function tracer can trace preempt_count_sub()
+		 * and it also uses preempt_enable/disable_notrace(), if
+		 * NEED_RESCHED is set, the preempt_enable_notrace() called
+		 * by the function tracer will call this function again and
+		 * cause infinite recursion.
+		 *
+		 * Preemption must be disabled here before the function
+		 * tracer can trace. Break up preempt_disable() into two
+		 * calls. One to disable preemption without fear of being
+		 * traced. The other to still record the preemption latency,
+		 * which can also be traced by the function tracer.
+		 */
+		preempt_disable_notrace();
+		preempt_latency_start(1);
+		/*
+		 * Needs preempt disabled in case user_exit() is traced
+		 * and the tracer calls preempt_enable_notrace() causing
+		 * an infinite recursion.
+		 */
+		prev_ctx = exception_enter();
+		__schedule(SM_PREEMPT);
+		exception_exit(prev_ctx);
+
+		preempt_latency_stop(1);
+		preempt_enable_no_resched_notrace();
+	} while (need_resched());
+}
+EXPORT_SYMBOL_GPL(preempt_schedule_notrace);
+
+#ifdef CONFIG_PREEMPT_DYNAMIC
+# ifdef CONFIG_HAVE_PREEMPT_DYNAMIC_CALL
+#  ifndef preempt_schedule_notrace_dynamic_enabled
+#   define preempt_schedule_notrace_dynamic_enabled	preempt_schedule_notrace
+#   define preempt_schedule_notrace_dynamic_disabled	NULL
+#  endif
+DEFINE_STATIC_CALL(preempt_schedule_notrace, preempt_schedule_notrace_dynamic_enabled);
+EXPORT_STATIC_CALL_TRAMP(preempt_schedule_notrace);
+# elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY)
+static DEFINE_STATIC_KEY_TRUE(sk_dynamic_preempt_schedule_notrace);
+void __sched notrace dynamic_preempt_schedule_notrace(void)
+{
+	if (!static_branch_unlikely(&sk_dynamic_preempt_schedule_notrace))
+		return;
+	preempt_schedule_notrace();
+}
+NOKPROBE_SYMBOL(dynamic_preempt_schedule_notrace);
+EXPORT_SYMBOL(dynamic_preempt_schedule_notrace);
+# endif
+#endif /* CONFIG_PREEMPT_DYNAMIC */
+
+#endif /* CONFIG_PREEMPTION */
+
+/*
+ * This is the entry point to schedule() from kernel preemption
+ * off of IRQ context.
+ * Note, that this is called and return with IRQs disabled. This will
+ * protect us against recursive calling from IRQ contexts.
+ */
+asmlinkage __visible void __sched preempt_schedule_irq(void)
+{
+	enum ctx_state prev_state;
+
+	/* Catch callers which need to be fixed */
+	BUG_ON(preempt_count() || !irqs_disabled());
+
+	prev_state = exception_enter();
+
+	do {
+		preempt_disable();
+		local_irq_enable();
+		__schedule(SM_PREEMPT);
+		local_irq_disable();
+		sched_preempt_enable_no_resched();
+	} while (need_resched());
+
+	exception_exit(prev_state);
+}
+
+int default_wake_function(wait_queue_entry_t *curr, unsigned mode, int wake_flags,
+			  void *key)
+{
+	WARN_ON_ONCE(wake_flags & ~(WF_SYNC|WF_CURRENT_CPU));
+	return try_to_wake_up(curr->private, mode, wake_flags);
+}
+EXPORT_SYMBOL(default_wake_function);
+
+#ifdef CONFIG_RT_MUTEXES
+
+/*
+ * Would be more useful with typeof()/auto_type but they don't mix with
+ * bit-fields. Since it's a local thing, use int. Keep the generic sounding
+ * name such that if someone were to implement this function we get to compare
+ * notes.
+ */
+#define fetch_and_set(x, v) ({ int _x = (x); (x) = (v); _x; })
+
+void rt_mutex_pre_schedule(void)
+{
+	lockdep_assert(!fetch_and_set(current->sched_rt_mutex, 1));
+	sched_submit_work(current);
+}
+
+void rt_mutex_schedule(void)
+{
+	lockdep_assert(current->sched_rt_mutex);
+	__schedule_loop(SM_NONE);
+}
+
+void rt_mutex_post_schedule(void)
+{
+	sched_update_worker(current);
+	lockdep_assert(fetch_and_set(current->sched_rt_mutex, 0));
+}
+
+/*
+ * rt_mutex_setprio - set the current priority of a task
+ * @p: task to boost
+ * @pi_task: donor task
+ *
+ * This function changes the 'effective' priority of a task. It does
+ * not touch ->normal_prio like __setscheduler().
+ *
+ * Used by the rt_mutex code to implement priority inheritance
+ * logic. Call site only calls if the priority of the task changed.
+ */
+void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task)
+{
+	int prio, queue_flag = DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK;
+	struct rq *rq;
+	struct rq_flags rf;
+
+	/* XXX used to be waiter->prio, not waiter->task->prio */
+	prio = __rt_effective_prio(pi_task, p->normal_prio);
+
+	/*
+	 * If nothing changed; bail early.
+	 */
+	if (p->pi_top_task == pi_task && prio == p->prio)
+		return;
+
+	rq = __task_modify_lock(p, &rf);
+	/*
+	 * Set under pi_lock && rq->lock, such that the value can be used under
+	 * either lock.
+	 *
+	 * Note that there is loads of tricky to make this pointer cache work
+	 * right. rt_mutex_slowunlock()+rt_mutex_postunlock() work together to
+	 * ensure a task is de-boosted (pi_task is set to NULL) before the
+	 * task is allowed to run again (and can exit). This ensures the pointer
+	 * points to a blocked task -- which guarantees the task is present.
+	 */
+	p->pi_top_task = pi_task;
+
+	/*
+	 * For FIFO/RR we only need to set prio, if that matches we're done.
+	 */
+	if (prio == p->prio)
+		goto out_unlock;
+
+	/*
+	 * Idle task boosting is a no-no in general. There is one
+	 * exception, when PREEMPT_RT and NOHZ is active:
+	 *
+	 * The idle task calls get_next_timer_interrupt() and holds
+	 * the timer wheel base->lock on the CPU and another CPU wants
+	 * to access the timer (probably to cancel it). We can safely
+	 * ignore the boosting request, as the idle CPU runs this code
+	 * with interrupts disabled and will complete the lock
+	 * protected section without being interrupted. So there is no
+	 * real need to boost.
+	 */
+	if (unlikely(p == rq->idle)) {
+		WARN_ON(p != rq->curr);
+		WARN_ON(p->pi_blocked_on);
+		goto out_unlock;
+	}
+
+	trace_sched_pi_setprio(p, pi_task);
+
+	scoped_guard (sched_change, p, queue_flag) {
+		p->prio = prio;
+	}
+
+out_unlock:
+	/* Caller holds task_struct::pi_lock, IRQs are still disabled */
+
+	__balance_callbacks(rq);
+	__task_modify_unlock(p, &rf);
+}
+#endif /* CONFIG_RT_MUTEXES */
+
+#if !defined(CONFIG_PREEMPTION) || defined(CONFIG_PREEMPT_DYNAMIC)
+int __sched __cond_resched(void)
+{
+	if (should_resched(0) && !irqs_disabled()) {
+		preempt_schedule_common();
+		return 1;
+	}
+	/*
+	 * In PREEMPT_RCU kernels, ->rcu_read_lock_nesting tells the tick
+	 * whether the current CPU is in an RCU read-side critical section,
+	 * so the tick can report quiescent states even for CPUs looping
+	 * in kernel context.  In contrast, in non-preemptible kernels,
+	 * RCU readers leave no in-memory hints, which means that CPU-bound
+	 * processes executing in kernel context might never report an
+	 * RCU quiescent state.  Therefore, the following code causes
+	 * cond_resched() to report a quiescent state, but only when RCU
+	 * is in urgent need of one.
+	 * A third case, preemptible, but non-PREEMPT_RCU provides for
+	 * urgently needed quiescent states via rcu_flavor_sched_clock_irq().
+	 */
+#ifndef CONFIG_PREEMPT_RCU
+	rcu_all_qs();
+#endif
+	return 0;
+}
+EXPORT_SYMBOL(__cond_resched);
+#endif
+
+#ifdef CONFIG_PREEMPT_DYNAMIC
+# ifdef CONFIG_HAVE_PREEMPT_DYNAMIC_CALL
+#  define cond_resched_dynamic_enabled	__cond_resched
+#  define cond_resched_dynamic_disabled	((void *)&__static_call_return0)
+DEFINE_STATIC_CALL_RET0(cond_resched, __cond_resched);
+EXPORT_STATIC_CALL_TRAMP(cond_resched);
+
+#  define might_resched_dynamic_enabled		__cond_resched
+#  define might_resched_dynamic_disabled	((void *)&__static_call_return0)
+DEFINE_STATIC_CALL_RET0(might_resched, __cond_resched);
+EXPORT_STATIC_CALL_TRAMP(might_resched);
+# elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY)
+static DEFINE_STATIC_KEY_FALSE(sk_dynamic_cond_resched);
+int __sched dynamic_cond_resched(void)
+{
+	if (!static_branch_unlikely(&sk_dynamic_cond_resched))
+		return 0;
+	return __cond_resched();
+}
+EXPORT_SYMBOL(dynamic_cond_resched);
+
+static DEFINE_STATIC_KEY_FALSE(sk_dynamic_might_resched);
+int __sched dynamic_might_resched(void)
+{
+	if (!static_branch_unlikely(&sk_dynamic_might_resched))
+		return 0;
+	return __cond_resched();
+}
+EXPORT_SYMBOL(dynamic_might_resched);
+# endif
+#endif /* CONFIG_PREEMPT_DYNAMIC */
+
+/*
+ * __cond_resched_lock() - if a reschedule is pending, drop the given lock,
+ * call schedule, and on return reacquire the lock.
+ *
+ * This works OK both with and without CONFIG_PREEMPTION.  We do strange low-level
+ * operations here to prevent schedule() from being called twice (once via
+ * spin_unlock(), once by hand).
+ */
+int __cond_resched_lock(spinlock_t *lock)
+{
+	int resched = should_resched(PREEMPT_LOCK_OFFSET);
+	int ret = 0;
+
+	lockdep_assert_held(lock);
+
+	if (spin_needbreak(lock) || resched) {
+		spin_unlock(lock);
+		if (!_cond_resched())
+			cpu_relax();
+		ret = 1;
+		spin_lock(lock);
+	}
+	return ret;
+}
+EXPORT_SYMBOL(__cond_resched_lock);
+
+int __cond_resched_rwlock_read(rwlock_t *lock)
+{
+	int resched = should_resched(PREEMPT_LOCK_OFFSET);
+	int ret = 0;
+
+	lockdep_assert_held_read(lock);
+
+	if (rwlock_needbreak(lock) || resched) {
+		read_unlock(lock);
+		if (!_cond_resched())
+			cpu_relax();
+		ret = 1;
+		read_lock(lock);
+	}
+	return ret;
+}
+EXPORT_SYMBOL(__cond_resched_rwlock_read);
+
+int __cond_resched_rwlock_write(rwlock_t *lock)
+{
+	int resched = should_resched(PREEMPT_LOCK_OFFSET);
+	int ret = 0;
+
+	lockdep_assert_held_write(lock);
+
+	if (rwlock_needbreak(lock) || resched) {
+		write_unlock(lock);
+		if (!_cond_resched())
+			cpu_relax();
+		ret = 1;
+		write_lock(lock);
+	}
+	return ret;
+}
+EXPORT_SYMBOL(__cond_resched_rwlock_write);
+
+#ifdef CONFIG_PREEMPT_DYNAMIC
+
+# ifdef CONFIG_GENERIC_ENTRY
+#  include <linux/entry-common.h>
+# endif
+
+/*
+ * SC:cond_resched
+ * SC:might_resched
+ * SC:preempt_schedule
+ * SC:preempt_schedule_notrace
+ * SC:irqentry_exit_cond_resched
+ *
+ *
+ * NONE:
+ *   cond_resched               <- __cond_resched
+ *   might_resched              <- RET0
+ *   preempt_schedule           <- NOP
+ *   preempt_schedule_notrace   <- NOP
+ *   irqentry_exit_cond_resched <- NOP
+ *   dynamic_preempt_lazy       <- false
+ *
+ * VOLUNTARY:
+ *   cond_resched               <- __cond_resched
+ *   might_resched              <- __cond_resched
+ *   preempt_schedule           <- NOP
+ *   preempt_schedule_notrace   <- NOP
+ *   irqentry_exit_cond_resched <- NOP
+ *   dynamic_preempt_lazy       <- false
+ *
+ * FULL:
+ *   cond_resched               <- RET0
+ *   might_resched              <- RET0
+ *   preempt_schedule           <- preempt_schedule
+ *   preempt_schedule_notrace   <- preempt_schedule_notrace
+ *   irqentry_exit_cond_resched <- irqentry_exit_cond_resched
+ *   dynamic_preempt_lazy       <- false
+ *
+ * LAZY:
+ *   cond_resched               <- RET0
+ *   might_resched              <- RET0
+ *   preempt_schedule           <- preempt_schedule
+ *   preempt_schedule_notrace   <- preempt_schedule_notrace
+ *   irqentry_exit_cond_resched <- irqentry_exit_cond_resched
+ *   dynamic_preempt_lazy       <- true
+ */
+
+enum {
+	preempt_dynamic_undefined = -1,
+	preempt_dynamic_none,
+	preempt_dynamic_voluntary,
+	preempt_dynamic_full,
+	preempt_dynamic_lazy,
+};
+
+int preempt_dynamic_mode = preempt_dynamic_undefined;
+
+int sched_dynamic_mode(const char *str)
+{
+# if !(defined(CONFIG_PREEMPT_RT) || defined(CONFIG_ARCH_HAS_PREEMPT_LAZY))
+	if (!strcmp(str, "none"))
+		return preempt_dynamic_none;
+
+	if (!strcmp(str, "voluntary"))
+		return preempt_dynamic_voluntary;
+# endif
+
+	if (!strcmp(str, "full"))
+		return preempt_dynamic_full;
+
+# ifdef CONFIG_ARCH_HAS_PREEMPT_LAZY
+	if (!strcmp(str, "lazy"))
+		return preempt_dynamic_lazy;
+# endif
+
+	return -EINVAL;
+}
+
+# define preempt_dynamic_key_enable(f)  static_key_enable(&sk_dynamic_##f.key)
+# define preempt_dynamic_key_disable(f) static_key_disable(&sk_dynamic_##f.key)
+
+# if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL)
+#  define preempt_dynamic_enable(f)	static_call_update(f, f##_dynamic_enabled)
+#  define preempt_dynamic_disable(f)	static_call_update(f, f##_dynamic_disabled)
+# elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY)
+#  define preempt_dynamic_enable(f)	preempt_dynamic_key_enable(f)
+#  define preempt_dynamic_disable(f)	preempt_dynamic_key_disable(f)
+# else
+#  error "Unsupported PREEMPT_DYNAMIC mechanism"
+# endif
+
+static DEFINE_MUTEX(sched_dynamic_mutex);
+
+static void __sched_dynamic_update(int mode)
+{
+	/*
+	 * Avoid {NONE,VOLUNTARY} -> FULL transitions from ever ending up in
+	 * the ZERO state, which is invalid.
+	 */
+	preempt_dynamic_enable(cond_resched);
+	preempt_dynamic_enable(cond_resched);
+	preempt_dynamic_enable(might_resched);
+	preempt_dynamic_enable(preempt_schedule);
+	preempt_dynamic_enable(preempt_schedule_notrace);
+	preempt_dynamic_enable(irqentry_exit_cond_resched);
+	preempt_dynamic_key_disable(preempt_lazy);
+
+	switch (mode) {
+	case preempt_dynamic_none:
+		preempt_dynamic_enable(cond_resched);
+		preempt_dynamic_disable(might_resched);
+		preempt_dynamic_disable(preempt_schedule);
+		preempt_dynamic_disable(preempt_schedule_notrace);
+		preempt_dynamic_disable(irqentry_exit_cond_resched);
+		preempt_dynamic_key_disable(preempt_lazy);
+		if (mode != preempt_dynamic_mode)
+			pr_info("Dynamic Preempt: none\n");
+		break;
+
+	case preempt_dynamic_voluntary:
+		preempt_dynamic_enable(cond_resched);
+		preempt_dynamic_enable(might_resched);
+		preempt_dynamic_disable(preempt_schedule);
+		preempt_dynamic_disable(preempt_schedule_notrace);
+		preempt_dynamic_disable(irqentry_exit_cond_resched);
+		preempt_dynamic_key_disable(preempt_lazy);
+		if (mode != preempt_dynamic_mode)
+			pr_info("Dynamic Preempt: voluntary\n");
+		break;
+
+	case preempt_dynamic_full:
+		preempt_dynamic_enable(cond_resched);
+		preempt_dynamic_disable(might_resched);
+		preempt_dynamic_enable(preempt_schedule);
+		preempt_dynamic_enable(preempt_schedule_notrace);
+		preempt_dynamic_enable(irqentry_exit_cond_resched);
+		preempt_dynamic_key_disable(preempt_lazy);
+		if (mode != preempt_dynamic_mode)
+			pr_info("Dynamic Preempt: full\n");
+		break;
+
+	case preempt_dynamic_lazy:
+		preempt_dynamic_disable(cond_resched);
+		preempt_dynamic_disable(might_resched);
+		preempt_dynamic_enable(preempt_schedule);
+		preempt_dynamic_enable(preempt_schedule_notrace);
+		preempt_dynamic_enable(irqentry_exit_cond_resched);
+		preempt_dynamic_key_enable(preempt_lazy);
+		if (mode != preempt_dynamic_mode)
+			pr_info("Dynamic Preempt: lazy\n");
+		break;
+	}
+
+	preempt_dynamic_mode = mode;
+}
+
+void sched_dynamic_update(int mode)
+{
+	mutex_lock(&sched_dynamic_mutex);
+	__sched_dynamic_update(mode);
+	mutex_unlock(&sched_dynamic_mutex);
+}
+
+static int __init setup_preempt_mode(char *str)
+{
+	int mode = sched_dynamic_mode(str);
+	if (mode < 0) {
+		pr_warn("Dynamic Preempt: unsupported mode: %s\n", str);
+		return 0;
+	}
+
+	sched_dynamic_update(mode);
+	return 1;
+}
+__setup("preempt=", setup_preempt_mode);
+
+static void __init preempt_dynamic_init(void)
+{
+	if (preempt_dynamic_mode == preempt_dynamic_undefined) {
+		if (IS_ENABLED(CONFIG_PREEMPT_NONE)) {
+			sched_dynamic_update(preempt_dynamic_none);
+		} else if (IS_ENABLED(CONFIG_PREEMPT_VOLUNTARY)) {
+			sched_dynamic_update(preempt_dynamic_voluntary);
+		} else if (IS_ENABLED(CONFIG_PREEMPT_LAZY)) {
+			sched_dynamic_update(preempt_dynamic_lazy);
+		} else {
+			/* Default static call setting, nothing to do */
+			WARN_ON_ONCE(!IS_ENABLED(CONFIG_PREEMPT));
+			preempt_dynamic_mode = preempt_dynamic_full;
+			pr_info("Dynamic Preempt: full\n");
+		}
+	}
+}
+
+# define PREEMPT_MODEL_ACCESSOR(mode) \
+	bool preempt_model_##mode(void)						 \
+	{									 \
+		WARN_ON_ONCE(preempt_dynamic_mode == preempt_dynamic_undefined); \
+		return preempt_dynamic_mode == preempt_dynamic_##mode;		 \
+	}									 \
+	EXPORT_SYMBOL_GPL(preempt_model_##mode)
+
+PREEMPT_MODEL_ACCESSOR(none);
+PREEMPT_MODEL_ACCESSOR(voluntary);
+PREEMPT_MODEL_ACCESSOR(full);
+PREEMPT_MODEL_ACCESSOR(lazy);
+
+#else /* !CONFIG_PREEMPT_DYNAMIC: */
+
+#define preempt_dynamic_mode -1
+
+static inline void preempt_dynamic_init(void) { }
+
+#endif /* CONFIG_PREEMPT_DYNAMIC */
+
+const char *preempt_modes[] = {
+	"none", "voluntary", "full", "lazy", NULL,
+};
+
+const char *preempt_model_str(void)
+{
+	bool brace = IS_ENABLED(CONFIG_PREEMPT_RT) &&
+		(IS_ENABLED(CONFIG_PREEMPT_DYNAMIC) ||
+		 IS_ENABLED(CONFIG_PREEMPT_LAZY));
+	static char buf[128];
+
+	if (IS_ENABLED(CONFIG_PREEMPT_BUILD)) {
+		struct seq_buf s;
+
+		seq_buf_init(&s, buf, sizeof(buf));
+		seq_buf_puts(&s, "PREEMPT");
+
+		if (IS_ENABLED(CONFIG_PREEMPT_RT))
+			seq_buf_printf(&s, "%sRT%s",
+				       brace ? "_{" : "_",
+				       brace ? "," : "");
+
+		if (IS_ENABLED(CONFIG_PREEMPT_DYNAMIC)) {
+			seq_buf_printf(&s, "(%s)%s",
+				       preempt_dynamic_mode >= 0 ?
+				       preempt_modes[preempt_dynamic_mode] : "undef",
+				       brace ? "}" : "");
+			return seq_buf_str(&s);
+		}
+
+		if (IS_ENABLED(CONFIG_PREEMPT_LAZY)) {
+			seq_buf_printf(&s, "LAZY%s",
+				       brace ? "}" : "");
+			return seq_buf_str(&s);
+		}
+
+		return seq_buf_str(&s);
+	}
+
+	if (IS_ENABLED(CONFIG_PREEMPT_VOLUNTARY_BUILD))
+		return "VOLUNTARY";
+
+	return "NONE";
+}
+
+int io_schedule_prepare(void)
+{
+	int old_iowait = current->in_iowait;
+
+	current->in_iowait = 1;
+	blk_flush_plug(current->plug, true);
+	return old_iowait;
+}
+
+void io_schedule_finish(int token)
+{
+	current->in_iowait = token;
+}
+
+/*
+ * This task is about to go to sleep on IO.  Increment rq->nr_iowait so
+ * that process accounting knows that this is a task in IO wait state.
+ *
+ * But don't do that if it is a deliberate, throttling IO wait (this task
+ * has set its backing_dev_info: the queue against which it should throttle)
+ */
+
+long __sched io_schedule_timeout(long timeout)
+{
+	int token;
+	long ret;
+
+	token = io_schedule_prepare();
+	ret = schedule_timeout(timeout);
+	io_schedule_finish(token);
+
+	return ret;
+}
+EXPORT_SYMBOL(io_schedule_timeout);
+
+void __sched io_schedule(void)
+{
+	int token;
+
+	token = io_schedule_prepare();
+	schedule();
+	io_schedule_finish(token);
+}
+EXPORT_SYMBOL(io_schedule);
+
+void sched_show_task(struct task_struct *p)
+{
+	unsigned long free;
+	int ppid;
+
+	if (!try_get_task_stack(p))
+		return;
+
+	pr_info("task:%-15.15s state:%c", p->comm, task_state_to_char(p));
+
+	if (task_is_running(p))
+		pr_cont("  running task    ");
+	free = stack_not_used(p);
+	ppid = 0;
+	rcu_read_lock();
+	if (pid_alive(p))
+		ppid = task_pid_nr(rcu_dereference(p->real_parent));
+	rcu_read_unlock();
+	pr_cont(" stack:%-5lu pid:%-5d tgid:%-5d ppid:%-6d task_flags:0x%04x flags:0x%08lx\n",
+		free, task_pid_nr(p), task_tgid_nr(p),
+		ppid, p->flags, read_task_thread_flags(p));
+
+	print_worker_info(KERN_INFO, p);
+	print_stop_info(KERN_INFO, p);
+	show_stack(p, NULL, KERN_INFO);
+	put_task_stack(p);
+}
+EXPORT_SYMBOL_GPL(sched_show_task);
+
+static inline bool
+state_filter_match(unsigned long state_filter, struct task_struct *p)
+{
+	unsigned int state = READ_ONCE(p->__state);
+
+	/* no filter, everything matches */
+	if (!state_filter)
+		return true;
+
+	/* filter, but doesn't match */
+	if (!(state & state_filter))
+		return false;
+
+	/*
+	 * When looking for TASK_UNINTERRUPTIBLE skip TASK_IDLE (allows
+	 * TASK_KILLABLE).
+	 */
+	if (state_filter == TASK_UNINTERRUPTIBLE && (state & TASK_NOLOAD))
+		return false;
+
+	return true;
+}
+
+
+void show_state_filter(unsigned int state_filter)
+{
+	struct task_struct *g, *p;
+
+	rcu_read_lock();
+	for_each_process_thread(g, p) {
+		/*
+		 * reset the NMI-timeout, listing all files on a slow
+		 * console might take a lot of time:
+		 * Also, reset softlockup watchdogs on all CPUs, because
+		 * another CPU might be blocked waiting for us to process
+		 * an IPI.
+		 */
+		touch_nmi_watchdog();
+		touch_all_softlockup_watchdogs();
+		if (state_filter_match(state_filter, p))
+			sched_show_task(p);
+	}
+
+	/* TODO: Alt schedule FW should support this
+	if (!state_filter)
+		sysrq_sched_debug_show();
+	*/
+	rcu_read_unlock();
+	/*
+	 * Only show locks if all tasks are dumped:
+	 */
+	if (!state_filter)
+		debug_show_all_locks();
+}
+
+void dump_cpu_task(int cpu)
+{
+	if (in_hardirq() && cpu == smp_processor_id()) {
+		struct pt_regs *regs;
+
+		regs = get_irq_regs();
+		if (regs) {
+			show_regs(regs);
+			return;
+		}
+	}
+
+	if (trigger_single_cpu_backtrace(cpu))
+		return;
+
+	pr_info("Task dump for CPU %d:\n", cpu);
+	sched_show_task(cpu_curr(cpu));
+}
+
+/**
+ * init_idle - set up an idle thread for a given CPU
+ * @idle: task in question
+ * @cpu: CPU the idle task belongs to
+ *
+ * NOTE: this function does not set the idle thread's NEED_RESCHED
+ * flag, to make booting more robust.
+ */
+void __init init_idle(struct task_struct *idle, int cpu)
+{
+	struct affinity_context ac = (struct affinity_context) {
+		.new_mask  = cpumask_of(cpu),
+		.flags     = 0,
+	};
+	struct rq *rq = cpu_rq(cpu);
+	unsigned long flags;
+
+	raw_spin_lock_irqsave(&idle->pi_lock, flags);
+	raw_spin_lock(&rq->lock);
+
+	idle->prio = IDLE_TASK_PRIO;
+	idle->boost_prio = 0;
+	idle->time_slice = sysctl_sched_base_slice;
+	idle->last_ran = rq->clock_task;
+	idle->__state = TASK_RUNNING;
+	/*
+	 * PF_KTHREAD should already be set at this point; regardless, make it
+	 * look like a proper per-CPU kthread.
+	 */
+	idle->flags |= PF_KTHREAD | PF_NO_SETAFFINITY;
+	kthread_set_per_cpu(idle, cpu);
+
+	/*
+	 * No validation and serialization required at boot time and for
+	 * setting up the idle tasks of not yet online CPUs.
+	 */
+	set_cpus_allowed_common(idle, &ac);
+
+	/* Silence PROVE_RCU */
+	rcu_read_lock();
+	__set_task_cpu(idle, cpu);
+	rcu_read_unlock();
+
+	rq->idle = idle;
+	rcu_assign_pointer(rq->curr, idle);
+	idle->on_cpu = 1;
+
+	raw_spin_unlock(&rq->lock);
+	raw_spin_unlock_irqrestore(&idle->pi_lock, flags);
+
+	/* Set the preempt count _outside_ the spinlocks! */
+	init_idle_preempt_count(idle, cpu);
+
+	ftrace_graph_init_idle_task(idle, cpu);
+	vtime_init_idle(idle, cpu);
+	sprintf(idle->comm, "%s/%d", INIT_TASK_COMM, cpu);
+}
+
+int cpuset_cpumask_can_shrink(const struct cpumask __maybe_unused *cur,
+			      const struct cpumask __maybe_unused *trial)
+{
+	return 1;
+}
+
+int task_can_attach(struct task_struct *p)
+{
+	int ret = 0;
+
+	/*
+	 * Kthreads which disallow setaffinity shouldn't be moved
+	 * to a new cpuset; we don't want to change their CPU
+	 * affinity and isolating such threads by their set of
+	 * allowed nodes is unnecessary.  Thus, cpusets are not
+	 * applicable for such threads.  This prevents checking for
+	 * success of set_cpus_allowed_ptr() on all attached tasks
+	 * before cpus_mask may be changed.
+	 */
+	if (p->flags & PF_NO_SETAFFINITY)
+		ret = -EINVAL;
+
+	return ret;
+}
+
+bool sched_smp_initialized __read_mostly;
+
+#ifdef CONFIG_HOTPLUG_CPU
+/*
+ * Invoked on the outgoing CPU in context of the CPU hotplug thread
+ * after ensuring that there are no user space tasks left on the CPU.
+ *
+ * If there is a lazy mm in use on the hotplug thread, drop it and
+ * switch to init_mm.
+ *
+ * The reference count on init_mm is dropped in finish_cpu().
+ */
+static void sched_force_init_mm(void)
+{
+	struct mm_struct *mm = current->active_mm;
+
+	if (mm != &init_mm) {
+		mmgrab_lazy_tlb(&init_mm);
+		local_irq_disable();
+		current->active_mm = &init_mm;
+		switch_mm_irqs_off(mm, &init_mm, current);
+		local_irq_enable();
+		finish_arch_post_lock_switch();
+		mmdrop_lazy_tlb(mm);
+	}
+
+	/* finish_cpu(), as ran on the BP, will clean up the active_mm state */
+}
+
+static DEFINE_PER_CPU(struct cpu_stop_work, push_work);
+
+/*
+ * This is enabled below SCHED_AP_ACTIVE; when !cpu_active(), but only
+ * effective when the hotplug motion is down.
+ */
+static void balance_push(struct rq *rq)
+{
+	struct task_struct *push_task = rq->curr;
+
+	lockdep_assert_held(&rq->lock);
+
+	/*
+	 * Ensure the thing is persistent until balance_push_set(.on = false);
+	 */
+	rq->balance_callback = &balance_push_callback;
+
+	/*
+	 * Only active while going offline and when invoked on the outgoing
+	 * CPU.
+	 */
+	if (!cpu_dying(rq->cpu) || rq != this_rq())
+		return;
+
+	/*
+	 * Both the cpu-hotplug and stop task are in this case and are
+	 * required to complete the hotplug process.
+	 */
+	if (kthread_is_per_cpu(push_task) ||
+	    is_migration_disabled(push_task)) {
+
+		/*
+		 * If this is the idle task on the outgoing CPU try to wake
+		 * up the hotplug control thread which might wait for the
+		 * last task to vanish. The rcuwait_active() check is
+		 * accurate here because the waiter is pinned on this CPU
+		 * and can't obviously be running in parallel.
+		 *
+		 * On RT kernels this also has to check whether there are
+		 * pinned and scheduled out tasks on the runqueue. They
+		 * need to leave the migrate disabled section first.
+		 */
+		if (rq->curr == rq->idle && !rq_has_pinned_tasks(rq) &&
+		    rcuwait_active(&rq->hotplug_wait)) {
+			raw_spin_unlock(&rq->lock);
+			rcuwait_wake_up(&rq->hotplug_wait);
+			raw_spin_lock(&rq->lock);
+		}
+		return;
+	}
+
+	/*
+	 * Temporarily drop rq->lock such that we can wake-up the stop task.
+	 * Both preemption and IRQs are still disabled.
+	 */
+	preempt_disable();
+	raw_spin_unlock(&rq->lock);
+	stop_one_cpu_nowait(rq->cpu, dummy_cpu_stop, push_task, this_cpu_ptr(&push_work));
+	preempt_enable();
+	/*
+	 * At this point need_resched() is true and we'll take the loop in
+	 * schedule(). The next pick is obviously going to be the stop task
+	 * which kthread_is_per_cpu() and will push this task away.
+	 */
+	raw_spin_lock(&rq->lock);
+}
+
+static void balance_push_set(int cpu, bool on)
+{
+	struct rq *rq = cpu_rq(cpu);
+	struct rq_flags rf;
+
+	rq_lock_irqsave(rq, &rf);
+	if (on) {
+		WARN_ON_ONCE(rq->balance_callback);
+		rq->balance_callback = &balance_push_callback;
+	} else if (rq->balance_callback == &balance_push_callback) {
+		rq->balance_callback = NULL;
+	}
+	rq_unlock_irqrestore(rq, &rf);
+}
+
+/*
+ * Invoked from a CPUs hotplug control thread after the CPU has been marked
+ * inactive. All tasks which are not per CPU kernel threads are either
+ * pushed off this CPU now via balance_push() or placed on a different CPU
+ * during wakeup. Wait until the CPU is quiescent.
+ */
+static void balance_hotplug_wait(void)
+{
+	struct rq *rq = this_rq();
+	int cpu = cpu_of(rq);
+
+	rcuwait_wait_event(&rq->hotplug_wait, !rq_has_pinned_tasks(rq) &&	\
+			   is_preempt_list_empty(cpu), TASK_UNINTERRUPTIBLE);
+}
+
+#else /* !CONFIG_HOTPLUG_CPU: */
+
+static void balance_push(struct rq *rq)
+{
+}
+
+static void balance_push_set(int cpu, bool on)
+{
+}
+
+static inline void balance_hotplug_wait(void)
+{
+}
+#endif /* !CONFIG_HOTPLUG_CPU */
+
+static void set_rq_offline(struct rq *rq)
+{
+	if (rq->online) {
+		update_rq_clock(rq);
+		rq->online = false;
+	}
+}
+
+static void set_rq_online(struct rq *rq)
+{
+	if (!rq->online)
+		rq->online = true;
+}
+
+static inline void sched_set_rq_online(struct rq *rq, int cpu)
+{
+	unsigned long flags;
+
+	raw_spin_lock_irqsave(&rq->lock, flags);
+	set_rq_online(rq);
+	raw_spin_unlock_irqrestore(&rq->lock, flags);
+}
+
+static inline void sched_set_rq_offline(struct rq *rq, int cpu)
+{
+	unsigned long flags;
+
+	raw_spin_lock_irqsave(&rq->lock, flags);
+	set_rq_offline(rq);
+	raw_spin_unlock_irqrestore(&rq->lock, flags);
+}
+
+/*
+ * used to mark begin/end of suspend/resume:
+ */
+static int num_cpus_frozen;
+
+/*
+ * Update cpusets according to cpu_active mask.  If cpusets are
+ * disabled, cpuset_update_active_cpus() becomes a simple wrapper
+ * around partition_sched_domains().
+ *
+ * If we come here as part of a suspend/resume, don't touch cpusets because we
+ * want to restore it back to its original state upon resume anyway.
+ */
+static void cpuset_cpu_active(void)
+{
+	if (cpuhp_tasks_frozen) {
+		/*
+		 * num_cpus_frozen tracks how many CPUs are involved in suspend
+		 * resume sequence. As long as this is not the last online
+		 * operation in the resume sequence, just build a single sched
+		 * domain, ignoring cpusets.
+		 */
+		cpuset_reset_sched_domains();
+		if (--num_cpus_frozen)
+			return;
+		/*
+		 * This is the last CPU online operation. So fall through and
+		 * restore the original sched domains by considering the
+		 * cpuset configurations.
+		 */
+		cpuset_force_rebuild();
+	}
+
+	cpuset_update_active_cpus();
+}
+
+static void cpuset_cpu_inactive(unsigned int cpu)
+{
+	if (!cpuhp_tasks_frozen) {
+		cpuset_update_active_cpus();
+	} else {
+		num_cpus_frozen++;
+		cpuset_reset_sched_domains();
+	}
+}
+
+static inline void sched_smt_present_inc(int cpu)
+{
+#ifdef CONFIG_SCHED_SMT
+	if (cpumask_weight(cpu_smt_mask(cpu)) == 2) {
+		static_branch_inc_cpuslocked(&sched_smt_present);
+		cpumask_or(&sched_smt_mask, &sched_smt_mask, cpu_smt_mask(cpu));
+	}
+#endif /* CONFIG_SCHED_SMT */
+}
+
+static inline void sched_smt_present_dec(int cpu)
+{
+#ifdef CONFIG_SCHED_SMT
+	if (cpumask_weight(cpu_smt_mask(cpu)) == 2) {
+		static_branch_dec_cpuslocked(&sched_smt_present);
+		cpumask_andnot(&sched_smt_mask, &sched_smt_mask, cpu_smt_mask(cpu));
+	}
+#endif /* CONFIG_SCHED_SMT */
+}
+
+int sched_cpu_activate(unsigned int cpu)
+{
+	struct rq *rq = cpu_rq(cpu);
+
+	/*
+	 * Clear the balance_push callback and prepare to schedule
+	 * regular tasks.
+	 */
+	balance_push_set(cpu, false);
+
+	set_cpu_active(cpu, true);
+
+	if (sched_smp_initialized)
+		cpuset_cpu_active();
+
+	/*
+	 * Put the rq online, if not already. This happens:
+	 *
+	 * 1) In the early boot process, because we build the real domains
+	 *    after all cpus have been brought up.
+	 *
+	 * 2) At runtime, if cpuset_cpu_active() fails to rebuild the
+	 *    domains.
+	 */
+	sched_set_rq_online(rq, cpu);
+
+	/*
+	 * When going up, increment the number of cores with SMT present.
+	 */
+	sched_smt_present_inc(cpu);
+
+	return 0;
+}
+
+int sched_cpu_deactivate(unsigned int cpu)
+{
+	struct rq *rq = cpu_rq(cpu);
+
+	set_cpu_active(cpu, false);
+
+	/*
+	 * From this point forward, this CPU will refuse to run any task that
+	 * is not: migrate_disable() or KTHREAD_IS_PER_CPU, and will actively
+	 * push those tasks away until this gets cleared, see
+	 * sched_cpu_dying().
+	 */
+	balance_push_set(cpu, true);
+
+	/*
+	 * We've cleared cpu_active_mask, wait for all preempt-disabled and RCU
+	 * users of this state to go away such that all new such users will
+	 * observe it.
+	 *
+	 * Specifically, we rely on ttwu to no longer target this CPU, see
+	 * ttwu_queue_cond() and is_cpu_allowed().
+	 *
+	 * Do sync before park smpboot threads to take care the RCU boost case.
+	 */
+	synchronize_rcu();
+
+	sched_set_rq_offline(rq, cpu);
+
+	/*
+	 * When going down, decrement the number of cores with SMT present.
+	 */
+	sched_smt_present_dec(cpu);
+
+	if (!sched_smp_initialized)
+		return 0;
+
+	cpuset_cpu_inactive(cpu);
+
+	return 0;
+}
+
+static void sched_rq_cpu_starting(unsigned int cpu)
+{
+	struct rq *rq = cpu_rq(cpu);
+
+	rq->calc_load_update = calc_load_update;
+}
+
+int sched_cpu_starting(unsigned int cpu)
+{
+	sched_rq_cpu_starting(cpu);
+	sched_tick_start(cpu);
+	return 0;
+}
+
+#ifdef CONFIG_HOTPLUG_CPU
+
+/*
+ * Invoked immediately before the stopper thread is invoked to bring the
+ * CPU down completely. At this point all per CPU kthreads except the
+ * hotplug thread (current) and the stopper thread (inactive) have been
+ * either parked or have been unbound from the outgoing CPU. Ensure that
+ * any of those which might be on the way out are gone.
+ *
+ * If after this point a bound task is being woken on this CPU then the
+ * responsible hotplug callback has failed to do it's job.
+ * sched_cpu_dying() will catch it with the appropriate fireworks.
+ */
+int sched_cpu_wait_empty(unsigned int cpu)
+{
+	balance_hotplug_wait();
+	sched_force_init_mm();
+	return 0;
+}
+
+static void dump_rq_tasks(struct rq *rq, const char *loglvl)
+{
+	struct task_struct *g, *p;
+	int cpu = cpu_of(rq);
+
+	lockdep_assert_held(&rq->lock);
+
+	printk("%sCPU%d\n", loglvl, cpu);
+	for_each_process_thread(g, p) {
+		if (task_cpu(p) != cpu)
+			continue;
+
+		if (!task_on_rq_queued(p))
+			continue;
+
+		printk("%s\tpid: %d, name: %s\n", loglvl, p->pid, p->comm);
+	}
+}
+
+int sched_cpu_dying(unsigned int cpu)
+{
+	struct rq *rq = cpu_rq(cpu);
+	unsigned long flags;
+
+	/* Handle pending wakeups and then migrate everything off */
+	sched_tick_stop(cpu);
+
+	raw_spin_lock_irqsave(&rq->lock, flags);
+	if (rq_has_pinned_tasks(rq)) {
+		WARN(true, "Dying CPU not properly vacated!");
+		dump_rq_tasks(rq, KERN_WARNING);
+	}
+	raw_spin_unlock_irqrestore(&rq->lock, flags);
+
+	hrtick_clear(rq);
+	return 0;
+}
+#endif /* CONFIG_HOTPLUG_CPU */
+
+void __init sched_init_smp(void)
+{
+	/* Move init over to a non-isolated CPU */
+	if (set_cpus_allowed_ptr(current, housekeeping_cpumask(HK_TYPE_DOMAIN)) < 0)
+		BUG();
+	current->flags &= ~PF_NO_SETAFFINITY;
+
+	sched_init_topology();
+
+	sched_smp_initialized = true;
+}
+
+static int __init migration_init(void)
+{
+	sched_cpu_starting(smp_processor_id());
+	return 0;
+}
+early_initcall(migration_init);
+
+int in_sched_functions(unsigned long addr)
+{
+	return in_lock_functions(addr) ||
+		(addr >= (unsigned long)__sched_text_start
+		&& addr < (unsigned long)__sched_text_end);
+}
+
+#ifdef CONFIG_CGROUP_SCHED
+/*
+ * Default task group.
+ * Every task in system belongs to this group at bootup.
+ */
+struct task_group root_task_group;
+LIST_HEAD(task_groups);
+
+/* Cacheline aligned slab cache for task_group */
+static struct kmem_cache *task_group_cache __ro_after_init;
+#endif /* CONFIG_CGROUP_SCHED */
+
+void __init sched_init(void)
+{
+	int i;
+	struct rq *rq;
+
+	printk(KERN_INFO "sched/alt: "ALT_SCHED_NAME" CPU Scheduler "ALT_SCHED_VERSION\
+			 " by Alfred Chen.\n");
+
+	wait_bit_init();
+
+#ifdef CONFIG_CGROUP_SCHED
+	task_group_cache = KMEM_CACHE(task_group, 0);
+
+	list_add(&root_task_group.list, &task_groups);
+	INIT_LIST_HEAD(&root_task_group.children);
+	INIT_LIST_HEAD(&root_task_group.siblings);
+#endif /* CONFIG_CGROUP_SCHED */
+	sched_run_queue_init(&grq);
+	for_each_possible_cpu(i) {
+		WRITE_ONCE(cpu_prio[i], IDLE_TASK_SCHED_PRIO);
+
+		rq = cpu_rq(i);
+
+#ifdef CONFIG_SCHED_PDS
+		rq->prio_idx = rq->prio;
+#endif
+
+		raw_spin_lock_init(&rq->lock);
+		rq->calc_load_active = 0;
+		rq->calc_load_update = jiffies + LOAD_FREQ;
+		rq->online = false;
+		rq->cpu = i;
+
+		rq->balance_func = NULL;
+		rq->active_balance_arg.active = 0;
+
+#ifdef CONFIG_NO_HZ_COMMON
+		INIT_CSD(&rq->nohz_csd, nohz_csd_func, rq);
+#endif
+		rq->balance_callback = &balance_push_callback;
+#ifdef CONFIG_HOTPLUG_CPU
+		rcuwait_init(&rq->hotplug_wait);
+#endif
+		rq->nr_switches = 0;
+
+		hrtick_rq_init(rq);
+
+		zalloc_cpumask_var_node(&rq->scratch_mask, GFP_KERNEL, cpu_to_node(i));
+	}
+	/* Set rq->online for cpu 0 */
+	cpu_rq(0)->online = true;
+	/*
+	 * The boot idle thread does lazy MMU switching as well:
+	 */
+	mmgrab_lazy_tlb(&init_mm);
+	enter_lazy_tlb(&init_mm, current);
+
+	/*
+	 * The idle task doesn't need the kthread struct to function, but it
+	 * is dressed up as a per-CPU kthread and thus needs to play the part
+	 * if we want to avoid special-casing it in code that deals with per-CPU
+	 * kthreads.
+	 */
+	WARN_ON(!set_kthread_struct(current));
+
+	/*
+	 * Make us the idle thread. Technically, schedule() should not be
+	 * called from this thread, however somewhere below it might be,
+	 * but because we are the idle thread, we just pick up running again
+	 * when this runqueue becomes "idle".
+	 */
+	__sched_fork(0, current);
+	init_idle(current, smp_processor_id());
+
+	calc_load_update = jiffies + LOAD_FREQ;
+
+	idle_thread_set_boot_cpu();
+	balance_push_set(smp_processor_id(), false);
+
+	sched_init_topology_early();
+
+	preempt_dynamic_init();
+}
+
+#ifdef CONFIG_DEBUG_ATOMIC_SLEEP
+
+void __might_sleep(const char *file, int line)
+{
+	unsigned int state = get_current_state();
+	/*
+	 * Blocking primitives will set (and therefore destroy) current->state,
+	 * since we will exit with TASK_RUNNING make sure we enter with it,
+	 * otherwise we will destroy state.
+	 */
+	WARN_ONCE(state != TASK_RUNNING && current->task_state_change,
+			"do not call blocking ops when !TASK_RUNNING; "
+			"state=%x set at [<%p>] %pS\n", state,
+			(void *)current->task_state_change,
+			(void *)current->task_state_change);
+
+	__might_resched(file, line, 0);
+}
+EXPORT_SYMBOL(__might_sleep);
+
+static void print_preempt_disable_ip(int preempt_offset, unsigned long ip)
+{
+	if (!IS_ENABLED(CONFIG_DEBUG_PREEMPT))
+		return;
+
+	if (preempt_count() == preempt_offset)
+		return;
+
+	pr_err("Preemption disabled at:");
+	print_ip_sym(KERN_ERR, ip);
+}
+
+static inline bool resched_offsets_ok(unsigned int offsets)
+{
+	unsigned int nested = preempt_count();
+
+	nested += rcu_preempt_depth() << MIGHT_RESCHED_RCU_SHIFT;
+
+	return nested == offsets;
+}
+
+void __might_resched(const char *file, int line, unsigned int offsets)
+{
+	/* Ratelimiting timestamp: */
+	static unsigned long prev_jiffy;
+
+	unsigned long preempt_disable_ip;
+
+	/* WARN_ON_ONCE() by default, no rate limit required: */
+	rcu_sleep_check();
+
+	if ((resched_offsets_ok(offsets) && !irqs_disabled() &&
+	     !is_idle_task(current) && !current->non_block_count) ||
+	    system_state == SYSTEM_BOOTING || system_state > SYSTEM_RUNNING ||
+	    oops_in_progress)
+		return;
+	if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy)
+		return;
+	prev_jiffy = jiffies;
+
+	/* Save this before calling printk(), since that will clobber it: */
+	preempt_disable_ip = get_preempt_disable_ip(current);
+
+	pr_err("BUG: sleeping function called from invalid context at %s:%d\n",
+	       file, line);
+	pr_err("in_atomic(): %d, irqs_disabled(): %d, non_block: %d, pid: %d, name: %s\n",
+	       in_atomic(), irqs_disabled(), current->non_block_count,
+	       current->pid, current->comm);
+	pr_err("preempt_count: %x, expected: %x\n", preempt_count(),
+	       offsets & MIGHT_RESCHED_PREEMPT_MASK);
+
+	if (IS_ENABLED(CONFIG_PREEMPT_RCU)) {
+		pr_err("RCU nest depth: %d, expected: %u\n",
+		       rcu_preempt_depth(), offsets >> MIGHT_RESCHED_RCU_SHIFT);
+	}
+
+	if (task_stack_end_corrupted(current))
+		pr_emerg("Thread overran stack, or stack corrupted\n");
+
+	debug_show_held_locks(current);
+	if (irqs_disabled())
+		print_irqtrace_events(current);
+
+	print_preempt_disable_ip(offsets & MIGHT_RESCHED_PREEMPT_MASK,
+				 preempt_disable_ip);
+
+	dump_stack();
+	add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
+}
+EXPORT_SYMBOL(__might_resched);
+
+void __cant_sleep(const char *file, int line, int preempt_offset)
+{
+	static unsigned long prev_jiffy;
+
+	if (irqs_disabled())
+		return;
+
+	if (!IS_ENABLED(CONFIG_PREEMPT_COUNT))
+		return;
+
+	if (preempt_count() > preempt_offset)
+		return;
+
+	if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy)
+		return;
+	prev_jiffy = jiffies;
+
+	printk(KERN_ERR "BUG: assuming atomic context at %s:%d\n", file, line);
+	printk(KERN_ERR "in_atomic(): %d, irqs_disabled(): %d, pid: %d, name: %s\n",
+			in_atomic(), irqs_disabled(),
+			current->pid, current->comm);
+
+	debug_show_held_locks(current);
+	dump_stack();
+	add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
+}
+EXPORT_SYMBOL_GPL(__cant_sleep);
+
+void __cant_migrate(const char *file, int line)
+{
+	static unsigned long prev_jiffy;
+
+	if (irqs_disabled())
+		return;
+
+	if (is_migration_disabled(current))
+		return;
+
+	if (!IS_ENABLED(CONFIG_PREEMPT_COUNT))
+		return;
+
+	if (preempt_count() > 0)
+		return;
+
+	if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy)
+		return;
+	prev_jiffy = jiffies;
+
+	pr_err("BUG: assuming non migratable context at %s:%d\n", file, line);
+	pr_err("in_atomic(): %d, irqs_disabled(): %d, migration_disabled() %u pid: %d, name: %s\n",
+	       in_atomic(), irqs_disabled(), is_migration_disabled(current),
+	       current->pid, current->comm);
+
+	debug_show_held_locks(current);
+	dump_stack();
+	add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
+}
+EXPORT_SYMBOL_GPL(__cant_migrate);
+#endif /* CONFIG_DEBUG_ATOMIC_SLEEP */
+
+#ifdef CONFIG_MAGIC_SYSRQ
+void normalize_rt_tasks(void)
+{
+	struct task_struct *g, *p;
+	struct sched_attr attr = {
+		.sched_policy = SCHED_NORMAL,
+	};
+
+	read_lock(&tasklist_lock);
+	for_each_process_thread(g, p) {
+		/*
+		 * Only normalize user tasks:
+		 */
+		if (p->flags & PF_KTHREAD)
+			continue;
+
+		schedstat_set(p->stats.wait_start,  0);
+		schedstat_set(p->stats.sleep_start, 0);
+		schedstat_set(p->stats.block_start, 0);
+
+		if (!rt_or_dl_task(p)) {
+			/*
+			 * Renice negative nice level userspace
+			 * tasks back to 0:
+			 */
+			if (task_nice(p) < 0)
+				set_user_nice(p, 0);
+			continue;
+		}
+
+		__sched_setscheduler(p, &attr, false, false);
+	}
+	read_unlock(&tasklist_lock);
+}
+#endif /* CONFIG_MAGIC_SYSRQ */
+
+#ifdef CONFIG_KGDB_KDB
+/*
+ * These functions are only useful for KDB.
+ *
+ * They can only be called when the whole system has been
+ * stopped - every CPU needs to be quiescent, and no scheduling
+ * activity can take place. Using them for anything else would
+ * be a serious bug, and as a result, they aren't even visible
+ * under any other configuration.
+ */
+
+/**
+ * curr_task - return the current task for a given CPU.
+ * @cpu: the processor in question.
+ *
+ * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
+ *
+ * Return: The current task for @cpu.
+ */
+struct task_struct *curr_task(int cpu)
+{
+	return cpu_curr(cpu);
+}
+
+#endif /* CONFIG_KGDB_KDB */
+
+#ifdef CONFIG_CGROUP_SCHED
+static void sched_free_group(struct task_group *tg)
+{
+	kmem_cache_free(task_group_cache, tg);
+}
+
+static void sched_free_group_rcu(struct rcu_head *rhp)
+{
+	sched_free_group(container_of(rhp, struct task_group, rcu));
+}
+
+static void sched_unregister_group(struct task_group *tg)
+{
+	/*
+	 * We have to wait for yet another RCU grace period to expire, as
+	 * print_cfs_stats() might run concurrently.
+	 */
+	call_rcu(&tg->rcu, sched_free_group_rcu);
+}
+
+/* allocate runqueue etc for a new task group */
+struct task_group *sched_create_group(struct task_group *parent)
+{
+	struct task_group *tg;
+
+	tg = kmem_cache_alloc(task_group_cache, GFP_KERNEL | __GFP_ZERO);
+	if (!tg)
+		return ERR_PTR(-ENOMEM);
+
+	return tg;
+}
+
+void sched_online_group(struct task_group *tg, struct task_group *parent)
+{
+}
+
+/* RCU callback to free various structures associated with a task group */
+static void sched_unregister_group_rcu(struct rcu_head *rhp)
+{
+	/* Now it should be safe to free those cfs_rqs: */
+	sched_unregister_group(container_of(rhp, struct task_group, rcu));
+}
+
+void sched_destroy_group(struct task_group *tg)
+{
+	/* Wait for possible concurrent references to cfs_rqs complete: */
+	call_rcu(&tg->rcu, sched_unregister_group_rcu);
+}
+
+void sched_release_group(struct task_group *tg)
+{
+}
+
+static inline struct task_group *css_tg(struct cgroup_subsys_state *css)
+{
+	return css ? container_of(css, struct task_group, css) : NULL;
+}
+
+static struct cgroup_subsys_state *
+cpu_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
+{
+	struct task_group *parent = css_tg(parent_css);
+	struct task_group *tg;
+
+	if (!parent) {
+		/* This is early initialization for the top cgroup */
+		return &root_task_group.css;
+	}
+
+	tg = sched_create_group(parent);
+	if (IS_ERR(tg))
+		return ERR_PTR(-ENOMEM);
+	return &tg->css;
+}
+
+/* Expose task group only after completing cgroup initialization */
+static int cpu_cgroup_css_online(struct cgroup_subsys_state *css)
+{
+	struct task_group *tg = css_tg(css);
+	struct task_group *parent = css_tg(css->parent);
+
+	if (parent)
+		sched_online_group(tg, parent);
+	return 0;
+}
+
+static void cpu_cgroup_css_released(struct cgroup_subsys_state *css)
+{
+	struct task_group *tg = css_tg(css);
+
+	sched_release_group(tg);
+}
+
+static void cpu_cgroup_css_free(struct cgroup_subsys_state *css)
+{
+	struct task_group *tg = css_tg(css);
+
+	/*
+	 * Relies on the RCU grace period between css_released() and this.
+	 */
+	sched_unregister_group(tg);
+}
+
+#ifdef CONFIG_RT_GROUP_SCHED
+static int cpu_cgroup_can_attach(struct cgroup_taskset *tset)
+{
+	return 0;
+}
+#endif /* CONFIG_RT_GROUP_SCHED */
+
+static void cpu_cgroup_attach(struct cgroup_taskset *tset)
+{
+}
+
+#ifdef CONFIG_GROUP_SCHED_WEIGHT
+static int sched_group_set_shares(struct task_group *tg, unsigned long shares)
+{
+	return 0;
+}
+
+static int sched_group_set_idle(struct task_group *tg, long idle)
+{
+	return 0;
+}
+
+static int cpu_shares_write_u64(struct cgroup_subsys_state *css,
+				struct cftype *cftype, u64 shareval)
+{
+	return sched_group_set_shares(css_tg(css), shareval);
+}
+
+static u64 cpu_shares_read_u64(struct cgroup_subsys_state *css,
+			       struct cftype *cft)
+{
+	return 1024;
+}
+
+static s64 cpu_idle_read_s64(struct cgroup_subsys_state *css,
+			       struct cftype *cft)
+{
+	return 0;
+}
+
+static int cpu_idle_write_s64(struct cgroup_subsys_state *css,
+				struct cftype *cft, s64 idle)
+{
+	return sched_group_set_idle(css_tg(css), idle);
+}
+#endif /* CONFIG_GROUP_SCHED_WEIGHT */
+
+#ifdef CONFIG_CFS_BANDWIDTH
+static s64 cpu_cfs_quota_read_s64(struct cgroup_subsys_state *css,
+				  struct cftype *cft)
+{
+	return -1;
+}
+
+static int cpu_cfs_quota_write_s64(struct cgroup_subsys_state *css,
+				   struct cftype *cftype, s64 cfs_quota_us)
+{
+	return 0;
+}
+
+static u64 cpu_cfs_period_read_u64(struct cgroup_subsys_state *css,
+				   struct cftype *cft)
+{
+	return 100000ULL;
+}
+
+static int cpu_cfs_period_write_u64(struct cgroup_subsys_state *css,
+				    struct cftype *cftype, u64 cfs_period_us)
+{
+	return 0;
+}
+
+static u64 cpu_cfs_burst_read_u64(struct cgroup_subsys_state *css,
+				  struct cftype *cft)
+{
+	return 0;
+}
+
+static int cpu_cfs_burst_write_u64(struct cgroup_subsys_state *css,
+				   struct cftype *cftype, u64 cfs_burst_us)
+{
+	return 0;
+}
+
+static int cpu_cfs_stat_show(struct seq_file *sf, void *v)
+{
+	return 0;
+}
+
+static int cpu_cfs_local_stat_show(struct seq_file *sf, void *v)
+{
+	return 0;
+}
+#endif /* CONFIG_CFS_BANDWIDTH */
+
+#ifdef CONFIG_RT_GROUP_SCHED
+static int cpu_rt_runtime_write(struct cgroup_subsys_state *css,
+				struct cftype *cft, s64 val)
+{
+	return 0;
+}
+
+static s64 cpu_rt_runtime_read(struct cgroup_subsys_state *css,
+			       struct cftype *cft)
+{
+	return 0;
+}
+
+static int cpu_rt_period_write_uint(struct cgroup_subsys_state *css,
+				    struct cftype *cftype, u64 rt_period_us)
+{
+	return 0;
+}
+
+static u64 cpu_rt_period_read_uint(struct cgroup_subsys_state *css,
+				   struct cftype *cft)
+{
+	return 0;
+}
+#endif /* CONFIG_RT_GROUP_SCHED */
+
+#ifdef CONFIG_UCLAMP_TASK_GROUP
+static int cpu_uclamp_min_show(struct seq_file *sf, void *v)
+{
+	seq_puts(sf, "0\n");
+	return 0;
+}
+
+static int cpu_uclamp_max_show(struct seq_file *sf, void *v)
+{
+	seq_puts(sf, "max\n");
+	return 0;
+}
+
+static ssize_t cpu_uclamp_min_write(struct kernfs_open_file *of,
+				    char *buf, size_t nbytes,
+				    loff_t off)
+{
+	return nbytes;
+}
+
+static ssize_t cpu_uclamp_max_write(struct kernfs_open_file *of,
+				    char *buf, size_t nbytes,
+				    loff_t off)
+{
+	return nbytes;
+}
+#endif /* CONFIG_UCLAMP_TASK_GROUP */
+
+static struct cftype cpu_legacy_files[] = {
+#ifdef CONFIG_GROUP_SCHED_WEIGHT
+	{
+		.name = "shares",
+		.read_u64 = cpu_shares_read_u64,
+		.write_u64 = cpu_shares_write_u64,
+	},
+	{
+		.name = "idle",
+		.read_s64 = cpu_idle_read_s64,
+		.write_s64 = cpu_idle_write_s64,
+	},
+#endif /* CONFIG_GROUP_SCHED_WEIGHT */
+#ifdef CONFIG_CFS_BANDWIDTH
+	{
+		.name = "cfs_quota_us",
+		.read_s64 = cpu_cfs_quota_read_s64,
+		.write_s64 = cpu_cfs_quota_write_s64,
+	},
+	{
+		.name = "cfs_period_us",
+		.read_u64 = cpu_cfs_period_read_u64,
+		.write_u64 = cpu_cfs_period_write_u64,
+	},
+	{
+		.name = "cfs_burst_us",
+		.read_u64 = cpu_cfs_burst_read_u64,
+		.write_u64 = cpu_cfs_burst_write_u64,
+	},
+	{
+		.name = "stat",
+		.seq_show = cpu_cfs_stat_show,
+	},
+	{
+		.name = "stat.local",
+		.seq_show = cpu_cfs_local_stat_show,
+	},
+#endif /* CONFIG_CFS_BANDWIDTH */
+#ifdef CONFIG_RT_GROUP_SCHED
+	{
+		.name = "rt_runtime_us",
+		.read_s64 = cpu_rt_runtime_read,
+		.write_s64 = cpu_rt_runtime_write,
+	},
+	{
+		.name = "rt_period_us",
+		.read_u64 = cpu_rt_period_read_uint,
+		.write_u64 = cpu_rt_period_write_uint,
+	},
+#endif /* CONFIG_RT_GROUP_SCHED */
+#ifdef CONFIG_UCLAMP_TASK_GROUP
+	{
+		.name = "uclamp.min",
+		.flags = CFTYPE_NOT_ON_ROOT,
+		.seq_show = cpu_uclamp_min_show,
+		.write = cpu_uclamp_min_write,
+	},
+	{
+		.name = "uclamp.max",
+		.flags = CFTYPE_NOT_ON_ROOT,
+		.seq_show = cpu_uclamp_max_show,
+		.write = cpu_uclamp_max_write,
+	},
+#endif /* CONFIG_UCLAMP_TASK_GROUP */
+	{ }	/* Terminate */
+};
+
+#ifdef CONFIG_GROUP_SCHED_WEIGHT
+static u64 cpu_weight_read_u64(struct cgroup_subsys_state *css,
+			       struct cftype *cft)
+{
+	return 100;
+}
+
+static int cpu_weight_write_u64(struct cgroup_subsys_state *css,
+				struct cftype *cft, u64 weight)
+{
+	return 0;
+}
+
+static s64 cpu_weight_nice_read_s64(struct cgroup_subsys_state *css,
+				    struct cftype *cft)
+{
+	return 0;
+}
+
+static int cpu_weight_nice_write_s64(struct cgroup_subsys_state *css,
+				     struct cftype *cft, s64 nice)
+{
+	return 0;
+}
+#endif /* CONFIG_GROUP_SCHED_WEIGHT */
+
+#ifdef CONFIG_CFS_BANDWIDTH
+static int cpu_max_show(struct seq_file *sf, void *v)
+{
+	seq_puts(sf, "max 100000\n");
+	return 0;
+}
+
+static ssize_t cpu_max_write(struct kernfs_open_file *of,
+			     char *buf, size_t nbytes, loff_t off)
+{
+	return nbytes;
+}
+#endif /* CONFIG_CFS_BANDWIDTH */
+
+static struct cftype cpu_files[] = {
+#ifdef CONFIG_GROUP_SCHED_WEIGHT
+	{
+		.name = "weight",
+		.flags = CFTYPE_NOT_ON_ROOT,
+		.read_u64 = cpu_weight_read_u64,
+		.write_u64 = cpu_weight_write_u64,
+	},
+	{
+		.name = "weight.nice",
+		.flags = CFTYPE_NOT_ON_ROOT,
+		.read_s64 = cpu_weight_nice_read_s64,
+		.write_s64 = cpu_weight_nice_write_s64,
+	},
+	{
+		.name = "idle",
+		.flags = CFTYPE_NOT_ON_ROOT,
+		.read_s64 = cpu_idle_read_s64,
+		.write_s64 = cpu_idle_write_s64,
+	},
+#endif /* CONFIG_GROUP_SCHED_WEIGHT */
+#ifdef CONFIG_CFS_BANDWIDTH
+	{
+		.name = "max",
+		.flags = CFTYPE_NOT_ON_ROOT,
+		.seq_show = cpu_max_show,
+		.write = cpu_max_write,
+	},
+	{
+		.name = "max.burst",
+		.flags = CFTYPE_NOT_ON_ROOT,
+		.read_u64 = cpu_cfs_burst_read_u64,
+		.write_u64 = cpu_cfs_burst_write_u64,
+	},
+#endif /* CONFIG_CFS_BANDWIDTH */
+#ifdef CONFIG_UCLAMP_TASK_GROUP
+	{
+		.name = "uclamp.min",
+		.flags = CFTYPE_NOT_ON_ROOT,
+		.seq_show = cpu_uclamp_min_show,
+		.write = cpu_uclamp_min_write,
+	},
+	{
+		.name = "uclamp.max",
+		.flags = CFTYPE_NOT_ON_ROOT,
+		.seq_show = cpu_uclamp_max_show,
+		.write = cpu_uclamp_max_write,
+	},
+#endif /* CONFIG_UCLAMP_TASK_GROUP */
+	{ }	/* terminate */
+};
+
+static int cpu_extra_stat_show(struct seq_file *sf,
+			       struct cgroup_subsys_state *css)
+{
+	return 0;
+}
+
+static int cpu_local_stat_show(struct seq_file *sf,
+			       struct cgroup_subsys_state *css)
+{
+	return 0;
+}
+
+struct cgroup_subsys cpu_cgrp_subsys = {
+	.css_alloc	= cpu_cgroup_css_alloc,
+	.css_online	= cpu_cgroup_css_online,
+	.css_released	= cpu_cgroup_css_released,
+	.css_free	= cpu_cgroup_css_free,
+	.css_extra_stat_show = cpu_extra_stat_show,
+	.css_local_stat_show = cpu_local_stat_show,
+#ifdef CONFIG_RT_GROUP_SCHED
+	.can_attach	= cpu_cgroup_can_attach,
+#endif /* CONFIG_RT_GROUP_SCHED */
+	.attach		= cpu_cgroup_attach,
+	.legacy_cftypes	= cpu_legacy_files,
+	.dfl_cftypes	= cpu_files,
+	.early_init	= true,
+	.threaded	= true,
+};
+#endif	/* CONFIG_CGROUP_SCHED */
+
+#undef CREATE_TRACE_POINTS
+
+#ifdef CONFIG_SCHED_MM_CID
+/*
+ * Concurrency IDentifier management
+ *
+ * Serialization rules:
+ *
+ * mm::mm_cid::mutex:	Serializes fork() and exit() and therefore
+ * 			protects mm::mm_cid::users and mode switch
+ * 			transitions
+ *
+ * mm::mm_cid::lock:	Serializes mm_update_max_cids() and
+ *			mm_update_cpus_allowed(). Nests in mm_cid::mutex
+ *			and runqueue lock.
+ *
+ * The mm_cidmask bitmap is not protected by any of the mm::mm_cid locks
+ * and can only be modified with atomic operations.
+ *
+ * The mm::mm_cid:pcpu per CPU storage is protected by the CPUs runqueue
+ * lock.
+ *
+ * CID ownership:
+ *
+ * A CID is either owned by a task (stored in task_struct::mm_cid.cid) or
+ * by a CPU (stored in mm::mm_cid.pcpu::cid). CIDs owned by CPUs have the
+ * MM_CID_ONCPU bit set.
+ *
+ * During the transition of ownership mode, the MM_CID_TRANSIT bit is set
+ * on the CIDs. When this bit is set the tasks drop the CID back into the
+ * pool when scheduling out.
+ *
+ * Both bits (ONCPU and TRANSIT) are filtered out by task_cid() when the
+ * CID is actually handed over to user space in the RSEQ memory.
+ *
+ * Mode switching:
+ *
+ * The ownership mode is per process and stored in mm:mm_cid::mode with the
+ * following possible states:
+ *
+ *	0:				Per task ownership
+ *	0 | MM_CID_TRANSIT:		Transition from per CPU to per task
+ *	MM_CID_ONCPU:			Per CPU ownership
+ *	MM_CID_ONCPU | MM_CID_TRANSIT:	Transition from per task to per CPU
+ *
+ * All transitions of ownership mode happen in two phases:
+ *
+ *  1) mm:mm_cid::mode has the MM_CID_TRANSIT bit set. This is OR'ed on the
+ *     CIDs and denotes that the CID is only temporarily owned by a
+ *     task. When the task schedules out it drops the CID back into the
+ *     pool if this bit is set.
+ *
+ *  2) The initiating context walks the per CPU space or the tasks to fixup
+ *     or drop the CIDs and after completion it clears MM_CID_TRANSIT in
+ *     mm:mm_cid::mode. After that point the CIDs are strictly task or CPU
+ *     owned again.
+ *
+ * This two phase transition is required to prevent CID space exhaustion
+ * during the transition as a direct transfer of ownership would fail:
+ *
+ *   - On task to CPU mode switch if a task is scheduled in on one CPU and
+ *     then migrated to another CPU before the fixup freed enough per task
+ *     CIDs.
+ *
+ *   - On CPU to task mode switch if two tasks are scheduled in on the same
+ *     CPU before the fixup freed per CPU CIDs.
+ *
+ *   Both scenarios can result in a live lock because sched_in() is invoked
+ *   with runqueue lock held and loops in search of a CID and the fixup
+ *   thread can't make progress freeing them up because it is stuck on the
+ *   same runqueue lock.
+ *
+ * While MM_CID_TRANSIT is active during the transition phase the MM_CID
+ * bitmap can be contended, but that's a temporary contention bound to the
+ * transition period. After that everything goes back into steady state and
+ * nothing except fork() and exit() will touch the bitmap. This is an
+ * acceptable tradeoff as it completely avoids complex serialization,
+ * memory barriers and atomic operations for the common case.
+ *
+ * Aside of that this mechanism also ensures RT compability:
+ *
+ *   - The task which runs the fixup is fully preemptible except for the
+ *     short runqueue lock held sections.
+ *
+ *   - The transient impact of the bitmap contention is only problematic
+ *     when there is a thundering herd scenario of tasks scheduling in and
+ *     out concurrently. There is not much which can be done about that
+ *     except for avoiding mode switching by a proper overall system
+ *     configuration.
+ *
+ * Switching to per CPU mode happens when the user count becomes greater
+ * than the maximum number of CIDs, which is calculated by:
+ *
+ *	opt_cids = min(mm_cid::nr_cpus_allowed, mm_cid::users);
+ *	max_cids = min(1.25 * opt_cids, num_possible_cpus());
+ *
+ * The +25% allowance is useful for tight CPU masks in scenarios where only
+ * a few threads are created and destroyed to avoid frequent mode
+ * switches. Though this allowance shrinks, the closer opt_cids becomes to
+ * num_possible_cpus(), which is the (unfortunate) hard ABI limit.
+ *
+ * At the point of switching to per CPU mode the new user is not yet
+ * visible in the system, so the task which initiated the fork() runs the
+ * fixup function. mm_cid_fixup_tasks_to_cpu() walks the thread list and
+ * either marks each task owned CID with MM_CID_TRANSIT if the task is
+ * running on a CPU or drops it into the CID pool if a task is not on a
+ * CPU. Tasks which schedule in before the task walk reaches them do the
+ * handover in mm_cid_schedin(). When mm_cid_fixup_tasks_to_cpus()
+ * completes it is guaranteed that no task related to that MM owns a CID
+ * anymore.
+ *
+ * Switching back to task mode happens when the user count goes below the
+ * threshold which was recorded on the per CPU mode switch:
+ *
+ *	pcpu_thrs = min(opt_cids - (opt_cids / 4), num_possible_cpus() / 2);
+ *
+ * This threshold is updated when a affinity change increases the number of
+ * allowed CPUs for the MM, which might cause a switch back to per task
+ * mode.
+ *
+ * If the switch back was initiated by a exiting task, then that task runs
+ * the fixup function. If it was initiated by a affinity change, then it's
+ * run either in the deferred update function in context of a workqueue or
+ * by a task which forks a new one or by a task which exits. Whatever
+ * happens first. mm_cid_fixup_cpus_to_task() walks through the possible
+ * CPUs and either transfers the CPU owned CIDs to a related task which
+ * runs on the CPU or drops it into the pool. Tasks which schedule in on a
+ * CPU which the walk did not cover yet do the handover themself.
+ *
+ * This transition from CPU to per task ownership happens in two phases:
+ *
+ *  1) mm:mm_cid.transit contains MM_CID_TRANSIT This is OR'ed on the task
+ *     CID and denotes that the CID is only temporarily owned by the
+ *     task. When it schedules out the task drops the CID back into the
+ *     pool if this bit is set.
+ *
+ *  2) The initiating context walks the per CPU space and after completion
+ *     clears mm:mm_cid.transit. So after that point the CIDs are strictly
+ *     task owned again.
+ *
+ * This two phase transition is required to prevent CID space exhaustion
+ * during the transition as a direct transfer of ownership would fail if
+ * two tasks are scheduled in on the same CPU before the fixup freed per
+ * CPU CIDs.
+ *
+ * When mm_cid_fixup_cpus_to_tasks() completes it's guaranteed that no CID
+ * related to that MM is owned by a CPU anymore.
+ */
+
+/*
+ * Update the CID range properties when the constraints change. Invoked via
+ * fork(), exit() and affinity changes
+ */
+static void __mm_update_max_cids(struct mm_mm_cid *mc)
+{
+	unsigned int opt_cids, max_cids;
+
+	/* Calculate the new optimal constraint */
+	opt_cids = min(mc->nr_cpus_allowed, mc->users);
+
+	/* Adjust the maximum CIDs to +25% limited by the number of possible CPUs */
+	max_cids = min(opt_cids + (opt_cids / 4), num_possible_cpus());
+	WRITE_ONCE(mc->max_cids, max_cids);
+}
+
+static inline unsigned int mm_cid_calc_pcpu_thrs(struct mm_mm_cid *mc)
+{
+	unsigned int opt_cids;
+
+	opt_cids = min(mc->nr_cpus_allowed, mc->users);
+	/* Has to be at least 1 because 0 indicates PCPU mode off */
+	return max(min(opt_cids - opt_cids / 4, num_possible_cpus() / 2), 1);
+}
+
+static bool mm_update_max_cids(struct mm_struct *mm)
+{
+	struct mm_mm_cid *mc = &mm->mm_cid;
+	bool percpu = cid_on_cpu(mc->mode);
+
+	lockdep_assert_held(&mm->mm_cid.lock);
+
+	/* Clear deferred mode switch flag. A change is handled by the caller */
+	mc->update_deferred = false;
+	__mm_update_max_cids(mc);
+
+	/* Check whether owner mode must be changed */
+	if (!percpu) {
+		/* Enable per CPU mode when the number of users is above max_cids */
+		if (mc->users > mc->max_cids)
+			mc->pcpu_thrs = mm_cid_calc_pcpu_thrs(mc);
+	} else {
+		/* Switch back to per task if user count under threshold */
+		if (mc->users < mc->pcpu_thrs)
+			mc->pcpu_thrs = 0;
+	}
+
+	/* Mode change required? */
+	if (percpu == !!mc->pcpu_thrs)
+		return false;
+
+	/* Flip the mode and set the transition flag to bridge the transfer */
+	WRITE_ONCE(mc->mode, mc->mode ^ (MM_CID_TRANSIT | MM_CID_ONCPU));
+	/*
+	 * Order the store against the subsequent fixups so that
+	 * acquire(rq::lock) cannot be reordered by the CPU before the
+	 * store.
+	 */
+	smp_mb();
+	return true;
+}
+
+static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpumask *affmsk)
+{
+	struct cpumask *mm_allowed;
+	struct mm_mm_cid *mc;
+	unsigned int weight;
+
+	if (!mm || !READ_ONCE(mm->mm_cid.users))
+		return;
+	/*
+	 * mm::mm_cid::mm_cpus_allowed is the superset of each threads
+	 * allowed CPUs mask which means it can only grow.
+	 */
+	mc = &mm->mm_cid;
+	guard(raw_spinlock)(&mc->lock);
+	mm_allowed = mm_cpus_allowed(mm);
+	weight = cpumask_weighted_or(mm_allowed, mm_allowed, affmsk);
+	if (weight == mc->nr_cpus_allowed)
+		return;
+
+	WRITE_ONCE(mc->nr_cpus_allowed, weight);
+	__mm_update_max_cids(mc);
+	if (!cid_on_cpu(mc->mode))
+		return;
+
+	/* Adjust the threshold to the wider set */
+	mc->pcpu_thrs = mm_cid_calc_pcpu_thrs(mc);
+	/* Switch back to per task mode? */
+	if (mc->users >= mc->pcpu_thrs)
+		return;
+
+	/* Don't queue twice */
+	if (mc->update_deferred)
+		return;
+
+	/* Queue the irq work, which schedules the real work */
+	mc->update_deferred = true;
+	irq_work_queue(&mc->irq_work);
+}
+
+static inline void mm_cid_complete_transit(struct mm_struct *mm, unsigned int mode)
+{
+	/*
+	 * Ensure that the store removing the TRANSIT bit cannot be
+	 * reordered by the CPU before the fixups have been completed.
+	 */
+	smp_mb();
+	WRITE_ONCE(mm->mm_cid.mode, mode);
+}
+
+static inline void mm_cid_transit_to_task(struct task_struct *t, struct mm_cid_pcpu *pcp)
+{
+	if (cid_on_cpu(t->mm_cid.cid)) {
+		unsigned int cid = cpu_cid_to_cid(t->mm_cid.cid);
+
+		t->mm_cid.cid = cid_to_transit_cid(cid);
+		pcp->cid = t->mm_cid.cid;
+	}
+}
+
+static void mm_cid_fixup_cpus_to_tasks(struct mm_struct *mm)
+{
+	unsigned int cpu;
+
+	/* Walk the CPUs and fixup all stale CIDs */
+	for_each_possible_cpu(cpu) {
+		struct mm_cid_pcpu *pcp = per_cpu_ptr(mm->mm_cid.pcpu, cpu);
+		struct rq *rq = cpu_rq(cpu);
+
+		/* Remote access to mm::mm_cid::pcpu requires rq_lock */
+		guard(rq_lock_irq)(rq);
+		/* Is the CID still owned by the CPU? */
+		if (cid_on_cpu(pcp->cid)) {
+			/*
+			 * If rq->curr has @mm, transfer it with the
+			 * transition bit set. Otherwise drop it.
+			 */
+			if (rq->curr->mm == mm && rq->curr->mm_cid.active)
+				mm_cid_transit_to_task(rq->curr, pcp);
+			else
+				mm_drop_cid_on_cpu(mm, pcp);
+
+		} else if (rq->curr->mm == mm && rq->curr->mm_cid.active) {
+			unsigned int cid = rq->curr->mm_cid.cid;
+
+			/* Ensure it has the transition bit set */
+			if (!cid_in_transit(cid)) {
+				cid = cid_to_transit_cid(cid);
+				rq->curr->mm_cid.cid = cid;
+				pcp->cid = cid;
+			}
+		}
+	}
+	mm_cid_complete_transit(mm, 0);
+}
+
+static inline void mm_cid_transit_to_cpu(struct task_struct *t, struct mm_cid_pcpu *pcp)
+{
+	if (cid_on_task(t->mm_cid.cid)) {
+		t->mm_cid.cid = cid_to_transit_cid(t->mm_cid.cid);
+		pcp->cid = t->mm_cid.cid;
+	}
+}
+
+static void mm_cid_fixup_task_to_cpu(struct task_struct *t, struct mm_struct *mm)
+{
+	/* Remote access to mm::mm_cid::pcpu requires rq_lock */
+	guard(task_rq_lock)(t);
+	if (cid_on_task(t->mm_cid.cid)) {
+		/* If running on the CPU, put the CID in transit mode, otherwise drop it */
+		if (task_rq(t)->curr == t)
+			mm_cid_transit_to_cpu(t, per_cpu_ptr(mm->mm_cid.pcpu, task_cpu(t)));
+		else
+			mm_unset_cid_on_task(t);
+	}
+}
+
+static void mm_cid_fixup_tasks_to_cpus(void)
+{
+	struct mm_struct *mm = current->mm;
+	struct task_struct *t;
+
+	lockdep_assert_held(&mm->mm_cid.mutex);
+
+	hlist_for_each_entry(t, &mm->mm_cid.user_list, mm_cid.node) {
+		/* Current has already transferred before invoking the fixup. */
+		if (t != current)
+			mm_cid_fixup_task_to_cpu(t, mm);
+	}
+
+	mm_cid_complete_transit(mm, MM_CID_ONCPU);
+}
+
+static bool sched_mm_cid_add_user(struct task_struct *t, struct mm_struct *mm)
+{
+	lockdep_assert_held(&mm->mm_cid.lock);
+
+	t->mm_cid.active = 1;
+	hlist_add_head(&t->mm_cid.node, &mm->mm_cid.user_list);
+	mm->mm_cid.users++;
+	return mm_update_max_cids(mm);
+}
+
+static void sched_mm_cid_fork(struct task_struct *t)
+{
+	struct mm_struct *mm = t->mm;
+	bool percpu;
+
+	if (!mm)
+		return;
+
+	WARN_ON_ONCE(t->mm_cid.cid != MM_CID_UNSET);
+
+	guard(mutex)(&mm->mm_cid.mutex);
+	scoped_guard(raw_spinlock_irq, &mm->mm_cid.lock) {
+		struct mm_cid_pcpu *pcp = this_cpu_ptr(mm->mm_cid.pcpu);
+
+		/* First user ? */
+		if (!mm->mm_cid.users) {
+			sched_mm_cid_add_user(t, mm);
+			t->mm_cid.cid = mm_get_cid(mm);
+			/* Required for execve() */
+			pcp->cid = t->mm_cid.cid;
+			return;
+		}
+
+		if (!sched_mm_cid_add_user(t, mm)) {
+			if (!cid_on_cpu(mm->mm_cid.mode))
+				t->mm_cid.cid = mm_get_cid(mm);
+			return;
+		}
+
+		/* Handle the mode change and transfer current's CID */
+		percpu = cid_on_cpu(mm->mm_cid.mode);
+		if (!percpu)
+			mm_cid_transit_to_task(current, pcp);
+		else
+			mm_cid_transit_to_cpu(current, pcp);
+	}
+
+	if (percpu) {
+		mm_cid_fixup_tasks_to_cpus();
+	} else {
+		mm_cid_fixup_cpus_to_tasks(mm);
+		t->mm_cid.cid = mm_get_cid(mm);
+	}
+}
+
+static bool sched_mm_cid_remove_user(struct task_struct *t)
+{
+	lockdep_assert_held(&t->mm->mm_cid.lock);
+
+	t->mm_cid.active = 0;
+	/* Clear the transition bit */
+	t->mm_cid.cid = cid_from_transit_cid(t->mm_cid.cid);
+	mm_unset_cid_on_task(t);
+	hlist_del_init(&t->mm_cid.node);
+	t->mm->mm_cid.users--;
+	return mm_update_max_cids(t->mm);
+}
+
+static bool __sched_mm_cid_exit(struct task_struct *t)
+{
+	struct mm_struct *mm = t->mm;
+
+	if (!sched_mm_cid_remove_user(t))
+		return false;
+	/*
+	 * Contrary to fork() this only deals with a switch back to per
+	 * task mode either because the above decreased users or an
+	 * affinity change increased the number of allowed CPUs and the
+	 * deferred fixup did not run yet.
+	 */
+	if (WARN_ON_ONCE(cid_on_cpu(mm->mm_cid.mode)))
+		return false;
+	/*
+	 * A failed fork(2) cleanup never gets here, so @current must have
+	 * the same MM as @t. That's true for exit() and the failed
+	 * pthread_create() cleanup case.
+	 */
+	if (WARN_ON_ONCE(current->mm != mm))
+		return false;
+	return true;
+}
+
+/*
+ * When a task exits, the MM CID held by the task is not longer required as
+ * the task cannot return to user space.
+ */
+void sched_mm_cid_exit(struct task_struct *t)
+{
+	struct mm_struct *mm = t->mm;
+
+	if (!mm || !t->mm_cid.active)
+		return;
+	/*
+	 * Ensure that only one instance is doing MM CID operations within
+	 * a MM. The common case is uncontended. The rare fixup case adds
+	 * some overhead.
+	 */
+	scoped_guard(mutex, &mm->mm_cid.mutex) {
+		/* mm_cid::mutex is sufficient to protect mm_cid::users */
+		if (likely(mm->mm_cid.users > 1)) {
+			scoped_guard(raw_spinlock_irq, &mm->mm_cid.lock) {
+				if (!__sched_mm_cid_exit(t))
+					return;
+				/*
+				 * Mode change. The task has the CID unset
+				 * already and dealt with an eventually set
+				 * TRANSIT bit. If the CID is owned by the CPU
+				 * then drop it.
+				 */
+				mm_drop_cid_on_cpu(mm, this_cpu_ptr(mm->mm_cid.pcpu));
+			}
+			mm_cid_fixup_cpus_to_tasks(mm);
+			return;
+		}
+		/* Last user */
+		scoped_guard(raw_spinlock_irq, &mm->mm_cid.lock) {
+			/* Required across execve() */
+			if (t == current)
+				mm_cid_transit_to_task(t, this_cpu_ptr(mm->mm_cid.pcpu));
+			/* Ignore mode change. There is nothing to do. */
+			sched_mm_cid_remove_user(t);
+		}
+	}
+
+	/*
+	 * As this is the last user (execve(), process exit or failed
+	 * fork(2)) there is no concurrency anymore.
+	 *
+	 * Synchronize eventually pending work to ensure that there are no
+	 * dangling references left. @t->mm_cid.users is zero so nothing
+	 * can queue this work anymore.
+	 */
+	irq_work_sync(&mm->mm_cid.irq_work);
+	cancel_work_sync(&mm->mm_cid.work);
+}
+
+/* Deactivate MM CID allocation across execve() */
+void sched_mm_cid_before_execve(struct task_struct *t)
+{
+	sched_mm_cid_exit(t);
+}
+
+/* Reactivate MM CID after execve() */
+void sched_mm_cid_after_execve(struct task_struct *t)
+{
+	if (t->mm)
+		sched_mm_cid_fork(t);
+}
+
+static void mm_cid_work_fn(struct work_struct *work)
+{
+	struct mm_struct *mm = container_of(work, struct mm_struct, mm_cid.work);
+
+	guard(mutex)(&mm->mm_cid.mutex);
+	/* Did the last user task exit already? */
+	if (!mm->mm_cid.users)
+		return;
+
+	scoped_guard(raw_spinlock_irq, &mm->mm_cid.lock) {
+		/* Have fork() or exit() handled it already? */
+		if (!mm->mm_cid.update_deferred)
+			return;
+		/* This clears mm_cid::update_deferred */
+		if (!mm_update_max_cids(mm))
+			return;
+		/* Affinity changes can only switch back to task mode */
+		if (WARN_ON_ONCE(cid_on_cpu(mm->mm_cid.mode)))
+			return;
+	}
+	mm_cid_fixup_cpus_to_tasks(mm);
+}
+
+static void mm_cid_irq_work(struct irq_work *work)
+{
+	struct mm_struct *mm = container_of(work, struct mm_struct, mm_cid.irq_work);
+
+	/*
+	 * Needs to be unconditional because mm_cid::lock cannot be held
+	 * when scheduling work as mm_update_cpus_allowed() nests inside
+	 * rq::lock and schedule_work() might end up in wakeup...
+	 */
+	schedule_work(&mm->mm_cid.work);
+}
+
+void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
+{
+	mm->mm_cid.max_cids = 0;
+	mm->mm_cid.mode = 0;
+	mm->mm_cid.nr_cpus_allowed = p->nr_cpus_allowed;
+	mm->mm_cid.users = 0;
+	mm->mm_cid.pcpu_thrs = 0;
+	mm->mm_cid.update_deferred = 0;
+	raw_spin_lock_init(&mm->mm_cid.lock);
+	mutex_init(&mm->mm_cid.mutex);
+	mm->mm_cid.irq_work = IRQ_WORK_INIT_HARD(mm_cid_irq_work);
+	INIT_WORK(&mm->mm_cid.work, mm_cid_work_fn);
+	INIT_HLIST_HEAD(&mm->mm_cid.user_list);
+	cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);
+	bitmap_zero(mm_cidmask(mm), num_possible_cpus());
+}
+#else /* CONFIG_SCHED_MM_CID */
+static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpumask *affmsk) { }
+static inline void sched_mm_cid_fork(struct task_struct *t) { }
+#endif /* !CONFIG_SCHED_MM_CID */
+
+static DEFINE_PER_CPU(struct sched_change_ctx, sched_change_ctx);
+
+struct sched_change_ctx *sched_change_begin(struct task_struct *p, unsigned int flags)
+{
+	struct sched_change_ctx *ctx = this_cpu_ptr(&sched_change_ctx);
+	struct rq *rq = task_rq(p);
+
+	/*
+	 * Must exclusively use matched flags since this is both dequeue and
+	 * enqueue.
+	 */
+	WARN_ON_ONCE(flags & 0xFFFF0000);
+
+	*ctx = (struct sched_change_ctx){
+		.p = p,
+		.flags = flags,
+		.queued = task_on_rq_queued(p),
+		.running = (p == rq->curr),
+	};
+
+	return ctx;
+}
+
+void sched_change_end(struct sched_change_ctx *ctx)
+{
+	if (ctx->running) {
+		struct rq *rq = task_rq(ctx->p);
+		const int cpu = cpu_of(rq);
+
+		update_sched_cpu_prio(cpu, task_sched_prio(ctx->p));
+		if (find_first_bit(rq_srq(rq)->bitmap, SCHED_QUEUE_BITS) <
+		    READ_ONCE(cpu_prio[cpu]))
+			resched_curr(rq);
+	}
+}
diff --git a/kernel/sched/alt_core.h b/kernel/sched/alt_core.h
new file mode 100644
index 000000000..9eb967501
--- /dev/null
+++ b/kernel/sched/alt_core.h
@@ -0,0 +1,290 @@
+#ifndef _KERNEL_SCHED_ALT_CORE_H
+#define _KERNEL_SCHED_ALT_CORE_H
+
+/*
+ * Compile time debug macro
+ * #define ALT_SCHED_DEBUG
+ */
+#define ALT_SCHED_DEBUG
+
+#ifdef ALT_SCHED_DEBUG
+extern void alt_sched_debug(void);
+#else
+static inline void alt_sched_debug(void) {}
+#endif
+
+/* preempt list function */
+DECLARE_PER_CPU_SHARED_ALIGNED(struct llist_head, preempt_list);
+
+static inline bool is_preempt_list_empty(const int cpu)
+{
+	return llist_empty(per_cpu_ptr(&preempt_list, cpu));
+}
+
+/*
+ * Task related inlined functions
+ */
+static inline bool is_migration_disabled(const struct task_struct *p)
+{
+	return p->migration_disabled;
+}
+
+/* rt_prio(prio) defined in include/linux/sched/rt.h */
+#define rt_task(p)		rt_prio((p)->prio)
+#define rt_policy(policy)	((policy) == SCHED_FIFO || (policy) == SCHED_RR)
+#define task_has_rt_policy(p)	(rt_policy((p)->policy))
+
+#define fair_policy(policy)	((policy) == SCHED_NORMAL || (policy) == SCHED_BATCH)
+
+#define valid_policy(policy)	((policy) <= SCHED_IDLE)
+
+#define task_has_dl_policy(p)	(false)
+#define dl_prio(prio)		(false)
+
+struct affinity_context {
+	const struct cpumask	*new_mask;
+	struct cpumask		*user_mask;
+	unsigned int		flags;
+};
+
+/* CONFIG_SCHED_CLASS_EXT is not supported */
+#define scx_switched_all()	false
+
+#define SCA_CHECK		0x01
+/* SCA_MIGRATE_DISABLE & SCA_MIGRATE_ENABLE is not supported */
+//#define SCA_MIGRATE_DISABLE	0x02
+//#define SCA_MIGRATE_ENABLE	0x04
+#define SCA_USER		0x08
+
+extern int __set_cpus_allowed_ptr(struct task_struct *p, struct affinity_context *ctx);
+
+static inline cpumask_t *alloc_user_cpus_ptr(int node)
+{
+	/*
+	 * See set_cpus_allowed_force() above for the rcu_head usage.
+	 */
+	int size = max_t(int, cpumask_size(), sizeof(struct rcu_head));
+
+	return kmalloc_node(size, GFP_KERNEL, node);
+}
+
+#ifdef CONFIG_RT_MUTEXES
+
+static inline int __rt_effective_prio(struct task_struct *pi_task, int prio)
+{
+	if (pi_task)
+		prio = min(prio, pi_task->prio);
+
+	return prio;
+}
+
+static inline int rt_effective_prio(struct task_struct *p, int prio)
+{
+	struct task_struct *pi_task = rt_mutex_get_top_task(p);
+
+	return __rt_effective_prio(pi_task, prio);
+}
+
+#else /* !CONFIG_RT_MUTEXES: */
+
+static inline int rt_effective_prio(struct task_struct *p, int prio)
+{
+	return prio;
+}
+
+#endif /* !CONFIG_RT_MUTEXES */
+
+extern int __sched_setscheduler(struct task_struct *p, const struct sched_attr *attr, bool user, bool pi);
+extern int __sched_setaffinity(struct task_struct *p, struct affinity_context *ctx);
+
+extern void wakeup_modified_task(struct task_struct *p);
+
+/*
+ * run queue related inlined functions
+ */
+static __always_inline
+struct llist_node *sched_llist_del(struct llist_node **curr, struct llist_node *target)
+{
+	struct llist_node *entry;
+
+	while (*curr) {
+		entry = *curr;
+
+		if (entry == target) {
+			*curr = entry->next;
+			entry = container_of(curr, struct llist_node, next);
+			while (entry->next)
+				entry = entry->next;
+
+			return entry;
+		}
+		curr = &entry->next;
+	}
+	WARN_ONCE(true, "sched/alt: llist_del() target should be in list\n");
+	return NULL;
+}
+
+static __always_inline
+void sched_llist_merge(struct llist_head *head, struct llist_node *first, struct llist_node *last)
+{
+	while (!llist_add_batch(first, last, head)) {
+		struct llist_node *node, *new_first = llist_del_all(head);
+
+		/* move [first, last] to the tail */
+		for (node = last->next; node->next; node = node->next);
+		node->next = first;
+
+		/* merge new first if any */
+		if (new_first != first) {
+			for (node = new_first; first != node->next; node = node->next);
+			node->next = last->next;
+			first = new_first;
+		} else {
+			first = last->next;
+		}
+		/* terminal the tail */
+		last->next = NULL;
+	}
+}
+
+#define SRQ_DEQUEUE_TASK(srq, p, __modify_body__)					\
+{											\
+	int idx = READ_ONCE(p->__sched_prio);						\
+	struct llist_head *head = &srq->_head[idx];					\
+	struct llist_node *last, *first = llist_del_all(head);				\
+											\
+	last = sched_llist_del(&first, &p->pq_node);					\
+	if (first) {									\
+		sched_llist_merge(head, first, last);					\
+	} else if (llist_empty(head)) {							\
+		WARN_ONCE(task_sched_prio(p) != idx, "sched: srq en/dequeue bug.\n");	\
+		clear_bit(idx, srq->bitmap);						\
+		smp_rmb();								\
+		if (!llist_empty(head))							\
+			set_bit(idx, srq->bitmap);					\
+	}										\
+	__modify_body__									\
+	WRITE_ONCE(p->__sched_prio, -1);						\
+	atomic_dec(&srq->nr_queued);							\
+}
+
+#define SRQ_ENQUEUE_TASK(srq, p, __modify_body__)		\
+{								\
+	int sched_prio = task_sched_prio(p);			\
+								\
+	WRITE_ONCE(p->__sched_prio, sched_prio);		\
+	__modify_body__						\
+	if (llist_add(&p->pq_node, &(srq)->_head[sched_prio]))	\
+		set_bit(sched_prio, (srq)->bitmap);		\
+	atomic_inc(&(srq)->nr_queued);				\
+}
+
+/*
+ * Context API
+ */
+static inline struct rq *__task_modify_lock(struct task_struct *p, struct rq_flags *rf)
+{
+	for (;;) {
+		if (TASK_ON_RQ_WAKING == p->on_rq) {
+			struct rq *rq = cpu_rq(p->wake_cpu);
+
+			raw_spin_lock(&rq->lock);
+			if (likely(TASK_ON_RQ_WAKING == p->on_rq && rq == task_rq(p))) {
+				rf->lock = &rq->lock;
+				rf->queued = false;
+				return rq;
+			}
+			raw_spin_unlock(&rq->lock);
+		} else if (task_on_rq_queued(p)) {
+			int idx;
+			if (p->on_cpu) {
+				struct rq *rq = task_rq(p);
+
+				raw_spin_lock(&rq->lock);
+				if (likely(task_on_rq_queued(p) && p->on_cpu && rq == task_rq(p))) {
+					rf->lock = &rq->lock;
+					rf->queued = false;
+					return rq;
+				}
+				raw_spin_unlock(&rq->lock);
+			} else if ((idx = READ_ONCE(p->__sched_prio)) != -1) {
+				struct sched_run_queue *srq = cpu_srq(0);
+				raw_spinlock_t *lock = &srq->_lock[idx];
+
+				raw_spin_lock(lock);
+				if (task_on_rq_queued(p) && !p->on_cpu &&
+				    idx == READ_ONCE(p->__sched_prio)) {
+
+					SRQ_DEQUEUE_TASK(srq, p, {
+							 WRITE_ONCE(p->on_rq, TASK_ON_RQ_MIGRATING);
+							 });
+
+					raw_spin_unlock(lock);
+
+					rf->lock = NULL;
+					rf->queued = true;
+					return task_rq(p);
+				}
+				raw_spin_unlock(lock);
+			}
+		} else if (task_on_rq_migrating(p)) {
+			do {
+				cpu_relax();
+			} while (unlikely(task_on_rq_migrating(p)));
+		} else {
+			rf->lock = NULL;
+			rf->queued = false;
+			return task_rq(p);
+		}
+	}
+}
+
+static inline void __task_modify_unlock(struct task_struct *p, struct rq_flags *rf)
+{
+	if (NULL != rf->lock)
+		raw_spin_unlock(rf->lock);
+	if (rf->queued)
+		wakeup_modified_task(p);
+}
+
+static inline struct rq *task_access_lock(struct task_struct *p, struct rq_flags *rf)
+{
+	raw_spin_lock_irqsave(&p->pi_lock, rf->flags);
+	return __task_modify_lock(p, rf);
+}
+
+static inline void task_access_unlock(struct task_struct *p, struct rq_flags *rf)
+{
+	__task_modify_unlock(p, rf);
+	raw_spin_unlock_irqrestore(&p->pi_lock, rf->flags);
+}
+
+DEFINE_LOCK_GUARD_1(task_access_lock, struct task_struct,
+		    _T->rq = task_access_lock(_T->lock, &_T->rf),
+		    task_access_unlock(_T->lock, &_T->rf),
+		    struct rq *rq; struct rq_flags rf)
+
+#define task_rq_lock(...) _task_rq_lock(__VA_ARGS__)
+extern struct rq *_task_rq_lock(struct task_struct *p, struct rq_flags *rf);
+
+static inline void
+task_rq_unlock(struct rq *rq, struct task_struct *p, struct rq_flags *rf)
+{
+	raw_spin_unlock(&rq->lock);
+	raw_spin_unlock_irqrestore(&p->pi_lock, rf->flags);
+}
+
+DEFINE_LOCK_GUARD_1(task_rq_lock, struct task_struct,
+		    _T->rq = task_rq_lock(_T->lock, &_T->rf),
+		    task_rq_unlock(_T->rq, _T->lock, &_T->rf),
+		    struct rq *rq; struct rq_flags rf)
+
+extern void yield_task(struct rq *rq);
+
+DECLARE_STATIC_KEY_FALSE(sched_smt_present);
+
+/* balance callback */
+extern struct balance_callback *splice_balance_callbacks(struct rq *rq);
+extern void balance_callbacks(struct rq *rq, struct balance_callback *head);
+
+#endif /* _KERNEL_SCHED_ALT_CORE_H */
diff --git a/kernel/sched/alt_debug.c b/kernel/sched/alt_debug.c
new file mode 100644
index 000000000..1dbd7eb6a
--- /dev/null
+++ b/kernel/sched/alt_debug.c
@@ -0,0 +1,32 @@
+/*
+ * kernel/sched/alt_debug.c
+ *
+ * Print the alt scheduler debugging details
+ *
+ * Author: Alfred Chen
+ * Date  : 2020
+ */
+#include "sched.h"
+#include "linux/sched/debug.h"
+
+/*
+ * This allows printing both to /proc/sched_debug and
+ * to the console
+ */
+#define SEQ_printf(m, x...)			\
+ do {						\
+	if (m)					\
+		seq_printf(m, x);		\
+	else					\
+		pr_cont(x);			\
+ } while (0)
+
+void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns,
+			  struct seq_file *m)
+{
+	SEQ_printf(m, "%s (%d, #threads: %d)\n", p->comm, task_pid_nr_ns(p, ns),
+						get_nr_threads(p));
+}
+
+void proc_sched_set_task(struct task_struct *p)
+{}
diff --git a/kernel/sched/alt_sched.h b/kernel/sched/alt_sched.h
new file mode 100644
index 000000000..f0c0fe25a
--- /dev/null
+++ b/kernel/sched/alt_sched.h
@@ -0,0 +1,1001 @@
+#ifndef _KERNEL_SCHED_ALT_SCHED_H
+#define _KERNEL_SCHED_ALT_SCHED_H
+
+#include <linux/context_tracking.h>
+#include <linux/memblock.h>
+#include <linux/profile.h>
+#include <linux/sched/stat.h>
+#include <linux/stop_machine.h>
+#include <linux/sched/rseq_api.h>
+#include <linux/syscalls.h>
+#include <linux/tick.h>
+
+#include <trace/events/power.h>
+#include <trace/events/sched.h>
+
+#include "../workqueue_internal.h"
+
+#include "cpupri.h"
+
+#ifdef CONFIG_CGROUP_SCHED
+/* task group related information */
+struct task_group {
+	struct cgroup_subsys_state css;
+
+	struct rcu_head rcu;
+	struct list_head list;
+
+	struct task_group *parent;
+	struct list_head siblings;
+	struct list_head children;
+};
+
+extern struct task_group *sched_create_group(struct task_group *parent);
+extern void sched_online_group(struct task_group *tg,
+			       struct task_group *parent);
+extern void sched_destroy_group(struct task_group *tg);
+extern void sched_release_group(struct task_group *tg);
+#endif /* CONFIG_CGROUP_SCHED */
+
+#define MIN_SCHED_NORMAL_PRIO	(25)
+/*
+ * levels  : RT[0-24], NORMAL[25-56], cpu idle task[57]
+ * vlevels : ecore/pcore/ucore single idle task    -- [58,60]
+ *           ecore/pcore/ucore smt_group idle task -- [61,63]
+ * -- BMQ --
+ * NORMAL: (lower boost range 12, NICE_WIDTH 40, higher boost range 12) / 2
+ * -- PDS --
+ * NORMAL: SCHED_EDGE_DELTA + ((NICE_WIDTH 40) / 2)
+ */
+#define SCHED_LEVELS		(64)
+#define IDLE_TASK_SCHED_PRIO	(MIN_SCHED_NORMAL_PRIO + NORMAL_PRIO_NUM / 2)
+
+#define IDLE_TASK_PRIO		(MIN_NORMAL_PRIO + NORMAL_PRIO_NUM)
+
+/*
+ * Increase resolution of nice-level calculations for 64-bit architectures.
+ * The extra resolution improves shares distribution and load balancing of
+ * low-weight task groups (eg. nice +19 on an autogroup), deeper taskgroup
+ * hierarchies, especially on larger systems. This is not a user-visible change
+ * and does not change the user-interface for setting shares/weights.
+ *
+ * We increase resolution only if we have enough bits to allow this increased
+ * resolution (i.e. 64-bit). The costs for increasing resolution when 32-bit
+ * are pretty high and the returns do not justify the increased costs.
+ *
+ * Really only required when CONFIG_FAIR_GROUP_SCHED=y is also set, but to
+ * increase coverage and consistency always enable it on 64-bit platforms.
+ */
+#ifdef CONFIG_64BIT
+# define NICE_0_LOAD_SHIFT	(SCHED_FIXEDPOINT_SHIFT + SCHED_FIXEDPOINT_SHIFT)
+# define scale_load(w)		((w) << SCHED_FIXEDPOINT_SHIFT)
+# define scale_load_down(w) \
+({ \
+	unsigned long __w = (w); \
+	if (__w) \
+		__w = max(2UL, __w >> SCHED_FIXEDPOINT_SHIFT); \
+	__w; \
+})
+#else
+# define NICE_0_LOAD_SHIFT	(SCHED_FIXEDPOINT_SHIFT)
+# define scale_load(w)		(w)
+# define scale_load_down(w)	(w)
+#endif
+
+/* task_struct::on_rq states: */
+#define TASK_ON_RQ_QUEUED	1
+#define TASK_ON_RQ_MIGRATING	2
+#define TASK_ON_RQ_WAKING	11
+
+static inline int task_on_rq_queued(struct task_struct *p)
+{
+	return READ_ONCE(p->on_rq) == TASK_ON_RQ_QUEUED;
+}
+
+static inline int task_on_rq_migrating(struct task_struct *p)
+{
+	return READ_ONCE(p->on_rq) == TASK_ON_RQ_MIGRATING;
+}
+
+/* Wake flags. The first three directly map to some SD flag value */
+#define WF_EXEC         0x02 /* Wakeup after exec; maps to SD_BALANCE_EXEC */
+#define WF_FORK         0x04 /* Wakeup after fork; maps to SD_BALANCE_FORK */
+#define WF_TTWU         0x08 /* Wakeup;            maps to SD_BALANCE_WAKE */
+
+#define WF_SYNC         0x10 /* Waker goes to sleep after wakeup */
+#define WF_MIGRATED     0x20 /* Internal use, task got migrated */
+#define WF_CURRENT_CPU  0x40 /* Prefer to move the wakee to the current CPU. */
+
+static_assert(WF_EXEC == SD_BALANCE_EXEC);
+static_assert(WF_FORK == SD_BALANCE_FORK);
+static_assert(WF_TTWU == SD_BALANCE_WAKE);
+
+/*
+ * {de,en}queue flags:
+ *
+ * SLEEP/WAKEUP - task is no-longer/just-became runnable
+ *
+ * SAVE/RESTORE - an otherwise spurious dequeue/enqueue, done to ensure tasks
+ *                are in a known state which allows modification. Such pairs
+ *                should preserve as much state as possible.
+ *
+ * MOVE - paired with SAVE/RESTORE, explicitly does not preserve the location
+ *        in the runqueue.
+ *
+ * NOCLOCK - skip the update_rq_clock() (avoids double updates)
+ *
+ * MIGRATION - p->on_rq == TASK_ON_RQ_MIGRATING (used for DEADLINE)
+ *
+ * DELAYED - de/re-queue a sched_delayed task
+ *
+ * CLASS - going to update p->sched_class; makes sched_change call the
+ *         various switch methods.
+ *
+ * ENQUEUE_HEAD      - place at front of runqueue (tail if not specified)
+ * ENQUEUE_REPLENISH - CBS (replenish runtime and postpone deadline)
+ * ENQUEUE_MIGRATED  - the task was migrated during wakeup
+ * ENQUEUE_RQ_SELECTED - ->select_task_rq() was called
+ *
+ * XXX SAVE/RESTORE in combination with CLASS doesn't really make sense, but
+ * SCHED_DEADLINE seems to rely on this for now.
+ */
+
+#define DEQUEUE_SLEEP		0x0001 /* Matches ENQUEUE_WAKEUP */
+#define DEQUEUE_SAVE		0x0002 /* Matches ENQUEUE_RESTORE */
+#define DEQUEUE_MOVE		0x0004 /* Matches ENQUEUE_MOVE */
+#define DEQUEUE_NOCLOCK		0x0008 /* Matches ENQUEUE_NOCLOCK */
+
+#define DEQUEUE_MIGRATING	0x0010 /* Matches ENQUEUE_MIGRATING */
+#define DEQUEUE_DELAYED		0x0020 /* Matches ENQUEUE_DELAYED */
+#define DEQUEUE_CLASS		0x0040 /* Matches ENQUEUE_CLASS */
+
+#define DEQUEUE_SPECIAL		0x00010000
+#define DEQUEUE_THROTTLE	0x00020000
+
+#define ENQUEUE_WAKEUP		0x0001
+#define ENQUEUE_RESTORE		0x0002
+#define ENQUEUE_MOVE		0x0004
+#define ENQUEUE_NOCLOCK		0x0008
+
+#define ENQUEUE_MIGRATING	0x0010
+#define ENQUEUE_DELAYED		0x0020
+#define ENQUEUE_CLASS		0x0040
+
+#define ENQUEUE_HEAD		0x00010000
+#define ENQUEUE_REPLENISH	0x00020000
+#define ENQUEUE_MIGRATED	0x00040000
+#define ENQUEUE_INITIAL		0x00080000
+#define ENQUEUE_RQ_SELECTED	0x00100000
+
+struct rq;
+struct cpuidle_state;
+
+struct balance_callback {
+	struct balance_callback *next;
+	void (*func)(struct rq *rq);
+};
+
+typedef void (*balance_func_t)(struct rq *rq, int cpu);
+
+struct balance_arg {
+	int			active;
+};
+
+/*
+ * This is the main, per-CPU runqueue data structure.
+ * This data should only be modified by the local cpu.
+ */
+struct rq {
+	/* runqueue lock: */
+	raw_spinlock_t			lock;
+
+	struct task_struct __rcu	*curr;
+	struct task_struct		*idle;
+	struct task_struct		*stop;
+	struct mm_struct		*prev_mm;
+	bool				block;
+
+	/* switch count */
+	u64 nr_switches;
+
+	u64 last_seen_need_resched_ns;
+	int ticks_without_resched;
+
+#ifdef CONFIG_MEMBARRIER
+	int membarrier_state;
+#endif
+
+	int cpu;		/* cpu of this runqueue */
+	bool online;
+
+	unsigned int		ttwu_pending;
+	unsigned char		nohz_idle_balance;
+	unsigned char		idle_balance;
+
+#ifdef CONFIG_HAVE_SCHED_AVG_IRQ
+	struct sched_avg	avg_irq;
+#endif
+
+	balance_func_t		balance_func;
+	struct balance_arg	active_balance_arg		____cacheline_aligned;
+	struct cpu_stop_work	active_balance_work;
+
+	struct balance_callback	*balance_callback;
+
+#ifdef CONFIG_HOTPLUG_CPU
+	struct rcuwait		hotplug_wait;
+#endif
+	unsigned int		nr_pinned;
+
+#ifdef CONFIG_IRQ_TIME_ACCOUNTING
+	u64 prev_irq_time;
+#endif /* CONFIG_IRQ_TIME_ACCOUNTING */
+#ifdef CONFIG_PARAVIRT
+	u64 prev_steal_time;
+#endif /* CONFIG_PARAVIRT */
+#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
+	u64 prev_steal_time_rq;
+#endif /* CONFIG_PARAVIRT_TIME_ACCOUNTING */
+
+	/* For genenal cpu load util */
+	s32 load_history;
+	u64 load_block;
+	u64 load_stamp;
+
+	/* calc_load related fields */
+	unsigned long calc_load_update;
+	long calc_load_active;
+
+	/* Ensure that all clocks are in the same cache line */
+	u64			clock ____cacheline_aligned;
+	u64			clock_task;
+
+#ifdef CONFIG_SCHED_HRTICK
+	call_single_data_t hrtick_csd;
+	struct hrtimer		hrtick_timer;
+	ktime_t			hrtick_time;
+	ktime_t			hrtick_delay;
+	unsigned int		hrtick_sched;
+#endif
+
+#ifdef CONFIG_SCHEDSTATS
+
+	/* latency stats */
+	struct sched_info rq_sched_info;
+	unsigned long long rq_cpu_time;
+	/* could above be rq->cfs_rq.exec_clock + rq->rt_rq.rt_runtime ? */
+
+	/* sys_sched_yield() stats */
+	unsigned int yld_count;
+
+	/* schedule() stats */
+	unsigned int sched_switch;
+	unsigned int sched_count;
+	unsigned int sched_goidle;
+
+	/* try_to_wake_up() stats */
+	unsigned int ttwu_count;
+	unsigned int ttwu_local;
+#endif /* CONFIG_SCHEDSTATS */
+
+#ifdef CONFIG_CPU_IDLE
+	/* Must be inspected within a rcu lock section */
+	struct cpuidle_state *idle_state;
+#endif
+
+#ifdef CONFIG_NO_HZ_COMMON
+	call_single_data_t	nohz_csd;
+	atomic_t		nohz_flags;
+#endif /* CONFIG_NO_HZ_COMMON */
+
+	/* Scratch cpumask to be temporarily used under rq_lock */
+	cpumask_var_t		scratch_mask;
+};
+
+extern unsigned int sysctl_sched_base_slice;
+
+extern unsigned long rq_load_util(struct rq *rq, unsigned long max);
+
+extern unsigned long calc_load_update;
+extern atomic_long_t calc_load_tasks;
+
+DECLARE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
+#define cpu_rq(cpu)		(&per_cpu(runqueues, (cpu)))
+#define this_rq()		this_cpu_ptr(&runqueues)
+#define task_rq(p)		cpu_rq(task_cpu(p))
+#define cpu_curr(cpu)		(cpu_rq(cpu)->curr)
+#define raw_rq()		raw_cpu_ptr(&runqueues)
+
+#ifdef CONFIG_SYSCTL
+void register_sched_domain_sysctl(void);
+void unregister_sched_domain_sysctl(void);
+#else
+static inline void register_sched_domain_sysctl(void)
+{
+}
+static inline void unregister_sched_domain_sysctl(void)
+{
+}
+#endif
+
+extern bool sched_smp_initialized;
+
+enum {
+#ifdef CONFIG_SCHED_SMT
+	SMT_LEVEL_SPACE_HOLDER,
+#endif
+	CLUSTER_LEVEL_SPACE_HOLDER,
+	COREGROUP_LEVEL_SPACE_HOLDER,
+	CORE_LEVEL_SPACE_HOLDER,
+	OTHER_LEVEL_SPACE_HOLDER,
+	NR_CPU_AFFINITY_LEVELS
+};
+
+DECLARE_PER_CPU_ALIGNED(cpumask_t [NR_CPU_AFFINITY_LEVELS], cpu_affinity_masks);
+
+extern void resched_latency_warn(int cpu, u64 latency);
+
+#ifndef arch_scale_freq_tick
+static __always_inline
+void arch_scale_freq_tick(void)
+{
+}
+#endif
+
+#ifndef arch_scale_freq_capacity
+static __always_inline
+unsigned long arch_scale_freq_capacity(int cpu)
+{
+	return SCHED_CAPACITY_SCALE;
+}
+#endif
+
+static inline u64 __rq_clock_broken(struct rq *rq)
+{
+	return READ_ONCE(rq->clock);
+}
+
+static inline u64 rq_clock(struct rq *rq)
+{
+	/*
+	 * Relax lockdep_assert_held() checking as in VRQ, call to
+	 * sched_info_xxxx() may not held rq->lock
+	 * lockdep_assert_held(&rq->lock);
+	 */
+	return rq->clock;
+}
+
+static inline u64 rq_clock_task(struct rq *rq)
+{
+	/*
+	 * Relax lockdep_assert_held() checking as in VRQ, call to
+	 * sched_info_xxxx() may not held rq->lock
+	 * lockdep_assert_held(&rq->lock);
+	 */
+	return rq->clock_task;
+}
+
+/*
+ * Below are scheduler API which using in other kernel code
+ * It use the dummy rq_flags
+ * ToDo : BMQ need to support these APIs for compatibility with mainline
+ * scheduler code.
+ */
+struct rq_flags {
+	unsigned long	flags;
+	raw_spinlock_t	*lock;
+	bool		queued;
+};
+
+static inline void
+rq_lock_irq(struct rq *rq, struct rq_flags *rf)
+	__acquires(rq->lock)
+{
+	raw_spin_lock_irq(&rq->lock);
+}
+
+static inline void
+rq_unlock_irq(struct rq *rq, struct rq_flags *rf)
+	__releases(rq->lock)
+{
+	raw_spin_unlock_irq(&rq->lock);
+}
+
+DEFINE_LOCK_GUARD_1(rq_lock_irq, struct rq,
+		    rq_lock_irq(_T->lock, &_T->rf),
+		    rq_unlock_irq(_T->lock, &_T->rf),
+		    struct rq_flags rf)
+
+static inline struct rq *
+this_rq_lock_irq(struct rq_flags *rf)
+	__acquires(rq->lock)
+{
+	struct rq *rq;
+
+	local_irq_disable();
+	rq = this_rq();
+	raw_spin_lock(&rq->lock);
+
+	return rq;
+}
+
+static inline raw_spinlock_t *__rq_lockp(struct rq *rq)
+{
+	return &rq->lock;
+}
+
+static inline raw_spinlock_t *rq_lockp(struct rq *rq)
+{
+	return __rq_lockp(rq);
+}
+
+static inline void lockdep_assert_rq_held(struct rq *rq)
+{
+	lockdep_assert_held(__rq_lockp(rq));
+}
+
+static inline int task_current(struct rq *rq, struct task_struct *p)
+{
+	return rq->curr == p;
+}
+
+static inline bool task_on_cpu(struct task_struct *p)
+{
+	return p->on_cpu;
+}
+
+extern struct static_key_false sched_schedstats;
+
+#ifdef CONFIG_CPU_IDLE
+static inline void idle_set_state(struct rq *rq,
+				  struct cpuidle_state *idle_state)
+{
+	rq->idle_state = idle_state;
+}
+#else
+static inline void idle_set_state(struct rq *rq,
+				  struct cpuidle_state *idle_state)
+{
+}
+#endif
+
+static inline int cpu_of(const struct rq *rq)
+{
+	return rq->cpu;
+}
+
+extern void resched_cpu(int cpu);
+
+#define SCHED_QUEUE_BITS	(SCHED_LEVELS - 1)
+#define IDLE_TASK_PRIO		(MIN_NORMAL_PRIO + NORMAL_PRIO_NUM)
+
+struct sched_run_queue {
+	raw_spinlock_t		_lock[SCHED_QUEUE_BITS];
+	struct llist_head	_head[SCHED_QUEUE_BITS];
+	DECLARE_BITMAP(bitmap, SCHED_QUEUE_BITS);
+	atomic_t		nr_queued;
+	atomic_t		nr_uninterruptible;
+	atomic_t		nr_iowait;
+};
+
+extern struct sched_run_queue grq;
+
+/*
+ * schedule run queue functions
+ */
+static inline struct sched_run_queue *cpu_srq(const int cpu)
+{
+	return &grq;
+}
+
+static inline struct sched_run_queue *rq_srq(struct rq *rq)
+{
+	return cpu_srq(cpu_of(rq));
+}
+
+static inline int srq_nr_queued(const int cpu)
+{
+	return raw_atomic_read(&cpu_srq(cpu)->nr_queued);
+}
+
+static inline unsigned int nr_uninterruptible(void)
+{
+	return raw_atomic_read(&cpu_srq(0)->nr_uninterruptible);
+}
+
+static inline bool idle_rq(struct rq *rq)
+{
+	return rq->curr == rq->idle && !srq_nr_queued(cpu_of(rq)) && !rq->ttwu_pending;
+}
+
+/**
+ * available_idle_cpu - is a given CPU idle for enqueuing work.
+ * @cpu: the CPU in question.
+ *
+ * Return: 1 if the CPU is currently idle. 0 otherwise.
+ */
+static inline bool available_idle_cpu(int cpu)
+{
+	if (!idle_rq(cpu_rq(cpu)))
+		return 0;
+
+	if (vcpu_is_preempted(cpu))
+		return 0;
+
+	return 1;
+}
+
+#include "stats.h"
+
+#ifdef CONFIG_NO_HZ_COMMON
+#define NOHZ_BALANCE_KICK_BIT	0
+#define NOHZ_STATS_KICK_BIT	1
+
+#define NOHZ_BALANCE_KICK	BIT(NOHZ_BALANCE_KICK_BIT)
+#define NOHZ_STATS_KICK		BIT(NOHZ_STATS_KICK_BIT)
+
+#define NOHZ_KICK_MASK	(NOHZ_BALANCE_KICK | NOHZ_STATS_KICK)
+
+#define nohz_flags(cpu)	(&cpu_rq(cpu)->nohz_flags)
+
+/* TODO: needed?
+extern void nohz_balance_exit_idle(struct rq *rq);
+#else
+static inline void nohz_balance_exit_idle(struct rq *rq) { }
+*/
+#endif
+
+#ifdef CONFIG_IRQ_TIME_ACCOUNTING
+struct irqtime {
+	u64			total;
+	u64			tick_delta;
+	u64			irq_start_time;
+	struct u64_stats_sync	sync;
+};
+
+DECLARE_PER_CPU(struct irqtime, cpu_irqtime);
+DECLARE_STATIC_KEY_FALSE(sched_clock_irqtime);
+
+static inline int irqtime_enabled(void)
+{
+	return static_branch_likely(&sched_clock_irqtime);
+}
+
+/*
+ * Returns the irqtime minus the softirq time computed by ksoftirqd.
+ * Otherwise ksoftirqd's sum_exec_runtime is substracted its own runtime
+ * and never move forward.
+ */
+static inline u64 irq_time_read(int cpu)
+{
+	struct irqtime *irqtime = &per_cpu(cpu_irqtime, cpu);
+	unsigned int seq;
+	u64 total;
+
+	do {
+		seq = __u64_stats_fetch_begin(&irqtime->sync);
+		total = irqtime->total;
+	} while (__u64_stats_fetch_retry(&irqtime->sync, seq));
+
+	return total;
+}
+#else
+
+static inline int irqtime_enabled(void)
+{
+	return 0;
+}
+
+#endif /* CONFIG_IRQ_TIME_ACCOUNTING */
+
+#ifdef CONFIG_CPU_FREQ
+DECLARE_PER_CPU(struct update_util_data __rcu *, cpufreq_update_util_data);
+#endif /* CONFIG_CPU_FREQ */
+
+#ifdef CONFIG_NO_HZ_FULL
+extern int __init sched_tick_offload_init(void);
+#else
+static inline int sched_tick_offload_init(void) { return 0; }
+#endif
+
+#ifdef arch_scale_freq_capacity
+#ifndef arch_scale_freq_invariant
+#define arch_scale_freq_invariant()	(true)
+#endif
+#else /* arch_scale_freq_capacity */
+#define arch_scale_freq_invariant()	(false)
+#endif
+
+unsigned long sugov_effective_cpu_perf(int cpu, unsigned long actual,
+				 unsigned long min,
+				 unsigned long max);
+
+extern void schedule_idle(void);
+
+#define cap_scale(v, s) ((v)*(s) >> SCHED_CAPACITY_SHIFT)
+
+/*
+ * !! For sched_setattr_nocheck() (kernel) only !!
+ *
+ * This is actually gross. :(
+ *
+ * It is used to make schedutil kworker(s) higher priority than SCHED_DEADLINE
+ * tasks, but still be able to sleep. We need this on platforms that cannot
+ * atomically change clock frequency. Remove once fast switching will be
+ * available on such platforms.
+ *
+ * SUGOV stands for SchedUtil GOVernor.
+ */
+#define SCHED_FLAG_SUGOV	0x10000000
+
+#ifdef CONFIG_MEMBARRIER
+/*
+ * The scheduler provides memory barriers required by membarrier between:
+ * - prior user-space memory accesses and store to rq->membarrier_state,
+ * - store to rq->membarrier_state and following user-space memory accesses.
+ * In the same way it provides those guarantees around store to rq->curr.
+ */
+static inline void membarrier_switch_mm(struct rq *rq,
+					struct mm_struct *prev_mm,
+					struct mm_struct *next_mm)
+{
+	int membarrier_state;
+
+	if (prev_mm == next_mm)
+		return;
+
+	membarrier_state = atomic_read(&next_mm->membarrier_state);
+	if (READ_ONCE(rq->membarrier_state) == membarrier_state)
+		return;
+
+	WRITE_ONCE(rq->membarrier_state, membarrier_state);
+}
+#else
+static inline void membarrier_switch_mm(struct rq *rq,
+					struct mm_struct *prev_mm,
+					struct mm_struct *next_mm)
+{
+}
+#endif
+
+#ifdef CONFIG_NUMA
+extern int sched_numa_find_closest(const struct cpumask *cpus, int cpu);
+#else
+static inline int sched_numa_find_closest(const struct cpumask *cpus, int cpu)
+{
+	return nr_cpu_ids;
+}
+#endif
+
+extern void swake_up_all_locked(struct swait_queue_head *q);
+extern void __prepare_to_swait(struct swait_queue_head *q, struct swait_queue *wait);
+
+extern int try_to_wake_up(struct task_struct *tsk, unsigned int state, int wake_flags);
+
+#ifdef CONFIG_PREEMPT_DYNAMIC
+extern int preempt_dynamic_mode;
+extern int sched_dynamic_mode(const char *str);
+extern void sched_dynamic_update(int mode);
+#endif
+extern const char *preempt_modes[];
+
+static inline void nohz_run_idle_balance(int cpu) { }
+
+static inline unsigned long
+uclamp_eff_value(struct task_struct *p, enum uclamp_id clamp_id)
+{
+	if (clamp_id == UCLAMP_MIN)
+		return 0;
+
+	return SCHED_CAPACITY_SCALE;
+}
+
+static inline bool uclamp_rq_is_capped(struct rq *rq) { return false; }
+
+static inline bool uclamp_is_used(void)
+{
+	return false;
+}
+
+static inline unsigned long
+uclamp_rq_get(struct rq *rq, enum uclamp_id clamp_id)
+{
+	if (clamp_id == UCLAMP_MIN)
+		return 0;
+
+	return SCHED_CAPACITY_SCALE;
+}
+
+static inline void
+uclamp_rq_set(struct rq *rq, enum uclamp_id clamp_id, unsigned int value)
+{
+}
+
+static inline bool uclamp_rq_is_idle(struct rq *rq)
+{
+	return false;
+}
+
+#ifdef CONFIG_SCHED_MM_CID
+
+static __always_inline bool cid_on_cpu(unsigned int cid)
+{
+	return cid & MM_CID_ONCPU;
+}
+
+static __always_inline bool cid_in_transit(unsigned int cid)
+{
+	return cid & MM_CID_TRANSIT;
+}
+
+static __always_inline unsigned int cpu_cid_to_cid(unsigned int cid)
+{
+	return cid & ~MM_CID_ONCPU;
+}
+
+static __always_inline unsigned int cid_to_cpu_cid(unsigned int cid)
+{
+	return cid | MM_CID_ONCPU;
+}
+
+static __always_inline unsigned int cid_to_transit_cid(unsigned int cid)
+{
+	return cid | MM_CID_TRANSIT;
+}
+
+static __always_inline unsigned int cid_from_transit_cid(unsigned int cid)
+{
+	return cid & ~MM_CID_TRANSIT;
+}
+
+static __always_inline bool cid_on_task(unsigned int cid)
+{
+	/* True if none of the MM_CID_ONCPU, MM_CID_TRANSIT, MM_CID_UNSET bits is set */
+	return cid < MM_CID_TRANSIT;
+}
+
+static __always_inline void mm_drop_cid(struct mm_struct *mm, unsigned int cid)
+{
+	clear_bit(cid, mm_cidmask(mm));
+}
+
+static __always_inline void mm_unset_cid_on_task(struct task_struct *t)
+{
+	unsigned int cid = t->mm_cid.cid;
+
+	t->mm_cid.cid = MM_CID_UNSET;
+	if (cid_on_task(cid))
+		mm_drop_cid(t->mm, cid);
+}
+
+static __always_inline void mm_drop_cid_on_cpu(struct mm_struct *mm, struct mm_cid_pcpu *pcp)
+{
+	/* Clear the ONCPU bit, but do not set UNSET in the per CPU storage */
+	if (cid_on_cpu(pcp->cid)) {
+		pcp->cid = cpu_cid_to_cid(pcp->cid);
+		mm_drop_cid(mm, pcp->cid);
+	}
+}
+
+static inline unsigned int __mm_get_cid(struct mm_struct *mm, unsigned int max_cids)
+{
+	unsigned int cid = find_first_zero_bit(mm_cidmask(mm), max_cids);
+
+	if (cid >= max_cids)
+		return MM_CID_UNSET;
+	if (test_and_set_bit(cid, mm_cidmask(mm)))
+		return MM_CID_UNSET;
+	return cid;
+}
+
+static inline unsigned int mm_get_cid(struct mm_struct *mm)
+{
+	unsigned int cid = __mm_get_cid(mm, READ_ONCE(mm->mm_cid.max_cids));
+
+	while (cid == MM_CID_UNSET) {
+		cpu_relax();
+		cid = __mm_get_cid(mm, num_possible_cpus());
+	}
+	return cid;
+}
+
+static inline unsigned int mm_cid_converge(struct mm_struct *mm, unsigned int orig_cid,
+					   unsigned int max_cids)
+{
+	unsigned int new_cid, cid = cpu_cid_to_cid(orig_cid);
+
+	/* Is it in the optimal CID space? */
+	if (likely(cid < max_cids))
+		return orig_cid;
+
+	/* Try to find one in the optimal space. Otherwise keep the provided. */
+	new_cid = __mm_get_cid(mm, max_cids);
+	if (new_cid != MM_CID_UNSET) {
+		mm_drop_cid(mm, cid);
+		/* Preserve the ONCPU mode of the original CID */
+		return new_cid | (orig_cid & MM_CID_ONCPU);
+	}
+	return orig_cid;
+}
+
+static __always_inline void mm_cid_update_task_cid(struct task_struct *t, unsigned int cid)
+{
+	if (t->mm_cid.cid != cid) {
+		t->mm_cid.cid = cid;
+		rseq_sched_set_ids_changed(t);
+	}
+}
+
+static __always_inline void mm_cid_update_pcpu_cid(struct mm_struct *mm, unsigned int cid)
+{
+	__this_cpu_write(mm->mm_cid.pcpu->cid, cid);
+}
+
+static __always_inline void mm_cid_from_cpu(struct task_struct *t, unsigned int cpu_cid,
+					    unsigned int mode)
+{
+	unsigned int max_cids, tcid = t->mm_cid.cid;
+	struct mm_struct *mm = t->mm;
+
+	max_cids = READ_ONCE(mm->mm_cid.max_cids);
+	/* Optimize for the common case where both have the ONCPU bit set */
+	if (likely(cid_on_cpu(cpu_cid & tcid))) {
+		if (likely(cpu_cid_to_cid(cpu_cid) < max_cids)) {
+			mm_cid_update_task_cid(t, cpu_cid);
+			return;
+		}
+		/* Try to converge into the optimal CID space */
+		cpu_cid = mm_cid_converge(mm, cpu_cid, max_cids);
+	} else {
+		/* Hand over or drop the task owned CID */
+		if (cid_on_task(tcid)) {
+			if (cid_on_cpu(cpu_cid))
+				mm_unset_cid_on_task(t);
+			else
+				cpu_cid = cid_to_cpu_cid(tcid);
+		}
+		/* Still nothing, allocate a new one */
+		if (!cid_on_cpu(cpu_cid))
+			cpu_cid = cid_to_cpu_cid(mm_get_cid(mm));
+
+		/* Handle the transition mode flag if required */
+		if (mode & MM_CID_TRANSIT)
+			cpu_cid = cpu_cid_to_cid(cpu_cid) | MM_CID_TRANSIT;
+	}
+	mm_cid_update_pcpu_cid(mm, cpu_cid);
+	mm_cid_update_task_cid(t, cpu_cid);
+}
+
+static __always_inline void mm_cid_from_task(struct task_struct *t, unsigned int cpu_cid,
+					     unsigned int mode)
+{
+	unsigned int max_cids, tcid = t->mm_cid.cid;
+	struct mm_struct *mm = t->mm;
+
+	max_cids = READ_ONCE(mm->mm_cid.max_cids);
+	/* Optimize for the common case, where both have the ONCPU bit clear */
+	if (likely(cid_on_task(tcid | cpu_cid))) {
+		if (likely(tcid < max_cids)) {
+			mm_cid_update_pcpu_cid(mm, tcid);
+			return;
+		}
+		/* Try to converge into the optimal CID space */
+		tcid = mm_cid_converge(mm, tcid, max_cids);
+	} else {
+		/* Hand over or drop the CPU owned CID */
+		if (cid_on_cpu(cpu_cid)) {
+			if (cid_on_task(tcid))
+				mm_drop_cid_on_cpu(mm, this_cpu_ptr(mm->mm_cid.pcpu));
+			else
+				tcid = cpu_cid_to_cid(cpu_cid);
+		}
+		/* Still nothing, allocate a new one */
+		if (!cid_on_task(tcid))
+			tcid = mm_get_cid(mm);
+		/* Set the transition mode flag if required */
+		tcid |= mode & MM_CID_TRANSIT;
+	}
+	mm_cid_update_pcpu_cid(mm, tcid);
+	mm_cid_update_task_cid(t, tcid);
+}
+
+static __always_inline void mm_cid_schedin(struct task_struct *next)
+{
+	struct mm_struct *mm = next->mm;
+	unsigned int cpu_cid, mode;
+
+	if (!next->mm_cid.active)
+		return;
+
+	cpu_cid = __this_cpu_read(mm->mm_cid.pcpu->cid);
+	mode = READ_ONCE(mm->mm_cid.mode);
+	if (likely(!cid_on_cpu(mode)))
+		mm_cid_from_task(next, cpu_cid, mode);
+	else
+		mm_cid_from_cpu(next, cpu_cid, mode);
+}
+
+static __always_inline void mm_cid_schedout(struct task_struct *prev)
+{
+	/* During mode transitions CIDs are temporary and need to be dropped */
+	if (likely(!cid_in_transit(prev->mm_cid.cid)))
+		return;
+
+	mm_drop_cid(prev->mm, cid_from_transit_cid(prev->mm_cid.cid));
+	prev->mm_cid.cid = MM_CID_UNSET;
+}
+
+static inline void mm_cid_switch_to(struct task_struct *prev, struct task_struct *next)
+{
+	mm_cid_schedout(prev);
+	mm_cid_schedin(next);
+}
+
+#else /* !CONFIG_SCHED_MM_CID: */
+static inline void mm_cid_switch_to(struct task_struct *prev, struct task_struct *next) { }
+#endif /* !CONFIG_SCHED_MM_CID */
+
+extern struct balance_callback balance_push_callback;
+
+static inline void
+queue_balance_callback(struct rq *rq,
+		       struct balance_callback *head,
+		       void (*func)(struct rq *rq))
+{
+	lockdep_assert_rq_held(rq);
+
+	/*
+	 * Don't (re)queue an already queued item; nor queue anything when
+	 * balance_push() is active, see the comment with
+	 * balance_push_callback.
+	 */
+	if (unlikely(head->next || rq->balance_callback == &balance_push_callback))
+		return;
+
+	head->func = func;
+	head->next = rq->balance_callback;
+	rq->balance_callback = head;
+}
+
+/*
+ * The 'sched_change' pattern is the safe, easy and slow way of changing a
+ * task's scheduling properties. It dequeues a task, such that the scheduler
+ * is fully unaware of it; at which point its properties can be modified;
+ * after which it is enqueued again.
+ *
+ * Typically this must be called while holding task_rq_lock, since most/all
+ * properties are serialized under those locks. There is currently one
+ * exception to this rule in sched/ext which only holds rq->lock.
+ */
+
+/*
+ * This structure is a temporary, used to preserve/convey the queueing state
+ * of the task between sched_change_begin() and sched_change_end(). Ensuring
+ * the task's queueing state is idempotent across the operation.
+ */
+struct sched_change_ctx {
+	u64			prio;
+	struct task_struct	*p;
+	int			flags;
+	bool			queued;
+	bool			running;
+};
+
+struct sched_change_ctx *sched_change_begin(struct task_struct *p, unsigned int flags);
+void sched_change_end(struct sched_change_ctx *ctx);
+
+DEFINE_CLASS(sched_change, struct sched_change_ctx *,
+	     sched_change_end(_T),
+	     sched_change_begin(p, flags),
+	     struct task_struct *p, unsigned int flags)
+
+DEFINE_CLASS_IS_UNCONDITIONAL(sched_change)
+
+#ifdef CONFIG_SCHED_BMQ
+#include "bmq.h"
+#endif
+#ifdef CONFIG_SCHED_PDS
+#include "pds.h"
+#endif
+
+#endif /* _KERNEL_SCHED_ALT_SCHED_H */
diff --git a/kernel/sched/alt_topology.c b/kernel/sched/alt_topology.c
new file mode 100644
index 000000000..1b542b705
--- /dev/null
+++ b/kernel/sched/alt_topology.c
@@ -0,0 +1,389 @@
+#ifdef ALT_SCHED_DEBUG
+#define SCHED_DEBUG_INFO(...)	printk(KERN_INFO __VA_ARGS__)
+#else
+#define SCHED_DEBUG_INFO(...)	do { } while(0)
+#endif
+
+static cpumask_t cpu_topo_masks[6] ____cacheline_aligned_in_smp;
+
+#define sched_ucore_mask	(cpu_topo_masks)
+#define sched_pcore_mask	(cpu_topo_masks + 1)
+#define sched_ecore_mask	(cpu_topo_masks + 2)
+
+/*
+ * pcore_cpus kernel parameter
+ */
+static int __init pcore_cpus_setup(char *str)
+{
+	if (cpulist_parse(str, sched_pcore_mask))
+		pr_warn("sched/alt: pcore_cpus= incorrect CPU range\n");
+
+	return 1;
+}
+__setup("pcore_cpus=", pcore_cpus_setup);
+
+/*
+ * ultra_core_cpus kernel parameter
+ */
+static int __init ucore_cpus_setup(char *str)
+{
+	if (cpulist_parse(str, sched_ucore_mask))
+		pr_warn("sched/alt: ucore_cpus= incorrect CPU range\n");
+
+	return 1;
+}
+__setup("ucore_cpus=", ucore_cpus_setup);
+
+/*
+ * CPU topology type
+ */
+enum cpu_topo_type {
+	CPU_TOPOLOGY_DEFAULT	= 0,
+	CPU_TOPOLOGY_UCORE,
+	CPU_TOPOLOGY_PCORE,
+	CPU_TOPOLOGY_ECORE,
+	CPU_TOPOLOGY_NONE,
+#ifdef CONFIG_SCHED_SMT
+	CPU_TOPOLOGY_UCORE_SMT,
+	CPU_TOPOLOGY_PCORE_SMT,
+	CPU_TOPOLOGY_SMT	= CPU_TOPOLOGY_PCORE_SMT,
+	CPU_TOPOLOGY_ECORE_SMT,
+#endif
+};
+
+#define CPU_TOPOLOGY_MASK	(0x03)
+
+static DEFINE_PER_CPU_READ_MOSTLY(enum cpu_topo_type, sched_cpu_topo);
+
+static __always_inline void sched_set_idle_mask(const unsigned int cpu)
+{
+	int topo = per_cpu(sched_cpu_topo, cpu);
+
+	switch (topo) {
+	case CPU_TOPOLOGY_DEFAULT:
+	case CPU_TOPOLOGY_NONE:
+		break;
+	case CPU_TOPOLOGY_UCORE:
+	case CPU_TOPOLOGY_PCORE:
+	case CPU_TOPOLOGY_ECORE:
+		topo--;
+		cpumask_set_cpu(cpu, cpu_sched_prio_mask + topo);
+		set_bit(topo, cpu_sched_prio_bitmap);
+		break;
+#ifdef CONFIG_SCHED_SMT
+	case CPU_TOPOLOGY_UCORE_SMT:
+	case CPU_TOPOLOGY_PCORE_SMT:
+	case CPU_TOPOLOGY_ECORE_SMT:
+		topo -= CPU_TOPOLOGY_UCORE_SMT;
+		if (cpumask_intersects(cpu_smt_mask(cpu), sched_idle_mask)) {
+			cpumask_or(cpu_sched_prio_mask + topo, cpu_sched_prio_mask + topo,
+				   cpu_smt_mask(cpu));
+			set_bit(topo, cpu_sched_prio_bitmap);
+
+			topo += 3;
+			if (!cpumask_andnot(cpu_sched_prio_mask + topo, cpu_sched_prio_mask + topo,
+					    cpu_smt_mask(cpu)))
+				clear_bit(topo, cpu_sched_prio_bitmap);
+		} else {
+			topo += 3;
+			cpumask_set_cpu(cpu, cpu_sched_prio_mask + topo);
+			set_bit(topo, cpu_sched_prio_bitmap);
+		}
+		break;
+#endif
+	}
+
+	cpumask_set_cpu(cpu, sched_idle_mask);
+	set_bit(SCHED_LEVELS - 1 - IDLE_TASK_SCHED_PRIO, cpu_sched_prio_bitmap);
+}
+
+static __always_inline void sched_clear_idle_mask(const unsigned int cpu)
+{
+	int topo = per_cpu(sched_cpu_topo, cpu);
+
+	switch (topo) {
+	case CPU_TOPOLOGY_DEFAULT:
+	case CPU_TOPOLOGY_NONE:
+		break;
+	case CPU_TOPOLOGY_UCORE:
+	case CPU_TOPOLOGY_PCORE:
+	case CPU_TOPOLOGY_ECORE:
+		topo--;
+		cpumask_clear_cpu(cpu, cpu_sched_prio_mask + topo);
+		if (cpumask_empty(cpu_sched_prio_mask + topo))
+			clear_bit(topo, cpu_sched_prio_bitmap);
+		break;
+#ifdef CONFIG_SCHED_SMT
+	case CPU_TOPOLOGY_UCORE_SMT:
+	case CPU_TOPOLOGY_PCORE_SMT:
+	case CPU_TOPOLOGY_ECORE_SMT:
+		topo -= CPU_TOPOLOGY_UCORE_SMT;
+		if (cpumask_test_cpu(cpu, cpu_sched_prio_mask + topo)) {
+			if (!cpumask_andnot(cpu_sched_prio_mask + topo, cpu_sched_prio_mask + topo,
+					    cpu_smt_mask(cpu)))
+				clear_bit(topo, cpu_sched_prio_bitmap);
+
+			topo += 3;
+			cpumask_or(cpu_sched_prio_mask + topo, cpu_sched_prio_mask + topo,
+				   per_cpu(cpu_affinity_masks, cpu));
+			set_bit(topo, cpu_sched_prio_bitmap);
+		} else {
+			topo += 3;
+			cpumask_clear_cpu(cpu, cpu_sched_prio_mask + topo);
+			if (cpumask_empty(cpu_sched_prio_mask + topo))
+				clear_bit(topo, cpu_sched_prio_bitmap);
+		}
+		break;
+#endif
+	}
+
+	cpumask_clear_cpu(cpu, sched_idle_mask);
+	if (cpumask_empty(sched_idle_mask))
+		clear_bit(SCHED_LEVELS - 1 - IDLE_TASK_SCHED_PRIO, cpu_sched_prio_bitmap);
+}
+
+
+/*
+ * CPU topology balance type
+ */
+static DEFINE_PER_CPU(struct balance_callback, active_balance_head);
+
+/* common balance functions */
+static int active_balance_cpu_stop(void *data)
+{
+	struct balance_arg *arg = data;
+	unsigned long flags;
+
+	local_irq_save(flags);
+	arg->active = 0;
+	local_irq_restore(flags);
+
+	return 0;
+}
+
+/* trigger_active_balance - for @cpu */
+static __always_inline int
+trigger_active_balance(const int cpu, struct rq *src_rq, cpumask_t *target_mask)
+{
+	struct rq *rq = cpu_rq(cpu);
+	struct balance_arg *arg;
+	unsigned long flags;
+	struct task_struct *p;
+	int res;
+
+	if (!raw_spin_trylock_irqsave(&rq->lock, flags))
+		return 0;
+
+	arg = &rq->active_balance_arg;
+	p = rq->curr;
+	res = (p != rq->idle) && (0 == srq_nr_queued(cpu)) && is_preempt_list_empty(cpu) &&	\
+	      !is_migration_disabled(p) && cpumask_intersects(p->cpus_ptr, target_mask) &&	\
+	      !arg->active;
+	if (res)
+		arg->active = 1;
+
+	raw_spin_unlock_irqrestore(&rq->lock, flags);
+
+	if (res) {
+		preempt_disable();
+		raw_spin_unlock(&src_rq->lock);
+
+		stop_one_cpu_nowait(cpu, active_balance_cpu_stop, arg, &rq->active_balance_work);
+
+		preempt_enable();
+		raw_spin_lock(&src_rq->lock);
+	}
+
+	return res;
+}
+
+static __always_inline void cpu_topo_balance(struct rq *rq)
+{
+	int i, cpu = cpu_of(rq);
+	int idx = (per_cpu(sched_cpu_topo, cpu) & CPU_TOPOLOGY_MASK) - 1;
+	cpumask_t *target_mask = cpu_sched_prio_mask + idx;
+
+#ifdef CONFIG_SCHED_SMT
+	for (int level = 3; level < 6; level++) {
+		if (cpumask_empty(cpu_topo_masks + level))
+			continue;
+
+		for_each_cpu_andnot(i, cpu_topo_masks + level, sched_idle_mask) {
+			if (!cpumask_intersects(cpu_smt_mask(i), sched_idle_mask) &&
+			    trigger_active_balance(i, rq, target_mask))
+				return;
+		}
+	}
+#endif /* CONFIG_SCHED_SMT */
+
+	if (2 != idx) {
+		for_each_cpu_andnot(i, sched_ecore_mask, sched_idle_mask) {
+			if (trigger_active_balance(i, rq, target_mask))
+				return;
+		}
+	}
+}
+
+static inline void sched_cpu_topology_balance(const unsigned int cpu, struct rq *rq)
+{
+	const int topo = per_cpu(sched_cpu_topo, cpu);
+	switch (topo) {
+	case CPU_TOPOLOGY_DEFAULT:
+		break;
+	case CPU_TOPOLOGY_UCORE:
+	case CPU_TOPOLOGY_PCORE:
+#ifdef CONFIG_SCHED_SMT
+	case CPU_TOPOLOGY_ECORE:
+#endif
+		queue_balance_callback(rq, &per_cpu(active_balance_head, cpu), cpu_topo_balance);
+		break;
+	case CPU_TOPOLOGY_UCORE_SMT:
+	case CPU_TOPOLOGY_PCORE_SMT: /* as same as CPU_TOPOLOGY_SMT */
+	case CPU_TOPOLOGY_ECORE_SMT:
+		int idx = (topo & CPU_TOPOLOGY_MASK) - 1;
+
+		if (bitmap_empty(cpu_sched_prio_bitmap, idx) &&
+		    cpumask_test_cpu(cpu, cpu_sched_prio_mask + idx))
+			queue_balance_callback(rq, &per_cpu(active_balance_head, cpu),
+					       cpu_topo_balance);
+		break;
+	}
+}
+
+/*
+ * Keep a unique ID per domain (we use the first CPUs number in the cpumask of
+ * the domain), this allows us to quickly tell if two cpus are in the same cache
+ * domain, see cpus_share_cache().
+ */
+static DEFINE_PER_CPU_READ_MOSTLY(int, sd_llc_id);
+
+bool cpus_share_cache(int this_cpu, int that_cpu)
+{
+	if (this_cpu == that_cpu)
+		return true;
+
+	return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu);
+}
+
+
+/*
+ * CPU topology mask
+ */
+DEFINE_PER_CPU_ALIGNED(cpumask_t [NR_CPU_AFFINITY_LEVELS], cpu_affinity_masks);
+DEFINE_PER_CPU_ALIGNED(cpumask_t *, cpu_affinity_end_mask);
+
+static inline void sched_init_topology_early(void)
+{
+	int cpu;
+
+	for_each_possible_cpu(cpu) {
+		/* init topo masks */
+		cpumask_t *tmp = per_cpu(cpu_affinity_masks, cpu);
+
+		cpumask_copy(tmp, cpu_possible_mask);
+		per_cpu(cpu_affinity_end_mask, cpu) = ++tmp;
+	}
+}
+
+/* init schedule cpu topology masks: ucore/pcore/ecore and smt mask */
+static inline void sched_init_cpu_topology(void)
+{
+	bool cpu_topo_smt_present = false;
+
+#ifdef CONFIG_SCHED_SMT
+	if (!cpumask_empty(&sched_smt_mask)) {
+		printk(KERN_INFO "sched: (smt) mask : 0x%08lx\n", sched_smt_mask.bits[0]);
+		if (cpumask_empty(sched_pcore_mask))
+			cpumask_copy(sched_pcore_mask, &sched_smt_mask);
+	}
+#endif
+
+	if (cpumask_empty(sched_pcore_mask) && cpumask_empty(sched_ucore_mask))
+		return;
+
+	cpumask_andnot(sched_pcore_mask, sched_pcore_mask, sched_ucore_mask);
+
+	cpumask_andnot(sched_ecore_mask, cpu_online_mask, sched_pcore_mask);
+	cpumask_andnot(sched_ecore_mask, sched_ecore_mask, sched_ucore_mask);
+
+	printk(KERN_INFO "sched: topo  ucore: 0x%08lx, pcore: 0x%08lx, ecore: 0x%08lx\n",
+	       sched_ucore_mask->bits[0],
+	       sched_pcore_mask->bits[0],
+	       sched_ecore_mask->bits[0]);
+
+#ifdef CONFIG_SCHED_SMT
+	if (cpumask_empty(&sched_smt_mask))
+		return;
+
+	for (int i = 0; i < 3; i++)
+		if (cpumask_and(cpu_topo_masks + i + 3,
+				cpu_topo_masks + i, &sched_smt_mask))
+			cpu_topo_smt_present = true;
+
+	if (!cpu_topo_smt_present)
+		return;
+
+	printk(KERN_INFO "sched: (smt) ucore: 0x%08lx, pcore: 0x%08lx, ecore: 0x%08lx\n",
+	       (sched_ucore_mask + 3)->bits[0],
+	       (sched_pcore_mask + 3)->bits[0],
+	       (sched_ecore_mask + 3)->bits[0]);
+#endif
+}
+
+#define TOPOLOGY_CPUMASK(name, mask, last)					\
+	if (cpumask_and(topo, topo, mask)) {					\
+		printk(KERN_INFO "sched: cpu#%02d affinity: 0x%08lx - "#name,	\
+		       cpu, (topo++)->bits[0]);					\
+	}									\
+	if (!last)								\
+		bitmap_complement(cpumask_bits(topo), cpumask_bits(mask),	\
+				  nr_cpumask_bits);
+
+static inline void sched_init_topology(void)
+{
+	int cpu;
+
+	sched_init_cpu_topology();
+
+	/* CPU topology setup */
+	for_each_online_cpu(cpu) {
+		cpumask_t *topo;
+		int cpu_topo = 0;
+#ifdef CONFIG_SCHED_SMT
+		if (cpumask_weight(cpu_smt_mask(cpu)) > 1)
+			cpu_topo += CPU_TOPOLOGY_NONE;
+#endif
+		if (cpumask_test_cpu(cpu, sched_ucore_mask))
+			cpu_topo += CPU_TOPOLOGY_UCORE;
+		else if (cpumask_test_cpu(cpu, sched_pcore_mask))
+			cpu_topo += CPU_TOPOLOGY_PCORE;
+		else if (cpumask_test_cpu(cpu, sched_ecore_mask))
+			cpu_topo += CPU_TOPOLOGY_ECORE;
+
+		if (cpu_topo)
+			per_cpu(sched_cpu_topo, cpu) = cpu_topo;
+
+		topo = per_cpu(cpu_affinity_masks, cpu);
+
+		bitmap_complement(cpumask_bits(topo), cpumask_bits(cpumask_of(cpu)),
+				  nr_cpumask_bits);
+#ifdef CONFIG_SCHED_SMT
+		TOPOLOGY_CPUMASK(smt, topology_sibling_cpumask(cpu), false);
+#endif /* CONFIG_SCHED_SMT */
+		TOPOLOGY_CPUMASK(cluster, topology_cluster_cpumask(cpu), false);
+
+		per_cpu(sd_llc_id, cpu) = cpumask_first(cpu_coregroup_mask(cpu));
+		TOPOLOGY_CPUMASK(coregroup, cpu_coregroup_mask(cpu), false);
+
+		TOPOLOGY_CPUMASK(core, topology_core_cpumask(cpu), false);
+
+		TOPOLOGY_CPUMASK(others, cpu_online_mask, true);
+
+		per_cpu(cpu_affinity_end_mask, cpu) = topo;
+
+		SCHED_DEBUG_INFO("sched: cpu#%02d topology: %d%5s, llc_id = %d",
+				 cpu, cpu_topo & CPU_TOPOLOGY_MASK,
+				 (cpu_topo > CPU_TOPOLOGY_MASK) ? "(smt)" : "",
+				 per_cpu(sd_llc_id, cpu));
+	}
+}
diff --git a/kernel/sched/bmq.h b/kernel/sched/bmq.h
new file mode 100644
index 000000000..9b307fcc1
--- /dev/null
+++ b/kernel/sched/bmq.h
@@ -0,0 +1,84 @@
+#ifndef _KERNEL_SCHED_BMQ_H
+#define _KERNEL_SCHED_BMQ_H
+
+#define ALT_SCHED_NAME "BMQ"
+
+/*
+ * BMQ only routines
+ */
+static inline void boost_task(struct task_struct *p, int n)
+{
+	int limit;
+
+	switch (p->policy) {
+	case SCHED_NORMAL:
+		limit = -MAX_PRIORITY_ADJ;
+		break;
+	case SCHED_BATCH:
+		limit = 0;
+		break;
+	default:
+		return;
+	}
+
+	p->boost_prio = max(limit, p->boost_prio - n);
+}
+
+static inline void deboost_task(struct task_struct *p)
+{
+	if ((SCHED_NORMAL == p->policy || SCHED_BATCH == p->policy) &&
+	    p->boost_prio < MAX_PRIORITY_ADJ)
+		p->boost_prio++;
+}
+
+/*
+ * Common interfaces
+ */
+
+/* This API is used in task_prio(), return value readed by human users */
+static inline int
+task_sched_prio_normal(const struct task_struct *p, const struct rq *rq)
+{
+	return p->prio + p->boost_prio - MIN_NORMAL_PRIO;
+}
+
+static inline int task_sched_prio(const struct task_struct *p)
+{
+	return (p->prio < MIN_NORMAL_PRIO)? (p->prio >> 2) :
+		MIN_SCHED_NORMAL_PRIO + (p->prio + p->boost_prio - MIN_NORMAL_PRIO) / 2;
+}
+
+static inline int task_running_nice(struct task_struct *p)
+{
+	return (p->prio + p->boost_prio > DEFAULT_PRIO);
+}
+
+static inline void sched_update_rq_clock(struct rq *rq) {}
+
+static inline void sched_task_renew(struct task_struct *p, const struct rq *rq)
+{
+	deboost_task(p);
+}
+
+static inline void sched_task_sanity_check(struct task_struct *p, struct rq *rq) {}
+static inline void sched_task_fork(struct task_struct *p, struct rq *rq) {}
+
+static inline void do_sched_yield_type_1(struct task_struct *p, struct rq *rq)
+{
+	p->boost_prio = MAX_PRIORITY_ADJ;
+}
+
+static inline void sched_task_ttwu(struct task_struct *p)
+{
+	s64 delta = this_rq()->clock_task > p->last_ran;
+
+	if (likely(delta > 0))
+		boost_task(p, delta  >> 22);
+}
+
+static inline void sched_task_deactivate(struct task_struct *p, struct rq *rq)
+{
+	boost_task(p, 1);
+}
+
+#endif /* _KERNEL_SCHED_BMQ_H */
diff --git a/kernel/sched/build_policy.c b/kernel/sched/build_policy.c
index 755883faf..f50aae196 100644
--- a/kernel/sched/build_policy.c
+++ b/kernel/sched/build_policy.c
@@ -49,13 +49,17 @@
 
 #include "idle.c"
 
-#include "rt.c"
-#include "cpudeadline.c"
+#ifndef CONFIG_SCHED_ALT
+# include "rt.c"
+# include "cpudeadline.c"
+#endif
 
 #include "pelt.c"
 
 #include "cputime.c"
+#ifndef CONFIG_SCHED_ALT
 #include "deadline.c"
+#endif
 
 #ifdef CONFIG_SCHED_CLASS_EXT
 # include "ext_internal.h"
diff --git a/kernel/sched/build_utility.c b/kernel/sched/build_utility.c
index e2cf3b08d..92cf7cd1f 100644
--- a/kernel/sched/build_utility.c
+++ b/kernel/sched/build_utility.c
@@ -68,7 +68,7 @@
 # include "cpufreq_schedutil.c"
 #endif
 
-#include "debug.c"
+# include "debug.c"
 
 #ifdef CONFIG_SCHEDSTATS
 # include "stats.c"
@@ -81,7 +81,9 @@
 #include "wait.c"
 
 #include "cpupri.c"
-#include "stop_task.c"
+#ifndef CONFIG_SCHED_ALT
+# include "stop_task.c"
+#endif
 
 #include "topology.c"
 
diff --git a/kernel/sched/cpufreq_schedutil.c b/kernel/sched/cpufreq_schedutil.c
index ae9fd211c..1a3ecefdb 100644
--- a/kernel/sched/cpufreq_schedutil.c
+++ b/kernel/sched/cpufreq_schedutil.c
@@ -225,6 +225,7 @@ unsigned long sugov_effective_cpu_perf(int cpu, unsigned long actual,
 
 static void sugov_get_util(struct sugov_cpu *sg_cpu, unsigned long boost)
 {
+#ifndef CONFIG_SCHED_ALT
 	unsigned long min, max, util = scx_cpuperf_target(sg_cpu->cpu);
 
 	if (!scx_switched_all())
@@ -233,6 +234,10 @@ static void sugov_get_util(struct sugov_cpu *sg_cpu, unsigned long boost)
 	util = max(util, boost);
 	sg_cpu->bw_min = min;
 	sg_cpu->util = sugov_effective_cpu_perf(sg_cpu->cpu, util, min, max);
+#else /* CONFIG_SCHED_ALT */
+	sg_cpu->bw_min = 0;
+	sg_cpu->util = rq_load_util(cpu_rq(sg_cpu->cpu), arch_scale_cpu_capacity(sg_cpu->cpu));
+#endif /* CONFIG_SCHED_ALT */
 }
 
 /**
@@ -360,6 +365,7 @@ static bool sugov_hold_freq(struct sugov_cpu *sg_cpu)
 	unsigned long idle_calls;
 	bool ret;
 
+#ifndef CONFIG_SCHED_ALT
 	/*
 	 * The heuristics in this function is for the fair class. For SCX, the
 	 * performance target comes directly from the BPF scheduler. Let's just
@@ -367,6 +373,7 @@ static bool sugov_hold_freq(struct sugov_cpu *sg_cpu)
 	 */
 	if (scx_switched_all())
 		return false;
+#endif /* !CONFIG_SCHED_ALT */
 
 	/* if capped by uclamp_max, always update to be in compliance */
 	if (uclamp_rq_is_capped(cpu_rq(sg_cpu->cpu)))
@@ -392,8 +399,10 @@ static inline bool sugov_hold_freq(struct sugov_cpu *sg_cpu) { return false; }
  */
 static inline void ignore_dl_rate_limit(struct sugov_cpu *sg_cpu)
 {
+#ifndef CONFIG_SCHED_ALT
 	if (cpu_bw_dl(cpu_rq(sg_cpu->cpu)) > sg_cpu->bw_min)
-		sg_cpu->sg_policy->need_freq_update = true;
+		sg_cpu->sg_policy->limits_changed = true;
+#endif
 }
 
 static inline bool sugov_update_single_common(struct sugov_cpu *sg_cpu,
@@ -688,6 +697,7 @@ static int sugov_kthread_create(struct sugov_policy *sg_policy)
 	}
 
 	ret = sched_setattr_nocheck(thread, &attr);
+
 	if (ret) {
 		kthread_stop(thread);
 		pr_warn("%s: failed to set SCHED_DEADLINE\n", __func__);
diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c
index fbf31db0d..92f1e27e1 100644
--- a/kernel/sched/cputime.c
+++ b/kernel/sched/cputime.c
@@ -128,7 +128,7 @@ void account_user_time(struct task_struct *p, u64 cputime)
 	p->utime += cputime;
 	account_group_user_time(p, cputime);
 
-	index = (task_nice(p) > 0) ? CPUTIME_NICE : CPUTIME_USER;
+	index = task_running_nice(p) ? CPUTIME_NICE : CPUTIME_USER;
 
 	/* Add user time to cpustat. */
 	task_group_account_field(p, index, cputime);
@@ -152,7 +152,7 @@ void account_guest_time(struct task_struct *p, u64 cputime)
 	p->gtime += cputime;
 
 	/* Add guest time to cpustat. */
-	if (task_nice(p) > 0) {
+	if (task_running_nice(p)) {
 		task_group_account_field(p, CPUTIME_NICE, cputime);
 		cpustat[CPUTIME_GUEST_NICE] += cputime;
 	} else {
@@ -226,7 +226,7 @@ void account_idle_time(u64 cputime)
 	u64 *cpustat = kcpustat_this_cpu->cpustat;
 	struct rq *rq = this_rq();
 
-	if (atomic_read(&rq->nr_iowait) > 0)
+	if (nr_iowait_cpu(cpu_of(rq)) > 0)
 		cpustat[CPUTIME_IOWAIT] += cputime;
 	else
 		cpustat[CPUTIME_IDLE] += cputime;
@@ -303,7 +303,7 @@ static inline u64 account_other_time(u64 max)
 #ifdef CONFIG_64BIT
 static inline u64 read_sum_exec_runtime(struct task_struct *t)
 {
-	return t->se.sum_exec_runtime;
+	return tsk_seruntime(t);
 }
 #else /* !CONFIG_64BIT: */
 static u64 read_sum_exec_runtime(struct task_struct *t)
@@ -313,7 +313,7 @@ static u64 read_sum_exec_runtime(struct task_struct *t)
 	struct rq *rq;
 
 	rq = task_rq_lock(t, &rf);
-	ns = t->se.sum_exec_runtime;
+	ns = tsk_seruntime(t);
 	task_rq_unlock(rq, t, &rf);
 
 	return ns;
@@ -628,7 +628,7 @@ void cputime_adjust(struct task_cputime *curr, struct prev_cputime *prev,
 void task_cputime_adjusted(struct task_struct *p, u64 *ut, u64 *st)
 {
 	struct task_cputime cputime = {
-		.sum_exec_runtime = p->se.sum_exec_runtime,
+		.sum_exec_runtime = tsk_seruntime(p),
 	};
 
 	if (task_cputime(p, &cputime.utime, &cputime.stime))
diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c
index 74c1617cf..5b38a3ba7 100644
--- a/kernel/sched/debug.c
+++ b/kernel/sched/debug.c
@@ -11,6 +11,7 @@
 #include <linux/log2.h>
 #include "sched.h"
 
+#ifndef CONFIG_SCHED_ALT
 /*
  * This allows printing both to /sys/kernel/debug/sched/debug and
  * to the console
@@ -210,6 +211,8 @@ static const struct file_operations sched_scaling_fops = {
 	.release	= single_release,
 };
 
+#endif /* !CONFIG_SCHED_ALT */
+
 #ifdef CONFIG_PREEMPT_DYNAMIC
 
 static ssize_t sched_dynamic_write(struct file *filp, const char __user *ubuf,
@@ -275,6 +278,7 @@ static const struct file_operations sched_dynamic_fops = {
 
 #endif /* CONFIG_PREEMPT_DYNAMIC */
 
+#ifndef CONFIG_SCHED_ALT
 __read_mostly bool sched_debug_verbose;
 
 static struct dentry           *sd_dentry;
@@ -510,6 +514,7 @@ static const struct file_operations fair_server_period_fops = {
 	.llseek		= seq_lseek,
 	.release	= single_release,
 };
+#endif /* !CONFIG_SCHED_ALT */
 
 #ifdef CONFIG_SCHED_CLASS_EXT
 static ssize_t
@@ -547,6 +552,7 @@ static const struct file_operations ext_server_period_fops = {
 
 static struct dentry *debugfs_sched;
 
+#ifndef CONFIG_SCHED_ALT
 static void debugfs_fair_server_init(void)
 {
 	struct dentry *d_fair;
@@ -567,6 +573,7 @@ static void debugfs_fair_server_init(void)
 		debugfs_create_file("period", 0644, d_cpu, (void *) cpu, &fair_server_period_fops);
 	}
 }
+#endif /* !CONFIG_SCHED_ALT */
 
 #ifdef CONFIG_SCHED_CLASS_EXT
 static void debugfs_ext_server_init(void)
@@ -597,14 +604,17 @@ static __init int sched_init_debug(void)
 
 	debugfs_sched = debugfs_create_dir("sched", NULL);
 
+#ifndef CONFIG_SCHED_ALT
 	debugfs_create_file("features", 0644, debugfs_sched, NULL, &sched_feat_fops);
 	debugfs_create_file_unsafe("verbose", 0644, debugfs_sched, &sched_debug_verbose, &sched_verbose_fops);
+#endif /* !CONFIG_SCHED_ALT */
 #ifdef CONFIG_PREEMPT_DYNAMIC
 	debugfs_create_file("preempt", 0644, debugfs_sched, NULL, &sched_dynamic_fops);
 #endif
 
 	debugfs_create_u32("base_slice_ns", 0644, debugfs_sched, &sysctl_sched_base_slice);
 
+#ifndef CONFIG_SCHED_ALT
 	debugfs_create_u32("latency_warn_ms", 0644, debugfs_sched, &sysctl_resched_latency_warn_ms);
 	debugfs_create_u32("latency_warn_once", 0644, debugfs_sched, &sysctl_resched_latency_warn_once);
 
@@ -627,16 +637,21 @@ static __init int sched_init_debug(void)
 #endif /* CONFIG_NUMA_BALANCING */
 
 	debugfs_create_file("debug", 0444, debugfs_sched, NULL, &sched_debug_fops);
+#endif /* !CONFIG_SCHED_ALT */
 
+#ifndef CONFIG_SCHED_ALT
 	debugfs_fair_server_init();
 #ifdef CONFIG_SCHED_CLASS_EXT
 	debugfs_ext_server_init();
 #endif
+#endif /* !CONFIG_SCHED_ALT */
 
 	return 0;
 }
 late_initcall(sched_init_debug);
 
+#ifndef CONFIG_SCHED_ALT
+
 static cpumask_var_t		sd_sysctl_cpus;
 
 static int sd_flags_show(struct seq_file *m, void *v)
@@ -1382,6 +1397,11 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns,
 
 	sched_show_numa(p, m);
 }
+#else
+void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns,
+						  struct seq_file *m)
+{ }
+#endif /* !CONFIG_SCHED_ALT */
 
 void proc_sched_set_task(struct task_struct *p)
 {
diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c
index 912b2a846..3848a9c06 100644
--- a/kernel/sched/idle.c
+++ b/kernel/sched/idle.c
@@ -297,10 +297,10 @@ static void do_idle(void)
 	__current_set_polling();
 	tick_nohz_idle_enter();
 
-#ifdef CONFIG_SCHED_POC_SELECTOR
+#if defined(CONFIG_SCHED_POC_SELECTOR) && !defined(CONFIG_SCHED_ALT)
 	/* POC Selector: mark CPU as idle */
 	set_cpu_idle_state_poc(cpu, 1);
-#endif /* CONFIG_SCHED_POC_SELECTOR */
+#endif /* CONFIG_SCHED_POC_SELECTOR && !CONFIG_SCHED_ALT */
 
 	while (!need_resched()) {
 
@@ -360,10 +360,10 @@ static void do_idle(void)
 		arch_cpu_idle_exit();
 	}
 
-#ifdef CONFIG_SCHED_POC_SELECTOR
+#if defined(CONFIG_SCHED_POC_SELECTOR) && !defined(CONFIG_SCHED_ALT)
 	/* POC Selector: mark CPU as busy */
 	set_cpu_idle_state_poc(cpu, 0);
-#endif /* CONFIG_SCHED_POC_SELECTOR */
+#endif /* CONFIG_SCHED_POC_SELECTOR && !CONFIG_SCHED_ALT */
 
 	/*
 	 * Since we fell out of the loop above, we know TIF_NEED_RESCHED must
@@ -461,6 +461,8 @@ void cpu_startup_entry(enum cpuhp_state state)
 		do_idle();
 }
 
+#ifndef CONFIG_SCHED_ALT
+#ifndef CONFIG_SCHED_ALT
 /*
  * idle-task scheduling class.
  */
@@ -598,3 +600,4 @@ DEFINE_SCHED_CLASS(idle) = {
 	.switching_to		= switching_to_idle,
 	.update_curr		= update_curr_idle,
 };
+#endif
diff --git a/kernel/sched/loadavg.c b/kernel/sched/loadavg.c
index b601e0243..1a11c505d 100644
--- a/kernel/sched/loadavg.c
+++ b/kernel/sched/loadavg.c
@@ -77,6 +77,7 @@ void get_avenrun(unsigned long *loads, unsigned long offset, int shift)
 	loads[2] = (avenrun[2] + offset) << shift;
 }
 
+#ifndef CONFIG_SCHED_ALT
 long calc_load_fold_active(struct rq *this_rq, long adjust)
 {
 	long nr_active, delta = 0;
@@ -91,6 +92,7 @@ long calc_load_fold_active(struct rq *this_rq, long adjust)
 
 	return delta;
 }
+#endif
 
 /**
  * fixed_power_int - compute: x^n, in O(log n) time
@@ -161,7 +163,7 @@ calc_load_n(unsigned long load, unsigned long exp,
 	return calc_load(load, fixed_power_int(exp, FSHIFT, n), active);
 }
 
-#ifdef CONFIG_NO_HZ_COMMON
+#if defined(CONFIG_NO_HZ_COMMON) && !defined(CONFIG_SCHED_ALT)
 /*
  * Handle NO_HZ for the global load-average.
  *
@@ -351,12 +353,19 @@ static inline void calc_global_nohz(void) { }
 void calc_global_load(void)
 {
 	unsigned long sample_window;
+#ifdef CONFIG_SCHED_BMQ
+	long active;
+#else
 	long active, delta;
+#endif
 
 	sample_window = READ_ONCE(calc_load_update);
 	if (time_before(jiffies, sample_window + 10))
 		return;
 
+#ifdef CONFIG_SCHED_BMQ
+	active = nr_running() + nr_uninterruptible();
+#else
 	/*
 	 * Fold the 'old' NO_HZ-delta to include all NO_HZ CPUs.
 	 */
@@ -365,6 +374,7 @@ void calc_global_load(void)
 		atomic_long_add(delta, &calc_load_tasks);
 
 	active = atomic_long_read(&calc_load_tasks);
+#endif
 	active = active > 0 ? active * FIXED_1 : 0;
 
 	avenrun[0] = calc_load(avenrun[0], EXP_1, active);
@@ -380,6 +390,7 @@ void calc_global_load(void)
 	calc_global_nohz();
 }
 
+#ifndef CONFIG_SCHED_ALT
 /*
  * Called from sched_tick() to periodically update this CPU's
  * active count.
@@ -397,3 +408,4 @@ void calc_global_load_tick(struct rq *this_rq)
 
 	this_rq->calc_load_update += LOAD_FREQ;
 }
+#endif
diff --git a/kernel/sched/pds.h b/kernel/sched/pds.h
new file mode 100644
index 000000000..fe3099071
--- /dev/null
+++ b/kernel/sched/pds.h
@@ -0,0 +1,139 @@
+#ifndef _KERNEL_SCHED_PDS_H
+#define _KERNEL_SCHED_PDS_H
+
+#define ALT_SCHED_NAME "PDS"
+
+static const u64 RT_MASK = ((1ULL << MIN_SCHED_NORMAL_PRIO) - 1);
+
+#define SCHED_NORMAL_PRIO_NUM	(32)
+#define SCHED_EDGE_DELTA	(SCHED_NORMAL_PRIO_NUM - NICE_WIDTH / 2)
+
+/* PDS assume SCHED_NORMAL_PRIO_NUM is power of 2 */
+#define SCHED_NORMAL_PRIO_MOD(x)	((x) & (SCHED_NORMAL_PRIO_NUM - 1))
+
+/* default time slice 4ms -> shift 22, 2 time slice slots -> shift 23 */
+static __read_mostly int sched_timeslice_shift = 23;
+
+/*
+ * Common interfaces
+ */
+static inline int
+task_sched_prio_normal(const struct task_struct *p, const struct rq *rq)
+{
+	u64 sched_dl = max(p->deadline, rq->time_edge);
+
+#ifdef ALT_SCHED_DEBUG
+	if (WARN_ONCE(sched_dl - rq->time_edge > NORMAL_PRIO_NUM - 1,
+		      "pds: task_sched_prio_normal() delta %lld\n", sched_dl - rq->time_edge))
+		return SCHED_NORMAL_PRIO_NUM - 1;
+#endif
+
+	return sched_dl - rq->time_edge;
+}
+
+static inline int task_sched_prio(const struct task_struct *p)
+{
+	return (p->prio < MIN_NORMAL_PRIO) ? (p->prio >> 2) :
+		MIN_SCHED_NORMAL_PRIO + task_sched_prio_normal(p, task_rq(p));
+}
+
+#define TASK_SCHED_PRIO_IDX(p, rq, idx, prio)							\
+	if (p->prio < MIN_NORMAL_PRIO) {							\
+		prio = p->prio >> 2;								\
+		idx = prio;									\
+	} else {										\
+		u64 sched_dl = max(p->deadline, rq->time_edge);					\
+		prio = MIN_SCHED_NORMAL_PRIO + sched_dl - rq->time_edge;			\
+		idx = MIN_SCHED_NORMAL_PRIO + SCHED_NORMAL_PRIO_MOD(sched_dl);			\
+	}
+
+static inline int sched_prio2idx(int sched_prio, struct rq *rq)
+{
+	return (IDLE_TASK_SCHED_PRIO == sched_prio || sched_prio < MIN_SCHED_NORMAL_PRIO) ?
+		sched_prio :
+		MIN_SCHED_NORMAL_PRIO + SCHED_NORMAL_PRIO_MOD(sched_prio + rq->time_edge);
+}
+
+static inline int sched_idx2prio(int sched_idx, struct rq *rq)
+{
+	return (sched_idx < MIN_SCHED_NORMAL_PRIO) ?
+		sched_idx :
+		MIN_SCHED_NORMAL_PRIO + SCHED_NORMAL_PRIO_MOD(sched_idx - rq->time_edge);
+}
+
+static inline int sched_rq_prio_idx(struct rq *rq)
+{
+	return rq->prio_idx;
+}
+
+static inline int task_running_nice(struct task_struct *p)
+{
+	return (p->prio > DEFAULT_PRIO);
+}
+
+static inline void sched_update_rq_clock(struct rq *rq)
+{
+	struct list_head head;
+	u64 old = rq->time_edge;
+	u64 now = rq->clock >> sched_timeslice_shift;
+	u64 prio, delta;
+	DECLARE_BITMAP(normal, SCHED_QUEUE_BITS);
+
+	if (now == old)
+		return;
+
+	rq->time_edge = now;
+	delta = min_t(u64, SCHED_NORMAL_PRIO_NUM, now - old);
+	INIT_LIST_HEAD(&head);
+
+	prio = MIN_SCHED_NORMAL_PRIO;
+	for_each_set_bit_from(prio, rq->queue.bitmap, MIN_SCHED_NORMAL_PRIO + delta)
+		list_splice_tail_init(rq->queue.heads + MIN_SCHED_NORMAL_PRIO +
+				      SCHED_NORMAL_PRIO_MOD(prio + old), &head);
+
+	bitmap_shift_right(normal, rq->queue.bitmap, delta, SCHED_QUEUE_BITS);
+	if (!list_empty(&head)) {
+		u64 idx = MIN_SCHED_NORMAL_PRIO + SCHED_NORMAL_PRIO_MOD(now);
+
+		__list_splice(&head, rq->queue.heads + idx, rq->queue.heads[idx].next);
+		set_bit(MIN_SCHED_NORMAL_PRIO, normal);
+	}
+	bitmap_replace(rq->queue.bitmap, normal, rq->queue.bitmap,
+		       (const unsigned long *)&RT_MASK, SCHED_QUEUE_BITS);
+
+	if (rq->prio < MIN_SCHED_NORMAL_PRIO || IDLE_TASK_SCHED_PRIO == rq->prio)
+		return;
+
+	rq->prio = max_t(u64, MIN_SCHED_NORMAL_PRIO, rq->prio - delta);
+	rq->prio_idx = sched_prio2idx(rq->prio, rq);
+}
+
+static inline void sched_task_renew(struct task_struct *p, const struct rq *rq)
+{
+	if (p->prio >= MIN_NORMAL_PRIO)
+		p->deadline = rq->time_edge + SCHED_EDGE_DELTA +
+			      (p->static_prio - (MAX_PRIO - NICE_WIDTH)) / 2;
+}
+
+static inline void sched_task_sanity_check(struct task_struct *p, struct rq *rq)
+{
+	u64 max_dl = rq->time_edge + SCHED_EDGE_DELTA + NICE_WIDTH / 2 - 1;
+	if (unlikely(p->deadline > max_dl))
+		p->deadline = max_dl;
+}
+
+static inline void sched_task_fork(struct task_struct *p, struct rq *rq)
+{
+	sched_task_renew(p, rq);
+}
+
+static inline void do_sched_yield_type_1(struct task_struct *p, struct rq *rq)
+{
+	p->time_slice = sysctl_sched_base_slice;
+	sched_task_renew(p, rq);
+}
+
+static inline void sched_task_ttwu(struct task_struct *p) {}
+static inline void sched_task_deactivate(struct task_struct *p, struct rq *rq) {}
+
+#endif /* _KERNEL_SCHED_PDS_H */
diff --git a/kernel/sched/pelt.c b/kernel/sched/pelt.c
index 897790889..376f039d2 100644
--- a/kernel/sched/pelt.c
+++ b/kernel/sched/pelt.c
@@ -267,6 +267,7 @@ ___update_load_avg(struct sched_avg *sa, unsigned long load)
 	WRITE_ONCE(sa->util_avg, sa->util_sum / divider);
 }
 
+#ifndef CONFIG_SCHED_ALT
 /*
  * sched_entity:
  *
@@ -384,8 +385,9 @@ int update_dl_rq_load_avg(u64 now, struct rq *rq, int running)
 
 	return 0;
 }
+#endif
 
-#ifdef CONFIG_SCHED_HW_PRESSURE
+#if defined(CONFIG_SCHED_HW_PRESSURE) && !defined(CONFIG_SCHED_ALT)
 /*
  * hardware:
  *
@@ -469,6 +471,7 @@ int update_irq_load_avg(struct rq *rq, u64 running)
 }
 #endif /* CONFIG_HAVE_SCHED_AVG_IRQ */
 
+#ifndef CONFIG_SCHED_ALT
 /*
  * Load avg and utiliztion metrics need to be updated periodically and before
  * consumption. This function updates the metrics for all subsystems except for
@@ -488,3 +491,4 @@ bool update_other_load_avgs(struct rq *rq)
 		update_hw_load_avg(rq_clock_task(rq), rq, hw_pressure) |
 		update_irq_load_avg(rq, 0);
 }
+#endif /* !CONFIG_SCHED_ALT */
diff --git a/kernel/sched/pelt.h b/kernel/sched/pelt.h
index f921302dc..2fade7dab 100644
--- a/kernel/sched/pelt.h
+++ b/kernel/sched/pelt.h
@@ -5,14 +5,16 @@
 
 #include "sched-pelt.h"
 
+#ifndef CONFIG_SCHED_ALT
 int __update_load_avg_blocked_se(u64 now, struct sched_entity *se);
 int __update_load_avg_se(u64 now, struct cfs_rq *cfs_rq, struct sched_entity *se);
 int __update_load_avg_cfs_rq(u64 now, struct cfs_rq *cfs_rq);
 int update_rt_rq_load_avg(u64 now, struct rq *rq, int running);
 int update_dl_rq_load_avg(u64 now, struct rq *rq, int running);
 bool update_other_load_avgs(struct rq *rq);
+#endif
 
-#ifdef CONFIG_SCHED_HW_PRESSURE
+#if defined(CONFIG_SCHED_HW_PRESSURE) && !defined(CONFIG_SCHED_ALT)
 int update_hw_load_avg(u64 now, struct rq *rq, u64 capacity);
 
 static inline u64 hw_load_avg(struct rq *rq)
@@ -49,6 +51,7 @@ static inline u32 get_pelt_divider(struct sched_avg *avg)
 	return PELT_MIN_DIVIDER + avg->period_contrib;
 }
 
+#ifndef CONFIG_SCHED_ALT
 static inline void cfs_se_util_change(struct sched_avg *avg)
 {
 	unsigned int enqueued;
@@ -185,5 +188,6 @@ static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq)
 	return rq_clock_pelt(rq_of(cfs_rq));
 }
 #endif /* !CONFIG_CFS_BANDWIDTH */
+#endif /* CONFIG_SCHED_ALT */
 
 #endif /* _KERNEL_SCHED_PELT_H */
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index a4eb46e12..4bd19581d 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -5,6 +5,10 @@
 #ifndef _KERNEL_SCHED_SCHED_H
 #define _KERNEL_SCHED_SCHED_H
 
+#ifdef CONFIG_SCHED_ALT
+#include "alt_sched.h"
+#else
+
 #include <linux/prandom.h>
 #include <linux/sched/affinity.h>
 #include <linux/sched/autogroup.h>
@@ -4248,4 +4252,9 @@ DEFINE_CLASS_IS_UNCONDITIONAL(sched_change)
 
 #include "ext.h"
 
+static inline int task_running_nice(struct task_struct *p)
+{
+	return (task_nice(p) > 0);
+}
+#endif /* !CONFIG_SCHED_ALT */
 #endif /* _KERNEL_SCHED_SCHED_H */
diff --git a/kernel/sched/stats.c b/kernel/sched/stats.c
index d1c9429a4..cc3764073 100644
--- a/kernel/sched/stats.c
+++ b/kernel/sched/stats.c
@@ -115,8 +115,10 @@ static int show_schedstat(struct seq_file *seq, void *v)
 		seq_printf(seq, "timestamp %lu\n", jiffies);
 	} else {
 		struct rq *rq;
+#ifndef CONFIG_SCHED_ALT
 		struct sched_domain *sd;
 		int dcount = 0;
+#endif
 		cpu = (unsigned long)(v - 2);
 		rq = cpu_rq(cpu);
 
@@ -131,6 +133,7 @@ static int show_schedstat(struct seq_file *seq, void *v)
 
 		seq_printf(seq, "\n");
 
+#ifndef CONFIG_SCHED_ALT
 		/* domain-specific stats */
 		rcu_read_lock();
 		for_each_domain(cpu, sd) {
@@ -161,6 +164,7 @@ static int show_schedstat(struct seq_file *seq, void *v)
 			    sd->ttwu_move_balance);
 		}
 		rcu_read_unlock();
+#endif
 	}
 	return 0;
 }
diff --git a/kernel/sched/stats.h b/kernel/sched/stats.h
index a612cf253..1018d72ff 100644
--- a/kernel/sched/stats.h
+++ b/kernel/sched/stats.h
@@ -89,6 +89,7 @@ static inline void rq_sched_info_depart  (struct rq *rq, unsigned long long delt
 
 #endif /* CONFIG_SCHEDSTATS */
 
+#ifndef CONFIG_SCHED_ALT
 #ifdef CONFIG_FAIR_GROUP_SCHED
 struct sched_entity_stats {
 	struct sched_entity     se;
@@ -105,6 +106,7 @@ __schedstats_from_se(struct sched_entity *se)
 #endif
 	return &task_of(se)->stats;
 }
+#endif /* CONFIG_SCHED_ALT */
 
 #ifdef CONFIG_PSI
 void psi_task_change(struct task_struct *task, int clear, int set);
diff --git a/kernel/sched/syscalls.c b/kernel/sched/syscalls.c
index b215b0ead..33f41fe32 100644
--- a/kernel/sched/syscalls.c
+++ b/kernel/sched/syscalls.c
@@ -16,6 +16,14 @@
 #include "sched.h"
 #include "autogroup.h"
 
+#ifdef CONFIG_SCHED_ALT
+#include "alt_core.h"
+
+static inline int __normal_prio(int policy, int rt_prio, int static_prio)
+{
+	return rt_policy(policy) ? (MAX_RT_PRIO - 1 - rt_prio) : static_prio;
+}
+#else /* !CONFIG_SCHED_ALT */
 static inline int __normal_prio(int policy, int rt_prio, int nice)
 {
 	int prio;
@@ -29,6 +37,7 @@ static inline int __normal_prio(int policy, int rt_prio, int nice)
 
 	return prio;
 }
+#endif /* !CONFIG_SCHED_ALT */
 
 /*
  * Calculate the expected normal priority: i.e. priority
@@ -39,7 +48,11 @@ static inline int __normal_prio(int policy, int rt_prio, int nice)
  */
 static inline int normal_prio(struct task_struct *p)
 {
+#ifdef CONFIG_SCHED_ALT
+	return __normal_prio(p->policy, p->rt_priority, p->static_prio);
+#else /* !CONFIG_SCHED_ALT */
 	return __normal_prio(p->policy, p->rt_priority, PRIO_TO_NICE(p->static_prio));
+#endif /* !CONFIG_SCHED_ALT */
 }
 
 /*
@@ -64,7 +77,9 @@ static int effective_prio(struct task_struct *p)
 
 void set_user_nice(struct task_struct *p, long nice)
 {
+#ifndef CONFIG_SCHED_ALT
 	int old_prio;
+#endif
 
 	if (task_nice(p) == nice || nice < MIN_NICE || nice > MAX_NICE)
 		return;
@@ -72,7 +87,11 @@ void set_user_nice(struct task_struct *p, long nice)
 	 * We have to be careful, if called from sys_setpriority(),
 	 * the task might be in the middle of scheduling on another CPU.
 	 */
+#ifdef CONFIG_SCHED_ALT
+	guard(task_access_lock)(p);
+#else
 	guard(task_rq_lock)(p);
+#endif
 
 	/*
 	 * The RT priorities are set via sched_setscheduler(), but we still
@@ -87,8 +106,10 @@ void set_user_nice(struct task_struct *p, long nice)
 
 	scoped_guard (sched_change, p, DEQUEUE_SAVE) {
 		p->static_prio = NICE_TO_PRIO(nice);
+#ifndef CONFIG_SCHED_ALT
 		set_load_weight(p, true);
 		old_prio = p->prio;
+#endif
 		p->prio = effective_prio(p);
 	}
 }
@@ -169,7 +190,19 @@ SYSCALL_DEFINE1(nice, int, increment)
  */
 int task_prio(const struct task_struct *p)
 {
+#ifdef CONFIG_SCHED_ALT
+/*
+ * sched policy         return value   kernel prio    user prio/nice
+ *
+ * (BMQ)normal, batch, idle[0 ... 53]  [100 ... 139]          0/[-20 ... 19]/[-7 ... 7]
+ * (PDS)normal, batch, idle[0 ... 39]            100          0/[-20 ... 19]
+ * fifo, rr             [-1 ... -100]     [99 ... 0]  [0 ... 99]
+ */
+	return (p->prio < MAX_RT_PRIO) ? p->prio - MAX_RT_PRIO :
+		task_sched_prio_normal(p, task_rq(p));
+#else
 	return p->prio - MAX_RT_PRIO;
+#endif /* !CONFIG_SCHED_ALT */
 }
 
 /**
@@ -248,11 +281,16 @@ static void __setscheduler_params(struct task_struct *p,
 
 	p->policy = policy;
 
+#ifndef CONFIG_SCHED_ALT
 	if (dl_policy(policy))
 		__setparam_dl(p, attr);
 	else if (fair_policy(policy))
 		__setparam_fair(p, attr);
+#else	/* !CONFIG_SCHED_ALT */
+	p->static_prio = NICE_TO_PRIO(attr->sched_nice);
+#endif /* CONFIG_SCHED_ALT */
 
+#ifndef CONFIG_SCHED_ALT
 	/* rt-policy tasks do not have a timerslack */
 	if (rt_or_dl_task_policy(p)) {
 		p->timer_slack_ns = 0;
@@ -260,6 +298,7 @@ static void __setscheduler_params(struct task_struct *p,
 		/* when switching back to non-rt policy, restore timerslack */
 		p->timer_slack_ns = p->default_timer_slack_ns;
 	}
+#endif /* !CONFIG_SCHED_ALT */
 
 	/*
 	 * __sched_setscheduler() ensures attr->sched_priority == 0 when
@@ -268,7 +307,9 @@ static void __setscheduler_params(struct task_struct *p,
 	 */
 	p->rt_priority = attr->sched_priority;
 	p->normal_prio = normal_prio(p);
+#ifndef CONFIG_SCHED_ALT
 	set_load_weight(p, true);
+#endif /* !CONFIG_SCHED_ALT */
 }
 
 /*
@@ -284,7 +325,7 @@ static bool check_same_owner(struct task_struct *p)
 		uid_eq(cred->euid, pcred->uid));
 }
 
-#ifdef CONFIG_RT_MUTEXES
+#if defined(CONFIG_RT_MUTEXES) && !defined(CONFIG_SCHED_ALT)
 static inline void __setscheduler_dl_pi(int newprio, int policy,
 			      struct task_struct *p,
 			      struct sched_change_ctx *scope)
@@ -313,6 +354,7 @@ static inline void __setscheduler_dl_pi(int newprio, int policy,
 }
 #endif /* !CONFIG_RT_MUTEXES */
 
+#ifndef CONFIG_SCHED_ALT
 #ifdef CONFIG_UCLAMP_TASK
 
 static int uclamp_validate(struct task_struct *p,
@@ -426,6 +468,7 @@ static inline int uclamp_validate(struct task_struct *p,
 static void __setscheduler_uclamp(struct task_struct *p,
 				  const struct sched_attr *attr) { }
 #endif /* !CONFIG_UCLAMP_TASK */
+#endif /* !CONFIG_SCHED_ALT */
 
 /*
  * Allow unprivileged RT tasks to decrease priority.
@@ -436,11 +479,13 @@ static int user_check_sched_setscheduler(struct task_struct *p,
 					 const struct sched_attr *attr,
 					 int policy, int reset_on_fork)
 {
+#ifndef CONFIG_SCHED_ALT
 	if (fair_policy(policy)) {
 		if (attr->sched_nice < task_nice(p) &&
 		    !is_nice_reduction(p, attr->sched_nice))
 			goto req_priv;
 	}
+#endif /* !CONFIG_SCHED_ALT */
 
 	if (rt_policy(policy)) {
 		unsigned long rlim_rtprio = task_rlimit(p, RLIMIT_RTPRIO);
@@ -455,6 +500,7 @@ static int user_check_sched_setscheduler(struct task_struct *p,
 			goto req_priv;
 	}
 
+#ifndef CONFIG_SCHED_ALT
 	/*
 	 * Can't set/change SCHED_DEADLINE policy at all for now
 	 * (safest behavior); in the future we would like to allow
@@ -472,6 +518,7 @@ static int user_check_sched_setscheduler(struct task_struct *p,
 		if (!is_nice_reduction(p, task_nice(p)))
 			goto req_priv;
 	}
+#endif /* !CONFIG_SCHED_ALT */
 
 	/* Can't change other user's priorities: */
 	if (!check_same_owner(p))
@@ -491,12 +538,22 @@ static int user_check_sched_setscheduler(struct task_struct *p,
 }
 
 int __sched_setscheduler(struct task_struct *p,
-			 const struct sched_attr *attr,
-			 bool user, bool pi)
+			const struct sched_attr *attr,
+			bool user, bool pi)
 {
+#ifdef CONFIG_SCHED_ALT
+	const struct sched_attr dl_squash_attr = {
+		.size		= sizeof(struct sched_attr),
+		.sched_policy	= SCHED_FIFO,
+		.sched_nice	= 0,
+		.sched_priority = 99,
+	};
+#endif /* CONFIG_SCHED_ALT */
 	int oldpolicy = -1, policy = attr->sched_policy;
 	int retval, oldprio, newprio;
+#ifndef CONFIG_SCHED_ALT
 	const struct sched_class *prev_class, *next_class;
+#endif /* !CONFIG_SCHED_ALT */
 	struct balance_callback *head;
 	struct rq_flags rf;
 	int reset_on_fork;
@@ -506,6 +563,15 @@ int __sched_setscheduler(struct task_struct *p,
 
 	/* The pi code expects interrupts enabled */
 	BUG_ON(pi && in_interrupt());
+#ifdef CONFIG_SCHED_ALT
+	/*
+	 * Alt schedule FW supports SCHED_DEADLINE by squash it as prio 0 SCHED_FIFO
+	 */
+	if (unlikely(SCHED_DEADLINE == policy)) {
+		attr = &dl_squash_attr;
+		policy = attr->sched_policy;
+	}
+#endif /* CONFIG_SCHED_ALT */
 recheck:
 	/* Double check policy once rq lock held: */
 	if (policy < 0) {
@@ -528,8 +594,12 @@ int __sched_setscheduler(struct task_struct *p,
 	 */
 	if (attr->sched_priority > MAX_RT_PRIO-1)
 		return -EINVAL;
+#ifdef CONFIG_SCHED_ALT
+	if ((rt_policy(policy) != (attr->sched_priority != 0)))
+#else
 	if ((dl_policy(policy) && !__checkparam_dl(attr)) ||
 	    (rt_policy(policy) != (attr->sched_priority != 0)))
+#endif /* !CONFIG_SCHED_ALT */
 		return -EINVAL;
 
 	if (user) {
@@ -545,6 +615,7 @@ int __sched_setscheduler(struct task_struct *p,
 			return retval;
 	}
 
+#ifndef CONFIG_SCHED_ALT
 	/* Update task specific "requested" clamps */
 	if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP) {
 		retval = uclamp_validate(p, attr);
@@ -560,6 +631,7 @@ int __sched_setscheduler(struct task_struct *p,
 		cpuset_locked = true;
 		cpuset_lock();
 	}
+#endif /* !CONFIG_SCHED_ALT */
 
 	/*
 	 * Make sure no PI-waiters arrive (or leave) while we are
@@ -568,8 +640,12 @@ int __sched_setscheduler(struct task_struct *p,
 	 * To be able to change p->policy safely, the appropriate
 	 * runqueue lock must be held.
 	 */
+#ifdef CONFIG_SCHED_ALT
+	rq = task_access_lock(p, &rf);
+#else
 	rq = task_rq_lock(p, &rf);
 	update_rq_clock(rq);
+#endif /* !CONFIG_SCHED_ALT */
 
 	/*
 	 * Changing the policy of the stop threads its a very bad idea:
@@ -579,23 +655,31 @@ int __sched_setscheduler(struct task_struct *p,
 		goto unlock;
 	}
 
+#ifndef CONFIG_SCHED_ALT
 	retval = scx_check_setscheduler(p, policy);
 	if (retval)
 		goto unlock;
+#endif /* !CONFIG_SCHED_ALT */
 
 	/*
 	 * If not changing anything there's no need to proceed further,
 	 * but store a possible modification of reset_on_fork.
 	 */
 	if (unlikely(policy == p->policy)) {
+#ifdef CONFIG_SCHED_ALT
+		if (!rt_policy(policy) && NICE_TO_PRIO(attr->sched_nice) != p->static_prio)
+#else
 		if (fair_policy(policy) &&
 		    (attr->sched_nice != task_nice(p) ||
 		     (attr->sched_runtime != p->se.slice)))
+#endif /* !CONFIG_SCHED_ALT */
 			goto change;
 		if (rt_policy(policy) && attr->sched_priority != p->rt_priority)
 			goto change;
+#ifndef CONFIG_SCHED_ALT
 		if (dl_policy(policy) && dl_param_changed(p, attr))
 			goto change;
+#endif /* !CONFIG_SCHED_ALT */
 		if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP)
 			goto change;
 
@@ -605,6 +689,7 @@ int __sched_setscheduler(struct task_struct *p,
 	}
 change:
 
+#ifndef CONFIG_SCHED_ALT
 	if (user) {
 #ifdef CONFIG_RT_GROUP_SCHED
 		/*
@@ -635,16 +720,22 @@ int __sched_setscheduler(struct task_struct *p,
 			}
 		}
 	}
+#endif /* !CONFIG_SCHED_ALT */
 
 	/* Re-check policy now with rq lock held: */
 	if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) {
 		policy = oldpolicy = -1;
+#ifdef CONFIG_SCHED_ALT
+		task_access_unlock(p, &rf);
+#else
 		task_rq_unlock(rq, p, &rf);
+#endif /* !CONFIG_SCHED_ALT */
 		if (cpuset_locked)
 			cpuset_unlock();
 		goto recheck;
 	}
 
+#ifndef CONFIG_SCHED_ALT
 	/*
 	 * If setscheduling to SCHED_DEADLINE (or changing the parameters
 	 * of a SCHED_DEADLINE task) we need to check if enough bandwidth
@@ -654,11 +745,16 @@ int __sched_setscheduler(struct task_struct *p,
 		retval = -EBUSY;
 		goto unlock;
 	}
+#endif /* !CONFIG_SCHED_ALT */
 
 	p->sched_reset_on_fork = reset_on_fork;
 	oldprio = p->prio;
 
+#ifdef CONFIG_SCHED_ALT
+	newprio = __normal_prio(policy, attr->sched_priority, NICE_TO_PRIO(attr->sched_nice));
+#else
 	newprio = __normal_prio(policy, attr->sched_priority, attr->sched_nice);
+#endif /* !CONFIG_SCHED_ALT */
 	if (pi) {
 		/*
 		 * Take priority boosted tasks into account. If the new
@@ -672,21 +768,27 @@ int __sched_setscheduler(struct task_struct *p,
 			queue_flags &= ~DEQUEUE_MOVE;
 	}
 
+#ifndef CONFIG_SCHED_ALT
 	prev_class = p->sched_class;
 	next_class = __setscheduler_class(policy, newprio);
 
 	if (prev_class != next_class)
 		queue_flags |= DEQUEUE_CLASS;
+#endif /* !CONFIG_SCHED_ALT */
 
 	scoped_guard (sched_change, p, queue_flags) {
 
 		if (!(attr->sched_flags & SCHED_FLAG_KEEP_PARAMS)) {
 			__setscheduler_params(p, attr);
+#ifndef CONFIG_SCHED_ALT
 			p->sched_class = next_class;
+#endif /* !CONFIG_SCHED_ALT */
 			p->prio = newprio;
 			__setscheduler_dl_pi(newprio, policy, p, scope);
 		}
+#ifndef CONFIG_SCHED_ALT
 		__setscheduler_uclamp(p, attr);
+#endif /* !CONFIG_SCHED_ALT */
 
 		if (scope->queued) {
 			/*
@@ -701,7 +803,11 @@ int __sched_setscheduler(struct task_struct *p,
 	/* Avoid rq from going away on us: */
 	preempt_disable();
 	head = splice_balance_callbacks(rq);
+#ifdef CONFIG_SCHED_ALT
+	task_access_unlock(p, &rf);
+#else
 	task_rq_unlock(rq, p, &rf);
+#endif /* !CONFIG_SCHED_ALT */
 
 	if (pi) {
 		if (cpuset_locked)
@@ -716,7 +822,11 @@ int __sched_setscheduler(struct task_struct *p,
 	return 0;
 
 unlock:
+#ifdef CONFIG_SCHED_ALT
+	task_access_unlock(p, &rf);
+#else
 	task_rq_unlock(rq, p, &rf);
+#endif /* !CONFIG_SCHED_ALT */
 	if (cpuset_locked)
 		cpuset_unlock();
 	return retval;
@@ -731,8 +841,10 @@ static int _sched_setscheduler(struct task_struct *p, int policy,
 		.sched_nice	= PRIO_TO_NICE(p->static_prio),
 	};
 
+#ifndef CONFIG_SCHED_ALT
 	if (p->se.custom_slice)
 		attr.sched_runtime = p->se.slice;
+#endif /* !CONFIG_SCHED_ALT */
 
 	/* Fixup the legacy SCHED_RESET_ON_FORK hack. */
 	if ((policy != SETPARAM_POLICY) && (policy & SCHED_RESET_ON_FORK)) {
@@ -913,13 +1025,18 @@ static int sched_copy_attr(struct sched_attr __user *uattr, struct sched_attr *a
 
 static void get_params(struct task_struct *p, struct sched_attr *attr, unsigned int flags)
 {
+#ifndef CONFIG_SCHED_ALT
 	if (task_has_dl_policy(p)) {
 		__getparam_dl(p, attr, flags);
-	} else if (task_has_rt_policy(p)) {
+	} else
+#endif
+	if (task_has_rt_policy(p)) {
 		attr->sched_priority = p->rt_priority;
 	} else {
 		attr->sched_nice = task_nice(p);
+#ifndef CONFIG_SCHED_ALT
 		attr->sched_runtime = p->se.slice;
+#endif
 	}
 }
 
@@ -1106,6 +1223,7 @@ SYSCALL_DEFINE4(sched_getattr, pid_t, pid, struct sched_attr __user *, uattr,
 
 int dl_task_check_affinity(struct task_struct *p, const struct cpumask *mask)
 {
+#ifndef CONFIG_SCHED_ALT
 	/*
 	 * If the task isn't a deadline task or admission control is
 	 * disabled then we don't care about affinity changes.
@@ -1129,6 +1247,7 @@ int dl_task_check_affinity(struct task_struct *p, const struct cpumask *mask)
 	guard(rcu)();
 	if (!cpumask_subset(task_rq(p)->rd->span, mask))
 		return -EBUSY;
+#endif
 
 	return 0;
 }
@@ -1152,9 +1271,11 @@ int __sched_setaffinity(struct task_struct *p, struct affinity_context *ctx)
 	ctx->new_mask = new_mask;
 	ctx->flags |= SCA_CHECK;
 
+#ifndef CONFIG_SCHED_ALT
 	retval = dl_task_check_affinity(p, new_mask);
 	if (retval)
 		goto out_free_new_mask;
+#endif
 
 	retval = __set_cpus_allowed_ptr(p, ctx);
 	if (retval)
@@ -1334,13 +1455,20 @@ SYSCALL_DEFINE3(sched_getaffinity, pid_t, pid, unsigned int, len,
 
 static void do_sched_yield(void)
 {
-	struct rq_flags rf;
 	struct rq *rq;
+	struct rq_flags rf;
 
 	rq = this_rq_lock_irq(&rf);
 
+	schedstat_inc(rq->yld_count);
+#ifdef CONFIG_SCHED_ALT
+	yield_task(rq);
+#else /* !CONFIG_SCHED_ALT */
+	rq = this_rq_lock_irq(&rf);
+
 	schedstat_inc(rq->yld_count);
 	rq->donor->sched_class->yield_task(rq);
+#endif /* !CONFIG_SCHED_ALT */
 
 	preempt_disable();
 	rq_unlock_irq(rq, &rf);
@@ -1409,6 +1537,9 @@ EXPORT_SYMBOL(yield);
  */
 int __sched yield_to(struct task_struct *p, bool preempt)
 {
+#ifdef CONFIG_SCHED_ALT
+	return 0;
+#else /* !CONFIG_SCHED_ALT */
 	struct task_struct *curr;
 	struct rq *rq, *p_rq;
 	int yielded = 0;
@@ -1455,6 +1586,7 @@ int __sched yield_to(struct task_struct *p, bool preempt)
 		schedule();
 
 	return yielded;
+#endif /* !CONFIG_SCHED_ALT */
 }
 EXPORT_SYMBOL_GPL(yield_to);
 
@@ -1475,7 +1607,9 @@ SYSCALL_DEFINE1(sched_get_priority_max, int, policy)
 	case SCHED_RR:
 		ret = MAX_RT_PRIO-1;
 		break;
+#ifndef CONFIG_SCHED_ALT
 	case SCHED_DEADLINE:
+#endif
 	case SCHED_NORMAL:
 	case SCHED_BATCH:
 	case SCHED_IDLE:
@@ -1503,7 +1637,9 @@ SYSCALL_DEFINE1(sched_get_priority_min, int, policy)
 	case SCHED_RR:
 		ret = 1;
 		break;
+#ifndef CONFIG_SCHED_ALT
 	case SCHED_DEADLINE:
+#endif
 	case SCHED_NORMAL:
 	case SCHED_BATCH:
 	case SCHED_IDLE:
@@ -1515,7 +1651,9 @@ SYSCALL_DEFINE1(sched_get_priority_min, int, policy)
 
 static int sched_rr_get_interval(pid_t pid, struct timespec64 *t)
 {
+#ifndef CONFIG_SCHED_ALT
 	unsigned int time_slice = 0;
+#endif
 	int retval;
 
 	if (pid < 0)
@@ -1530,6 +1668,7 @@ static int sched_rr_get_interval(pid_t pid, struct timespec64 *t)
 		if (retval)
 			return retval;
 
+#ifndef CONFIG_SCHED_ALT
 		scoped_guard (task_rq_lock, p) {
 			struct rq *rq = scope.rq;
 			if (p->sched_class->get_rr_interval)
@@ -1538,6 +1677,13 @@ static int sched_rr_get_interval(pid_t pid, struct timespec64 *t)
 	}
 
 	jiffies_to_timespec64(time_slice, t);
+#else
+	}
+
+	alt_sched_debug();
+
+	*t = ns_to_timespec64(sysctl_sched_base_slice);
+#endif /* !CONFIG_SCHED_ALT */
 	return 0;
 }
 
diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c
index e4e84f92c..0eda3b3cc 100644
--- a/kernel/sched/topology.c
+++ b/kernel/sched/topology.c
@@ -3,6 +3,7 @@
  * Scheduler topology setup/handling methods
  */
 
+#ifndef CONFIG_SCHED_ALT
 #include <linux/sched/isolation.h>
 #include <linux/sched/clock.h>
 #include <linux/bsearch.h>
@@ -1514,8 +1515,10 @@ static void asym_cpu_capacity_scan(void)
  */
 
 static int default_relax_domain_level = -1;
+#endif /* CONFIG_SCHED_ALT */
 int sched_domain_level_max;
 
+#ifndef CONFIG_SCHED_ALT
 static int __init setup_relax_domain_level(char *str)
 {
 	if (kstrtoint(str, 0, &default_relax_domain_level))
@@ -1765,6 +1768,7 @@ sd_init(struct sched_domain_topology_level *tl,
 
 	return sd;
 }
+#endif /* CONFIG_SCHED_ALT */
 
 #ifdef CONFIG_SCHED_SMT
 int cpu_smt_flags(void)
@@ -1842,6 +1846,7 @@ void __init set_sched_topology(struct sched_domain_topology_level *tl)
 	sched_domain_topology_saved = NULL;
 }
 
+#ifndef CONFIG_SCHED_ALT
 #ifdef CONFIG_NUMA
 static int cpu_numa_flags(void)
 {
@@ -3027,3 +3032,40 @@ void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
 	partition_sched_domains_locked(ndoms_new, doms_new, dattr_new);
 	sched_domains_mutex_unlock();
 }
+#else /* CONFIG_SCHED_ALT */
+DEFINE_STATIC_KEY_FALSE(sched_asym_cpucapacity);
+
+void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
+			     struct sched_domain_attr *dattr_new)
+{}
+
+#ifdef CONFIG_NUMA
+int sched_numa_find_closest(const struct cpumask *cpus, int cpu)
+{
+	struct cpumask *mask;
+
+	if (cpumask_test_cpu(cpu, cpus))
+		return cpu;
+
+	mask = per_cpu(cpu_affinity_masks, cpu);
+	while ((cpu = cpumask_any_and(cpus, mask)) >= nr_cpu_ids)
+		mask++;
+
+	return cpu;
+}
+
+int sched_numa_find_nth_cpu(const struct cpumask *cpus, int cpu, int node)
+{
+	return cpumask_nth(cpu, cpus);
+}
+
+const struct cpumask *sched_numa_hop_mask(unsigned int node, unsigned int hops)
+{
+	return ERR_PTR(-EOPNOTSUPP);
+}
+EXPORT_SYMBOL_GPL(sched_numa_hop_mask);
+#endif /* CONFIG_NUMA */
+
+void sched_update_asym_prefer_cpu(int cpu, int old_prio, int new_prio)
+{}
+#endif
diff --git a/kernel/sysctl.c b/kernel/sysctl.c
index c9efb17cc..efe0de047 100644
--- a/kernel/sysctl.c
+++ b/kernel/sysctl.c
@@ -35,6 +35,10 @@ EXPORT_SYMBOL_GPL(sysctl_long_vals);
 static const int ngroups_max = NGROUPS_MAX;
 static const int cap_last_cap = CAP_LAST_CAP;
 
+#ifdef CONFIG_SCHED_ALT
+extern int sched_yield_type;
+#endif
+
 #ifdef CONFIG_PROC_SYSCTL
 
 /**
@@ -1406,6 +1410,17 @@ static const struct ctl_table sysctl_subsys_table[] = {
 		.proc_handler	= proc_dointvec,
 	},
 #endif
+#ifdef CONFIG_SCHED_ALT
+	{
+		.procname	= "yield_type",
+		.data		= &sched_yield_type,
+		.maxlen		= sizeof (int),
+		.mode		= 0644,
+		.proc_handler	= &proc_dointvec_minmax,
+		.extra1		= SYSCTL_ZERO,
+		.extra2		= SYSCTL_TWO,
+	},
+#endif
 #ifdef CONFIG_SYSCTL_ARCH_UNALIGN_NO_WARN
 	{
 		.procname	= "ignore-unaligned-usertrap",
diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c
index 0de2bb7cb..36ba69ac1 100644
--- a/kernel/time/posix-cpu-timers.c
+++ b/kernel/time/posix-cpu-timers.c
@@ -223,7 +223,7 @@ static void task_sample_cputime(struct task_struct *p, u64 *samples)
 	u64 stime, utime;
 
 	task_cputime(p, &utime, &stime);
-	store_samples(samples, stime, utime, p->se.sum_exec_runtime);
+	store_samples(samples, stime, utime, tsk_seruntime(p));
 }
 
 static void proc_sample_cputime_atomic(struct task_cputime_atomic *at,
@@ -835,6 +835,7 @@ static void collect_posix_cputimers(struct posix_cputimers *pct, u64 *samples,
 	}
 }
 
+#ifndef CONFIG_SCHED_ALT
 static inline void check_dl_overrun(struct task_struct *tsk)
 {
 	if (tsk->dl.dl_overrun) {
@@ -842,6 +843,7 @@ static inline void check_dl_overrun(struct task_struct *tsk)
 		send_signal_locked(SIGXCPU, SEND_SIG_PRIV, tsk, PIDTYPE_TGID);
 	}
 }
+#endif
 
 static bool check_rlimit(u64 time, u64 limit, int signo, bool rt, bool hard)
 {
@@ -869,8 +871,10 @@ static void check_thread_timers(struct task_struct *tsk,
 	u64 samples[CPUCLOCK_MAX];
 	unsigned long soft;
 
+#ifndef CONFIG_SCHED_ALT
 	if (dl_task(tsk))
 		check_dl_overrun(tsk);
+#endif
 
 	if (expiry_cache_is_inactive(pct))
 		return;
@@ -884,7 +888,7 @@ static void check_thread_timers(struct task_struct *tsk,
 	soft = task_rlimit(tsk, RLIMIT_RTTIME);
 	if (soft != RLIM_INFINITY) {
 		/* Task RT timeout is accounted in jiffies. RTTIME is usec */
-		unsigned long rttime = tsk->rt.timeout * (USEC_PER_SEC / HZ);
+		unsigned long rttime = tsk_rttimeout(tsk) * (USEC_PER_SEC / HZ);
 		unsigned long hard = task_rlimit_max(tsk, RLIMIT_RTTIME);
 
 		/* At the hard limit, send SIGKILL. No further action. */
@@ -1120,8 +1124,10 @@ static inline bool fastpath_timer_check(struct task_struct *tsk)
 			return true;
 	}
 
+#ifndef CONFIG_SCHED_ALT
 	if (dl_task(tsk) && tsk->dl.dl_overrun)
 		return true;
+#endif
 
 	return false;
 }
diff --git a/kernel/trace/trace_osnoise.c b/kernel/trace/trace_osnoise.c
index 75678053b..268e42381 100644
--- a/kernel/trace/trace_osnoise.c
+++ b/kernel/trace/trace_osnoise.c
@@ -1662,6 +1662,9 @@ static void osnoise_sleep(bool skip_period)
  */
 static inline int osnoise_migration_pending(void)
 {
+#ifdef CONFIG_SCHED_ALT
+	return 0;
+#else
 	if (!current->migration_pending)
 		return 0;
 
@@ -1683,6 +1686,7 @@ static inline int osnoise_migration_pending(void)
 	mutex_unlock(&interface_lock);
 
 	return 1;
+#endif
 }
 
 /*
diff --git a/kernel/trace/trace_selftest.c b/kernel/trace/trace_selftest.c
index 929c84075..0cba8b036 100644
--- a/kernel/trace/trace_selftest.c
+++ b/kernel/trace/trace_selftest.c
@@ -1423,10 +1423,15 @@ static int trace_wakeup_test_thread(void *data)
 {
 	/* Make this a -deadline thread */
 	static const struct sched_attr attr = {
+#ifdef CONFIG_SCHED_ALT
+		/* No deadline on BMQ/PDS, use RR */
+		.sched_policy = SCHED_RR,
+#else
 		.sched_policy = SCHED_DEADLINE,
 		.sched_runtime = 100000ULL,
 		.sched_deadline = 10000000ULL,
 		.sched_period = 10000000ULL
+#endif
 	};
 	struct wakeup_test_data *x = data;
 
diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index 33b721a9a..2aeb5b697 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -1281,6 +1281,7 @@ static bool kick_pool(struct worker_pool *pool)
 
 	p = worker->task;
 
+#ifndef CONFIG_SCHED_ALT
 #ifdef CONFIG_SMP
 	/*
 	 * Idle @worker is about to execute @work and waking up provides an
@@ -1310,6 +1311,8 @@ static bool kick_pool(struct worker_pool *pool)
 		}
 	}
 #endif
+#endif /* !CONFIG_SCHED_ALT */
+
 	wake_up_process(p);
 	return true;
 }
@@ -1438,7 +1441,11 @@ void wq_worker_running(struct task_struct *task)
 	 * CPU intensive auto-detection cares about how long a work item hogged
 	 * CPU without sleeping. Reset the starting timestamp on wakeup.
 	 */
+#ifdef CONFIG_SCHED_ALT
+	worker->current_at = worker->task->sched_time;
+#else
 	worker->current_at = worker->task->se.sum_exec_runtime;
+#endif
 
 	WRITE_ONCE(worker->sleeping, 0);
 }
@@ -1523,7 +1530,11 @@ void wq_worker_tick(struct task_struct *task)
 	 * We probably want to make this prettier in the future.
 	 */
 	if ((worker->flags & WORKER_NOT_RUNNING) || READ_ONCE(worker->sleeping) ||
+#ifdef CONFIG_SCHED_ALT
+	    worker->task->sched_time - worker->current_at <
+#else
 	    worker->task->se.sum_exec_runtime - worker->current_at <
+#endif
 	    wq_cpu_intensive_thresh_us * NSEC_PER_USEC)
 		return;
 
@@ -3241,7 +3252,11 @@ __acquires(&pool->lock)
 	worker->current_func = work->func;
 	worker->current_pwq = pwq;
 	if (worker->task)
+#ifdef CONFIG_SCHED_ALT
+		worker->current_at = worker->task->sched_time;
+#else
 		worker->current_at = worker->task->se.sum_exec_runtime;
+#endif
 	worker->current_start = jiffies;
 	work_data = *work_data_bits(work);
 	worker->current_color = get_work_color(work_data);
-- 
2.54.0