mirror of
https://github.com/torvalds/linux
synced 2024-07-22 19:21:07 +00:00
a9ab775bca
Rebinding workers of a per-cpu pool after a CPU comes online involves a lot of back-and-forth mostly because only the task itself could adjust CPU affinity if PF_THREAD_BOUND was set. As CPU_ONLINE itself couldn't adjust affinity, it had to somehow coerce the workers themselves to perform set_cpus_allowed_ptr(). Due to the various states a worker can be in, this led to three different paths a worker may be rebound. worker->rebind_work is queued to busy workers. Idle ones are signaled by unlinking worker->entry and call idle_worker_rebind(). The manager isn't covered by either and implements its own mechanism. PF_THREAD_BOUND has been relaced with PF_NO_SETAFFINITY and CPU_ONLINE itself now can manipulate CPU affinity of workers. This patch replaces the existing rebind mechanism with direct one where CPU_ONLINE iterates over all workers using for_each_pool_worker(), restores CPU affinity, and clears WORKER_UNBOUND. There are a couple subtleties. All bound idle workers should have their runqueues set to that of the bound CPU; however, if the target task isn't running, set_cpus_allowed_ptr() just updates the cpus_allowed mask deferring the actual migration to when the task wakes up. This is worked around by waking up idle workers after restoring CPU affinity before any workers can become bound. Another subtlety is stems from matching @pool->nr_running with the number of running unbound workers. While DISASSOCIATED, all workers are unbound and nr_running is zero. As workers become bound again, nr_running needs to be adjusted accordingly; however, there is no good way to tell whether a given worker is running without poking into scheduler internals. Instead of clearing UNBOUND directly, rebind_workers() replaces UNBOUND with another new NOT_RUNNING flag - REBOUND, which will later be cleared by the workers themselves while preparing for the next round of work item execution. The only change needed for the workers is clearing REBOUND along with PREP. * This patch leaves for_each_busy_worker() without any user. Removed. * idle_worker_rebind(), busy_worker_rebind_fn(), worker->rebind_work and rebind logic in manager_workers() removed. * worker_thread() now looks at WORKER_DIE instead of testing whether @worker->entry is empty to determine whether it needs to do something special as dying is the only special thing now. Signed-off-by: Tejun Heo <tj@kernel.org> Reviewed-by: Lai Jiangshan <laijs@cn.fujitsu.com>
63 lines
1.9 KiB
C
63 lines
1.9 KiB
C
/*
|
|
* kernel/workqueue_internal.h
|
|
*
|
|
* Workqueue internal header file. Only to be included by workqueue and
|
|
* core kernel subsystems.
|
|
*/
|
|
#ifndef _KERNEL_WORKQUEUE_INTERNAL_H
|
|
#define _KERNEL_WORKQUEUE_INTERNAL_H
|
|
|
|
#include <linux/workqueue.h>
|
|
#include <linux/kthread.h>
|
|
|
|
struct worker_pool;
|
|
|
|
/*
|
|
* The poor guys doing the actual heavy lifting. All on-duty workers are
|
|
* either serving the manager role, on idle list or on busy hash. For
|
|
* details on the locking annotation (L, I, X...), refer to workqueue.c.
|
|
*
|
|
* Only to be used in workqueue and async.
|
|
*/
|
|
struct worker {
|
|
/* on idle list while idle, on busy hash table while busy */
|
|
union {
|
|
struct list_head entry; /* L: while idle */
|
|
struct hlist_node hentry; /* L: while busy */
|
|
};
|
|
|
|
struct work_struct *current_work; /* L: work being processed */
|
|
work_func_t current_func; /* L: current_work's fn */
|
|
struct pool_workqueue *current_pwq; /* L: current_work's pwq */
|
|
struct list_head scheduled; /* L: scheduled works */
|
|
struct task_struct *task; /* I: worker task */
|
|
struct worker_pool *pool; /* I: the associated pool */
|
|
/* L: for rescuers */
|
|
/* 64 bytes boundary on 64bit, 32 on 32bit */
|
|
unsigned long last_active; /* L: last active timestamp */
|
|
unsigned int flags; /* X: flags */
|
|
int id; /* I: worker id */
|
|
|
|
/* used only by rescuers to point to the target workqueue */
|
|
struct workqueue_struct *rescue_wq; /* I: the workqueue to rescue */
|
|
};
|
|
|
|
/**
|
|
* current_wq_worker - return struct worker if %current is a workqueue worker
|
|
*/
|
|
static inline struct worker *current_wq_worker(void)
|
|
{
|
|
if (current->flags & PF_WQ_WORKER)
|
|
return kthread_data(current);
|
|
return NULL;
|
|
}
|
|
|
|
/*
|
|
* Scheduler hooks for concurrency managed workqueue. Only to be used from
|
|
* sched.c and workqueue.c.
|
|
*/
|
|
void wq_worker_waking_up(struct task_struct *task, int cpu);
|
|
struct task_struct *wq_worker_sleeping(struct task_struct *task, int cpu);
|
|
|
|
#endif /* _KERNEL_WORKQUEUE_INTERNAL_H */
|