#ifndef _LINUX_SCHED_H #define _LINUX_SCHED_H #include struct sched_param { int sched_priority; }; #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define SCHED_ATTR_SIZE_VER0 48 struct sched_attr { u32 size; u32 sched_policy; u64 sched_flags; s32 sched_nice; u32 sched_priority; u64 sched_runtime; u64 sched_deadline; u64 sched_period; }; struct exec_domain; struct futex_pi_state; struct robust_list_head; struct bio_list; struct fs_struct; struct perf_event_context; struct blk_plug; #define CLONE_KERNEL (CLONE_FS | CLONE_FILES | CLONE_SIGHAND) extern unsigned long avenrun[]; extern void get_avenrun(unsigned long *loads, unsigned long offset, int shift); #define FSHIFT 11 #define FIXED_1 (1<>= FSHIFT; extern unsigned long total_forks; extern int nr_threads; DECLARE_PER_CPU(unsigned long, process_counts); extern int nr_processes(void); extern unsigned long nr_running(void); extern unsigned long nr_iowait(void); extern unsigned long nr_iowait_cpu(int cpu); extern unsigned long this_cpu_load(void); extern void sched_update_nr_prod(int cpu, long delta, bool inc); extern void sched_get_nr_running_avg(int *avg, int *iowait_avg, int *big_avg); extern void calc_global_load(unsigned long ticks); extern void update_cpu_load_nohz(void); struct task_migration_notifier { struct task_struct *task; int from_cpu; int to_cpu; }; extern void register_task_migration_notifier(struct notifier_block *n); extern unsigned long get_parent_ip(unsigned long addr); extern void dump_cpu_task(int cpu); struct seq_file; struct cfs_rq; struct task_group; #ifdef CONFIG_SCHED_DEBUG extern void proc_sched_show_task(struct task_struct *p, struct seq_file *m); extern void proc_sched_set_task(struct task_struct *p); extern void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq); #endif #define TASK_RUNNING 0 #define TASK_INTERRUPTIBLE 1 #define TASK_UNINTERRUPTIBLE 2 #define __TASK_STOPPED 4 #define __TASK_TRACED 8 #define EXIT_ZOMBIE 16 #define EXIT_DEAD 32 #define TASK_DEAD 64 #define TASK_WAKEKILL 128 #define TASK_WAKING 256 #define TASK_PARKED 512 #define TASK_STATE_MAX 1024 #define TASK_STATE_TO_CHAR_STR "RSDTtZXxKWP" extern char ___assert_task_state[1 - 2*!!( sizeof(TASK_STATE_TO_CHAR_STR)-1 != ilog2(TASK_STATE_MAX)+1)]; #define TASK_KILLABLE (TASK_WAKEKILL | TASK_UNINTERRUPTIBLE) #define TASK_STOPPED (TASK_WAKEKILL | __TASK_STOPPED) #define TASK_TRACED (TASK_WAKEKILL | __TASK_TRACED) #define TASK_NORMAL (TASK_INTERRUPTIBLE | TASK_UNINTERRUPTIBLE) #define TASK_ALL (TASK_NORMAL | __TASK_STOPPED | __TASK_TRACED) #define TASK_REPORT (TASK_RUNNING | TASK_INTERRUPTIBLE | \ TASK_UNINTERRUPTIBLE | __TASK_STOPPED | \ __TASK_TRACED) #define task_is_traced(task) ((task->state & __TASK_TRACED) != 0) #define task_is_stopped(task) ((task->state & __TASK_STOPPED) != 0) #define task_is_dead(task) ((task)->exit_state != 0) #define task_is_stopped_or_traced(task) \ ((task->state & (__TASK_STOPPED | __TASK_TRACED)) != 0) #define task_contributes_to_load(task) \ ((task->state & TASK_UNINTERRUPTIBLE) != 0 && \ (task->flags & PF_FROZEN) == 0) #define __set_task_state(tsk, state_value) \ do { (tsk)->state = (state_value); } while (0) #define set_task_state(tsk, state_value) \ set_mb((tsk)->state, (state_value)) #define __set_current_state(state_value) \ do { current->state = (state_value); } while (0) #define set_current_state(state_value) \ set_mb(current->state, (state_value)) #define TASK_COMM_LEN 16 extern const char *sched_window_reset_reasons[]; enum task_event { PUT_PREV_TASK = 0, PICK_NEXT_TASK = 1, TASK_WAKE = 2, TASK_MIGRATE = 3, TASK_UPDATE = 4, IRQ_UPDATE = 5, }; #include extern rwlock_t tasklist_lock; extern spinlock_t mmlist_lock; struct task_struct; #ifdef CONFIG_PROVE_RCU extern int lockdep_tasklist_lock_is_held(void); #endif extern void sched_init(void); extern void sched_init_smp(void); extern asmlinkage void schedule_tail(struct task_struct *prev); extern void init_idle(struct task_struct *idle, int cpu); extern void init_idle_bootup_task(struct task_struct *idle); extern int runqueue_is_locked(int cpu); #if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON) extern void nohz_balance_enter_idle(int cpu); extern void set_cpu_sd_state_idle(void); extern int get_nohz_timer_target(void); #else static inline void nohz_balance_enter_idle(int cpu) { } static inline void set_cpu_sd_state_idle(void) { } #endif extern void show_state_filter(unsigned long state_filter); extern void show_thread_group_state_filter(const char *tg_comm, unsigned long state_filter); static inline void show_state(void) { show_state_filter(0); } extern void show_regs(struct pt_regs *); extern void show_stack(struct task_struct *task, unsigned long *sp); void io_schedule(void); long io_schedule_timeout(long timeout); extern void cpu_init (void); extern void trap_init(void); extern void update_process_times(int user); extern void scheduler_tick(void); extern void sched_show_task(struct task_struct *p); #ifdef CONFIG_LOCKUP_DETECTOR extern void touch_softlockup_watchdog(void); extern void touch_softlockup_watchdog_sync(void); extern void touch_all_softlockup_watchdogs(void); extern int proc_dowatchdog_thresh(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos); extern unsigned int softlockup_panic; void lockup_detector_init(void); #else static inline void touch_softlockup_watchdog(void) { } static inline void touch_softlockup_watchdog_sync(void) { } static inline void touch_all_softlockup_watchdogs(void) { } static inline void lockup_detector_init(void) { } #endif #define __sched __attribute__((__section__(".sched.text"))) extern char __sched_text_start[], __sched_text_end[]; extern int in_sched_functions(unsigned long addr); #define MAX_SCHEDULE_TIMEOUT LONG_MAX extern signed long schedule_timeout(signed long timeout); extern signed long schedule_timeout_interruptible(signed long timeout); extern signed long schedule_timeout_killable(signed long timeout); extern signed long schedule_timeout_uninterruptible(signed long timeout); asmlinkage void schedule(void); extern void schedule_preempt_disabled(void); struct nsproxy; struct user_namespace; #ifdef CONFIG_MMU extern void arch_pick_mmap_layout(struct mm_struct *mm); extern unsigned long arch_get_unmapped_area(struct file *, unsigned long, unsigned long, unsigned long, unsigned long); extern unsigned long arch_get_unmapped_area_topdown(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags); #else static inline void arch_pick_mmap_layout(struct mm_struct *mm) {} #endif extern void set_dumpable(struct mm_struct *mm, int value); extern int get_dumpable(struct mm_struct *mm); #define SUID_DUMP_DISABLE 0 #define SUID_DUMP_USER 1 #define SUID_DUMP_ROOT 2 #define MMF_DUMPABLE 0 #define MMF_DUMP_SECURELY 1 #define MMF_DUMPABLE_BITS 2 #define MMF_DUMPABLE_MASK ((1 << MMF_DUMPABLE_BITS) - 1) #define MMF_DUMP_ANON_PRIVATE 2 #define MMF_DUMP_ANON_SHARED 3 #define MMF_DUMP_MAPPED_PRIVATE 4 #define MMF_DUMP_MAPPED_SHARED 5 #define MMF_DUMP_ELF_HEADERS 6 #define MMF_DUMP_HUGETLB_PRIVATE 7 #define MMF_DUMP_HUGETLB_SHARED 8 #define MMF_DUMP_FILTER_SHIFT MMF_DUMPABLE_BITS #define MMF_DUMP_FILTER_BITS 7 #define MMF_DUMP_FILTER_MASK \ (((1 << MMF_DUMP_FILTER_BITS) - 1) << MMF_DUMP_FILTER_SHIFT) #define MMF_DUMP_FILTER_DEFAULT \ ((1 << MMF_DUMP_ANON_PRIVATE) | (1 << MMF_DUMP_ANON_SHARED) |\ (1 << MMF_DUMP_HUGETLB_PRIVATE) | MMF_DUMP_MASK_DEFAULT_ELF) #ifdef CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS # define MMF_DUMP_MASK_DEFAULT_ELF (1 << MMF_DUMP_ELF_HEADERS) #else # define MMF_DUMP_MASK_DEFAULT_ELF 0 #endif #define MMF_VM_MERGEABLE 16 #define MMF_VM_HUGEPAGE 17 #define MMF_EXE_FILE_CHANGED 18 #define MMF_HAS_UPROBES 19 #define MMF_RECALC_UPROBES 20 #define MMF_INIT_MASK (MMF_DUMPABLE_MASK | MMF_DUMP_FILTER_MASK) struct sighand_struct { atomic_t count; struct k_sigaction action[_NSIG]; spinlock_t siglock; wait_queue_head_t signalfd_wqh; }; struct pacct_struct { int ac_flag; long ac_exitcode; unsigned long ac_mem; cputime_t ac_utime, ac_stime; unsigned long ac_minflt, ac_majflt; }; struct cpu_itimer { cputime_t expires; cputime_t incr; u32 error; u32 incr_error; }; struct cputime { cputime_t utime; cputime_t stime; }; struct task_cputime { cputime_t utime; cputime_t stime; unsigned long long sum_exec_runtime; }; #define prof_exp stime #define virt_exp utime #define sched_exp sum_exec_runtime #define INIT_CPUTIME \ (struct task_cputime) { \ .utime = 0, \ .stime = 0, \ .sum_exec_runtime = 0, \ } #define INIT_PREEMPT_COUNT (1 + PREEMPT_ACTIVE) struct thread_group_cputimer { struct task_cputime cputime; int running; raw_spinlock_t lock; }; #include struct autogroup; struct signal_struct { atomic_t sigcnt; atomic_t live; int nr_threads; struct list_head thread_head; wait_queue_head_t wait_chldexit; struct task_struct *curr_target; struct sigpending shared_pending; int group_exit_code; int notify_count; struct task_struct *group_exit_task; int group_stop_count; unsigned int flags; unsigned int is_child_subreaper:1; unsigned int has_child_subreaper:1; int posix_timer_id; struct list_head posix_timers; struct hrtimer real_timer; struct pid *leader_pid; ktime_t it_real_incr; struct cpu_itimer it[2]; struct thread_group_cputimer cputimer; struct task_cputime cputime_expires; struct list_head cpu_timers[3]; struct pid *tty_old_pgrp; int leader; struct tty_struct *tty; #ifdef CONFIG_SCHED_AUTOGROUP struct autogroup *autogroup; #endif cputime_t utime, stime, cutime, cstime; cputime_t gtime; cputime_t cgtime; #ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE struct cputime prev_cputime; #endif unsigned long nvcsw, nivcsw, cnvcsw, cnivcsw; unsigned long min_flt, maj_flt, cmin_flt, cmaj_flt; unsigned long inblock, oublock, cinblock, coublock; unsigned long maxrss, cmaxrss; struct task_io_accounting ioac; unsigned long long sum_sched_runtime; struct rlimit rlim[RLIM_NLIMITS]; #ifdef CONFIG_BSD_PROCESS_ACCT struct pacct_struct pacct; #endif #ifdef CONFIG_TASKSTATS struct taskstats *stats; #endif #ifdef CONFIG_AUDIT unsigned audit_tty; unsigned audit_tty_log_passwd; struct tty_audit_buf *tty_audit_buf; #endif #ifdef CONFIG_CGROUPS struct rw_semaphore group_rwsem; #endif oom_flags_t oom_flags; short oom_score_adj; short oom_score_adj_min; struct mutex cred_guard_mutex; }; #define SIGNAL_STOP_STOPPED 0x00000001 #define SIGNAL_STOP_CONTINUED 0x00000002 #define SIGNAL_GROUP_EXIT 0x00000004 #define SIGNAL_GROUP_COREDUMP 0x00000008 #define SIGNAL_CLD_STOPPED 0x00000010 #define SIGNAL_CLD_CONTINUED 0x00000020 #define SIGNAL_CLD_MASK (SIGNAL_CLD_STOPPED|SIGNAL_CLD_CONTINUED) #define SIGNAL_UNKILLABLE 0x00000040 static inline int signal_group_exit(const struct signal_struct *sig) { return (sig->flags & SIGNAL_GROUP_EXIT) || (sig->group_exit_task != NULL); } struct user_struct { atomic_t __count; atomic_t processes; atomic_t files; atomic_t sigpending; #ifdef CONFIG_INOTIFY_USER atomic_t inotify_watches; atomic_t inotify_devs; #endif #ifdef CONFIG_FANOTIFY atomic_t fanotify_listeners; #endif #ifdef CONFIG_EPOLL atomic_long_t epoll_watches; #endif #ifdef CONFIG_POSIX_MQUEUE unsigned long mq_bytes; #endif unsigned long locked_shm; /* How many pages of mlocked shm ? */ unsigned long unix_inflight; /* How many files in flight in unix sockets */ atomic_long_t pipe_bufs; /* how many pages are allocated in pipe buffers */ #ifdef CONFIG_KEYS struct key *uid_keyring; struct key *session_keyring; #endif struct hlist_node uidhash_node; kuid_t uid; #ifdef CONFIG_PERF_EVENTS atomic_long_t locked_vm; #endif }; extern int uids_sysfs_init(void); extern struct user_struct *find_user(kuid_t); extern struct user_struct root_user; #define INIT_USER (&root_user) struct backing_dev_info; struct reclaim_state; #if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT) struct sched_info { unsigned long pcount; unsigned long long run_delay; unsigned long long last_arrival, last_queued; }; #endif #ifdef CONFIG_TASK_DELAY_ACCT struct task_delay_info { spinlock_t lock; unsigned int flags; struct timespec blkio_start, blkio_end; u64 blkio_delay; u64 swapin_delay; u32 blkio_count; u32 swapin_count; struct timespec freepages_start, freepages_end; u64 freepages_delay; u32 freepages_count; }; #endif static inline int sched_info_on(void) { #ifdef CONFIG_SCHEDSTATS return 1; #elif defined(CONFIG_TASK_DELAY_ACCT) extern int delayacct_on; return delayacct_on; #else return 0; #endif } enum cpu_idle_type { CPU_IDLE, CPU_NOT_IDLE, CPU_NEWLY_IDLE, CPU_MAX_IDLE_TYPES }; #define SCHED_POWER_SHIFT 10 #define SCHED_POWER_SCALE (1L << SCHED_POWER_SHIFT) #ifdef CONFIG_SMP #define SD_LOAD_BALANCE 0x0001 #define SD_BALANCE_NEWIDLE 0x0002 #define SD_BALANCE_EXEC 0x0004 #define SD_BALANCE_FORK 0x0008 #define SD_BALANCE_WAKE 0x0010 #define SD_WAKE_AFFINE 0x0020 #define SD_SHARE_CPUPOWER 0x0080 #define SD_SHARE_PKG_RESOURCES 0x0200 #define SD_SERIALIZE 0x0400 #define SD_ASYM_PACKING 0x0800 #define SD_PREFER_SIBLING 0x1000 #define SD_OVERLAP 0x2000 extern int __weak arch_sd_sibiling_asym_packing(void); struct sched_domain_attr { int relax_domain_level; }; #define SD_ATTR_INIT (struct sched_domain_attr) { \ .relax_domain_level = -1, \ } extern int sched_domain_level_max; struct sched_group; struct sched_domain { struct sched_domain *parent; struct sched_domain *child; struct sched_group *groups; unsigned long min_interval; unsigned long max_interval; unsigned int busy_factor; unsigned int imbalance_pct; unsigned int cache_nice_tries; unsigned int busy_idx; unsigned int idle_idx; unsigned int newidle_idx; unsigned int wake_idx; unsigned int forkexec_idx; unsigned int smt_gain; int nohz_idle; int flags; int level; unsigned long last_balance; unsigned int balance_interval; unsigned int nr_balance_failed; u64 last_update; #ifdef CONFIG_SCHEDSTATS unsigned int lb_count[CPU_MAX_IDLE_TYPES]; unsigned int lb_failed[CPU_MAX_IDLE_TYPES]; unsigned int lb_balanced[CPU_MAX_IDLE_TYPES]; unsigned int lb_imbalance[CPU_MAX_IDLE_TYPES]; unsigned int lb_gained[CPU_MAX_IDLE_TYPES]; unsigned int lb_hot_gained[CPU_MAX_IDLE_TYPES]; unsigned int lb_nobusyg[CPU_MAX_IDLE_TYPES]; unsigned int lb_nobusyq[CPU_MAX_IDLE_TYPES]; unsigned int alb_count; unsigned int alb_failed; unsigned int alb_pushed; unsigned int sbe_count; unsigned int sbe_balanced; unsigned int sbe_pushed; unsigned int sbf_count; unsigned int sbf_balanced; unsigned int sbf_pushed; unsigned int ttwu_wake_remote; unsigned int ttwu_move_affine; unsigned int ttwu_move_balance; #endif #ifdef CONFIG_SCHED_DEBUG char *name; #endif union { void *private; struct rcu_head rcu; }; unsigned int span_weight; unsigned long span[0]; }; static inline struct cpumask *sched_domain_span(struct sched_domain *sd) { return to_cpumask(sd->span); } extern void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[], struct sched_domain_attr *dattr_new); cpumask_var_t *alloc_sched_domains(unsigned int ndoms); void free_sched_domains(cpumask_var_t doms[], unsigned int ndoms); bool cpus_share_cache(int this_cpu, int that_cpu); #else struct sched_domain_attr; static inline void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[], struct sched_domain_attr *dattr_new) { } static inline bool cpus_share_cache(int this_cpu, int that_cpu) { return true; } #endif struct io_context; #ifdef ARCH_HAS_PREFETCH_SWITCH_STACK extern void prefetch_stack(struct task_struct *t); #else static inline void prefetch_stack(struct task_struct *t) { } #endif struct audit_context; struct mempolicy; struct pipe_inode_info; struct uts_namespace; struct load_weight { unsigned long weight, inv_weight; }; struct sched_avg { u32 runnable_avg_sum, runnable_avg_period; #ifdef CONFIG_SCHED_HMP u32 runnable_avg_sum_scaled; #endif u64 last_runnable_update; s64 decay_count; unsigned long load_avg_contrib; }; #ifdef CONFIG_SCHEDSTATS struct sched_statistics { u64 wait_start; u64 wait_max; u64 wait_count; u64 wait_sum; u64 iowait_count; u64 iowait_sum; u64 sleep_start; u64 sleep_max; s64 sum_sleep_runtime; u64 block_start; u64 block_max; u64 exec_max; u64 slice_max; u64 nr_migrations_cold; u64 nr_failed_migrations_affine; u64 nr_failed_migrations_running; u64 nr_failed_migrations_hot; u64 nr_forced_migrations; u64 nr_wakeups; u64 nr_wakeups_sync; u64 nr_wakeups_migrate; u64 nr_wakeups_local; u64 nr_wakeups_remote; u64 nr_wakeups_affine; u64 nr_wakeups_affine_attempts; u64 nr_wakeups_passive; u64 nr_wakeups_idle; }; #endif #define RAVG_HIST_SIZE_MAX 5 struct ravg { u64 mark_start; u32 sum, demand; u32 sum_history[RAVG_HIST_SIZE_MAX]; #ifdef CONFIG_SCHED_FREQ_INPUT u32 curr_window, prev_window; #endif }; struct sched_entity { struct load_weight load; struct rb_node run_node; struct list_head group_node; unsigned int on_rq; u64 exec_start; u64 sum_exec_runtime; u64 vruntime; u64 prev_sum_exec_runtime; u64 nr_migrations; #ifdef CONFIG_SCHEDSTATS struct sched_statistics statistics; #endif #ifdef CONFIG_FAIR_GROUP_SCHED struct sched_entity *parent; struct cfs_rq *cfs_rq; struct cfs_rq *my_q; #endif #if defined(CONFIG_SMP) && defined(CONFIG_FAIR_GROUP_SCHED) struct sched_avg avg; #endif }; struct sched_rt_entity { struct list_head run_list; unsigned long timeout; unsigned long watchdog_stamp; unsigned int time_slice; struct sched_rt_entity *back; #ifdef CONFIG_RT_GROUP_SCHED struct sched_rt_entity *parent; struct rt_rq *rt_rq; struct rt_rq *my_q; #endif }; struct rcu_node; enum perf_event_task_context { perf_invalid_context = -1, perf_hw_context = 0, perf_sw_context, perf_nr_task_contexts, }; struct task_struct { volatile long state; void *stack; atomic_t usage; unsigned int flags; unsigned int ptrace; #ifdef CONFIG_SMP struct llist_node wake_entry; int on_cpu; #endif int on_rq; int prio, static_prio, normal_prio; unsigned int rt_priority; const struct sched_class *sched_class; struct sched_entity se; struct sched_rt_entity rt; #ifdef CONFIG_SCHED_HMP struct ravg ravg; #endif #ifdef CONFIG_CGROUP_SCHED struct task_group *sched_task_group; #endif #ifdef CONFIG_PREEMPT_NOTIFIERS struct hlist_head preempt_notifiers; #endif unsigned char fpu_counter; #ifdef CONFIG_BLK_DEV_IO_TRACE unsigned int btrace_seq; #endif unsigned int policy; int nr_cpus_allowed; cpumask_t cpus_allowed; #ifdef CONFIG_PREEMPT_RCU int rcu_read_lock_nesting; char rcu_read_unlock_special; struct list_head rcu_node_entry; #endif #ifdef CONFIG_TREE_PREEMPT_RCU struct rcu_node *rcu_blocked_node; #endif #ifdef CONFIG_RCU_BOOST struct rt_mutex *rcu_boost_mutex; #endif #if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT) struct sched_info sched_info; #endif struct list_head tasks; #ifdef CONFIG_SMP struct plist_node pushable_tasks; #endif struct mm_struct *mm, *active_mm; #ifdef CONFIG_COMPAT_BRK unsigned brk_randomized:1; #endif #if defined(SPLIT_RSS_COUNTING) struct task_rss_stat rss_stat; #endif int exit_state; int exit_code, exit_signal; int pdeath_signal; unsigned int jobctl; unsigned int personality; unsigned did_exec:1; unsigned in_execve:1; unsigned in_iowait:1; unsigned sched_reset_on_fork:1; unsigned sched_contributes_to_load:1; unsigned long atomic_flags; pid_t pid; pid_t tgid; #ifdef CONFIG_CC_STACKPROTECTOR unsigned long stack_canary; #endif struct task_struct __rcu *real_parent; struct task_struct __rcu *parent; struct list_head children; struct list_head sibling; struct task_struct *group_leader; struct list_head ptraced; struct list_head ptrace_entry; struct pid_link pids[PIDTYPE_MAX]; struct list_head thread_group; struct list_head thread_node; struct completion *vfork_done; int __user *set_child_tid; int __user *clear_child_tid; cputime_t utime, stime, utimescaled, stimescaled; cputime_t gtime; unsigned long long cpu_power; #ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE struct cputime prev_cputime; #endif #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN seqlock_t vtime_seqlock; unsigned long long vtime_snap; enum { VTIME_SLEEPING = 0, VTIME_USER, VTIME_SYS, } vtime_snap_whence; #endif unsigned long nvcsw, nivcsw; struct timespec start_time; struct timespec real_start_time; unsigned long min_flt, maj_flt; struct task_cputime cputime_expires; struct list_head cpu_timers[3]; const struct cred __rcu *real_cred; const struct cred __rcu *cred; char comm[TASK_COMM_LEN]; int link_count, total_link_count; #ifdef CONFIG_SYSVIPC struct sysv_sem sysvsem; #endif #ifdef CONFIG_DETECT_HUNG_TASK unsigned long last_switch_count; #endif struct thread_struct thread; struct fs_struct *fs; struct files_struct *files; struct nsproxy *nsproxy; struct signal_struct *signal; struct sighand_struct *sighand; sigset_t blocked, real_blocked; sigset_t saved_sigmask; struct sigpending pending; unsigned long sas_ss_sp; size_t sas_ss_size; int (*notifier)(void *priv); void *notifier_data; sigset_t *notifier_mask; struct callback_head *task_works; struct audit_context *audit_context; #ifdef CONFIG_AUDITSYSCALL kuid_t loginuid; unsigned int sessionid; #endif struct seccomp seccomp; u32 parent_exec_id; u32 self_exec_id; spinlock_t alloc_lock; raw_spinlock_t pi_lock; #ifdef CONFIG_RT_MUTEXES struct plist_head pi_waiters; struct rt_mutex_waiter *pi_blocked_on; #endif #ifdef CONFIG_DEBUG_MUTEXES struct mutex_waiter *blocked_on; struct task_struct *blocked_by; unsigned long blocked_since; #endif #ifdef CONFIG_TRACE_IRQFLAGS unsigned int irq_events; unsigned long hardirq_enable_ip; unsigned long hardirq_disable_ip; unsigned int hardirq_enable_event; unsigned int hardirq_disable_event; int hardirqs_enabled; int hardirq_context; unsigned long softirq_disable_ip; unsigned long softirq_enable_ip; unsigned int softirq_disable_event; unsigned int softirq_enable_event; int softirqs_enabled; int softirq_context; #endif #ifdef CONFIG_LOCKDEP # define MAX_LOCK_DEPTH 48UL u64 curr_chain_key; int lockdep_depth; unsigned int lockdep_recursion; struct held_lock held_locks[MAX_LOCK_DEPTH]; gfp_t lockdep_reclaim_gfp; #endif void *journal_info; struct bio_list *bio_list; #ifdef CONFIG_BLOCK struct blk_plug *plug; #endif struct reclaim_state *reclaim_state; struct backing_dev_info *backing_dev_info; struct io_context *io_context; unsigned long ptrace_message; siginfo_t *last_siginfo; struct task_io_accounting ioac; #if defined(CONFIG_TASK_XACCT) u64 acct_rss_mem1; u64 acct_vm_mem1; cputime_t acct_timexpd; #endif #ifdef CONFIG_CPUSETS nodemask_t mems_allowed; seqcount_t mems_allowed_seq; int cpuset_mem_spread_rotor; int cpuset_slab_spread_rotor; #endif #ifdef CONFIG_CGROUPS struct css_set __rcu *cgroups; struct list_head cg_list; #endif #ifdef CONFIG_FUTEX struct robust_list_head __user *robust_list; #ifdef CONFIG_COMPAT struct compat_robust_list_head __user *compat_robust_list; #endif struct list_head pi_state_list; struct futex_pi_state *pi_state_cache; #endif #ifdef CONFIG_PERF_EVENTS struct perf_event_context *perf_event_ctxp[perf_nr_task_contexts]; struct mutex perf_event_mutex; struct list_head perf_event_list; #endif #ifdef CONFIG_NUMA struct mempolicy *mempolicy; short il_next; short pref_node_fork; #endif #ifdef CONFIG_NUMA_BALANCING int numa_scan_seq; int numa_migrate_seq; unsigned int numa_scan_period; u64 node_stamp; struct callback_head numa_work; #endif struct rcu_head rcu; struct pipe_inode_info *splice_pipe; struct page_frag task_frag; #ifdef CONFIG_TASK_DELAY_ACCT struct task_delay_info *delays; #endif #ifdef CONFIG_FAULT_INJECTION int make_it_fail; #endif int nr_dirtied; int nr_dirtied_pause; unsigned long dirty_paused_when; #ifdef CONFIG_LATENCYTOP int latency_record_count; struct latency_record latency_record[LT_SAVECOUNT]; #endif unsigned long timer_slack_ns; unsigned long default_timer_slack_ns; #ifdef CONFIG_FUNCTION_GRAPH_TRACER int curr_ret_stack; struct ftrace_ret_stack *ret_stack; unsigned long long ftrace_timestamp; atomic_t trace_overrun; atomic_t tracing_graph_pause; #endif #ifdef CONFIG_TRACING unsigned long trace; unsigned long trace_recursion; #endif #ifdef CONFIG_MEMCG struct memcg_batch_info { int do_batch; struct mem_cgroup *memcg; unsigned long nr_pages; unsigned long memsw_nr_pages; } memcg_batch; unsigned int memcg_kmem_skip_account; struct memcg_oom_info { struct mem_cgroup *memcg; gfp_t gfp_mask; int order; unsigned int may_oom:1; } memcg_oom; #endif #ifdef CONFIG_HAVE_HW_BREAKPOINT atomic_t ptrace_bp_refcnt; #endif #ifdef CONFIG_UPROBES struct uprobe_task *utask; #endif #if defined(CONFIG_BCACHE) || defined(CONFIG_BCACHE_MODULE) unsigned int sequential_io; unsigned int sequential_io_avg; #endif }; #define tsk_cpus_allowed(tsk) (&(tsk)->cpus_allowed) #ifdef CONFIG_NUMA_BALANCING extern void task_numa_fault(int node, int pages, bool migrated); extern void set_numabalancing_state(bool enabled); #else static inline void task_numa_fault(int node, int pages, bool migrated) { } static inline void set_numabalancing_state(bool enabled) { } #endif static inline struct pid *task_pid(struct task_struct *task) { return task->pids[PIDTYPE_PID].pid; } static inline struct pid *task_tgid(struct task_struct *task) { return task->group_leader->pids[PIDTYPE_PID].pid; } static inline struct pid *task_pgrp(struct task_struct *task) { return task->group_leader->pids[PIDTYPE_PGID].pid; } static inline struct pid *task_session(struct task_struct *task) { return task->group_leader->pids[PIDTYPE_SID].pid; } struct pid_namespace; pid_t __task_pid_nr_ns(struct task_struct *task, enum pid_type type, struct pid_namespace *ns); static inline pid_t task_pid_nr(struct task_struct *tsk) { return tsk->pid; } static inline pid_t task_pid_nr_ns(struct task_struct *tsk, struct pid_namespace *ns) { return __task_pid_nr_ns(tsk, PIDTYPE_PID, ns); } static inline pid_t task_pid_vnr(struct task_struct *tsk) { return __task_pid_nr_ns(tsk, PIDTYPE_PID, NULL); } static inline pid_t task_tgid_nr(struct task_struct *tsk) { return tsk->tgid; } pid_t task_tgid_nr_ns(struct task_struct *tsk, struct pid_namespace *ns); static inline pid_t task_tgid_vnr(struct task_struct *tsk) { return pid_vnr(task_tgid(tsk)); } static inline pid_t task_pgrp_nr_ns(struct task_struct *tsk, struct pid_namespace *ns) { return __task_pid_nr_ns(tsk, PIDTYPE_PGID, ns); } static inline pid_t task_pgrp_vnr(struct task_struct *tsk) { return __task_pid_nr_ns(tsk, PIDTYPE_PGID, NULL); } static inline pid_t task_session_nr_ns(struct task_struct *tsk, struct pid_namespace *ns) { return __task_pid_nr_ns(tsk, PIDTYPE_SID, ns); } static inline pid_t task_session_vnr(struct task_struct *tsk) { return __task_pid_nr_ns(tsk, PIDTYPE_SID, NULL); } static inline pid_t task_pgrp_nr(struct task_struct *tsk) { return task_pgrp_nr_ns(tsk, &init_pid_ns); } static inline int pid_alive(struct task_struct *p) { return p->pids[PIDTYPE_PID].pid != NULL; } static inline int is_global_init(struct task_struct *tsk) { return tsk->pid == 1; } extern struct pid *cad_pid; extern void free_task(struct task_struct *tsk); #define get_task_struct(tsk) do { atomic_inc(&(tsk)->usage); } while(0) extern void __put_task_struct(struct task_struct *t); static inline void put_task_struct(struct task_struct *t) { if (atomic_dec_and_test(&t->usage)) __put_task_struct(t); } #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN extern void task_cputime(struct task_struct *t, cputime_t *utime, cputime_t *stime); extern void task_cputime_scaled(struct task_struct *t, cputime_t *utimescaled, cputime_t *stimescaled); extern cputime_t task_gtime(struct task_struct *t); #else static inline void task_cputime(struct task_struct *t, cputime_t *utime, cputime_t *stime) { if (utime) *utime = t->utime; if (stime) *stime = t->stime; } static inline void task_cputime_scaled(struct task_struct *t, cputime_t *utimescaled, cputime_t *stimescaled) { if (utimescaled) *utimescaled = t->utimescaled; if (stimescaled) *stimescaled = t->stimescaled; } static inline cputime_t task_gtime(struct task_struct *t) { return t->gtime; } #endif extern void task_cputime_adjusted(struct task_struct *p, cputime_t *ut, cputime_t *st); extern void thread_group_cputime_adjusted(struct task_struct *p, cputime_t *ut, cputime_t *st); extern int task_free_register(struct notifier_block *n); extern int task_free_unregister(struct notifier_block *n); #ifdef CONFIG_SCHED_FREQ_INPUT extern int sched_set_window(u64 window_start, unsigned int window_size); extern unsigned long sched_get_busy(int cpu); extern void sched_set_io_is_busy(int val); #else static inline int sched_set_window(u64 window_start, unsigned int window_size) { return -EINVAL; } static inline unsigned long sched_get_busy(int cpu) { return 0; } static inline void sched_set_io_is_busy(int val) {}; #endif #define PF_EXITING 0x00000004 #define PF_EXITPIDONE 0x00000008 #define PF_VCPU 0x00000010 #define PF_WQ_WORKER 0x00000020 #define PF_FORKNOEXEC 0x00000040 #define PF_MCE_PROCESS 0x00000080 #define PF_SUPERPRIV 0x00000100 #define PF_DUMPCORE 0x00000200 #define PF_SIGNALED 0x00000400 #define PF_MEMALLOC 0x00000800 #define PF_NPROC_EXCEEDED 0x00001000 #define PF_USED_MATH 0x00002000 #define PF_USED_ASYNC 0x00004000 #define PF_NOFREEZE 0x00008000 #define PF_FROZEN 0x00010000 #define PF_FSTRANS 0x00020000 #define PF_KSWAPD 0x00040000 #define PF_MEMALLOC_NOIO 0x00080000 #define PF_LESS_THROTTLE 0x00100000 #define PF_KTHREAD 0x00200000 #define PF_RANDOMIZE 0x00400000 #define PF_SWAPWRITE 0x00800000 #define PF_SPREAD_PAGE 0x01000000 #define PF_SPREAD_SLAB 0x02000000 #define PF_NO_SETAFFINITY 0x04000000 #define PF_MCE_EARLY 0x08000000 #define PF_MEMPOLICY 0x10000000 #define PF_MUTEX_TESTER 0x20000000 #define PF_FREEZER_SKIP 0x40000000 #define PF_WAKE_UP_IDLE 0x80000000 #define clear_stopped_child_used_math(child) do { (child)->flags &= ~PF_USED_MATH; } while (0) #define set_stopped_child_used_math(child) do { (child)->flags |= PF_USED_MATH; } while (0) #define clear_used_math() clear_stopped_child_used_math(current) #define set_used_math() set_stopped_child_used_math(current) #define conditional_stopped_child_used_math(condition, child) \ do { (child)->flags &= ~PF_USED_MATH, (child)->flags |= (condition) ? PF_USED_MATH : 0; } while (0) #define conditional_used_math(condition) \ conditional_stopped_child_used_math(condition, current) #define copy_to_stopped_child_used_math(child) \ do { (child)->flags &= ~PF_USED_MATH, (child)->flags |= current->flags & PF_USED_MATH; } while (0) #define tsk_used_math(p) ((p)->flags & PF_USED_MATH) #define used_math() tsk_used_math(current) /* __GFP_IO isn't allowed if PF_MEMALLOC_NOIO is set in current->flags * __GFP_FS is also cleared as it implies __GFP_IO. */ static inline gfp_t memalloc_noio_flags(gfp_t flags) { if (unlikely(current->flags & PF_MEMALLOC_NOIO)) flags &= ~(__GFP_IO | __GFP_FS); return flags; } static inline unsigned int memalloc_noio_save(void) { unsigned int flags = current->flags & PF_MEMALLOC_NOIO; current->flags |= PF_MEMALLOC_NOIO; return flags; } static inline void memalloc_noio_restore(unsigned int flags) { current->flags = (current->flags & ~PF_MEMALLOC_NOIO) | flags; } #define PFA_NO_NEW_PRIVS 0x00000001 static inline bool task_no_new_privs(struct task_struct *p) { return test_bit(PFA_NO_NEW_PRIVS, &p->atomic_flags); } static inline void task_set_no_new_privs(struct task_struct *p) { set_bit(PFA_NO_NEW_PRIVS, &p->atomic_flags); } #define JOBCTL_STOP_SIGMASK 0xffff #define JOBCTL_STOP_DEQUEUED_BIT 16 #define JOBCTL_STOP_PENDING_BIT 17 #define JOBCTL_STOP_CONSUME_BIT 18 #define JOBCTL_TRAP_STOP_BIT 19 #define JOBCTL_TRAP_NOTIFY_BIT 20 #define JOBCTL_TRAPPING_BIT 21 #define JOBCTL_LISTENING_BIT 22 #define JOBCTL_STOP_DEQUEUED (1 << JOBCTL_STOP_DEQUEUED_BIT) #define JOBCTL_STOP_PENDING (1 << JOBCTL_STOP_PENDING_BIT) #define JOBCTL_STOP_CONSUME (1 << JOBCTL_STOP_CONSUME_BIT) #define JOBCTL_TRAP_STOP (1 << JOBCTL_TRAP_STOP_BIT) #define JOBCTL_TRAP_NOTIFY (1 << JOBCTL_TRAP_NOTIFY_BIT) #define JOBCTL_TRAPPING (1 << JOBCTL_TRAPPING_BIT) #define JOBCTL_LISTENING (1 << JOBCTL_LISTENING_BIT) #define JOBCTL_TRAP_MASK (JOBCTL_TRAP_STOP | JOBCTL_TRAP_NOTIFY) #define JOBCTL_PENDING_MASK (JOBCTL_STOP_PENDING | JOBCTL_TRAP_MASK) extern bool task_set_jobctl_pending(struct task_struct *task, unsigned int mask); extern void task_clear_jobctl_trapping(struct task_struct *task); extern void task_clear_jobctl_pending(struct task_struct *task, unsigned int mask); #ifdef CONFIG_PREEMPT_RCU #define RCU_READ_UNLOCK_BLOCKED (1 << 0) #define RCU_READ_UNLOCK_NEED_QS (1 << 1) static inline void rcu_copy_process(struct task_struct *p) { p->rcu_read_lock_nesting = 0; p->rcu_read_unlock_special = 0; #ifdef CONFIG_TREE_PREEMPT_RCU p->rcu_blocked_node = NULL; #endif #ifdef CONFIG_RCU_BOOST p->rcu_boost_mutex = NULL; #endif INIT_LIST_HEAD(&p->rcu_node_entry); } #else static inline void rcu_copy_process(struct task_struct *p) { } #endif static inline void tsk_restore_flags(struct task_struct *task, unsigned long orig_flags, unsigned long flags) { task->flags &= ~flags; task->flags |= orig_flags & flags; } #ifdef CONFIG_SMP extern void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask); extern int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask); extern void sched_set_cpu_cstate(int cpu, int cstate, int wakeup_energy, int wakeup_latency); #else static inline void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask) { } static inline int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask) { if (!cpumask_test_cpu(0, new_mask)) return -EINVAL; return 0; } static inline void sched_set_cpu_cstate(int cpu, int cstate, int wakeup_energy, int wakeup_latency) { } #endif #ifdef CONFIG_SCHED_HMP extern int sched_set_boost(int enable); #else static inline int sched_set_boost(int enable) { return -EINVAL; } #endif #ifdef CONFIG_NO_HZ_COMMON void calc_load_enter_idle(void); void calc_load_exit_idle(void); #else static inline void calc_load_enter_idle(void) { } static inline void calc_load_exit_idle(void) { } #endif static inline void set_wake_up_idle(bool enabled) { if (enabled) current->flags |= PF_WAKE_UP_IDLE; else current->flags &= ~PF_WAKE_UP_IDLE; } #ifndef CONFIG_CPUMASK_OFFSTACK static inline int set_cpus_allowed(struct task_struct *p, cpumask_t new_mask) { return set_cpus_allowed_ptr(p, &new_mask); } #endif extern unsigned long long notrace sched_clock(void); extern u64 cpu_clock(int cpu); extern u64 local_clock(void); extern u64 sched_clock_cpu(int cpu); extern void sched_clock_init(void); #ifndef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK static inline void sched_clock_tick(void) { } static inline void sched_clock_idle_sleep_event(void) { } static inline void sched_clock_idle_wakeup_event(u64 delta_ns) { } #else extern int sched_clock_stable; extern void sched_clock_tick(void); extern void sched_clock_idle_sleep_event(void); extern void sched_clock_idle_wakeup_event(u64 delta_ns); #endif #ifdef CONFIG_IRQ_TIME_ACCOUNTING extern void enable_sched_clock_irqtime(void); extern void disable_sched_clock_irqtime(void); #else static inline void enable_sched_clock_irqtime(void) {} static inline void disable_sched_clock_irqtime(void) {} #endif extern unsigned long long task_sched_runtime(struct task_struct *task); #if defined(CONFIG_SMP) extern void sched_exec(void); #else #define sched_exec() {} #endif extern void sched_clock_idle_sleep_event(void); extern void sched_clock_idle_wakeup_event(u64 delta_ns); #ifdef CONFIG_HOTPLUG_CPU extern void idle_task_exit(void); #else static inline void idle_task_exit(void) {} #endif #if defined(CONFIG_NO_HZ_COMMON) && defined(CONFIG_SMP) extern void wake_up_nohz_cpu(int cpu); #else static inline void wake_up_nohz_cpu(int cpu) { } #endif #ifdef CONFIG_NO_HZ_FULL extern bool sched_can_stop_tick(void); extern u64 scheduler_tick_max_deferment(void); #else static inline bool sched_can_stop_tick(void) { return false; } #endif #ifdef CONFIG_SCHED_AUTOGROUP extern void sched_autogroup_create_attach(struct task_struct *p); extern void sched_autogroup_detach(struct task_struct *p); extern void sched_autogroup_fork(struct signal_struct *sig); extern void sched_autogroup_exit(struct signal_struct *sig); #ifdef CONFIG_PROC_FS extern void proc_sched_autogroup_show_task(struct task_struct *p, struct seq_file *m); extern int proc_sched_autogroup_set_nice(struct task_struct *p, int nice); #endif #else static inline void sched_autogroup_create_attach(struct task_struct *p) { } static inline void sched_autogroup_detach(struct task_struct *p) { } static inline void sched_autogroup_fork(struct signal_struct *sig) { } static inline void sched_autogroup_exit(struct signal_struct *sig) { } #endif extern bool yield_to(struct task_struct *p, bool preempt); extern void set_user_nice(struct task_struct *p, long nice); extern int task_prio(const struct task_struct *p); extern int task_nice(const struct task_struct *p); extern int can_nice(const struct task_struct *p, const int nice); extern int task_curr(const struct task_struct *p); extern int idle_cpu(int cpu); extern int sched_setscheduler(struct task_struct *, int, const struct sched_param *); extern int sched_setscheduler_nocheck(struct task_struct *, int, const struct sched_param *); extern int sched_setattr(struct task_struct *, const struct sched_attr *); extern struct task_struct *idle_task(int cpu); static inline bool is_idle_task(const struct task_struct *p) { return p->pid == 0; } extern struct task_struct *curr_task(int cpu); extern void set_curr_task(int cpu, struct task_struct *p); void yield(void); extern struct exec_domain default_exec_domain; union thread_union { struct thread_info thread_info; unsigned long stack[THREAD_SIZE/sizeof(long)]; }; #ifndef __HAVE_ARCH_KSTACK_END static inline int kstack_end(void *addr) { return !(((unsigned long)addr+sizeof(void*)-1) & (THREAD_SIZE-sizeof(void*))); } #endif extern union thread_union init_thread_union; extern struct task_struct init_task; extern struct mm_struct init_mm; extern struct pid_namespace init_pid_ns; extern struct task_struct *find_task_by_vpid(pid_t nr); extern struct task_struct *find_task_by_pid_ns(pid_t nr, struct pid_namespace *ns); extern void __set_special_pids(struct pid *pid); extern struct user_struct * alloc_uid(kuid_t); static inline struct user_struct *get_uid(struct user_struct *u) { atomic_inc(&u->__count); return u; } extern void free_uid(struct user_struct *); #include extern void xtime_update(unsigned long ticks); extern int wake_up_state(struct task_struct *tsk, unsigned int state); extern int wake_up_process(struct task_struct *tsk); extern void wake_up_new_task(struct task_struct *tsk); #ifdef CONFIG_SMP extern void kick_process(struct task_struct *tsk); #else static inline void kick_process(struct task_struct *tsk) { } #endif extern void sched_fork(struct task_struct *p); extern void sched_dead(struct task_struct *p); #ifdef CONFIG_SCHED_HMP extern void sched_exit(struct task_struct *p); #else static inline void sched_exit(struct task_struct *p) { } #endif extern void proc_caches_init(void); extern void flush_signals(struct task_struct *); extern void __flush_signals(struct task_struct *); extern void ignore_signals(struct task_struct *); extern void flush_signal_handlers(struct task_struct *, int force_default); extern int dequeue_signal(struct task_struct *tsk, sigset_t *mask, siginfo_t *info); static inline int dequeue_signal_lock(struct task_struct *tsk, sigset_t *mask, siginfo_t *info) { unsigned long flags; int ret; spin_lock_irqsave(&tsk->sighand->siglock, flags); ret = dequeue_signal(tsk, mask, info); spin_unlock_irqrestore(&tsk->sighand->siglock, flags); return ret; } extern void block_all_signals(int (*notifier)(void *priv), void *priv, sigset_t *mask); extern void unblock_all_signals(void); extern void release_task(struct task_struct * p); extern int send_sig_info(int, struct siginfo *, struct task_struct *); extern int force_sigsegv(int, struct task_struct *); extern int force_sig_info(int, struct siginfo *, struct task_struct *); extern int __kill_pgrp_info(int sig, struct siginfo *info, struct pid *pgrp); extern int kill_pid_info(int sig, struct siginfo *info, struct pid *pid); extern int kill_pid_info_as_cred(int, struct siginfo *, struct pid *, const struct cred *, u32); extern int kill_pgrp(struct pid *pid, int sig, int priv); extern int kill_pid(struct pid *pid, int sig, int priv); extern int kill_proc_info(int, struct siginfo *, pid_t); extern __must_check bool do_notify_parent(struct task_struct *, int); extern void __wake_up_parent(struct task_struct *p, struct task_struct *parent); extern void force_sig(int, struct task_struct *); extern int send_sig(int, struct task_struct *, int); extern int zap_other_threads(struct task_struct *p); extern struct sigqueue *sigqueue_alloc(void); extern void sigqueue_free(struct sigqueue *); extern int send_sigqueue(struct sigqueue *, struct task_struct *, int group); extern int do_sigaction(int, struct k_sigaction *, struct k_sigaction *); static inline void restore_saved_sigmask(void) { if (test_and_clear_restore_sigmask()) __set_current_blocked(¤t->saved_sigmask); } static inline sigset_t *sigmask_to_save(void) { sigset_t *res = ¤t->blocked; if (unlikely(test_restore_sigmask())) res = ¤t->saved_sigmask; return res; } static inline int kill_cad_pid(int sig, int priv) { return kill_pid(cad_pid, sig, priv); } #define SEND_SIG_NOINFO ((struct siginfo *) 0) #define SEND_SIG_PRIV ((struct siginfo *) 1) #define SEND_SIG_FORCED ((struct siginfo *) 2) static inline int on_sig_stack(unsigned long sp) { #ifdef CONFIG_STACK_GROWSUP return sp >= current->sas_ss_sp && sp - current->sas_ss_sp < current->sas_ss_size; #else return sp > current->sas_ss_sp && sp - current->sas_ss_sp <= current->sas_ss_size; #endif } static inline int sas_ss_flags(unsigned long sp) { return (current->sas_ss_size == 0 ? SS_DISABLE : on_sig_stack(sp) ? SS_ONSTACK : 0); } static inline unsigned long sigsp(unsigned long sp, struct ksignal *ksig) { if (unlikely((ksig->ka.sa.sa_flags & SA_ONSTACK)) && ! sas_ss_flags(sp)) #ifdef CONFIG_STACK_GROWSUP return current->sas_ss_sp; #else return current->sas_ss_sp + current->sas_ss_size; #endif return sp; } extern struct mm_struct * mm_alloc(void); extern void __mmdrop(struct mm_struct *); static inline void mmdrop(struct mm_struct * mm) { if (unlikely(atomic_dec_and_test(&mm->mm_count))) __mmdrop(mm); } extern int mmput(struct mm_struct *); extern struct mm_struct *get_task_mm(struct task_struct *task); extern struct mm_struct *mm_access(struct task_struct *task, unsigned int mode); extern void mm_release(struct task_struct *, struct mm_struct *); extern struct mm_struct *dup_mm(struct task_struct *tsk); extern int copy_thread(unsigned long, unsigned long, unsigned long, struct task_struct *); extern void flush_thread(void); extern void exit_thread(void); extern void exit_files(struct task_struct *); extern void __cleanup_sighand(struct sighand_struct *); extern void exit_itimers(struct signal_struct *); extern void flush_itimer_signals(void); extern void do_group_exit(int); extern int allow_signal(int); extern int disallow_signal(int); extern int do_execve(const char *, const char __user * const __user *, const char __user * const __user *); extern long do_fork(unsigned long, unsigned long, unsigned long, int __user *, int __user *); struct task_struct *fork_idle(int); extern pid_t kernel_thread(int (*fn)(void *), void *arg, unsigned long flags); extern void set_task_comm(struct task_struct *tsk, char *from); extern char *get_task_comm(char *to, struct task_struct *tsk); #ifdef CONFIG_SMP void scheduler_ipi(void); extern unsigned long wait_task_inactive(struct task_struct *, long match_state); #else static inline void scheduler_ipi(void) { } static inline unsigned long wait_task_inactive(struct task_struct *p, long match_state) { return 1; } #endif #define next_task(p) \ list_entry_rcu((p)->tasks.next, struct task_struct, tasks) #define for_each_process(p) \ for (p = &init_task ; (p = next_task(p)) != &init_task ; ) extern bool current_is_single_threaded(void); #define do_each_thread(g, t) \ for (g = t = &init_task ; (g = t = next_task(g)) != &init_task ; ) do #define while_each_thread(g, t) \ while ((t = next_thread(t)) != g) #define __for_each_thread(signal, t) \ list_for_each_entry_rcu(t, &(signal)->thread_head, thread_node) #define for_each_thread(p, t) \ __for_each_thread((p)->signal, t) /* Careful: this is a double loop, 'break' won't work as expected. */ #define for_each_process_thread(p, t) \ for_each_process(p) for_each_thread(p, t) static inline int get_nr_threads(struct task_struct *tsk) { return tsk->signal->nr_threads; } static inline bool thread_group_leader(struct task_struct *p) { return p->exit_signal >= 0; } /* Do to the insanities of de_thread it is possible for a process * to have the pid of the thread group leader without actually being * the thread group leader. For iteration through the pids in proc * all we care about is that we have a task with the appropriate * pid, we don't actually care if we have the right task. */ static inline bool has_group_leader_pid(struct task_struct *p) { return task_pid(p) == p->signal->leader_pid; } static inline bool same_thread_group(struct task_struct *p1, struct task_struct *p2) { return p1->signal == p2->signal; } static inline struct task_struct *next_thread(const struct task_struct *p) { return list_entry_rcu(p->thread_group.next, struct task_struct, thread_group); } static inline int thread_group_empty(struct task_struct *p) { return list_empty(&p->thread_group); } #define delay_group_leader(p) \ (thread_group_leader(p) && !thread_group_empty(p)) static inline void task_lock(struct task_struct *p) { spin_lock(&p->alloc_lock); } static inline void task_unlock(struct task_struct *p) { spin_unlock(&p->alloc_lock); } extern struct sighand_struct *__lock_task_sighand(struct task_struct *tsk, unsigned long *flags); static inline struct sighand_struct *lock_task_sighand(struct task_struct *tsk, unsigned long *flags) { struct sighand_struct *ret; ret = __lock_task_sighand(tsk, flags); (void)__cond_lock(&tsk->sighand->siglock, ret); return ret; } static inline void unlock_task_sighand(struct task_struct *tsk, unsigned long *flags) { spin_unlock_irqrestore(&tsk->sighand->siglock, *flags); } #ifdef CONFIG_CGROUPS static inline void threadgroup_change_begin(struct task_struct *tsk) { down_read(&tsk->signal->group_rwsem); } static inline void threadgroup_change_end(struct task_struct *tsk) { up_read(&tsk->signal->group_rwsem); } static inline void threadgroup_lock(struct task_struct *tsk) { down_write(&tsk->signal->group_rwsem); } static inline void threadgroup_unlock(struct task_struct *tsk) { up_write(&tsk->signal->group_rwsem); } #else static inline void threadgroup_change_begin(struct task_struct *tsk) {} static inline void threadgroup_change_end(struct task_struct *tsk) {} static inline void threadgroup_lock(struct task_struct *tsk) {} static inline void threadgroup_unlock(struct task_struct *tsk) {} #endif #ifndef __HAVE_THREAD_FUNCTIONS #define task_thread_info(task) ((struct thread_info *)(task)->stack) #define task_stack_page(task) ((task)->stack) static inline void setup_thread_stack(struct task_struct *p, struct task_struct *org) { *task_thread_info(p) = *task_thread_info(org); task_thread_info(p)->task = p; } static inline unsigned long *end_of_stack(struct task_struct *p) { return (unsigned long *)(task_thread_info(p) + 1); } #endif static inline int object_is_on_stack(void *obj) { void *stack = task_stack_page(current); return (obj >= stack) && (obj < (stack + THREAD_SIZE)); } extern void thread_info_cache_init(void); #ifdef CONFIG_DEBUG_STACK_USAGE static inline unsigned long stack_not_used(struct task_struct *p) { unsigned long *n = end_of_stack(p); do { n++; } while (!*n); return (unsigned long)n - (unsigned long)end_of_stack(p); } #endif static inline void set_tsk_thread_flag(struct task_struct *tsk, int flag) { set_ti_thread_flag(task_thread_info(tsk), flag); } static inline void clear_tsk_thread_flag(struct task_struct *tsk, int flag) { clear_ti_thread_flag(task_thread_info(tsk), flag); } static inline int test_and_set_tsk_thread_flag(struct task_struct *tsk, int flag) { return test_and_set_ti_thread_flag(task_thread_info(tsk), flag); } static inline int test_and_clear_tsk_thread_flag(struct task_struct *tsk, int flag) { return test_and_clear_ti_thread_flag(task_thread_info(tsk), flag); } static inline int test_tsk_thread_flag(struct task_struct *tsk, int flag) { return test_ti_thread_flag(task_thread_info(tsk), flag); } static inline void set_tsk_need_resched(struct task_struct *tsk) { set_tsk_thread_flag(tsk,TIF_NEED_RESCHED); } static inline void clear_tsk_need_resched(struct task_struct *tsk) { clear_tsk_thread_flag(tsk,TIF_NEED_RESCHED); } static inline int test_tsk_need_resched(struct task_struct *tsk) { return unlikely(test_tsk_thread_flag(tsk,TIF_NEED_RESCHED)); } static inline int restart_syscall(void) { set_tsk_thread_flag(current, TIF_SIGPENDING); return -ERESTARTNOINTR; } static inline int signal_pending(struct task_struct *p) { return unlikely(test_tsk_thread_flag(p,TIF_SIGPENDING)); } static inline int __fatal_signal_pending(struct task_struct *p) { return unlikely(sigismember(&p->pending.signal, SIGKILL)); } static inline int fatal_signal_pending(struct task_struct *p) { return signal_pending(p) && __fatal_signal_pending(p); } static inline int signal_pending_state(long state, struct task_struct *p) { if (!(state & (TASK_INTERRUPTIBLE | TASK_WAKEKILL))) return 0; if (!signal_pending(p)) return 0; return (state & TASK_INTERRUPTIBLE) || __fatal_signal_pending(p); } static inline int need_resched(void) { return unlikely(test_thread_flag(TIF_NEED_RESCHED)); } extern int _cond_resched(void); #define cond_resched() ({ \ __might_sleep(__FILE__, __LINE__, 0); \ _cond_resched(); \ }) extern int __cond_resched_lock(spinlock_t *lock); #ifdef CONFIG_PREEMPT_COUNT #define PREEMPT_LOCK_OFFSET PREEMPT_OFFSET #else #define PREEMPT_LOCK_OFFSET 0 #endif #define cond_resched_lock(lock) ({ \ __might_sleep(__FILE__, __LINE__, PREEMPT_LOCK_OFFSET); \ __cond_resched_lock(lock); \ }) extern int __cond_resched_softirq(void); #define cond_resched_softirq() ({ \ __might_sleep(__FILE__, __LINE__, SOFTIRQ_DISABLE_OFFSET); \ __cond_resched_softirq(); \ }) static inline int spin_needbreak(spinlock_t *lock) { #ifdef CONFIG_PREEMPT return spin_is_contended(lock); #else return 0; #endif } #ifdef TS_POLLING static inline int tsk_is_polling(struct task_struct *p) { return task_thread_info(p)->status & TS_POLLING; } static inline void __current_set_polling(void) { current_thread_info()->status |= TS_POLLING; } static inline bool __must_check current_set_polling_and_test(void) { __current_set_polling(); smp_mb(); return unlikely(tif_need_resched()); } static inline void __current_clr_polling(void) { current_thread_info()->status &= ~TS_POLLING; } static inline bool __must_check current_clr_polling_and_test(void) { __current_clr_polling(); smp_mb(); return unlikely(tif_need_resched()); } #elif defined(TIF_POLLING_NRFLAG) static inline int tsk_is_polling(struct task_struct *p) { return test_tsk_thread_flag(p, TIF_POLLING_NRFLAG); } static inline void __current_set_polling(void) { set_thread_flag(TIF_POLLING_NRFLAG); } static inline bool __must_check current_set_polling_and_test(void) { __current_set_polling(); smp_mb__after_atomic(); return unlikely(tif_need_resched()); } static inline void __current_clr_polling(void) { clear_thread_flag(TIF_POLLING_NRFLAG); } static inline bool __must_check current_clr_polling_and_test(void) { __current_clr_polling(); smp_mb__after_atomic(); return unlikely(tif_need_resched()); } #else static inline int tsk_is_polling(struct task_struct *p) { return 0; } static inline void __current_set_polling(void) { } static inline void __current_clr_polling(void) { } static inline bool __must_check current_set_polling_and_test(void) { return unlikely(tif_need_resched()); } static inline bool __must_check current_clr_polling_and_test(void) { return unlikely(tif_need_resched()); } #endif void thread_group_cputime(struct task_struct *tsk, struct task_cputime *times); void thread_group_cputimer(struct task_struct *tsk, struct task_cputime *times); static inline void thread_group_cputime_init(struct signal_struct *sig) { raw_spin_lock_init(&sig->cputimer.lock); } extern void recalc_sigpending_and_wake(struct task_struct *t); extern void recalc_sigpending(void); extern void signal_wake_up_state(struct task_struct *t, unsigned int state); static inline void signal_wake_up(struct task_struct *t, bool resume) { signal_wake_up_state(t, resume ? TASK_WAKEKILL : 0); } static inline void ptrace_signal_wake_up(struct task_struct *t, bool resume) { signal_wake_up_state(t, resume ? __TASK_TRACED : 0); } #ifdef CONFIG_SMP static inline unsigned int task_cpu(const struct task_struct *p) { return task_thread_info(p)->cpu; } extern void set_task_cpu(struct task_struct *p, unsigned int cpu); #else static inline unsigned int task_cpu(const struct task_struct *p) { return 0; } static inline void set_task_cpu(struct task_struct *p, unsigned int cpu) { } #endif extern struct atomic_notifier_head migration_notifier_head; struct migration_notify_data { int src_cpu; int dest_cpu; int load; }; extern struct atomic_notifier_head load_alert_notifier_head; extern long sched_setaffinity(pid_t pid, const struct cpumask *new_mask); extern long sched_getaffinity(pid_t pid, struct cpumask *mask); #ifdef CONFIG_CGROUP_SCHED extern struct task_group root_task_group; #endif extern int task_can_switch_user(struct user_struct *up, struct task_struct *tsk); #ifdef CONFIG_TASK_XACCT static inline void add_rchar(struct task_struct *tsk, ssize_t amt) { tsk->ioac.rchar += amt; } static inline void add_wchar(struct task_struct *tsk, ssize_t amt) { tsk->ioac.wchar += amt; } static inline void inc_syscr(struct task_struct *tsk) { tsk->ioac.syscr++; } static inline void inc_syscw(struct task_struct *tsk) { tsk->ioac.syscw++; } #else static inline void add_rchar(struct task_struct *tsk, ssize_t amt) { } static inline void add_wchar(struct task_struct *tsk, ssize_t amt) { } static inline void inc_syscr(struct task_struct *tsk) { } static inline void inc_syscw(struct task_struct *tsk) { } #endif #ifndef TASK_SIZE_OF #define TASK_SIZE_OF(tsk) TASK_SIZE #endif #ifdef CONFIG_MM_OWNER extern void mm_update_next_owner(struct mm_struct *mm); extern void mm_init_owner(struct mm_struct *mm, struct task_struct *p); #else static inline void mm_update_next_owner(struct mm_struct *mm) { } static inline void mm_init_owner(struct mm_struct *mm, struct task_struct *p) { } #endif static inline unsigned long task_rlimit(const struct task_struct *tsk, unsigned int limit) { return ACCESS_ONCE(tsk->signal->rlim[limit].rlim_cur); } static inline unsigned long task_rlimit_max(const struct task_struct *tsk, unsigned int limit) { return ACCESS_ONCE(tsk->signal->rlim[limit].rlim_max); } static inline unsigned long rlimit(unsigned int limit) { return task_rlimit(current, limit); } static inline unsigned long rlimit_max(unsigned int limit) { return task_rlimit_max(current, limit); } #endif