1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
|
/*
* Shadow Call Stack support.
*
* Copyright (C) 2018 Google LLC
*/
#include <linux/cpuhotplug.h>
#include <linux/mm.h>
#include <linux/mmzone.h>
#include <linux/slab.h>
#include <linux/scs.h>
#include <linux/vmstat.h>
#include <asm/scs.h>
#define SCS_END_MAGIC 0xaf0194819b1635f6UL
static inline void *__scs_base(struct task_struct *tsk)
{
return (void *)((uintptr_t)task_scs(tsk) & ~(SCS_SIZE - 1));
}
static inline void *scs_alloc(int node)
{
return kmalloc(SCS_SIZE, SCS_GFP);
}
static inline void scs_free(void *s)
{
kfree(s);
}
static struct page *__scs_page(struct task_struct *tsk)
{
return virt_to_page(__scs_base(tsk));
}
static inline unsigned long *scs_magic(struct task_struct *tsk)
{
return (unsigned long *)(__scs_base(tsk) + SCS_SIZE - sizeof(long));
}
static inline void scs_set_magic(struct task_struct *tsk)
{
*scs_magic(tsk) = SCS_END_MAGIC;
}
void scs_task_init(struct task_struct *tsk)
{
task_set_scs(tsk, NULL);
}
void scs_task_reset(struct task_struct *tsk)
{
task_set_scs(tsk, __scs_base(tsk));
}
void scs_set_init_magic(struct task_struct *tsk)
{
scs_save(tsk);
scs_set_magic(tsk);
scs_load(tsk);
}
static void scs_account(struct task_struct *tsk, int account)
{
mod_zone_page_state(page_zone(__scs_page(tsk)), NR_KERNEL_SCS_BYTES,
account * SCS_SIZE);
}
int scs_prepare(struct task_struct *tsk, int node)
{
void *s;
s = scs_alloc(node);
if (!s)
return -ENOMEM;
task_set_scs(tsk, s);
scs_set_magic(tsk);
scs_account(tsk, 1);
return 0;
}
#ifdef CONFIG_DEBUG_STACK_USAGE
static inline unsigned long scs_used(struct task_struct *tsk)
{
unsigned long *p = __scs_base(tsk);
unsigned long *end = scs_magic(tsk);
uintptr_t s = (uintptr_t)p;
while (p < end && *p)
p++;
return (uintptr_t)p - s;
}
static void scs_check_usage(struct task_struct *tsk)
{
static DEFINE_SPINLOCK(lock);
static unsigned long highest;
unsigned long used = scs_used(tsk);
if (used <= highest)
return;
spin_lock(&lock);
if (used > highest) {
pr_info("%s: highest shadow stack usage %lu bytes\n",
__func__, used);
highest = used;
}
spin_unlock(&lock);
}
#else
static inline void scs_check_usage(struct task_struct *tsk)
{
}
#endif
bool scs_corrupted(struct task_struct *tsk)
{
return *scs_magic(tsk) != SCS_END_MAGIC;
}
void scs_release(struct task_struct *tsk)
{
void *s;
s = __scs_base(tsk);
if (!s)
return;
BUG_ON(scs_corrupted(tsk));
scs_check_usage(tsk);
scs_account(tsk, -1);
scs_task_init(tsk);
scs_free(s);
}
|
