1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
|
/*
* linux/arch/arm/mm/context.c
*
* Copyright (C) 2002-2003 Deep Blue Solutions Ltd, all rights reserved.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*/
#include <linux/init.h>
#include <linux/sched.h>
#include <linux/mm.h>
#include <linux/smp.h>
#include <linux/percpu.h>
#include <asm/mmu_context.h>
#include <asm/thread_notify.h>
#include <asm/tlbflush.h>
#include <mach/msm_rtb.h>
static DEFINE_RAW_SPINLOCK(cpu_asid_lock);
unsigned int cpu_last_asid = ASID_FIRST_VERSION;
#ifdef CONFIG_SMP
DEFINE_PER_CPU(struct mm_struct *, current_mm);
#endif
#ifdef CONFIG_ARM_LPAE
#define cpu_set_asid(asid) { \
unsigned long ttbl, ttbh; \
asm volatile( \
" mrrc p15, 0, %0, %1, c2 @ read TTBR0\n" \
" mov %1, %2, lsl #(48 - 32) @ set ASID\n" \
" mcrr p15, 0, %0, %1, c2 @ set TTBR0\n" \
: "=&r" (ttbl), "=&r" (ttbh) \
: "r" (asid & ~ASID_MASK)); \
}
#else
#define cpu_set_asid(asid) \
asm(" mcr p15, 0, %0, c13, c0, 1\n" : : "r" (asid))
#endif
static void write_contextidr(u32 contextidr)
{
uncached_logk(LOGK_CTXID, (void *)contextidr);
asm("mcr p15, 0, %0, c13, c0, 1" : : "r" (contextidr));
isb();
}
#ifdef CONFIG_PID_IN_CONTEXTIDR
static u32 read_contextidr(void)
{
u32 contextidr;
asm("mrc p15, 0, %0, c13, c0, 1" : "=r" (contextidr));
return contextidr;
}
static int contextidr_notifier(struct notifier_block *unused, unsigned long cmd,
void *t)
{
unsigned long flags;
u32 contextidr;
pid_t pid;
struct thread_info *thread = t;
if (cmd != THREAD_NOTIFY_SWITCH)
return NOTIFY_DONE;
pid = task_pid_nr(thread->task);
local_irq_save(flags);
contextidr = read_contextidr();
contextidr &= ~ASID_MASK;
contextidr |= pid << ASID_BITS;
write_contextidr(contextidr);
local_irq_restore(flags);
return NOTIFY_OK;
}
static struct notifier_block contextidr_notifier_block = {
.notifier_call = contextidr_notifier,
};
static int __init contextidr_notifier_init(void)
{
return thread_register_notifier(&contextidr_notifier_block);
}
arch_initcall(contextidr_notifier_init);
static void set_asid(unsigned int asid)
{
u32 contextidr = read_contextidr();
contextidr &= ASID_MASK;
contextidr |= asid & ~ASID_MASK;
write_contextidr(contextidr);
}
#else
static void set_asid(unsigned int asid)
{
write_contextidr(asid);
}
#endif
/*
* We fork()ed a process, and we need a new context for the child
* to run in. We reserve version 0 for initial tasks so we will
* always allocate an ASID. The ASID 0 is reserved for the TTBR
* register changing sequence.
*/
void __init_new_context(struct task_struct *tsk, struct mm_struct *mm)
{
mm->context.id = 0;
raw_spin_lock_init(&mm->context.id_lock);
}
static void flush_context(void)
{
/* set the reserved ASID before flushing the TLB */
set_asid(0);
local_flush_tlb_all();
if (icache_is_vivt_asid_tagged()) {
__flush_icache_all();
dsb();
}
}
#ifdef CONFIG_SMP
static void set_mm_context(struct mm_struct *mm, unsigned int asid)
{
unsigned long flags;
/*
* Locking needed for multi-threaded applications where the
* same mm->context.id could be set from different CPUs during
* the broadcast. This function is also called via IPI so the
* mm->context.id_lock has to be IRQ-safe.
*/
raw_spin_lock_irqsave(&mm->context.id_lock, flags);
if (likely((mm->context.id ^ cpu_last_asid) >> ASID_BITS)) {
/*
* Old version of ASID found. Set the new one and
* reset mm_cpumask(mm).
*/
mm->context.id = asid;
cpumask_clear(mm_cpumask(mm));
}
raw_spin_unlock_irqrestore(&mm->context.id_lock, flags);
/*
* Set the mm_cpumask(mm) bit for the current CPU.
*/
cpumask_set_cpu(smp_processor_id(), mm_cpumask(mm));
}
/*
* Reset the ASID on the current CPU. This function call is broadcast
* from the CPU handling the ASID rollover and holding cpu_asid_lock.
*/
static void reset_context(void *info)
{
unsigned int asid;
unsigned int cpu = smp_processor_id();
struct mm_struct *mm = per_cpu(current_mm, cpu);
/*
* Check if a current_mm was set on this CPU as it might still
* be in the early booting stages and using the reserved ASID.
*/
if (!mm)
return;
smp_rmb();
asid = cpu_last_asid + cpu + 1;
flush_context();
set_mm_context(mm, asid);
/* set the new ASID */
set_asid(mm->context.id);
}
#else
static inline void set_mm_context(struct mm_struct *mm, unsigned int asid)
{
mm->context.id = asid;
cpumask_copy(mm_cpumask(mm), cpumask_of(smp_processor_id()));
}
#endif
void __new_context(struct mm_struct *mm)
{
unsigned int asid;
raw_spin_lock(&cpu_asid_lock);
#ifdef CONFIG_SMP
/*
* Check the ASID again, in case the change was broadcast from
* another CPU before we acquired the lock.
*/
if (unlikely(((mm->context.id ^ cpu_last_asid) >> ASID_BITS) == 0)) {
cpumask_set_cpu(smp_processor_id(), mm_cpumask(mm));
raw_spin_unlock(&cpu_asid_lock);
return;
}
#endif
/*
* At this point, it is guaranteed that the current mm (with
* an old ASID) isn't active on any other CPU since the ASIDs
* are changed simultaneously via IPI.
*/
asid = ++cpu_last_asid;
if (asid == 0)
asid = cpu_last_asid = ASID_FIRST_VERSION;
/*
* If we've used up all our ASIDs, we need
* to start a new version and flush the TLB.
*/
if (unlikely((asid & ~ASID_MASK) == 0)) {
asid = cpu_last_asid + smp_processor_id() + 1;
flush_context();
#ifdef CONFIG_SMP
smp_wmb();
smp_call_function(reset_context, NULL, 1);
#endif
cpu_last_asid += NR_CPUS;
}
set_mm_context(mm, asid);
raw_spin_unlock(&cpu_asid_lock);
}
|
