1 | /* $NetBSD: xen_pmap.c,v 1.23 2016/11/21 04:10:05 ozaki-r Exp $ */ |
2 | |
3 | /* |
4 | * Copyright (c) 2007 Manuel Bouyer. |
5 | * |
6 | * Redistribution and use in source and binary forms, with or without |
7 | * modification, are permitted provided that the following conditions |
8 | * are met: |
9 | * 1. Redistributions of source code must retain the above copyright |
10 | * notice, this list of conditions and the following disclaimer. |
11 | * 2. Redistributions in binary form must reproduce the above copyright |
12 | * notice, this list of conditions and the following disclaimer in the |
13 | * documentation and/or other materials provided with the distribution. |
14 | * |
15 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR |
16 | * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES |
17 | * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. |
18 | * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, |
19 | * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT |
20 | * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
21 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
22 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
23 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF |
24 | * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
25 | * |
26 | */ |
27 | |
28 | /* |
29 | * Copyright (c) 2006 Mathieu Ropert <mro@adviseo.fr> |
30 | * |
31 | * Permission to use, copy, modify, and distribute this software for any |
32 | * purpose with or without fee is hereby granted, provided that the above |
33 | * copyright notice and this permission notice appear in all copies. |
34 | * |
35 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES |
36 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF |
37 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR |
38 | * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES |
39 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN |
40 | * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF |
41 | * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. |
42 | */ |
43 | |
44 | /* |
45 | * Copyright (c) 1997 Charles D. Cranor and Washington University. |
46 | * All rights reserved. |
47 | * |
48 | * Redistribution and use in source and binary forms, with or without |
49 | * modification, are permitted provided that the following conditions |
50 | * are met: |
51 | * 1. Redistributions of source code must retain the above copyright |
52 | * notice, this list of conditions and the following disclaimer. |
53 | * 2. Redistributions in binary form must reproduce the above copyright |
54 | * notice, this list of conditions and the following disclaimer in the |
55 | * documentation and/or other materials provided with the distribution. |
56 | * |
57 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR |
58 | * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES |
59 | * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. |
60 | * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, |
61 | * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT |
62 | * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
63 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
64 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
65 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF |
66 | * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
67 | */ |
68 | |
69 | /* |
70 | * Copyright 2001 (c) Wasabi Systems, Inc. |
71 | * All rights reserved. |
72 | * |
73 | * Written by Frank van der Linden for Wasabi Systems, Inc. |
74 | * |
75 | * Redistribution and use in source and binary forms, with or without |
76 | * modification, are permitted provided that the following conditions |
77 | * are met: |
78 | * 1. Redistributions of source code must retain the above copyright |
79 | * notice, this list of conditions and the following disclaimer. |
80 | * 2. Redistributions in binary form must reproduce the above copyright |
81 | * notice, this list of conditions and the following disclaimer in the |
82 | * documentation and/or other materials provided with the distribution. |
83 | * 3. All advertising materials mentioning features or use of this software |
84 | * must display the following acknowledgement: |
85 | * This product includes software developed for the NetBSD Project by |
86 | * Wasabi Systems, Inc. |
87 | * 4. The name of Wasabi Systems, Inc. may not be used to endorse |
88 | * or promote products derived from this software without specific prior |
89 | * written permission. |
90 | * |
91 | * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND |
92 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED |
93 | * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
94 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC |
95 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR |
96 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF |
97 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS |
98 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN |
99 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
100 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
101 | * POSSIBILITY OF SUCH DAMAGE. |
102 | */ |
103 | |
104 | #include <sys/cdefs.h> |
105 | __KERNEL_RCSID(0, "$NetBSD: xen_pmap.c,v 1.23 2016/11/21 04:10:05 ozaki-r Exp $" ); |
106 | |
107 | #include "opt_user_ldt.h" |
108 | #include "opt_lockdebug.h" |
109 | #include "opt_multiprocessor.h" |
110 | #include "opt_xen.h" |
111 | #if !defined(__x86_64__) |
112 | #include "opt_kstack_dr0.h" |
113 | #endif /* !defined(__x86_64__) */ |
114 | |
115 | #include <sys/param.h> |
116 | #include <sys/systm.h> |
117 | #include <sys/proc.h> |
118 | #include <sys/pool.h> |
119 | #include <sys/kernel.h> |
120 | #include <sys/atomic.h> |
121 | #include <sys/cpu.h> |
122 | #include <sys/intr.h> |
123 | |
124 | #include <uvm/uvm.h> |
125 | |
126 | #include <dev/isa/isareg.h> |
127 | |
128 | #include <machine/specialreg.h> |
129 | #include <machine/gdt.h> |
130 | #include <machine/isa_machdep.h> |
131 | #include <machine/cpuvar.h> |
132 | |
133 | #include <x86/pmap.h> |
134 | #include <x86/pmap_pv.h> |
135 | |
136 | #include <x86/i82489reg.h> |
137 | #include <x86/i82489var.h> |
138 | |
139 | #include <xen/xen-public/xen.h> |
140 | #include <xen/hypervisor.h> |
141 | #include <xen/xenpmap.h> |
142 | |
143 | #define COUNT(x) /* nothing */ |
144 | |
145 | extern pd_entry_t * const normal_pdes[]; |
146 | |
147 | extern paddr_t pmap_pa_start; /* PA of first physical page for this domain */ |
148 | extern paddr_t pmap_pa_end; /* PA of last physical page for this domain */ |
149 | |
150 | int |
151 | pmap_enter(struct pmap *pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) |
152 | { |
153 | paddr_t ma; |
154 | |
155 | if (__predict_false(pa < pmap_pa_start || pmap_pa_end <= pa)) { |
156 | ma = pa; /* XXX hack */ |
157 | } else { |
158 | ma = xpmap_ptom(pa); |
159 | } |
160 | |
161 | return pmap_enter_ma(pmap, va, ma, pa, prot, flags, DOMID_SELF); |
162 | } |
163 | |
164 | /* |
165 | * pmap_kenter_ma: enter a kernel mapping without R/M (pv_entry) tracking |
166 | * |
167 | * => no need to lock anything, assume va is already allocated |
168 | * => should be faster than normal pmap enter function |
169 | * => we expect a MACHINE address |
170 | */ |
171 | |
172 | void |
173 | pmap_kenter_ma(vaddr_t va, paddr_t ma, vm_prot_t prot, u_int flags) |
174 | { |
175 | pt_entry_t *pte, opte, npte; |
176 | |
177 | if (va < VM_MIN_KERNEL_ADDRESS) |
178 | pte = vtopte(va); |
179 | else |
180 | pte = kvtopte(va); |
181 | |
182 | npte = ma | ((prot & VM_PROT_WRITE) ? PG_RW : PG_RO) | |
183 | PG_V | PG_k; |
184 | if (flags & PMAP_NOCACHE) |
185 | npte |= PG_N; |
186 | |
187 | if ((cpu_feature[2] & CPUID_NOX) && !(prot & VM_PROT_EXECUTE)) |
188 | npte |= PG_NX; |
189 | |
190 | opte = pmap_pte_testset (pte, npte); /* zap! */ |
191 | |
192 | if (pmap_valid_entry(opte)) { |
193 | #if defined(MULTIPROCESSOR) |
194 | kpreempt_disable(); |
195 | pmap_tlb_shootdown(pmap_kernel(), va, opte, TLBSHOOT_KENTER); |
196 | kpreempt_enable(); |
197 | #else |
198 | /* Don't bother deferring in the single CPU case. */ |
199 | pmap_update_pg(va); |
200 | #endif |
201 | } |
202 | } |
203 | |
204 | /* |
205 | * pmap_extract_ma: extract a MA for the given VA |
206 | */ |
207 | |
208 | bool |
209 | (struct pmap *pmap, vaddr_t va, paddr_t *pap) |
210 | { |
211 | pt_entry_t *ptes, pte; |
212 | pd_entry_t pde; |
213 | pd_entry_t * const *pdes; |
214 | struct pmap *pmap2; |
215 | |
216 | kpreempt_disable(); |
217 | pmap_map_ptes(pmap, &pmap2, &ptes, &pdes); |
218 | if (!pmap_pdes_valid(va, pdes, &pde)) { |
219 | pmap_unmap_ptes(pmap, pmap2); |
220 | kpreempt_enable(); |
221 | return false; |
222 | } |
223 | |
224 | pte = ptes[pl1_i(va)]; |
225 | pmap_unmap_ptes(pmap, pmap2); |
226 | kpreempt_enable(); |
227 | |
228 | if (__predict_true((pte & PG_V) != 0)) { |
229 | if (pap != NULL) |
230 | *pap = (pte & PG_FRAME) | (va & (NBPD_L1 - 1)); |
231 | return true; |
232 | } |
233 | |
234 | return false; |
235 | } |
236 | |
237 | /* |
238 | * Xen pmap's handlers for save/restore |
239 | */ |
240 | void |
241 | pmap_xen_suspend(void) |
242 | { |
243 | pmap_unmap_recursive_entries(); |
244 | |
245 | xpq_flush_queue(); |
246 | } |
247 | |
248 | void |
249 | pmap_xen_resume(void) |
250 | { |
251 | pmap_map_recursive_entries(); |
252 | |
253 | xpq_flush_queue(); |
254 | } |
255 | |
256 | /* |
257 | * NetBSD uses L2 shadow pages to support PAE with Xen. However, Xen does not |
258 | * handle them correctly during save/restore, leading to incorrect page |
259 | * tracking and pinning during restore. |
260 | * For save/restore to succeed, two functions are introduced: |
261 | * - pmap_map_recursive_entries(), used by resume code to set the recursive |
262 | * mapping entries to their correct value |
263 | * - pmap_unmap_recursive_entries(), used by suspend code to clear all |
264 | * PDIR_SLOT_PTE entries |
265 | */ |
266 | void |
267 | pmap_map_recursive_entries(void) |
268 | { |
269 | int i; |
270 | struct pmap *pm; |
271 | |
272 | mutex_enter(&pmaps_lock); |
273 | LIST_FOREACH(pm, &pmaps, pm_list) { |
274 | for (i = 0; i < PDP_SIZE; i++) { |
275 | xpq_queue_pte_update( |
276 | xpmap_ptom(pmap_pdirpa(pm, PDIR_SLOT_PTE + i)), |
277 | xpmap_ptom((pm)->pm_pdirpa[i]) | PG_V); |
278 | } |
279 | } |
280 | mutex_exit(&pmaps_lock); |
281 | |
282 | for (i = 0; i < PDP_SIZE; i++) { |
283 | xpq_queue_pte_update( |
284 | xpmap_ptom(pmap_pdirpa(pmap_kernel(), PDIR_SLOT_PTE + i)), |
285 | xpmap_ptom(pmap_kernel()->pm_pdirpa[i]) | PG_V); |
286 | } |
287 | } |
288 | |
289 | /* |
290 | * Unmap recursive entries found in pmaps. Required during Xen |
291 | * save/restore operations, as Xen does not handle recursive mappings |
292 | * properly. |
293 | */ |
294 | void |
295 | pmap_unmap_recursive_entries(void) |
296 | { |
297 | int i; |
298 | struct pmap *pm; |
299 | |
300 | /* |
301 | * Invalidate pmap_pdp_cache as it contains L2-pinned objects with |
302 | * recursive entries. |
303 | * XXX jym@ : find a way to drain per-CPU caches to. pool_cache_inv |
304 | * does not do that. |
305 | */ |
306 | pool_cache_invalidate(&pmap_pdp_cache); |
307 | |
308 | mutex_enter(&pmaps_lock); |
309 | LIST_FOREACH(pm, &pmaps, pm_list) { |
310 | for (i = 0; i < PDP_SIZE; i++) { |
311 | xpq_queue_pte_update( |
312 | xpmap_ptom(pmap_pdirpa(pm, PDIR_SLOT_PTE + i)), 0); |
313 | } |
314 | } |
315 | mutex_exit(&pmaps_lock); |
316 | |
317 | /* do it for pmap_kernel() too! */ |
318 | for (i = 0; i < PDP_SIZE; i++) { |
319 | xpq_queue_pte_update( |
320 | xpmap_ptom(pmap_pdirpa(pmap_kernel(), PDIR_SLOT_PTE + i)), |
321 | 0); |
322 | } |
323 | } |
324 | |
325 | #if defined(PAE) || defined(__x86_64__) |
326 | |
327 | static __inline void |
328 | pmap_kpm_setpte(struct cpu_info *ci, struct pmap *pmap, int index) |
329 | { |
330 | KASSERT(mutex_owned(pmap->pm_lock)); |
331 | KASSERT(mutex_owned(&ci->ci_kpm_mtx)); |
332 | if (pmap == pmap_kernel()) { |
333 | KASSERT(index >= PDIR_SLOT_KERN); |
334 | } |
335 | #ifdef PAE |
336 | xpq_queue_pte_update( |
337 | xpmap_ptetomach(&ci->ci_kpm_pdir[l2tol2(index)]), |
338 | pmap->pm_pdir[index]); |
339 | #elif defined(__x86_64__) |
340 | xpq_queue_pte_update( |
341 | xpmap_ptetomach(&ci->ci_kpm_pdir[index]), |
342 | pmap->pm_pdir[index]); |
343 | #endif /* PAE */ |
344 | xpq_flush_queue(); |
345 | } |
346 | |
347 | /* |
348 | * Synchronise shadow pdir with the pmap on all cpus on which it is |
349 | * loaded. |
350 | */ |
351 | void |
352 | xen_kpm_sync(struct pmap *pmap, int index) |
353 | { |
354 | CPU_INFO_ITERATOR cii; |
355 | struct cpu_info *ci; |
356 | |
357 | KASSERT(pmap != NULL); |
358 | KASSERT(kpreempt_disabled()); |
359 | |
360 | pmap_pte_flush(); |
361 | |
362 | for (CPU_INFO_FOREACH(cii, ci)) { |
363 | if (ci == NULL) { |
364 | continue; |
365 | } |
366 | cpuid_t cid = cpu_index(ci); |
367 | if (pmap != pmap_kernel() && |
368 | !kcpuset_isset(pmap->pm_xen_ptp_cpus, cid)) |
369 | continue; |
370 | |
371 | /* take the lock and check again */ |
372 | mutex_enter(&ci->ci_kpm_mtx); |
373 | if (pmap == pmap_kernel() || |
374 | kcpuset_isset(pmap->pm_xen_ptp_cpus, cid)) { |
375 | pmap_kpm_setpte(ci, pmap, index); |
376 | } |
377 | mutex_exit(&ci->ci_kpm_mtx); |
378 | } |
379 | } |
380 | |
381 | #endif /* PAE || __x86_64__ */ |
382 | |