1/* $NetBSD: xen_pmap.c,v 1.23 2016/11/21 04:10:05 ozaki-r Exp $ */
2
3/*
4 * Copyright (c) 2007 Manuel Bouyer.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 *
26 */
27
28/*
29 * Copyright (c) 2006 Mathieu Ropert <mro@adviseo.fr>
30 *
31 * Permission to use, copy, modify, and distribute this software for any
32 * purpose with or without fee is hereby granted, provided that the above
33 * copyright notice and this permission notice appear in all copies.
34 *
35 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
36 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
37 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
38 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
39 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
40 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
41 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
42 */
43
44/*
45 * Copyright (c) 1997 Charles D. Cranor and Washington University.
46 * All rights reserved.
47 *
48 * Redistribution and use in source and binary forms, with or without
49 * modification, are permitted provided that the following conditions
50 * are met:
51 * 1. Redistributions of source code must retain the above copyright
52 * notice, this list of conditions and the following disclaimer.
53 * 2. Redistributions in binary form must reproduce the above copyright
54 * notice, this list of conditions and the following disclaimer in the
55 * documentation and/or other materials provided with the distribution.
56 *
57 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
58 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
59 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
60 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
61 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
62 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
63 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
64 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
65 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
66 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
67 */
68
69/*
70 * Copyright 2001 (c) Wasabi Systems, Inc.
71 * All rights reserved.
72 *
73 * Written by Frank van der Linden for Wasabi Systems, Inc.
74 *
75 * Redistribution and use in source and binary forms, with or without
76 * modification, are permitted provided that the following conditions
77 * are met:
78 * 1. Redistributions of source code must retain the above copyright
79 * notice, this list of conditions and the following disclaimer.
80 * 2. Redistributions in binary form must reproduce the above copyright
81 * notice, this list of conditions and the following disclaimer in the
82 * documentation and/or other materials provided with the distribution.
83 * 3. All advertising materials mentioning features or use of this software
84 * must display the following acknowledgement:
85 * This product includes software developed for the NetBSD Project by
86 * Wasabi Systems, Inc.
87 * 4. The name of Wasabi Systems, Inc. may not be used to endorse
88 * or promote products derived from this software without specific prior
89 * written permission.
90 *
91 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND
92 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
93 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
94 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC
95 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
96 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
97 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
98 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
99 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
100 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
101 * POSSIBILITY OF SUCH DAMAGE.
102 */
103
104#include <sys/cdefs.h>
105__KERNEL_RCSID(0, "$NetBSD: xen_pmap.c,v 1.23 2016/11/21 04:10:05 ozaki-r Exp $");
106
107#include "opt_user_ldt.h"
108#include "opt_lockdebug.h"
109#include "opt_multiprocessor.h"
110#include "opt_xen.h"
111#if !defined(__x86_64__)
112#include "opt_kstack_dr0.h"
113#endif /* !defined(__x86_64__) */
114
115#include <sys/param.h>
116#include <sys/systm.h>
117#include <sys/proc.h>
118#include <sys/pool.h>
119#include <sys/kernel.h>
120#include <sys/atomic.h>
121#include <sys/cpu.h>
122#include <sys/intr.h>
123
124#include <uvm/uvm.h>
125
126#include <dev/isa/isareg.h>
127
128#include <machine/specialreg.h>
129#include <machine/gdt.h>
130#include <machine/isa_machdep.h>
131#include <machine/cpuvar.h>
132
133#include <x86/pmap.h>
134#include <x86/pmap_pv.h>
135
136#include <x86/i82489reg.h>
137#include <x86/i82489var.h>
138
139#include <xen/xen-public/xen.h>
140#include <xen/hypervisor.h>
141#include <xen/xenpmap.h>
142
143#define COUNT(x) /* nothing */
144
145extern pd_entry_t * const normal_pdes[];
146
147extern paddr_t pmap_pa_start; /* PA of first physical page for this domain */
148extern paddr_t pmap_pa_end; /* PA of last physical page for this domain */
149
150int
151pmap_enter(struct pmap *pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags)
152{
153 paddr_t ma;
154
155 if (__predict_false(pa < pmap_pa_start || pmap_pa_end <= pa)) {
156 ma = pa; /* XXX hack */
157 } else {
158 ma = xpmap_ptom(pa);
159 }
160
161 return pmap_enter_ma(pmap, va, ma, pa, prot, flags, DOMID_SELF);
162}
163
164/*
165 * pmap_kenter_ma: enter a kernel mapping without R/M (pv_entry) tracking
166 *
167 * => no need to lock anything, assume va is already allocated
168 * => should be faster than normal pmap enter function
169 * => we expect a MACHINE address
170 */
171
172void
173pmap_kenter_ma(vaddr_t va, paddr_t ma, vm_prot_t prot, u_int flags)
174{
175 pt_entry_t *pte, opte, npte;
176
177 if (va < VM_MIN_KERNEL_ADDRESS)
178 pte = vtopte(va);
179 else
180 pte = kvtopte(va);
181
182 npte = ma | ((prot & VM_PROT_WRITE) ? PG_RW : PG_RO) |
183 PG_V | PG_k;
184 if (flags & PMAP_NOCACHE)
185 npte |= PG_N;
186
187 if ((cpu_feature[2] & CPUID_NOX) && !(prot & VM_PROT_EXECUTE))
188 npte |= PG_NX;
189
190 opte = pmap_pte_testset (pte, npte); /* zap! */
191
192 if (pmap_valid_entry(opte)) {
193#if defined(MULTIPROCESSOR)
194 kpreempt_disable();
195 pmap_tlb_shootdown(pmap_kernel(), va, opte, TLBSHOOT_KENTER);
196 kpreempt_enable();
197#else
198 /* Don't bother deferring in the single CPU case. */
199 pmap_update_pg(va);
200#endif
201 }
202}
203
204/*
205 * pmap_extract_ma: extract a MA for the given VA
206 */
207
208bool
209pmap_extract_ma(struct pmap *pmap, vaddr_t va, paddr_t *pap)
210{
211 pt_entry_t *ptes, pte;
212 pd_entry_t pde;
213 pd_entry_t * const *pdes;
214 struct pmap *pmap2;
215
216 kpreempt_disable();
217 pmap_map_ptes(pmap, &pmap2, &ptes, &pdes);
218 if (!pmap_pdes_valid(va, pdes, &pde)) {
219 pmap_unmap_ptes(pmap, pmap2);
220 kpreempt_enable();
221 return false;
222 }
223
224 pte = ptes[pl1_i(va)];
225 pmap_unmap_ptes(pmap, pmap2);
226 kpreempt_enable();
227
228 if (__predict_true((pte & PG_V) != 0)) {
229 if (pap != NULL)
230 *pap = (pte & PG_FRAME) | (va & (NBPD_L1 - 1));
231 return true;
232 }
233
234 return false;
235}
236
237/*
238 * Xen pmap's handlers for save/restore
239 */
240void
241pmap_xen_suspend(void)
242{
243 pmap_unmap_recursive_entries();
244
245 xpq_flush_queue();
246}
247
248void
249pmap_xen_resume(void)
250{
251 pmap_map_recursive_entries();
252
253 xpq_flush_queue();
254}
255
256/*
257 * NetBSD uses L2 shadow pages to support PAE with Xen. However, Xen does not
258 * handle them correctly during save/restore, leading to incorrect page
259 * tracking and pinning during restore.
260 * For save/restore to succeed, two functions are introduced:
261 * - pmap_map_recursive_entries(), used by resume code to set the recursive
262 * mapping entries to their correct value
263 * - pmap_unmap_recursive_entries(), used by suspend code to clear all
264 * PDIR_SLOT_PTE entries
265 */
266void
267pmap_map_recursive_entries(void)
268{
269 int i;
270 struct pmap *pm;
271
272 mutex_enter(&pmaps_lock);
273 LIST_FOREACH(pm, &pmaps, pm_list) {
274 for (i = 0; i < PDP_SIZE; i++) {
275 xpq_queue_pte_update(
276 xpmap_ptom(pmap_pdirpa(pm, PDIR_SLOT_PTE + i)),
277 xpmap_ptom((pm)->pm_pdirpa[i]) | PG_V);
278 }
279 }
280 mutex_exit(&pmaps_lock);
281
282 for (i = 0; i < PDP_SIZE; i++) {
283 xpq_queue_pte_update(
284 xpmap_ptom(pmap_pdirpa(pmap_kernel(), PDIR_SLOT_PTE + i)),
285 xpmap_ptom(pmap_kernel()->pm_pdirpa[i]) | PG_V);
286 }
287}
288
289/*
290 * Unmap recursive entries found in pmaps. Required during Xen
291 * save/restore operations, as Xen does not handle recursive mappings
292 * properly.
293 */
294void
295pmap_unmap_recursive_entries(void)
296{
297 int i;
298 struct pmap *pm;
299
300 /*
301 * Invalidate pmap_pdp_cache as it contains L2-pinned objects with
302 * recursive entries.
303 * XXX jym@ : find a way to drain per-CPU caches to. pool_cache_inv
304 * does not do that.
305 */
306 pool_cache_invalidate(&pmap_pdp_cache);
307
308 mutex_enter(&pmaps_lock);
309 LIST_FOREACH(pm, &pmaps, pm_list) {
310 for (i = 0; i < PDP_SIZE; i++) {
311 xpq_queue_pte_update(
312 xpmap_ptom(pmap_pdirpa(pm, PDIR_SLOT_PTE + i)), 0);
313 }
314 }
315 mutex_exit(&pmaps_lock);
316
317 /* do it for pmap_kernel() too! */
318 for (i = 0; i < PDP_SIZE; i++) {
319 xpq_queue_pte_update(
320 xpmap_ptom(pmap_pdirpa(pmap_kernel(), PDIR_SLOT_PTE + i)),
321 0);
322 }
323}
324
325#if defined(PAE) || defined(__x86_64__)
326
327static __inline void
328pmap_kpm_setpte(struct cpu_info *ci, struct pmap *pmap, int index)
329{
330 KASSERT(mutex_owned(pmap->pm_lock));
331 KASSERT(mutex_owned(&ci->ci_kpm_mtx));
332 if (pmap == pmap_kernel()) {
333 KASSERT(index >= PDIR_SLOT_KERN);
334 }
335#ifdef PAE
336 xpq_queue_pte_update(
337 xpmap_ptetomach(&ci->ci_kpm_pdir[l2tol2(index)]),
338 pmap->pm_pdir[index]);
339#elif defined(__x86_64__)
340 xpq_queue_pte_update(
341 xpmap_ptetomach(&ci->ci_kpm_pdir[index]),
342 pmap->pm_pdir[index]);
343#endif /* PAE */
344 xpq_flush_queue();
345}
346
347/*
348 * Synchronise shadow pdir with the pmap on all cpus on which it is
349 * loaded.
350 */
351void
352xen_kpm_sync(struct pmap *pmap, int index)
353{
354 CPU_INFO_ITERATOR cii;
355 struct cpu_info *ci;
356
357 KASSERT(pmap != NULL);
358 KASSERT(kpreempt_disabled());
359
360 pmap_pte_flush();
361
362 for (CPU_INFO_FOREACH(cii, ci)) {
363 if (ci == NULL) {
364 continue;
365 }
366 cpuid_t cid = cpu_index(ci);
367 if (pmap != pmap_kernel() &&
368 !kcpuset_isset(pmap->pm_xen_ptp_cpus, cid))
369 continue;
370
371 /* take the lock and check again */
372 mutex_enter(&ci->ci_kpm_mtx);
373 if (pmap == pmap_kernel() ||
374 kcpuset_isset(pmap->pm_xen_ptp_cpus, cid)) {
375 pmap_kpm_setpte(ci, pmap, index);
376 }
377 mutex_exit(&ci->ci_kpm_mtx);
378 }
379}
380
381#endif /* PAE || __x86_64__ */
382