1/* $NetBSD: xen_bus_dma.c,v 1.26 2012/06/30 23:36:20 jym Exp $ */
2/* NetBSD bus_dma.c,v 1.21 2005/04/16 07:53:35 yamt Exp */
3
4/*-
5 * Copyright (c) 1996, 1997, 1998 The NetBSD Foundation, Inc.
6 * All rights reserved.
7 *
8 * This code is derived from software contributed to The NetBSD Foundation
9 * by Charles M. Hannum and by Jason R. Thorpe of the Numerical Aerospace
10 * Simulation Facility, NASA Ames Research Center.
11 *
12 * Redistribution and use in source and binary forms, with or without
13 * modification, are permitted provided that the following conditions
14 * are met:
15 * 1. Redistributions of source code must retain the above copyright
16 * notice, this list of conditions and the following disclaimer.
17 * 2. Redistributions in binary form must reproduce the above copyright
18 * notice, this list of conditions and the following disclaimer in the
19 * documentation and/or other materials provided with the distribution.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
23 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
24 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
25 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
31 * POSSIBILITY OF SUCH DAMAGE.
32 */
33
34#include <sys/cdefs.h>
35__KERNEL_RCSID(0, "$NetBSD: xen_bus_dma.c,v 1.26 2012/06/30 23:36:20 jym Exp $");
36
37#include <sys/param.h>
38#include <sys/systm.h>
39#include <sys/kernel.h>
40#include <sys/mbuf.h>
41#include <sys/proc.h>
42
43#include <sys/bus.h>
44#include <machine/bus_private.h>
45
46#include <uvm/uvm.h>
47
48extern paddr_t avail_end;
49
50/* Pure 2^n version of get_order */
51static inline int get_order(unsigned long size)
52{
53 int order = -1;
54 size = (size - 1) >> (PAGE_SHIFT - 1);
55 do {
56 size >>= 1;
57 order++;
58 } while (size);
59 return order;
60}
61
62static int
63_xen_alloc_contig(bus_size_t size, bus_size_t alignment,
64 struct pglist *mlistp, int flags, bus_addr_t low, bus_addr_t high)
65{
66 int order, i;
67 unsigned long npagesreq, npages, mfn;
68 bus_addr_t pa;
69 struct vm_page *pg, *pgnext;
70 int s, error;
71 struct xen_memory_reservation res;
72
73 /*
74 * When requesting a contigous memory region, the hypervisor will
75 * return a memory range aligned on size.
76 * The only way to enforce alignment is to request a memory region
77 * of size max(alignment, size).
78 */
79 order = max(get_order(size), get_order(alignment));
80 npages = (1 << order);
81 npagesreq = (size >> PAGE_SHIFT);
82 KASSERT(npages >= npagesreq);
83
84 /* get npages from UVM, and give them back to the hypervisor */
85 error = uvm_pglistalloc(((psize_t)npages) << PAGE_SHIFT,
86 0, avail_end, 0, 0, mlistp, npages, (flags & BUS_DMA_NOWAIT) == 0);
87 if (error)
88 return (error);
89
90 for (pg = mlistp->tqh_first; pg != NULL; pg = pg->pageq.queue.tqe_next) {
91 pa = VM_PAGE_TO_PHYS(pg);
92 mfn = xpmap_ptom(pa) >> PAGE_SHIFT;
93 xpmap_ptom_unmap(pa);
94 set_xen_guest_handle(res.extent_start, &mfn);
95 res.nr_extents = 1;
96 res.extent_order = 0;
97 res.address_bits = 0;
98 res.domid = DOMID_SELF;
99 error = HYPERVISOR_memory_op(XENMEM_decrease_reservation, &res);
100 if (error != 1) {
101#ifdef DEBUG
102 printf("xen_alloc_contig: XENMEM_decrease_reservation "
103 "failed: err %d (pa %#" PRIxPADDR " mfn %#lx)\n",
104 error, pa, mfn);
105#endif
106 xpmap_ptom_map(pa, ptoa(mfn));
107
108 error = ENOMEM;
109 goto failed;
110 }
111 }
112 /* Get the new contiguous memory extent */
113 set_xen_guest_handle(res.extent_start, &mfn);
114 res.nr_extents = 1;
115 res.extent_order = order;
116 res.address_bits = get_order(high) + PAGE_SHIFT;
117 res.domid = DOMID_SELF;
118 error = HYPERVISOR_memory_op(XENMEM_increase_reservation, &res);
119 if (error != 1) {
120#ifdef DEBUG
121 printf("xen_alloc_contig: XENMEM_increase_reservation "
122 "failed: %d (order %d address_bits %d)\n",
123 error, order, res.address_bits);
124#endif
125 error = ENOMEM;
126 pg = NULL;
127 goto failed;
128 }
129 s = splvm();
130 /* Map the new extent in place of the old pages */
131 for (pg = mlistp->tqh_first, i = 0; pg != NULL; pg = pgnext, i++) {
132 pgnext = pg->pageq.queue.tqe_next;
133 pa = VM_PAGE_TO_PHYS(pg);
134 xpmap_ptom_map(pa, ptoa(mfn+i));
135 xpq_queue_machphys_update(((paddr_t)(mfn+i)) << PAGE_SHIFT, pa);
136 /* while here, give extra pages back to UVM */
137 if (i >= npagesreq) {
138 TAILQ_REMOVE(mlistp, pg, pageq.queue);
139 uvm_pagefree(pg);
140 }
141 }
142 /* Flush updates through and flush the TLB */
143 xpq_queue_tlb_flush();
144 splx(s);
145 return 0;
146
147failed:
148 /*
149 * Attempt to recover from a failed decrease or increase reservation:
150 * if decrease_reservation failed, we don't have given all pages
151 * back to Xen; give them back to UVM, and get the missing pages
152 * from Xen.
153 * if increase_reservation failed, we expect pg to be NULL and we just
154 * get back the missing pages from Xen one by one.
155 */
156 /* give back remaining pages to UVM */
157 for (; pg != NULL; pg = pgnext) {
158 pgnext = pg->pageq.queue.tqe_next;
159 TAILQ_REMOVE(mlistp, pg, pageq.queue);
160 uvm_pagefree(pg);
161 }
162 /* remplace the pages that we already gave to Xen */
163 s = splvm();
164 for (pg = mlistp->tqh_first; pg != NULL; pg = pgnext) {
165 pgnext = pg->pageq.queue.tqe_next;
166 set_xen_guest_handle(res.extent_start, &mfn);
167 res.nr_extents = 1;
168 res.extent_order = 0;
169 res.address_bits = 32;
170 res.domid = DOMID_SELF;
171 if (HYPERVISOR_memory_op(XENMEM_increase_reservation, &res)
172 < 0) {
173 printf("xen_alloc_contig: recovery "
174 "XENMEM_increase_reservation failed!\n");
175 break;
176 }
177 pa = VM_PAGE_TO_PHYS(pg);
178 xpmap_ptom_map(pa, ptoa(mfn));
179 xpq_queue_machphys_update(((paddr_t)mfn) << PAGE_SHIFT, pa);
180 TAILQ_REMOVE(mlistp, pg, pageq.queue);
181 uvm_pagefree(pg);
182 }
183 /* Flush updates through and flush the TLB */
184 xpq_queue_tlb_flush();
185 splx(s);
186 return error;
187}
188
189
190/*
191 * Allocate physical memory from the given physical address range.
192 * Called by DMA-safe memory allocation methods.
193 * We need our own version to deal with physical vs machine addresses.
194 */
195int
196_xen_bus_dmamem_alloc_range(bus_dma_tag_t t, bus_size_t size,
197 bus_size_t alignment, bus_size_t boundary, bus_dma_segment_t *segs,
198 int nsegs, int *rsegs, int flags, bus_addr_t low, bus_addr_t high)
199{
200 bus_addr_t curaddr, lastaddr;
201 struct vm_page *m;
202 struct pglist mlist;
203 int curseg, error;
204 int doingrealloc = 0;
205 bus_size_t uboundary;
206
207 /* Always round the size. */
208 size = round_page(size);
209
210 KASSERT((alignment & (alignment - 1)) == 0);
211 KASSERT((boundary & (boundary - 1)) == 0);
212 KASSERT(boundary >= PAGE_SIZE || boundary == 0);
213
214 if (alignment < PAGE_SIZE)
215 alignment = PAGE_SIZE;
216
217 /*
218 * Allocate pages from the VM system.
219 * We accept boundaries < size, splitting in multiple segments
220 * if needed. uvm_pglistalloc does not, so compute an appropriate
221 * boundary: next power of 2 >= size
222 */
223 if (boundary == 0)
224 uboundary = 0;
225 else {
226 uboundary = boundary;
227 while (uboundary < size)
228 uboundary = uboundary << 1;
229 }
230 error = uvm_pglistalloc(size, 0, avail_end, alignment, uboundary,
231 &mlist, nsegs, (flags & BUS_DMA_NOWAIT) == 0);
232 if (error)
233 return (error);
234again:
235
236 /*
237 * Compute the location, size, and number of segments actually
238 * returned by the VM code.
239 */
240 m = mlist.tqh_first;
241 curseg = 0;
242 curaddr = lastaddr = segs[curseg].ds_addr = _BUS_VM_PAGE_TO_BUS(m);
243 if (curaddr < low || curaddr >= high)
244 goto badaddr;
245 segs[curseg].ds_len = PAGE_SIZE;
246 m = m->pageq.queue.tqe_next;
247 if ((segs[curseg].ds_addr & (alignment - 1)) != 0)
248 goto dorealloc;
249
250 for (; m != NULL; m = m->pageq.queue.tqe_next) {
251 curaddr = _BUS_VM_PAGE_TO_BUS(m);
252 if (curaddr < low || curaddr >= high)
253 goto badaddr;
254 if (curaddr == (lastaddr + PAGE_SIZE) &&
255 (lastaddr & boundary) == (curaddr & boundary)) {
256 segs[curseg].ds_len += PAGE_SIZE;
257 } else {
258 curseg++;
259 if (curseg >= nsegs ||
260 (curaddr & (alignment - 1)) != 0) {
261 if (doingrealloc)
262 return EFBIG;
263 else
264 goto dorealloc;
265 }
266 segs[curseg].ds_addr = curaddr;
267 segs[curseg].ds_len = PAGE_SIZE;
268 }
269 lastaddr = curaddr;
270 }
271
272 *rsegs = curseg + 1;
273 return (0);
274
275badaddr:
276 if (doingrealloc == 0)
277 goto dorealloc;
278 if (curaddr < low) {
279 /* no way to enforce this */
280 printf("_xen_bus_dmamem_alloc_range: no way to "
281 "enforce address range (0x%" PRIx64 " - 0x%" PRIx64 ")\n",
282 (uint64_t)low, (uint64_t)high);
283 uvm_pglistfree(&mlist);
284 return EINVAL;
285 }
286 printf("xen_bus_dmamem_alloc_range: "
287 "curraddr=0x%lx > high=0x%lx\n",
288 (u_long)curaddr, (u_long)high);
289 panic("xen_bus_dmamem_alloc_range 1");
290dorealloc:
291 if (doingrealloc == 1)
292 panic("_xen_bus_dmamem_alloc_range: "
293 "xen_alloc_contig returned "
294 "too much segments");
295 doingrealloc = 1;
296 /*
297 * Too much segments, or memory doesn't fit
298 * constraints. Free this memory and
299 * get a contigous segment from the hypervisor.
300 */
301 uvm_pglistfree(&mlist);
302 for (curseg = 0; curseg < nsegs; curseg++) {
303 segs[curseg].ds_addr = 0;
304 segs[curseg].ds_len = 0;
305 }
306 error = _xen_alloc_contig(size, alignment,
307 &mlist, flags, low, high);
308 if (error)
309 return error;
310 goto again;
311}
312