4 * This file contains the various mmu fetch and update operations.
5 * The most important job they must perform is the mapping between the
6 * domain's pfn and the overall machine mfns.
8 * Xen allows guests to directly update the pagetable, in a controlled
9 * fashion. In other words, the guest modifies the same pagetable
10 * that the CPU actually uses, which eliminates the overhead of having
11 * a separate shadow pagetable.
13 * In order to allow this, it falls on the guest domain to map its
14 * notion of a "physical" pfn - which is just a domain-local linear
15 * address - into a real "machine address" which the CPU's MMU can
18 * A pgd_t/pmd_t/pte_t will typically contain an mfn, and so can be
19 * inserted directly into the pagetable. When creating a new
20 * pte/pmd/pgd, it converts the passed pfn into an mfn. Conversely,
21 * when reading the content back with __(pgd|pmd|pte)_val, it converts
22 * the mfn back into a pfn.
24 * The other constraint is that all pages which make up a pagetable
25 * must be mapped read-only in the guest. This prevents uncontrolled
26 * guest updates to the pagetable. Xen strictly enforces this, and
27 * will disallow any pagetable update which will end up mapping a
28 * pagetable page RW, and will disallow using any writable page as a
31 * Naively, when loading %cr3 with the base of a new pagetable, Xen
32 * would need to validate the whole pagetable before going on.
33 * Naturally, this is quite slow. The solution is to "pin" a
34 * pagetable, which enforces all the constraints on the pagetable even
35 * when it is not actively in use. This menas that Xen can be assured
36 * that it is still valid when you do load it into %cr3, and doesn't
37 * need to revalidate it.
39 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
41 #include <linux/sched.h>
42 #include <linux/highmem.h>
43 #include <linux/debugfs.h>
44 #include <linux/bug.h>
45 #include <linux/vmalloc.h>
46 #include <linux/export.h>
47 #include <linux/init.h>
48 #include <linux/gfp.h>
49 #include <linux/memblock.h>
50 #include <linux/seq_file.h>
51 #include <linux/crash_dump.h>
53 #include <trace/events/xen.h>
55 #include <asm/pgtable.h>
56 #include <asm/tlbflush.h>
57 #include <asm/fixmap.h>
58 #include <asm/mmu_context.h>
59 #include <asm/setup.h>
60 #include <asm/paravirt.h>
62 #include <asm/linkage.h>
68 #include <asm/xen/hypercall.h>
69 #include <asm/xen/hypervisor.h>
73 #include <xen/interface/xen.h>
74 #include <xen/interface/hvm/hvm_op.h>
75 #include <xen/interface/version.h>
76 #include <xen/interface/memory.h>
77 #include <xen/hvc-console.h>
79 #include "multicalls.h"
84 * Protects atomic reservation decrease/increase against concurrent increases.
85 * Also protects non-atomic updates of current_pages and balloon lists.
87 DEFINE_SPINLOCK(xen_reservation_lock);
91 * Identity map, in addition to plain kernel map. This needs to be
92 * large enough to allocate page table pages to allocate the rest.
93 * Each page can map 2MB.
95 #define LEVEL1_IDENT_ENTRIES (PTRS_PER_PTE * 4)
96 static RESERVE_BRK_ARRAY(pte_t, level1_ident_pgt, LEVEL1_IDENT_ENTRIES);
99 /* l3 pud for userspace vsyscall mapping */
100 static pud_t level3_user_vsyscall[PTRS_PER_PUD] __page_aligned_bss;
101 #endif /* CONFIG_X86_64 */
104 * Note about cr3 (pagetable base) values:
106 * xen_cr3 contains the current logical cr3 value; it contains the
107 * last set cr3. This may not be the current effective cr3, because
108 * its update may be being lazily deferred. However, a vcpu looking
109 * at its own cr3 can use this value knowing that it everything will
110 * be self-consistent.
112 * xen_current_cr3 contains the actual vcpu cr3; it is set once the
113 * hypercall to set the vcpu cr3 is complete (so it may be a little
114 * out of date, but it will never be set early). If one vcpu is
115 * looking at another vcpu's cr3 value, it should use this variable.
117 DEFINE_PER_CPU(unsigned long, xen_cr3); /* cr3 stored as physaddr */
118 DEFINE_PER_CPU(unsigned long, xen_current_cr3); /* actual vcpu cr3 */
120 static phys_addr_t xen_pt_base, xen_pt_size __initdata;
123 * Just beyond the highest usermode address. STACK_TOP_MAX has a
124 * redzone above it, so round it up to a PGD boundary.
126 #define USER_LIMIT ((STACK_TOP_MAX + PGDIR_SIZE - 1) & PGDIR_MASK)
128 unsigned long arbitrary_virt_to_mfn(void *vaddr)
130 xmaddr_t maddr = arbitrary_virt_to_machine(vaddr);
132 return PFN_DOWN(maddr.maddr);
135 xmaddr_t arbitrary_virt_to_machine(void *vaddr)
137 unsigned long address = (unsigned long)vaddr;
143 * if the PFN is in the linear mapped vaddr range, we can just use
144 * the (quick) virt_to_machine() p2m lookup
146 if (virt_addr_valid(vaddr))
147 return virt_to_machine(vaddr);
149 /* otherwise we have to do a (slower) full page-table walk */
151 pte = lookup_address(address, &level);
153 offset = address & ~PAGE_MASK;
154 return XMADDR(((phys_addr_t)pte_mfn(*pte) << PAGE_SHIFT) + offset);
156 EXPORT_SYMBOL_GPL(arbitrary_virt_to_machine);
158 void make_lowmem_page_readonly(void *vaddr)
161 unsigned long address = (unsigned long)vaddr;
164 pte = lookup_address(address, &level);
166 return; /* vaddr missing */
168 ptev = pte_wrprotect(*pte);
170 if (HYPERVISOR_update_va_mapping(address, ptev, 0))
174 void make_lowmem_page_readwrite(void *vaddr)
177 unsigned long address = (unsigned long)vaddr;
180 pte = lookup_address(address, &level);
182 return; /* vaddr missing */
184 ptev = pte_mkwrite(*pte);
186 if (HYPERVISOR_update_va_mapping(address, ptev, 0))
191 static bool xen_page_pinned(void *ptr)
193 struct page *page = virt_to_page(ptr);
195 return PagePinned(page);
198 void xen_set_domain_pte(pte_t *ptep, pte_t pteval, unsigned domid)
200 struct multicall_space mcs;
201 struct mmu_update *u;
203 trace_xen_mmu_set_domain_pte(ptep, pteval, domid);
205 mcs = xen_mc_entry(sizeof(*u));
208 /* ptep might be kmapped when using 32-bit HIGHPTE */
209 u->ptr = virt_to_machine(ptep).maddr;
210 u->val = pte_val_ma(pteval);
212 MULTI_mmu_update(mcs.mc, mcs.args, 1, NULL, domid);
214 xen_mc_issue(PARAVIRT_LAZY_MMU);
216 EXPORT_SYMBOL_GPL(xen_set_domain_pte);
218 static void xen_extend_mmu_update(const struct mmu_update *update)
220 struct multicall_space mcs;
221 struct mmu_update *u;
223 mcs = xen_mc_extend_args(__HYPERVISOR_mmu_update, sizeof(*u));
225 if (mcs.mc != NULL) {
228 mcs = __xen_mc_entry(sizeof(*u));
229 MULTI_mmu_update(mcs.mc, mcs.args, 1, NULL, DOMID_SELF);
236 static void xen_extend_mmuext_op(const struct mmuext_op *op)
238 struct multicall_space mcs;
241 mcs = xen_mc_extend_args(__HYPERVISOR_mmuext_op, sizeof(*u));
243 if (mcs.mc != NULL) {
246 mcs = __xen_mc_entry(sizeof(*u));
247 MULTI_mmuext_op(mcs.mc, mcs.args, 1, NULL, DOMID_SELF);
254 static void xen_set_pmd_hyper(pmd_t *ptr, pmd_t val)
262 /* ptr may be ioremapped for 64-bit pagetable setup */
263 u.ptr = arbitrary_virt_to_machine(ptr).maddr;
264 u.val = pmd_val_ma(val);
265 xen_extend_mmu_update(&u);
267 xen_mc_issue(PARAVIRT_LAZY_MMU);
272 static void xen_set_pmd(pmd_t *ptr, pmd_t val)
274 trace_xen_mmu_set_pmd(ptr, val);
276 /* If page is not pinned, we can just update the entry
278 if (!xen_page_pinned(ptr)) {
283 xen_set_pmd_hyper(ptr, val);
287 * Associate a virtual page frame with a given physical page frame
288 * and protection flags for that frame.
290 void set_pte_mfn(unsigned long vaddr, unsigned long mfn, pgprot_t flags)
292 set_pte_vaddr(vaddr, mfn_pte(mfn, flags));
295 static bool xen_batched_set_pte(pte_t *ptep, pte_t pteval)
299 if (paravirt_get_lazy_mode() != PARAVIRT_LAZY_MMU)
304 u.ptr = virt_to_machine(ptep).maddr | MMU_NORMAL_PT_UPDATE;
305 u.val = pte_val_ma(pteval);
306 xen_extend_mmu_update(&u);
308 xen_mc_issue(PARAVIRT_LAZY_MMU);
313 static inline void __xen_set_pte(pte_t *ptep, pte_t pteval)
315 if (!xen_batched_set_pte(ptep, pteval)) {
317 * Could call native_set_pte() here and trap and
318 * emulate the PTE write but with 32-bit guests this
319 * needs two traps (one for each of the two 32-bit
320 * words in the PTE) so do one hypercall directly
325 u.ptr = virt_to_machine(ptep).maddr | MMU_NORMAL_PT_UPDATE;
326 u.val = pte_val_ma(pteval);
327 HYPERVISOR_mmu_update(&u, 1, NULL, DOMID_SELF);
331 static void xen_set_pte(pte_t *ptep, pte_t pteval)
333 trace_xen_mmu_set_pte(ptep, pteval);
334 __xen_set_pte(ptep, pteval);
337 static void xen_set_pte_at(struct mm_struct *mm, unsigned long addr,
338 pte_t *ptep, pte_t pteval)
340 trace_xen_mmu_set_pte_at(mm, addr, ptep, pteval);
341 __xen_set_pte(ptep, pteval);
344 pte_t xen_ptep_modify_prot_start(struct mm_struct *mm,
345 unsigned long addr, pte_t *ptep)
347 /* Just return the pte as-is. We preserve the bits on commit */
348 trace_xen_mmu_ptep_modify_prot_start(mm, addr, ptep, *ptep);
352 void xen_ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr,
353 pte_t *ptep, pte_t pte)
357 trace_xen_mmu_ptep_modify_prot_commit(mm, addr, ptep, pte);
360 u.ptr = virt_to_machine(ptep).maddr | MMU_PT_UPDATE_PRESERVE_AD;
361 u.val = pte_val_ma(pte);
362 xen_extend_mmu_update(&u);
364 xen_mc_issue(PARAVIRT_LAZY_MMU);
367 /* Assume pteval_t is equivalent to all the other *val_t types. */
368 static pteval_t pte_mfn_to_pfn(pteval_t val)
370 if (val & _PAGE_PRESENT) {
371 unsigned long mfn = (val & PTE_PFN_MASK) >> PAGE_SHIFT;
372 unsigned long pfn = mfn_to_pfn(mfn);
374 pteval_t flags = val & PTE_FLAGS_MASK;
375 if (unlikely(pfn == ~0))
376 val = flags & ~_PAGE_PRESENT;
378 val = ((pteval_t)pfn << PAGE_SHIFT) | flags;
384 static pteval_t pte_pfn_to_mfn(pteval_t val)
386 if (val & _PAGE_PRESENT) {
387 unsigned long pfn = (val & PTE_PFN_MASK) >> PAGE_SHIFT;
388 pteval_t flags = val & PTE_FLAGS_MASK;
391 if (!xen_feature(XENFEAT_auto_translated_physmap))
392 mfn = __pfn_to_mfn(pfn);
396 * If there's no mfn for the pfn, then just create an
397 * empty non-present pte. Unfortunately this loses
398 * information about the original pfn, so
399 * pte_mfn_to_pfn is asymmetric.
401 if (unlikely(mfn == INVALID_P2M_ENTRY)) {
405 mfn &= ~(FOREIGN_FRAME_BIT | IDENTITY_FRAME_BIT);
406 val = ((pteval_t)mfn << PAGE_SHIFT) | flags;
412 __visible pteval_t xen_pte_val(pte_t pte)
414 pteval_t pteval = pte.pte;
416 return pte_mfn_to_pfn(pteval);
418 PV_CALLEE_SAVE_REGS_THUNK(xen_pte_val);
420 __visible pgdval_t xen_pgd_val(pgd_t pgd)
422 return pte_mfn_to_pfn(pgd.pgd);
424 PV_CALLEE_SAVE_REGS_THUNK(xen_pgd_val);
426 __visible pte_t xen_make_pte(pteval_t pte)
428 pte = pte_pfn_to_mfn(pte);
430 return native_make_pte(pte);
432 PV_CALLEE_SAVE_REGS_THUNK(xen_make_pte);
434 __visible pgd_t xen_make_pgd(pgdval_t pgd)
436 pgd = pte_pfn_to_mfn(pgd);
437 return native_make_pgd(pgd);
439 PV_CALLEE_SAVE_REGS_THUNK(xen_make_pgd);
441 __visible pmdval_t xen_pmd_val(pmd_t pmd)
443 return pte_mfn_to_pfn(pmd.pmd);
445 PV_CALLEE_SAVE_REGS_THUNK(xen_pmd_val);
447 static void xen_set_pud_hyper(pud_t *ptr, pud_t val)
455 /* ptr may be ioremapped for 64-bit pagetable setup */
456 u.ptr = arbitrary_virt_to_machine(ptr).maddr;
457 u.val = pud_val_ma(val);
458 xen_extend_mmu_update(&u);
460 xen_mc_issue(PARAVIRT_LAZY_MMU);
465 static void xen_set_pud(pud_t *ptr, pud_t val)
467 trace_xen_mmu_set_pud(ptr, val);
469 /* If page is not pinned, we can just update the entry
471 if (!xen_page_pinned(ptr)) {
476 xen_set_pud_hyper(ptr, val);
479 #ifdef CONFIG_X86_PAE
480 static void xen_set_pte_atomic(pte_t *ptep, pte_t pte)
482 trace_xen_mmu_set_pte_atomic(ptep, pte);
483 set_64bit((u64 *)ptep, native_pte_val(pte));
486 static void xen_pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
488 trace_xen_mmu_pte_clear(mm, addr, ptep);
489 if (!xen_batched_set_pte(ptep, native_make_pte(0)))
490 native_pte_clear(mm, addr, ptep);
493 static void xen_pmd_clear(pmd_t *pmdp)
495 trace_xen_mmu_pmd_clear(pmdp);
496 set_pmd(pmdp, __pmd(0));
498 #endif /* CONFIG_X86_PAE */
500 __visible pmd_t xen_make_pmd(pmdval_t pmd)
502 pmd = pte_pfn_to_mfn(pmd);
503 return native_make_pmd(pmd);
505 PV_CALLEE_SAVE_REGS_THUNK(xen_make_pmd);
507 #if CONFIG_PGTABLE_LEVELS == 4
508 __visible pudval_t xen_pud_val(pud_t pud)
510 return pte_mfn_to_pfn(pud.pud);
512 PV_CALLEE_SAVE_REGS_THUNK(xen_pud_val);
514 __visible pud_t xen_make_pud(pudval_t pud)
516 pud = pte_pfn_to_mfn(pud);
518 return native_make_pud(pud);
520 PV_CALLEE_SAVE_REGS_THUNK(xen_make_pud);
522 static pgd_t *xen_get_user_pgd(pgd_t *pgd)
524 pgd_t *pgd_page = (pgd_t *)(((unsigned long)pgd) & PAGE_MASK);
525 unsigned offset = pgd - pgd_page;
526 pgd_t *user_ptr = NULL;
528 if (offset < pgd_index(USER_LIMIT)) {
529 struct page *page = virt_to_page(pgd_page);
530 user_ptr = (pgd_t *)page->private;
538 static void __xen_set_pgd_hyper(pgd_t *ptr, pgd_t val)
542 u.ptr = virt_to_machine(ptr).maddr;
543 u.val = pgd_val_ma(val);
544 xen_extend_mmu_update(&u);
548 * Raw hypercall-based set_pgd, intended for in early boot before
549 * there's a page structure. This implies:
550 * 1. The only existing pagetable is the kernel's
551 * 2. It is always pinned
552 * 3. It has no user pagetable attached to it
554 static void __init xen_set_pgd_hyper(pgd_t *ptr, pgd_t val)
560 __xen_set_pgd_hyper(ptr, val);
562 xen_mc_issue(PARAVIRT_LAZY_MMU);
567 static void xen_set_pgd(pgd_t *ptr, pgd_t val)
569 pgd_t *user_ptr = xen_get_user_pgd(ptr);
571 trace_xen_mmu_set_pgd(ptr, user_ptr, val);
573 /* If page is not pinned, we can just update the entry
575 if (!xen_page_pinned(ptr)) {
578 WARN_ON(xen_page_pinned(user_ptr));
584 /* If it's pinned, then we can at least batch the kernel and
585 user updates together. */
588 __xen_set_pgd_hyper(ptr, val);
590 __xen_set_pgd_hyper(user_ptr, val);
592 xen_mc_issue(PARAVIRT_LAZY_MMU);
594 #endif /* CONFIG_PGTABLE_LEVELS == 4 */
597 * (Yet another) pagetable walker. This one is intended for pinning a
598 * pagetable. This means that it walks a pagetable and calls the
599 * callback function on each page it finds making up the page table,
600 * at every level. It walks the entire pagetable, but it only bothers
601 * pinning pte pages which are below limit. In the normal case this
602 * will be STACK_TOP_MAX, but at boot we need to pin up to
605 * For 32-bit the important bit is that we don't pin beyond there,
606 * because then we start getting into Xen's ptes.
608 * For 64-bit, we must skip the Xen hole in the middle of the address
609 * space, just after the big x86-64 virtual hole.
611 static int __xen_pgd_walk(struct mm_struct *mm, pgd_t *pgd,
612 int (*func)(struct mm_struct *mm, struct page *,
617 unsigned hole_low, hole_high;
618 unsigned pgdidx_limit, pudidx_limit, pmdidx_limit;
619 unsigned pgdidx, pudidx, pmdidx;
621 /* The limit is the last byte to be touched */
623 BUG_ON(limit >= FIXADDR_TOP);
625 if (xen_feature(XENFEAT_auto_translated_physmap))
629 * 64-bit has a great big hole in the middle of the address
630 * space, which contains the Xen mappings. On 32-bit these
631 * will end up making a zero-sized hole and so is a no-op.
633 hole_low = pgd_index(USER_LIMIT);
634 hole_high = pgd_index(PAGE_OFFSET);
636 pgdidx_limit = pgd_index(limit);
638 pudidx_limit = pud_index(limit);
643 pmdidx_limit = pmd_index(limit);
648 for (pgdidx = 0; pgdidx <= pgdidx_limit; pgdidx++) {
651 if (pgdidx >= hole_low && pgdidx < hole_high)
654 if (!pgd_val(pgd[pgdidx]))
657 pud = pud_offset(&pgd[pgdidx], 0);
659 if (PTRS_PER_PUD > 1) /* not folded */
660 flush |= (*func)(mm, virt_to_page(pud), PT_PUD);
662 for (pudidx = 0; pudidx < PTRS_PER_PUD; pudidx++) {
665 if (pgdidx == pgdidx_limit &&
666 pudidx > pudidx_limit)
669 if (pud_none(pud[pudidx]))
672 pmd = pmd_offset(&pud[pudidx], 0);
674 if (PTRS_PER_PMD > 1) /* not folded */
675 flush |= (*func)(mm, virt_to_page(pmd), PT_PMD);
677 for (pmdidx = 0; pmdidx < PTRS_PER_PMD; pmdidx++) {
680 if (pgdidx == pgdidx_limit &&
681 pudidx == pudidx_limit &&
682 pmdidx > pmdidx_limit)
685 if (pmd_none(pmd[pmdidx]))
688 pte = pmd_page(pmd[pmdidx]);
689 flush |= (*func)(mm, pte, PT_PTE);
695 /* Do the top level last, so that the callbacks can use it as
696 a cue to do final things like tlb flushes. */
697 flush |= (*func)(mm, virt_to_page(pgd), PT_PGD);
702 static int xen_pgd_walk(struct mm_struct *mm,
703 int (*func)(struct mm_struct *mm, struct page *,
707 return __xen_pgd_walk(mm, mm->pgd, func, limit);
710 /* If we're using split pte locks, then take the page's lock and
711 return a pointer to it. Otherwise return NULL. */
712 static spinlock_t *xen_pte_lock(struct page *page, struct mm_struct *mm)
714 spinlock_t *ptl = NULL;
716 #if USE_SPLIT_PTE_PTLOCKS
717 ptl = ptlock_ptr(page);
718 spin_lock_nest_lock(ptl, &mm->page_table_lock);
724 static void xen_pte_unlock(void *v)
730 static void xen_do_pin(unsigned level, unsigned long pfn)
735 op.arg1.mfn = pfn_to_mfn(pfn);
737 xen_extend_mmuext_op(&op);
740 static int xen_pin_page(struct mm_struct *mm, struct page *page,
743 unsigned pgfl = TestSetPagePinned(page);
747 flush = 0; /* already pinned */
748 else if (PageHighMem(page))
749 /* kmaps need flushing if we found an unpinned
753 void *pt = lowmem_page_address(page);
754 unsigned long pfn = page_to_pfn(page);
755 struct multicall_space mcs = __xen_mc_entry(0);
761 * We need to hold the pagetable lock between the time
762 * we make the pagetable RO and when we actually pin
763 * it. If we don't, then other users may come in and
764 * attempt to update the pagetable by writing it,
765 * which will fail because the memory is RO but not
766 * pinned, so Xen won't do the trap'n'emulate.
768 * If we're using split pte locks, we can't hold the
769 * entire pagetable's worth of locks during the
770 * traverse, because we may wrap the preempt count (8
771 * bits). The solution is to mark RO and pin each PTE
772 * page while holding the lock. This means the number
773 * of locks we end up holding is never more than a
774 * batch size (~32 entries, at present).
776 * If we're not using split pte locks, we needn't pin
777 * the PTE pages independently, because we're
778 * protected by the overall pagetable lock.
782 ptl = xen_pte_lock(page, mm);
784 MULTI_update_va_mapping(mcs.mc, (unsigned long)pt,
785 pfn_pte(pfn, PAGE_KERNEL_RO),
786 level == PT_PGD ? UVMF_TLB_FLUSH : 0);
789 xen_do_pin(MMUEXT_PIN_L1_TABLE, pfn);
791 /* Queue a deferred unlock for when this batch
793 xen_mc_callback(xen_pte_unlock, ptl);
800 /* This is called just after a mm has been created, but it has not
801 been used yet. We need to make sure that its pagetable is all
802 read-only, and can be pinned. */
803 static void __xen_pgd_pin(struct mm_struct *mm, pgd_t *pgd)
805 trace_xen_mmu_pgd_pin(mm, pgd);
809 if (__xen_pgd_walk(mm, pgd, xen_pin_page, USER_LIMIT)) {
810 /* re-enable interrupts for flushing */
820 pgd_t *user_pgd = xen_get_user_pgd(pgd);
822 xen_do_pin(MMUEXT_PIN_L4_TABLE, PFN_DOWN(__pa(pgd)));
825 xen_pin_page(mm, virt_to_page(user_pgd), PT_PGD);
826 xen_do_pin(MMUEXT_PIN_L4_TABLE,
827 PFN_DOWN(__pa(user_pgd)));
830 #else /* CONFIG_X86_32 */
831 #ifdef CONFIG_X86_PAE
832 /* Need to make sure unshared kernel PMD is pinnable */
833 xen_pin_page(mm, pgd_page(pgd[pgd_index(TASK_SIZE)]),
836 xen_do_pin(MMUEXT_PIN_L3_TABLE, PFN_DOWN(__pa(pgd)));
837 #endif /* CONFIG_X86_64 */
841 static void xen_pgd_pin(struct mm_struct *mm)
843 __xen_pgd_pin(mm, mm->pgd);
847 * On save, we need to pin all pagetables to make sure they get their
848 * mfns turned into pfns. Search the list for any unpinned pgds and pin
849 * them (unpinned pgds are not currently in use, probably because the
850 * process is under construction or destruction).
852 * Expected to be called in stop_machine() ("equivalent to taking
853 * every spinlock in the system"), so the locking doesn't really
854 * matter all that much.
856 void xen_mm_pin_all(void)
860 spin_lock(&pgd_lock);
862 list_for_each_entry(page, &pgd_list, lru) {
863 if (!PagePinned(page)) {
864 __xen_pgd_pin(&init_mm, (pgd_t *)page_address(page));
865 SetPageSavePinned(page);
869 spin_unlock(&pgd_lock);
873 * The init_mm pagetable is really pinned as soon as its created, but
874 * that's before we have page structures to store the bits. So do all
875 * the book-keeping now.
877 static int __init xen_mark_pinned(struct mm_struct *mm, struct page *page,
884 static void __init xen_mark_init_mm_pinned(void)
886 xen_pgd_walk(&init_mm, xen_mark_pinned, FIXADDR_TOP);
889 static int xen_unpin_page(struct mm_struct *mm, struct page *page,
892 unsigned pgfl = TestClearPagePinned(page);
894 if (pgfl && !PageHighMem(page)) {
895 void *pt = lowmem_page_address(page);
896 unsigned long pfn = page_to_pfn(page);
897 spinlock_t *ptl = NULL;
898 struct multicall_space mcs;
901 * Do the converse to pin_page. If we're using split
902 * pte locks, we must be holding the lock for while
903 * the pte page is unpinned but still RO to prevent
904 * concurrent updates from seeing it in this
905 * partially-pinned state.
907 if (level == PT_PTE) {
908 ptl = xen_pte_lock(page, mm);
911 xen_do_pin(MMUEXT_UNPIN_TABLE, pfn);
914 mcs = __xen_mc_entry(0);
916 MULTI_update_va_mapping(mcs.mc, (unsigned long)pt,
917 pfn_pte(pfn, PAGE_KERNEL),
918 level == PT_PGD ? UVMF_TLB_FLUSH : 0);
921 /* unlock when batch completed */
922 xen_mc_callback(xen_pte_unlock, ptl);
926 return 0; /* never need to flush on unpin */
929 /* Release a pagetables pages back as normal RW */
930 static void __xen_pgd_unpin(struct mm_struct *mm, pgd_t *pgd)
932 trace_xen_mmu_pgd_unpin(mm, pgd);
936 xen_do_pin(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(pgd)));
940 pgd_t *user_pgd = xen_get_user_pgd(pgd);
943 xen_do_pin(MMUEXT_UNPIN_TABLE,
944 PFN_DOWN(__pa(user_pgd)));
945 xen_unpin_page(mm, virt_to_page(user_pgd), PT_PGD);
950 #ifdef CONFIG_X86_PAE
951 /* Need to make sure unshared kernel PMD is unpinned */
952 xen_unpin_page(mm, pgd_page(pgd[pgd_index(TASK_SIZE)]),
956 __xen_pgd_walk(mm, pgd, xen_unpin_page, USER_LIMIT);
961 static void xen_pgd_unpin(struct mm_struct *mm)
963 __xen_pgd_unpin(mm, mm->pgd);
967 * On resume, undo any pinning done at save, so that the rest of the
968 * kernel doesn't see any unexpected pinned pagetables.
970 void xen_mm_unpin_all(void)
974 spin_lock(&pgd_lock);
976 list_for_each_entry(page, &pgd_list, lru) {
977 if (PageSavePinned(page)) {
978 BUG_ON(!PagePinned(page));
979 __xen_pgd_unpin(&init_mm, (pgd_t *)page_address(page));
980 ClearPageSavePinned(page);
984 spin_unlock(&pgd_lock);
987 static void xen_activate_mm(struct mm_struct *prev, struct mm_struct *next)
989 spin_lock(&next->page_table_lock);
991 spin_unlock(&next->page_table_lock);
994 static void xen_dup_mmap(struct mm_struct *oldmm, struct mm_struct *mm)
996 spin_lock(&mm->page_table_lock);
998 spin_unlock(&mm->page_table_lock);
1003 /* Another cpu may still have their %cr3 pointing at the pagetable, so
1004 we need to repoint it somewhere else before we can unpin it. */
1005 static void drop_other_mm_ref(void *info)
1007 struct mm_struct *mm = info;
1008 struct mm_struct *active_mm;
1010 active_mm = this_cpu_read(cpu_tlbstate.active_mm);
1012 if (active_mm == mm && this_cpu_read(cpu_tlbstate.state) != TLBSTATE_OK)
1013 leave_mm(smp_processor_id());
1015 /* If this cpu still has a stale cr3 reference, then make sure
1016 it has been flushed. */
1017 if (this_cpu_read(xen_current_cr3) == __pa(mm->pgd))
1018 load_cr3(swapper_pg_dir);
1021 static void xen_drop_mm_ref(struct mm_struct *mm)
1026 if (current->active_mm == mm) {
1027 if (current->mm == mm)
1028 load_cr3(swapper_pg_dir);
1030 leave_mm(smp_processor_id());
1033 /* Get the "official" set of cpus referring to our pagetable. */
1034 if (!alloc_cpumask_var(&mask, GFP_ATOMIC)) {
1035 for_each_online_cpu(cpu) {
1036 if (!cpumask_test_cpu(cpu, mm_cpumask(mm))
1037 && per_cpu(xen_current_cr3, cpu) != __pa(mm->pgd))
1039 smp_call_function_single(cpu, drop_other_mm_ref, mm, 1);
1043 cpumask_copy(mask, mm_cpumask(mm));
1045 /* It's possible that a vcpu may have a stale reference to our
1046 cr3, because its in lazy mode, and it hasn't yet flushed
1047 its set of pending hypercalls yet. In this case, we can
1048 look at its actual current cr3 value, and force it to flush
1050 for_each_online_cpu(cpu) {
1051 if (per_cpu(xen_current_cr3, cpu) == __pa(mm->pgd))
1052 cpumask_set_cpu(cpu, mask);
1055 if (!cpumask_empty(mask))
1056 smp_call_function_many(mask, drop_other_mm_ref, mm, 1);
1057 free_cpumask_var(mask);
1060 static void xen_drop_mm_ref(struct mm_struct *mm)
1062 if (current->active_mm == mm)
1063 load_cr3(swapper_pg_dir);
1068 * While a process runs, Xen pins its pagetables, which means that the
1069 * hypervisor forces it to be read-only, and it controls all updates
1070 * to it. This means that all pagetable updates have to go via the
1071 * hypervisor, which is moderately expensive.
1073 * Since we're pulling the pagetable down, we switch to use init_mm,
1074 * unpin old process pagetable and mark it all read-write, which
1075 * allows further operations on it to be simple memory accesses.
1077 * The only subtle point is that another CPU may be still using the
1078 * pagetable because of lazy tlb flushing. This means we need need to
1079 * switch all CPUs off this pagetable before we can unpin it.
1081 static void xen_exit_mmap(struct mm_struct *mm)
1083 get_cpu(); /* make sure we don't move around */
1084 xen_drop_mm_ref(mm);
1087 spin_lock(&mm->page_table_lock);
1089 /* pgd may not be pinned in the error exit path of execve */
1090 if (xen_page_pinned(mm->pgd))
1093 spin_unlock(&mm->page_table_lock);
1096 static void xen_post_allocator_init(void);
1098 static void __init pin_pagetable_pfn(unsigned cmd, unsigned long pfn)
1100 struct mmuext_op op;
1103 op.arg1.mfn = pfn_to_mfn(pfn);
1104 if (HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF))
1108 #ifdef CONFIG_X86_64
1109 static void __init xen_cleanhighmap(unsigned long vaddr,
1110 unsigned long vaddr_end)
1112 unsigned long kernel_end = roundup((unsigned long)_brk_end, PMD_SIZE) - 1;
1113 pmd_t *pmd = level2_kernel_pgt + pmd_index(vaddr);
1115 /* NOTE: The loop is more greedy than the cleanup_highmap variant.
1116 * We include the PMD passed in on _both_ boundaries. */
1117 for (; vaddr <= vaddr_end && (pmd < (level2_kernel_pgt + PTRS_PER_PMD));
1118 pmd++, vaddr += PMD_SIZE) {
1121 if (vaddr < (unsigned long) _text || vaddr > kernel_end)
1122 set_pmd(pmd, __pmd(0));
1124 /* In case we did something silly, we should crash in this function
1125 * instead of somewhere later and be confusing. */
1130 * Make a page range writeable and free it.
1132 static void __init xen_free_ro_pages(unsigned long paddr, unsigned long size)
1134 void *vaddr = __va(paddr);
1135 void *vaddr_end = vaddr + size;
1137 for (; vaddr < vaddr_end; vaddr += PAGE_SIZE)
1138 make_lowmem_page_readwrite(vaddr);
1140 memblock_free(paddr, size);
1143 static void __init xen_cleanmfnmap_free_pgtbl(void *pgtbl, bool unpin)
1145 unsigned long pa = __pa(pgtbl) & PHYSICAL_PAGE_MASK;
1148 pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, PFN_DOWN(pa));
1149 ClearPagePinned(virt_to_page(__va(pa)));
1150 xen_free_ro_pages(pa, PAGE_SIZE);
1154 * Since it is well isolated we can (and since it is perhaps large we should)
1155 * also free the page tables mapping the initial P->M table.
1157 static void __init xen_cleanmfnmap(unsigned long vaddr)
1159 unsigned long va = vaddr & PMD_MASK;
1161 pgd_t *pgd = pgd_offset_k(va);
1162 pud_t *pud_page = pud_offset(pgd, 0);
1169 unpin = (vaddr == 2 * PGDIR_SIZE);
1170 set_pgd(pgd, __pgd(0));
1172 pud = pud_page + pud_index(va);
1173 if (pud_none(*pud)) {
1175 } else if (pud_large(*pud)) {
1176 pa = pud_val(*pud) & PHYSICAL_PAGE_MASK;
1177 xen_free_ro_pages(pa, PUD_SIZE);
1180 pmd = pmd_offset(pud, va);
1181 if (pmd_large(*pmd)) {
1182 pa = pmd_val(*pmd) & PHYSICAL_PAGE_MASK;
1183 xen_free_ro_pages(pa, PMD_SIZE);
1184 } else if (!pmd_none(*pmd)) {
1185 pte = pte_offset_kernel(pmd, va);
1186 set_pmd(pmd, __pmd(0));
1187 for (i = 0; i < PTRS_PER_PTE; ++i) {
1188 if (pte_none(pte[i]))
1190 pa = pte_pfn(pte[i]) << PAGE_SHIFT;
1191 xen_free_ro_pages(pa, PAGE_SIZE);
1193 xen_cleanmfnmap_free_pgtbl(pte, unpin);
1198 set_pud(pud, __pud(0));
1199 xen_cleanmfnmap_free_pgtbl(pmd, unpin);
1202 } while (pud_index(va) || pmd_index(va));
1203 xen_cleanmfnmap_free_pgtbl(pud_page, unpin);
1206 static void __init xen_pagetable_p2m_free(void)
1211 size = PAGE_ALIGN(xen_start_info->nr_pages * sizeof(unsigned long));
1213 /* No memory or already called. */
1214 if ((unsigned long)xen_p2m_addr == xen_start_info->mfn_list)
1217 /* using __ka address and sticking INVALID_P2M_ENTRY! */
1218 memset((void *)xen_start_info->mfn_list, 0xff, size);
1220 addr = xen_start_info->mfn_list;
1222 * We could be in __ka space.
1223 * We roundup to the PMD, which means that if anybody at this stage is
1224 * using the __ka address of xen_start_info or
1225 * xen_start_info->shared_info they are in going to crash. Fortunatly
1226 * we have already revectored in xen_setup_kernel_pagetable and in
1227 * xen_setup_shared_info.
1229 size = roundup(size, PMD_SIZE);
1231 if (addr >= __START_KERNEL_map) {
1232 xen_cleanhighmap(addr, addr + size);
1233 size = PAGE_ALIGN(xen_start_info->nr_pages *
1234 sizeof(unsigned long));
1235 memblock_free(__pa(addr), size);
1237 xen_cleanmfnmap(addr);
1241 static void __init xen_pagetable_cleanhighmap(void)
1246 /* At this stage, cleanup_highmap has already cleaned __ka space
1247 * from _brk_limit way up to the max_pfn_mapped (which is the end of
1248 * the ramdisk). We continue on, erasing PMD entries that point to page
1249 * tables - do note that they are accessible at this stage via __va.
1250 * For good measure we also round up to the PMD - which means that if
1251 * anybody is using __ka address to the initial boot-stack - and try
1252 * to use it - they are going to crash. The xen_start_info has been
1253 * taken care of already in xen_setup_kernel_pagetable. */
1254 addr = xen_start_info->pt_base;
1255 size = roundup(xen_start_info->nr_pt_frames * PAGE_SIZE, PMD_SIZE);
1257 xen_cleanhighmap(addr, addr + size);
1258 xen_start_info->pt_base = (unsigned long)__va(__pa(xen_start_info->pt_base));
1260 /* This is superfluous and is not necessary, but you know what
1261 * lets do it. The MODULES_VADDR -> MODULES_END should be clear of
1262 * anything at this stage. */
1263 xen_cleanhighmap(MODULES_VADDR, roundup(MODULES_VADDR, PUD_SIZE) - 1);
1268 static void __init xen_pagetable_p2m_setup(void)
1270 if (xen_feature(XENFEAT_auto_translated_physmap))
1273 xen_vmalloc_p2m_tree();
1275 #ifdef CONFIG_X86_64
1276 xen_pagetable_p2m_free();
1278 xen_pagetable_cleanhighmap();
1280 /* And revector! Bye bye old array */
1281 xen_start_info->mfn_list = (unsigned long)xen_p2m_addr;
1284 static void __init xen_pagetable_init(void)
1287 xen_post_allocator_init();
1289 xen_pagetable_p2m_setup();
1291 /* Allocate and initialize top and mid mfn levels for p2m structure */
1292 xen_build_mfn_list_list();
1294 /* Remap memory freed due to conflicts with E820 map */
1295 if (!xen_feature(XENFEAT_auto_translated_physmap))
1298 xen_setup_shared_info();
1300 static void xen_write_cr2(unsigned long cr2)
1302 this_cpu_read(xen_vcpu)->arch.cr2 = cr2;
1305 static unsigned long xen_read_cr2(void)
1307 return this_cpu_read(xen_vcpu)->arch.cr2;
1310 unsigned long xen_read_cr2_direct(void)
1312 return this_cpu_read(xen_vcpu_info.arch.cr2);
1315 void xen_flush_tlb_all(void)
1317 struct mmuext_op *op;
1318 struct multicall_space mcs;
1322 mcs = xen_mc_entry(sizeof(*op));
1325 op->cmd = MMUEXT_TLB_FLUSH_ALL;
1326 MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
1328 xen_mc_issue(PARAVIRT_LAZY_MMU);
1332 static void xen_flush_tlb(void)
1334 struct mmuext_op *op;
1335 struct multicall_space mcs;
1339 mcs = xen_mc_entry(sizeof(*op));
1342 op->cmd = MMUEXT_TLB_FLUSH_LOCAL;
1343 MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
1345 xen_mc_issue(PARAVIRT_LAZY_MMU);
1350 static void xen_flush_tlb_single(unsigned long addr)
1352 struct mmuext_op *op;
1353 struct multicall_space mcs;
1355 trace_xen_mmu_flush_tlb_single(addr);
1359 mcs = xen_mc_entry(sizeof(*op));
1361 op->cmd = MMUEXT_INVLPG_LOCAL;
1362 op->arg1.linear_addr = addr & PAGE_MASK;
1363 MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
1365 xen_mc_issue(PARAVIRT_LAZY_MMU);
1370 static void xen_flush_tlb_others(const struct cpumask *cpus,
1371 struct mm_struct *mm, unsigned long start,
1375 struct mmuext_op op;
1377 DECLARE_BITMAP(mask, num_processors);
1379 DECLARE_BITMAP(mask, NR_CPUS);
1382 struct multicall_space mcs;
1384 trace_xen_mmu_flush_tlb_others(cpus, mm, start, end);
1386 if (cpumask_empty(cpus))
1387 return; /* nothing to do */
1389 mcs = xen_mc_entry(sizeof(*args));
1391 args->op.arg2.vcpumask = to_cpumask(args->mask);
1393 /* Remove us, and any offline CPUS. */
1394 cpumask_and(to_cpumask(args->mask), cpus, cpu_online_mask);
1395 cpumask_clear_cpu(smp_processor_id(), to_cpumask(args->mask));
1397 args->op.cmd = MMUEXT_TLB_FLUSH_MULTI;
1398 if (end != TLB_FLUSH_ALL && (end - start) <= PAGE_SIZE) {
1399 args->op.cmd = MMUEXT_INVLPG_MULTI;
1400 args->op.arg1.linear_addr = start;
1403 MULTI_mmuext_op(mcs.mc, &args->op, 1, NULL, DOMID_SELF);
1405 xen_mc_issue(PARAVIRT_LAZY_MMU);
1408 static unsigned long xen_read_cr3(void)
1410 return this_cpu_read(xen_cr3);
1413 static void set_current_cr3(void *v)
1415 this_cpu_write(xen_current_cr3, (unsigned long)v);
1418 static void __xen_write_cr3(bool kernel, unsigned long cr3)
1420 struct mmuext_op op;
1423 trace_xen_mmu_write_cr3(kernel, cr3);
1426 mfn = pfn_to_mfn(PFN_DOWN(cr3));
1430 WARN_ON(mfn == 0 && kernel);
1432 op.cmd = kernel ? MMUEXT_NEW_BASEPTR : MMUEXT_NEW_USER_BASEPTR;
1435 xen_extend_mmuext_op(&op);
1438 this_cpu_write(xen_cr3, cr3);
1440 /* Update xen_current_cr3 once the batch has actually
1442 xen_mc_callback(set_current_cr3, (void *)cr3);
1445 static void xen_write_cr3(unsigned long cr3)
1447 BUG_ON(preemptible());
1449 xen_mc_batch(); /* disables interrupts */
1451 /* Update while interrupts are disabled, so its atomic with
1453 this_cpu_write(xen_cr3, cr3);
1455 __xen_write_cr3(true, cr3);
1457 #ifdef CONFIG_X86_64
1459 pgd_t *user_pgd = xen_get_user_pgd(__va(cr3));
1461 __xen_write_cr3(false, __pa(user_pgd));
1463 __xen_write_cr3(false, 0);
1467 xen_mc_issue(PARAVIRT_LAZY_CPU); /* interrupts restored */
1470 #ifdef CONFIG_X86_64
1472 * At the start of the day - when Xen launches a guest, it has already
1473 * built pagetables for the guest. We diligently look over them
1474 * in xen_setup_kernel_pagetable and graft as appropriate them in the
1475 * init_level4_pgt and its friends. Then when we are happy we load
1476 * the new init_level4_pgt - and continue on.
1478 * The generic code starts (start_kernel) and 'init_mem_mapping' sets
1479 * up the rest of the pagetables. When it has completed it loads the cr3.
1480 * N.B. that baremetal would start at 'start_kernel' (and the early
1481 * #PF handler would create bootstrap pagetables) - so we are running
1482 * with the same assumptions as what to do when write_cr3 is executed
1485 * Since there are no user-page tables at all, we have two variants
1486 * of xen_write_cr3 - the early bootup (this one), and the late one
1487 * (xen_write_cr3). The reason we have to do that is that in 64-bit
1488 * the Linux kernel and user-space are both in ring 3 while the
1489 * hypervisor is in ring 0.
1491 static void __init xen_write_cr3_init(unsigned long cr3)
1493 BUG_ON(preemptible());
1495 xen_mc_batch(); /* disables interrupts */
1497 /* Update while interrupts are disabled, so its atomic with
1499 this_cpu_write(xen_cr3, cr3);
1501 __xen_write_cr3(true, cr3);
1503 xen_mc_issue(PARAVIRT_LAZY_CPU); /* interrupts restored */
1507 static int xen_pgd_alloc(struct mm_struct *mm)
1509 pgd_t *pgd = mm->pgd;
1512 BUG_ON(PagePinned(virt_to_page(pgd)));
1514 #ifdef CONFIG_X86_64
1516 struct page *page = virt_to_page(pgd);
1519 BUG_ON(page->private != 0);
1523 user_pgd = (pgd_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO);
1524 page->private = (unsigned long)user_pgd;
1526 if (user_pgd != NULL) {
1527 #ifdef CONFIG_X86_VSYSCALL_EMULATION
1528 user_pgd[pgd_index(VSYSCALL_ADDR)] =
1529 __pgd(__pa(level3_user_vsyscall) | _PAGE_TABLE);
1534 BUG_ON(PagePinned(virt_to_page(xen_get_user_pgd(pgd))));
1541 static void xen_pgd_free(struct mm_struct *mm, pgd_t *pgd)
1543 #ifdef CONFIG_X86_64
1544 pgd_t *user_pgd = xen_get_user_pgd(pgd);
1547 free_page((unsigned long)user_pgd);
1552 * Init-time set_pte while constructing initial pagetables, which
1553 * doesn't allow RO page table pages to be remapped RW.
1555 * If there is no MFN for this PFN then this page is initially
1556 * ballooned out so clear the PTE (as in decrease_reservation() in
1557 * drivers/xen/balloon.c).
1559 * Many of these PTE updates are done on unpinned and writable pages
1560 * and doing a hypercall for these is unnecessary and expensive. At
1561 * this point it is not possible to tell if a page is pinned or not,
1562 * so always write the PTE directly and rely on Xen trapping and
1563 * emulating any updates as necessary.
1565 __visible pte_t xen_make_pte_init(pteval_t pte)
1567 #ifdef CONFIG_X86_64
1571 * Pages belonging to the initial p2m list mapped outside the default
1572 * address range must be mapped read-only. This region contains the
1573 * page tables for mapping the p2m list, too, and page tables MUST be
1576 pfn = (pte & PTE_PFN_MASK) >> PAGE_SHIFT;
1577 if (xen_start_info->mfn_list < __START_KERNEL_map &&
1578 pfn >= xen_start_info->first_p2m_pfn &&
1579 pfn < xen_start_info->first_p2m_pfn + xen_start_info->nr_p2m_frames)
1582 pte = pte_pfn_to_mfn(pte);
1583 return native_make_pte(pte);
1585 PV_CALLEE_SAVE_REGS_THUNK(xen_make_pte_init);
1587 static void __init xen_set_pte_init(pte_t *ptep, pte_t pte)
1589 #ifdef CONFIG_X86_32
1590 /* If there's an existing pte, then don't allow _PAGE_RW to be set */
1591 if (pte_mfn(pte) != INVALID_P2M_ENTRY
1592 && pte_val_ma(*ptep) & _PAGE_PRESENT)
1593 pte = __pte_ma(((pte_val_ma(*ptep) & _PAGE_RW) | ~_PAGE_RW) &
1596 native_set_pte(ptep, pte);
1599 /* Early in boot, while setting up the initial pagetable, assume
1600 everything is pinned. */
1601 static void __init xen_alloc_pte_init(struct mm_struct *mm, unsigned long pfn)
1603 #ifdef CONFIG_FLATMEM
1604 BUG_ON(mem_map); /* should only be used early */
1606 make_lowmem_page_readonly(__va(PFN_PHYS(pfn)));
1607 pin_pagetable_pfn(MMUEXT_PIN_L1_TABLE, pfn);
1610 /* Used for pmd and pud */
1611 static void __init xen_alloc_pmd_init(struct mm_struct *mm, unsigned long pfn)
1613 #ifdef CONFIG_FLATMEM
1614 BUG_ON(mem_map); /* should only be used early */
1616 make_lowmem_page_readonly(__va(PFN_PHYS(pfn)));
1619 /* Early release_pte assumes that all pts are pinned, since there's
1620 only init_mm and anything attached to that is pinned. */
1621 static void __init xen_release_pte_init(unsigned long pfn)
1623 pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, pfn);
1624 make_lowmem_page_readwrite(__va(PFN_PHYS(pfn)));
1627 static void __init xen_release_pmd_init(unsigned long pfn)
1629 make_lowmem_page_readwrite(__va(PFN_PHYS(pfn)));
1632 static inline void __pin_pagetable_pfn(unsigned cmd, unsigned long pfn)
1634 struct multicall_space mcs;
1635 struct mmuext_op *op;
1637 mcs = __xen_mc_entry(sizeof(*op));
1640 op->arg1.mfn = pfn_to_mfn(pfn);
1642 MULTI_mmuext_op(mcs.mc, mcs.args, 1, NULL, DOMID_SELF);
1645 static inline void __set_pfn_prot(unsigned long pfn, pgprot_t prot)
1647 struct multicall_space mcs;
1648 unsigned long addr = (unsigned long)__va(pfn << PAGE_SHIFT);
1650 mcs = __xen_mc_entry(0);
1651 MULTI_update_va_mapping(mcs.mc, (unsigned long)addr,
1652 pfn_pte(pfn, prot), 0);
1655 /* This needs to make sure the new pte page is pinned iff its being
1656 attached to a pinned pagetable. */
1657 static inline void xen_alloc_ptpage(struct mm_struct *mm, unsigned long pfn,
1660 bool pinned = PagePinned(virt_to_page(mm->pgd));
1662 trace_xen_mmu_alloc_ptpage(mm, pfn, level, pinned);
1665 struct page *page = pfn_to_page(pfn);
1667 SetPagePinned(page);
1669 if (!PageHighMem(page)) {
1672 __set_pfn_prot(pfn, PAGE_KERNEL_RO);
1674 if (level == PT_PTE && USE_SPLIT_PTE_PTLOCKS)
1675 __pin_pagetable_pfn(MMUEXT_PIN_L1_TABLE, pfn);
1677 xen_mc_issue(PARAVIRT_LAZY_MMU);
1679 /* make sure there are no stray mappings of
1681 kmap_flush_unused();
1686 static void xen_alloc_pte(struct mm_struct *mm, unsigned long pfn)
1688 xen_alloc_ptpage(mm, pfn, PT_PTE);
1691 static void xen_alloc_pmd(struct mm_struct *mm, unsigned long pfn)
1693 xen_alloc_ptpage(mm, pfn, PT_PMD);
1696 /* This should never happen until we're OK to use struct page */
1697 static inline void xen_release_ptpage(unsigned long pfn, unsigned level)
1699 struct page *page = pfn_to_page(pfn);
1700 bool pinned = PagePinned(page);
1702 trace_xen_mmu_release_ptpage(pfn, level, pinned);
1705 if (!PageHighMem(page)) {
1708 if (level == PT_PTE && USE_SPLIT_PTE_PTLOCKS)
1709 __pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, pfn);
1711 __set_pfn_prot(pfn, PAGE_KERNEL);
1713 xen_mc_issue(PARAVIRT_LAZY_MMU);
1715 ClearPagePinned(page);
1719 static void xen_release_pte(unsigned long pfn)
1721 xen_release_ptpage(pfn, PT_PTE);
1724 static void xen_release_pmd(unsigned long pfn)
1726 xen_release_ptpage(pfn, PT_PMD);
1729 #if CONFIG_PGTABLE_LEVELS == 4
1730 static void xen_alloc_pud(struct mm_struct *mm, unsigned long pfn)
1732 xen_alloc_ptpage(mm, pfn, PT_PUD);
1735 static void xen_release_pud(unsigned long pfn)
1737 xen_release_ptpage(pfn, PT_PUD);
1741 void __init xen_reserve_top(void)
1743 #ifdef CONFIG_X86_32
1744 unsigned long top = HYPERVISOR_VIRT_START;
1745 struct xen_platform_parameters pp;
1747 if (HYPERVISOR_xen_version(XENVER_platform_parameters, &pp) == 0)
1748 top = pp.virt_start;
1750 reserve_top_address(-top);
1751 #endif /* CONFIG_X86_32 */
1755 * Like __va(), but returns address in the kernel mapping (which is
1756 * all we have until the physical memory mapping has been set up.
1758 static void * __init __ka(phys_addr_t paddr)
1760 #ifdef CONFIG_X86_64
1761 return (void *)(paddr + __START_KERNEL_map);
1767 /* Convert a machine address to physical address */
1768 static unsigned long __init m2p(phys_addr_t maddr)
1772 maddr &= PTE_PFN_MASK;
1773 paddr = mfn_to_pfn(maddr >> PAGE_SHIFT) << PAGE_SHIFT;
1778 /* Convert a machine address to kernel virtual */
1779 static void * __init m2v(phys_addr_t maddr)
1781 return __ka(m2p(maddr));
1784 /* Set the page permissions on an identity-mapped pages */
1785 static void __init set_page_prot_flags(void *addr, pgprot_t prot,
1786 unsigned long flags)
1788 unsigned long pfn = __pa(addr) >> PAGE_SHIFT;
1789 pte_t pte = pfn_pte(pfn, prot);
1791 /* For PVH no need to set R/O or R/W to pin them or unpin them. */
1792 if (xen_feature(XENFEAT_auto_translated_physmap))
1795 if (HYPERVISOR_update_va_mapping((unsigned long)addr, pte, flags))
1798 static void __init set_page_prot(void *addr, pgprot_t prot)
1800 return set_page_prot_flags(addr, prot, UVMF_NONE);
1802 #ifdef CONFIG_X86_32
1803 static void __init xen_map_identity_early(pmd_t *pmd, unsigned long max_pfn)
1805 unsigned pmdidx, pteidx;
1809 level1_ident_pgt = extend_brk(sizeof(pte_t) * LEVEL1_IDENT_ENTRIES,
1814 for (pmdidx = 0; pmdidx < PTRS_PER_PMD && pfn < max_pfn; pmdidx++) {
1817 /* Reuse or allocate a page of ptes */
1818 if (pmd_present(pmd[pmdidx]))
1819 pte_page = m2v(pmd[pmdidx].pmd);
1821 /* Check for free pte pages */
1822 if (ident_pte == LEVEL1_IDENT_ENTRIES)
1825 pte_page = &level1_ident_pgt[ident_pte];
1826 ident_pte += PTRS_PER_PTE;
1828 pmd[pmdidx] = __pmd(__pa(pte_page) | _PAGE_TABLE);
1831 /* Install mappings */
1832 for (pteidx = 0; pteidx < PTRS_PER_PTE; pteidx++, pfn++) {
1835 if (pfn > max_pfn_mapped)
1836 max_pfn_mapped = pfn;
1838 if (!pte_none(pte_page[pteidx]))
1841 pte = pfn_pte(pfn, PAGE_KERNEL_EXEC);
1842 pte_page[pteidx] = pte;
1846 for (pteidx = 0; pteidx < ident_pte; pteidx += PTRS_PER_PTE)
1847 set_page_prot(&level1_ident_pgt[pteidx], PAGE_KERNEL_RO);
1849 set_page_prot(pmd, PAGE_KERNEL_RO);
1852 void __init xen_setup_machphys_mapping(void)
1854 struct xen_machphys_mapping mapping;
1856 if (HYPERVISOR_memory_op(XENMEM_machphys_mapping, &mapping) == 0) {
1857 machine_to_phys_mapping = (unsigned long *)mapping.v_start;
1858 machine_to_phys_nr = mapping.max_mfn + 1;
1860 machine_to_phys_nr = MACH2PHYS_NR_ENTRIES;
1862 #ifdef CONFIG_X86_32
1863 WARN_ON((machine_to_phys_mapping + (machine_to_phys_nr - 1))
1864 < machine_to_phys_mapping);
1868 #ifdef CONFIG_X86_64
1869 static void __init convert_pfn_mfn(void *v)
1874 /* All levels are converted the same way, so just treat them
1876 for (i = 0; i < PTRS_PER_PTE; i++)
1877 pte[i] = xen_make_pte(pte[i].pte);
1879 static void __init check_pt_base(unsigned long *pt_base, unsigned long *pt_end,
1882 if (*pt_base == PFN_DOWN(__pa(addr))) {
1883 set_page_prot_flags((void *)addr, PAGE_KERNEL, UVMF_INVLPG);
1884 clear_page((void *)addr);
1887 if (*pt_end == PFN_DOWN(__pa(addr))) {
1888 set_page_prot_flags((void *)addr, PAGE_KERNEL, UVMF_INVLPG);
1889 clear_page((void *)addr);
1894 * Set up the initial kernel pagetable.
1896 * We can construct this by grafting the Xen provided pagetable into
1897 * head_64.S's preconstructed pagetables. We copy the Xen L2's into
1898 * level2_ident_pgt, and level2_kernel_pgt. This means that only the
1899 * kernel has a physical mapping to start with - but that's enough to
1900 * get __va working. We need to fill in the rest of the physical
1901 * mapping once some sort of allocator has been set up. NOTE: for
1902 * PVH, the page tables are native.
1904 void __init xen_setup_kernel_pagetable(pgd_t *pgd, unsigned long max_pfn)
1908 unsigned long addr[3];
1909 unsigned long pt_base, pt_end;
1912 /* max_pfn_mapped is the last pfn mapped in the initial memory
1913 * mappings. Considering that on Xen after the kernel mappings we
1914 * have the mappings of some pages that don't exist in pfn space, we
1915 * set max_pfn_mapped to the last real pfn mapped. */
1916 if (xen_start_info->mfn_list < __START_KERNEL_map)
1917 max_pfn_mapped = xen_start_info->first_p2m_pfn;
1919 max_pfn_mapped = PFN_DOWN(__pa(xen_start_info->mfn_list));
1921 pt_base = PFN_DOWN(__pa(xen_start_info->pt_base));
1922 pt_end = pt_base + xen_start_info->nr_pt_frames;
1924 /* Zap identity mapping */
1925 init_level4_pgt[0] = __pgd(0);
1927 if (!xen_feature(XENFEAT_auto_translated_physmap)) {
1928 /* Pre-constructed entries are in pfn, so convert to mfn */
1929 /* L4[272] -> level3_ident_pgt
1930 * L4[511] -> level3_kernel_pgt */
1931 convert_pfn_mfn(init_level4_pgt);
1933 /* L3_i[0] -> level2_ident_pgt */
1934 convert_pfn_mfn(level3_ident_pgt);
1935 /* L3_k[510] -> level2_kernel_pgt
1936 * L3_k[511] -> level2_fixmap_pgt */
1937 convert_pfn_mfn(level3_kernel_pgt);
1939 /* L3_k[511][506] -> level1_fixmap_pgt */
1940 convert_pfn_mfn(level2_fixmap_pgt);
1942 /* We get [511][511] and have Xen's version of level2_kernel_pgt */
1943 l3 = m2v(pgd[pgd_index(__START_KERNEL_map)].pgd);
1944 l2 = m2v(l3[pud_index(__START_KERNEL_map)].pud);
1946 addr[0] = (unsigned long)pgd;
1947 addr[1] = (unsigned long)l3;
1948 addr[2] = (unsigned long)l2;
1949 /* Graft it onto L4[272][0]. Note that we creating an aliasing problem:
1950 * Both L4[272][0] and L4[511][510] have entries that point to the same
1951 * L2 (PMD) tables. Meaning that if you modify it in __va space
1952 * it will be also modified in the __ka space! (But if you just
1953 * modify the PMD table to point to other PTE's or none, then you
1954 * are OK - which is what cleanup_highmap does) */
1955 copy_page(level2_ident_pgt, l2);
1956 /* Graft it onto L4[511][510] */
1957 copy_page(level2_kernel_pgt, l2);
1959 /* Copy the initial P->M table mappings if necessary. */
1960 i = pgd_index(xen_start_info->mfn_list);
1961 if (i && i < pgd_index(__START_KERNEL_map))
1962 init_level4_pgt[i] = ((pgd_t *)xen_start_info->pt_base)[i];
1964 if (!xen_feature(XENFEAT_auto_translated_physmap)) {
1965 /* Make pagetable pieces RO */
1966 set_page_prot(init_level4_pgt, PAGE_KERNEL_RO);
1967 set_page_prot(level3_ident_pgt, PAGE_KERNEL_RO);
1968 set_page_prot(level3_kernel_pgt, PAGE_KERNEL_RO);
1969 set_page_prot(level3_user_vsyscall, PAGE_KERNEL_RO);
1970 set_page_prot(level2_ident_pgt, PAGE_KERNEL_RO);
1971 set_page_prot(level2_kernel_pgt, PAGE_KERNEL_RO);
1972 set_page_prot(level2_fixmap_pgt, PAGE_KERNEL_RO);
1973 set_page_prot(level1_fixmap_pgt, PAGE_KERNEL_RO);
1975 /* Pin down new L4 */
1976 pin_pagetable_pfn(MMUEXT_PIN_L4_TABLE,
1977 PFN_DOWN(__pa_symbol(init_level4_pgt)));
1979 /* Unpin Xen-provided one */
1980 pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(pgd)));
1983 * At this stage there can be no user pgd, and no page
1984 * structure to attach it to, so make sure we just set kernel
1988 __xen_write_cr3(true, __pa(init_level4_pgt));
1989 xen_mc_issue(PARAVIRT_LAZY_CPU);
1991 native_write_cr3(__pa(init_level4_pgt));
1993 /* We can't that easily rip out L3 and L2, as the Xen pagetables are
1994 * set out this way: [L4], [L1], [L2], [L3], [L1], [L1] ... for
1995 * the initial domain. For guests using the toolstack, they are in:
1996 * [L4], [L3], [L2], [L1], [L1], order .. So for dom0 we can only
1997 * rip out the [L4] (pgd), but for guests we shave off three pages.
1999 for (i = 0; i < ARRAY_SIZE(addr); i++)
2000 check_pt_base(&pt_base, &pt_end, addr[i]);
2002 /* Our (by three pages) smaller Xen pagetable that we are using */
2003 xen_pt_base = PFN_PHYS(pt_base);
2004 xen_pt_size = (pt_end - pt_base) * PAGE_SIZE;
2005 memblock_reserve(xen_pt_base, xen_pt_size);
2007 /* Revector the xen_start_info */
2008 xen_start_info = (struct start_info *)__va(__pa(xen_start_info));
2012 * Read a value from a physical address.
2014 static unsigned long __init xen_read_phys_ulong(phys_addr_t addr)
2016 unsigned long *vaddr;
2019 vaddr = early_memremap_ro(addr, sizeof(val));
2021 early_memunmap(vaddr, sizeof(val));
2026 * Translate a virtual address to a physical one without relying on mapped
2027 * page tables. Don't rely on big pages being aligned in (guest) physical
2030 static phys_addr_t __init xen_early_virt_to_phys(unsigned long vaddr)
2039 pgd = native_make_pgd(xen_read_phys_ulong(pa + pgd_index(vaddr) *
2041 if (!pgd_present(pgd))
2044 pa = pgd_val(pgd) & PTE_PFN_MASK;
2045 pud = native_make_pud(xen_read_phys_ulong(pa + pud_index(vaddr) *
2047 if (!pud_present(pud))
2049 pa = pud_val(pud) & PTE_PFN_MASK;
2051 return pa + (vaddr & ~PUD_MASK);
2053 pmd = native_make_pmd(xen_read_phys_ulong(pa + pmd_index(vaddr) *
2055 if (!pmd_present(pmd))
2057 pa = pmd_val(pmd) & PTE_PFN_MASK;
2059 return pa + (vaddr & ~PMD_MASK);
2061 pte = native_make_pte(xen_read_phys_ulong(pa + pte_index(vaddr) *
2063 if (!pte_present(pte))
2065 pa = pte_pfn(pte) << PAGE_SHIFT;
2067 return pa | (vaddr & ~PAGE_MASK);
2071 * Find a new area for the hypervisor supplied p2m list and relocate the p2m to
2074 void __init xen_relocate_p2m(void)
2076 phys_addr_t size, new_area, pt_phys, pmd_phys, pud_phys;
2077 unsigned long p2m_pfn, p2m_pfn_end, n_frames, pfn, pfn_end;
2078 int n_pte, n_pt, n_pmd, n_pud, idx_pte, idx_pt, idx_pmd, idx_pud;
2083 unsigned long *new_p2m;
2085 size = PAGE_ALIGN(xen_start_info->nr_pages * sizeof(unsigned long));
2086 n_pte = roundup(size, PAGE_SIZE) >> PAGE_SHIFT;
2087 n_pt = roundup(size, PMD_SIZE) >> PMD_SHIFT;
2088 n_pmd = roundup(size, PUD_SIZE) >> PUD_SHIFT;
2089 n_pud = roundup(size, PGDIR_SIZE) >> PGDIR_SHIFT;
2090 n_frames = n_pte + n_pt + n_pmd + n_pud;
2092 new_area = xen_find_free_area(PFN_PHYS(n_frames));
2094 xen_raw_console_write("Can't find new memory area for p2m needed due to E820 map conflict\n");
2099 * Setup the page tables for addressing the new p2m list.
2100 * We have asked the hypervisor to map the p2m list at the user address
2101 * PUD_SIZE. It may have done so, or it may have used a kernel space
2102 * address depending on the Xen version.
2103 * To avoid any possible virtual address collision, just use
2104 * 2 * PUD_SIZE for the new area.
2106 pud_phys = new_area;
2107 pmd_phys = pud_phys + PFN_PHYS(n_pud);
2108 pt_phys = pmd_phys + PFN_PHYS(n_pmd);
2109 p2m_pfn = PFN_DOWN(pt_phys) + n_pt;
2111 pgd = __va(read_cr3());
2112 new_p2m = (unsigned long *)(2 * PGDIR_SIZE);
2113 for (idx_pud = 0; idx_pud < n_pud; idx_pud++) {
2114 pud = early_memremap(pud_phys, PAGE_SIZE);
2116 for (idx_pmd = 0; idx_pmd < min(n_pmd, PTRS_PER_PUD);
2118 pmd = early_memremap(pmd_phys, PAGE_SIZE);
2120 for (idx_pt = 0; idx_pt < min(n_pt, PTRS_PER_PMD);
2122 pt = early_memremap(pt_phys, PAGE_SIZE);
2125 idx_pte < min(n_pte, PTRS_PER_PTE);
2127 set_pte(pt + idx_pte,
2128 pfn_pte(p2m_pfn, PAGE_KERNEL));
2131 n_pte -= PTRS_PER_PTE;
2132 early_memunmap(pt, PAGE_SIZE);
2133 make_lowmem_page_readonly(__va(pt_phys));
2134 pin_pagetable_pfn(MMUEXT_PIN_L1_TABLE,
2136 set_pmd(pmd + idx_pt,
2137 __pmd(_PAGE_TABLE | pt_phys));
2138 pt_phys += PAGE_SIZE;
2140 n_pt -= PTRS_PER_PMD;
2141 early_memunmap(pmd, PAGE_SIZE);
2142 make_lowmem_page_readonly(__va(pmd_phys));
2143 pin_pagetable_pfn(MMUEXT_PIN_L2_TABLE,
2144 PFN_DOWN(pmd_phys));
2145 set_pud(pud + idx_pmd, __pud(_PAGE_TABLE | pmd_phys));
2146 pmd_phys += PAGE_SIZE;
2148 n_pmd -= PTRS_PER_PUD;
2149 early_memunmap(pud, PAGE_SIZE);
2150 make_lowmem_page_readonly(__va(pud_phys));
2151 pin_pagetable_pfn(MMUEXT_PIN_L3_TABLE, PFN_DOWN(pud_phys));
2152 set_pgd(pgd + 2 + idx_pud, __pgd(_PAGE_TABLE | pud_phys));
2153 pud_phys += PAGE_SIZE;
2156 /* Now copy the old p2m info to the new area. */
2157 memcpy(new_p2m, xen_p2m_addr, size);
2158 xen_p2m_addr = new_p2m;
2160 /* Release the old p2m list and set new list info. */
2161 p2m_pfn = PFN_DOWN(xen_early_virt_to_phys(xen_start_info->mfn_list));
2163 p2m_pfn_end = p2m_pfn + PFN_DOWN(size);
2165 if (xen_start_info->mfn_list < __START_KERNEL_map) {
2166 pfn = xen_start_info->first_p2m_pfn;
2167 pfn_end = xen_start_info->first_p2m_pfn +
2168 xen_start_info->nr_p2m_frames;
2169 set_pgd(pgd + 1, __pgd(0));
2172 pfn_end = p2m_pfn_end;
2175 memblock_free(PFN_PHYS(pfn), PAGE_SIZE * (pfn_end - pfn));
2176 while (pfn < pfn_end) {
2177 if (pfn == p2m_pfn) {
2181 make_lowmem_page_readwrite(__va(PFN_PHYS(pfn)));
2185 xen_start_info->mfn_list = (unsigned long)xen_p2m_addr;
2186 xen_start_info->first_p2m_pfn = PFN_DOWN(new_area);
2187 xen_start_info->nr_p2m_frames = n_frames;
2190 #else /* !CONFIG_X86_64 */
2191 static RESERVE_BRK_ARRAY(pmd_t, initial_kernel_pmd, PTRS_PER_PMD);
2192 static RESERVE_BRK_ARRAY(pmd_t, swapper_kernel_pmd, PTRS_PER_PMD);
2194 static void __init xen_write_cr3_init(unsigned long cr3)
2196 unsigned long pfn = PFN_DOWN(__pa(swapper_pg_dir));
2198 BUG_ON(read_cr3() != __pa(initial_page_table));
2199 BUG_ON(cr3 != __pa(swapper_pg_dir));
2202 * We are switching to swapper_pg_dir for the first time (from
2203 * initial_page_table) and therefore need to mark that page
2204 * read-only and then pin it.
2206 * Xen disallows sharing of kernel PMDs for PAE
2207 * guests. Therefore we must copy the kernel PMD from
2208 * initial_page_table into a new kernel PMD to be used in
2211 swapper_kernel_pmd =
2212 extend_brk(sizeof(pmd_t) * PTRS_PER_PMD, PAGE_SIZE);
2213 copy_page(swapper_kernel_pmd, initial_kernel_pmd);
2214 swapper_pg_dir[KERNEL_PGD_BOUNDARY] =
2215 __pgd(__pa(swapper_kernel_pmd) | _PAGE_PRESENT);
2216 set_page_prot(swapper_kernel_pmd, PAGE_KERNEL_RO);
2218 set_page_prot(swapper_pg_dir, PAGE_KERNEL_RO);
2220 pin_pagetable_pfn(MMUEXT_PIN_L3_TABLE, pfn);
2222 pin_pagetable_pfn(MMUEXT_UNPIN_TABLE,
2223 PFN_DOWN(__pa(initial_page_table)));
2224 set_page_prot(initial_page_table, PAGE_KERNEL);
2225 set_page_prot(initial_kernel_pmd, PAGE_KERNEL);
2227 pv_mmu_ops.write_cr3 = &xen_write_cr3;
2231 * For 32 bit domains xen_start_info->pt_base is the pgd address which might be
2232 * not the first page table in the page table pool.
2233 * Iterate through the initial page tables to find the real page table base.
2235 static phys_addr_t xen_find_pt_base(pmd_t *pmd)
2237 phys_addr_t pt_base, paddr;
2240 pt_base = min(__pa(xen_start_info->pt_base), __pa(pmd));
2242 for (pmdidx = 0; pmdidx < PTRS_PER_PMD; pmdidx++)
2243 if (pmd_present(pmd[pmdidx]) && !pmd_large(pmd[pmdidx])) {
2244 paddr = m2p(pmd[pmdidx].pmd);
2245 pt_base = min(pt_base, paddr);
2251 void __init xen_setup_kernel_pagetable(pgd_t *pgd, unsigned long max_pfn)
2255 kernel_pmd = m2v(pgd[KERNEL_PGD_BOUNDARY].pgd);
2257 xen_pt_base = xen_find_pt_base(kernel_pmd);
2258 xen_pt_size = xen_start_info->nr_pt_frames * PAGE_SIZE;
2260 initial_kernel_pmd =
2261 extend_brk(sizeof(pmd_t) * PTRS_PER_PMD, PAGE_SIZE);
2263 max_pfn_mapped = PFN_DOWN(xen_pt_base + xen_pt_size + 512 * 1024);
2265 copy_page(initial_kernel_pmd, kernel_pmd);
2267 xen_map_identity_early(initial_kernel_pmd, max_pfn);
2269 copy_page(initial_page_table, pgd);
2270 initial_page_table[KERNEL_PGD_BOUNDARY] =
2271 __pgd(__pa(initial_kernel_pmd) | _PAGE_PRESENT);
2273 set_page_prot(initial_kernel_pmd, PAGE_KERNEL_RO);
2274 set_page_prot(initial_page_table, PAGE_KERNEL_RO);
2275 set_page_prot(empty_zero_page, PAGE_KERNEL_RO);
2277 pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(pgd)));
2279 pin_pagetable_pfn(MMUEXT_PIN_L3_TABLE,
2280 PFN_DOWN(__pa(initial_page_table)));
2281 xen_write_cr3(__pa(initial_page_table));
2283 memblock_reserve(xen_pt_base, xen_pt_size);
2285 #endif /* CONFIG_X86_64 */
2287 void __init xen_reserve_special_pages(void)
2291 memblock_reserve(__pa(xen_start_info), PAGE_SIZE);
2292 if (xen_start_info->store_mfn) {
2293 paddr = PFN_PHYS(mfn_to_pfn(xen_start_info->store_mfn));
2294 memblock_reserve(paddr, PAGE_SIZE);
2296 if (!xen_initial_domain()) {
2297 paddr = PFN_PHYS(mfn_to_pfn(xen_start_info->console.domU.mfn));
2298 memblock_reserve(paddr, PAGE_SIZE);
2302 void __init xen_pt_check_e820(void)
2304 if (xen_is_e820_reserved(xen_pt_base, xen_pt_size)) {
2305 xen_raw_console_write("Xen hypervisor allocated page table memory conflicts with E820 map\n");
2310 static unsigned char dummy_mapping[PAGE_SIZE] __page_aligned_bss;
2312 static void xen_set_fixmap(unsigned idx, phys_addr_t phys, pgprot_t prot)
2316 phys >>= PAGE_SHIFT;
2319 case FIX_BTMAP_END ... FIX_BTMAP_BEGIN:
2321 #ifdef CONFIG_X86_32
2323 # ifdef CONFIG_HIGHMEM
2324 case FIX_KMAP_BEGIN ... FIX_KMAP_END:
2326 #elif defined(CONFIG_X86_VSYSCALL_EMULATION)
2329 case FIX_TEXT_POKE0:
2330 case FIX_TEXT_POKE1:
2331 /* All local page mappings */
2332 pte = pfn_pte(phys, prot);
2335 #ifdef CONFIG_X86_LOCAL_APIC
2336 case FIX_APIC_BASE: /* maps dummy local APIC */
2337 pte = pfn_pte(PFN_DOWN(__pa(dummy_mapping)), PAGE_KERNEL);
2341 #ifdef CONFIG_X86_IO_APIC
2342 case FIX_IO_APIC_BASE_0 ... FIX_IO_APIC_BASE_END:
2344 * We just don't map the IO APIC - all access is via
2345 * hypercalls. Keep the address in the pte for reference.
2347 pte = pfn_pte(PFN_DOWN(__pa(dummy_mapping)), PAGE_KERNEL);
2351 case FIX_PARAVIRT_BOOTMAP:
2352 /* This is an MFN, but it isn't an IO mapping from the
2354 pte = mfn_pte(phys, prot);
2358 /* By default, set_fixmap is used for hardware mappings */
2359 pte = mfn_pte(phys, prot);
2363 __native_set_fixmap(idx, pte);
2365 #ifdef CONFIG_X86_VSYSCALL_EMULATION
2366 /* Replicate changes to map the vsyscall page into the user
2367 pagetable vsyscall mapping. */
2368 if (idx == VSYSCALL_PAGE) {
2369 unsigned long vaddr = __fix_to_virt(idx);
2370 set_pte_vaddr_pud(level3_user_vsyscall, vaddr, pte);
2375 static void __init xen_post_allocator_init(void)
2377 if (xen_feature(XENFEAT_auto_translated_physmap))
2380 pv_mmu_ops.set_pte = xen_set_pte;
2381 pv_mmu_ops.set_pmd = xen_set_pmd;
2382 pv_mmu_ops.set_pud = xen_set_pud;
2383 #if CONFIG_PGTABLE_LEVELS == 4
2384 pv_mmu_ops.set_pgd = xen_set_pgd;
2387 /* This will work as long as patching hasn't happened yet
2388 (which it hasn't) */
2389 pv_mmu_ops.alloc_pte = xen_alloc_pte;
2390 pv_mmu_ops.alloc_pmd = xen_alloc_pmd;
2391 pv_mmu_ops.release_pte = xen_release_pte;
2392 pv_mmu_ops.release_pmd = xen_release_pmd;
2393 #if CONFIG_PGTABLE_LEVELS == 4
2394 pv_mmu_ops.alloc_pud = xen_alloc_pud;
2395 pv_mmu_ops.release_pud = xen_release_pud;
2397 pv_mmu_ops.make_pte = PV_CALLEE_SAVE(xen_make_pte);
2399 #ifdef CONFIG_X86_64
2400 pv_mmu_ops.write_cr3 = &xen_write_cr3;
2401 SetPagePinned(virt_to_page(level3_user_vsyscall));
2403 xen_mark_init_mm_pinned();
2406 static void xen_leave_lazy_mmu(void)
2410 paravirt_leave_lazy_mmu();
2414 static const struct pv_mmu_ops xen_mmu_ops __initconst = {
2415 .read_cr2 = xen_read_cr2,
2416 .write_cr2 = xen_write_cr2,
2418 .read_cr3 = xen_read_cr3,
2419 .write_cr3 = xen_write_cr3_init,
2421 .flush_tlb_user = xen_flush_tlb,
2422 .flush_tlb_kernel = xen_flush_tlb,
2423 .flush_tlb_single = xen_flush_tlb_single,
2424 .flush_tlb_others = xen_flush_tlb_others,
2426 .pte_update = paravirt_nop,
2428 .pgd_alloc = xen_pgd_alloc,
2429 .pgd_free = xen_pgd_free,
2431 .alloc_pte = xen_alloc_pte_init,
2432 .release_pte = xen_release_pte_init,
2433 .alloc_pmd = xen_alloc_pmd_init,
2434 .release_pmd = xen_release_pmd_init,
2436 .set_pte = xen_set_pte_init,
2437 .set_pte_at = xen_set_pte_at,
2438 .set_pmd = xen_set_pmd_hyper,
2440 .ptep_modify_prot_start = __ptep_modify_prot_start,
2441 .ptep_modify_prot_commit = __ptep_modify_prot_commit,
2443 .pte_val = PV_CALLEE_SAVE(xen_pte_val),
2444 .pgd_val = PV_CALLEE_SAVE(xen_pgd_val),
2446 .make_pte = PV_CALLEE_SAVE(xen_make_pte_init),
2447 .make_pgd = PV_CALLEE_SAVE(xen_make_pgd),
2449 #ifdef CONFIG_X86_PAE
2450 .set_pte_atomic = xen_set_pte_atomic,
2451 .pte_clear = xen_pte_clear,
2452 .pmd_clear = xen_pmd_clear,
2453 #endif /* CONFIG_X86_PAE */
2454 .set_pud = xen_set_pud_hyper,
2456 .make_pmd = PV_CALLEE_SAVE(xen_make_pmd),
2457 .pmd_val = PV_CALLEE_SAVE(xen_pmd_val),
2459 #if CONFIG_PGTABLE_LEVELS == 4
2460 .pud_val = PV_CALLEE_SAVE(xen_pud_val),
2461 .make_pud = PV_CALLEE_SAVE(xen_make_pud),
2462 .set_pgd = xen_set_pgd_hyper,
2464 .alloc_pud = xen_alloc_pmd_init,
2465 .release_pud = xen_release_pmd_init,
2466 #endif /* CONFIG_PGTABLE_LEVELS == 4 */
2468 .activate_mm = xen_activate_mm,
2469 .dup_mmap = xen_dup_mmap,
2470 .exit_mmap = xen_exit_mmap,
2473 .enter = paravirt_enter_lazy_mmu,
2474 .leave = xen_leave_lazy_mmu,
2475 .flush = paravirt_flush_lazy_mmu,
2478 .set_fixmap = xen_set_fixmap,
2481 void __init xen_init_mmu_ops(void)
2483 x86_init.paging.pagetable_init = xen_pagetable_init;
2485 if (xen_feature(XENFEAT_auto_translated_physmap))
2488 pv_mmu_ops = xen_mmu_ops;
2490 memset(dummy_mapping, 0xff, PAGE_SIZE);
2493 /* Protected by xen_reservation_lock. */
2494 #define MAX_CONTIG_ORDER 9 /* 2MB */
2495 static unsigned long discontig_frames[1<<MAX_CONTIG_ORDER];
2497 #define VOID_PTE (mfn_pte(0, __pgprot(0)))
2498 static void xen_zap_pfn_range(unsigned long vaddr, unsigned int order,
2499 unsigned long *in_frames,
2500 unsigned long *out_frames)
2503 struct multicall_space mcs;
2506 for (i = 0; i < (1UL<<order); i++, vaddr += PAGE_SIZE) {
2507 mcs = __xen_mc_entry(0);
2510 in_frames[i] = virt_to_mfn(vaddr);
2512 MULTI_update_va_mapping(mcs.mc, vaddr, VOID_PTE, 0);
2513 __set_phys_to_machine(virt_to_pfn(vaddr), INVALID_P2M_ENTRY);
2516 out_frames[i] = virt_to_pfn(vaddr);
2522 * Update the pfn-to-mfn mappings for a virtual address range, either to
2523 * point to an array of mfns, or contiguously from a single starting
2526 static void xen_remap_exchanged_ptes(unsigned long vaddr, int order,
2527 unsigned long *mfns,
2528 unsigned long first_mfn)
2535 limit = 1u << order;
2536 for (i = 0; i < limit; i++, vaddr += PAGE_SIZE) {
2537 struct multicall_space mcs;
2540 mcs = __xen_mc_entry(0);
2544 mfn = first_mfn + i;
2546 if (i < (limit - 1))
2550 flags = UVMF_INVLPG | UVMF_ALL;
2552 flags = UVMF_TLB_FLUSH | UVMF_ALL;
2555 MULTI_update_va_mapping(mcs.mc, vaddr,
2556 mfn_pte(mfn, PAGE_KERNEL), flags);
2558 set_phys_to_machine(virt_to_pfn(vaddr), mfn);
2565 * Perform the hypercall to exchange a region of our pfns to point to
2566 * memory with the required contiguous alignment. Takes the pfns as
2567 * input, and populates mfns as output.
2569 * Returns a success code indicating whether the hypervisor was able to
2570 * satisfy the request or not.
2572 static int xen_exchange_memory(unsigned long extents_in, unsigned int order_in,
2573 unsigned long *pfns_in,
2574 unsigned long extents_out,
2575 unsigned int order_out,
2576 unsigned long *mfns_out,
2577 unsigned int address_bits)
2582 struct xen_memory_exchange exchange = {
2584 .nr_extents = extents_in,
2585 .extent_order = order_in,
2586 .extent_start = pfns_in,
2590 .nr_extents = extents_out,
2591 .extent_order = order_out,
2592 .extent_start = mfns_out,
2593 .address_bits = address_bits,
2598 BUG_ON(extents_in << order_in != extents_out << order_out);
2600 rc = HYPERVISOR_memory_op(XENMEM_exchange, &exchange);
2601 success = (exchange.nr_exchanged == extents_in);
2603 BUG_ON(!success && ((exchange.nr_exchanged != 0) || (rc == 0)));
2604 BUG_ON(success && (rc != 0));
2609 int xen_create_contiguous_region(phys_addr_t pstart, unsigned int order,
2610 unsigned int address_bits,
2611 dma_addr_t *dma_handle)
2613 unsigned long *in_frames = discontig_frames, out_frame;
2614 unsigned long flags;
2616 unsigned long vstart = (unsigned long)phys_to_virt(pstart);
2619 * Currently an auto-translated guest will not perform I/O, nor will
2620 * it require PAE page directories below 4GB. Therefore any calls to
2621 * this function are redundant and can be ignored.
2624 if (xen_feature(XENFEAT_auto_translated_physmap))
2627 if (unlikely(order > MAX_CONTIG_ORDER))
2630 memset((void *) vstart, 0, PAGE_SIZE << order);
2632 spin_lock_irqsave(&xen_reservation_lock, flags);
2634 /* 1. Zap current PTEs, remembering MFNs. */
2635 xen_zap_pfn_range(vstart, order, in_frames, NULL);
2637 /* 2. Get a new contiguous memory extent. */
2638 out_frame = virt_to_pfn(vstart);
2639 success = xen_exchange_memory(1UL << order, 0, in_frames,
2640 1, order, &out_frame,
2643 /* 3. Map the new extent in place of old pages. */
2645 xen_remap_exchanged_ptes(vstart, order, NULL, out_frame);
2647 xen_remap_exchanged_ptes(vstart, order, in_frames, 0);
2649 spin_unlock_irqrestore(&xen_reservation_lock, flags);
2651 *dma_handle = virt_to_machine(vstart).maddr;
2652 return success ? 0 : -ENOMEM;
2654 EXPORT_SYMBOL_GPL(xen_create_contiguous_region);
2656 void xen_destroy_contiguous_region(phys_addr_t pstart, unsigned int order)
2658 unsigned long *out_frames = discontig_frames, in_frame;
2659 unsigned long flags;
2661 unsigned long vstart;
2663 if (xen_feature(XENFEAT_auto_translated_physmap))
2666 if (unlikely(order > MAX_CONTIG_ORDER))
2669 vstart = (unsigned long)phys_to_virt(pstart);
2670 memset((void *) vstart, 0, PAGE_SIZE << order);
2672 spin_lock_irqsave(&xen_reservation_lock, flags);
2674 /* 1. Find start MFN of contiguous extent. */
2675 in_frame = virt_to_mfn(vstart);
2677 /* 2. Zap current PTEs. */
2678 xen_zap_pfn_range(vstart, order, NULL, out_frames);
2680 /* 3. Do the exchange for non-contiguous MFNs. */
2681 success = xen_exchange_memory(1, order, &in_frame, 1UL << order,
2684 /* 4. Map new pages in place of old pages. */
2686 xen_remap_exchanged_ptes(vstart, order, out_frames, 0);
2688 xen_remap_exchanged_ptes(vstart, order, NULL, in_frame);
2690 spin_unlock_irqrestore(&xen_reservation_lock, flags);
2692 EXPORT_SYMBOL_GPL(xen_destroy_contiguous_region);
2694 #ifdef CONFIG_XEN_PVHVM
2695 #ifdef CONFIG_PROC_VMCORE
2697 * This function is used in two contexts:
2698 * - the kdump kernel has to check whether a pfn of the crashed kernel
2699 * was a ballooned page. vmcore is using this function to decide
2700 * whether to access a pfn of the crashed kernel.
2701 * - the kexec kernel has to check whether a pfn was ballooned by the
2702 * previous kernel. If the pfn is ballooned, handle it properly.
2703 * Returns 0 if the pfn is not backed by a RAM page, the caller may
2704 * handle the pfn special in this case.
2706 static int xen_oldmem_pfn_is_ram(unsigned long pfn)
2708 struct xen_hvm_get_mem_type a = {
2709 .domid = DOMID_SELF,
2714 if (HYPERVISOR_hvm_op(HVMOP_get_mem_type, &a))
2717 switch (a.mem_type) {
2718 case HVMMEM_mmio_dm:
2732 static void xen_hvm_exit_mmap(struct mm_struct *mm)
2734 struct xen_hvm_pagetable_dying a;
2737 a.domid = DOMID_SELF;
2738 a.gpa = __pa(mm->pgd);
2739 rc = HYPERVISOR_hvm_op(HVMOP_pagetable_dying, &a);
2740 WARN_ON_ONCE(rc < 0);
2743 static int is_pagetable_dying_supported(void)
2745 struct xen_hvm_pagetable_dying a;
2748 a.domid = DOMID_SELF;
2750 rc = HYPERVISOR_hvm_op(HVMOP_pagetable_dying, &a);
2752 printk(KERN_DEBUG "HVMOP_pagetable_dying not supported\n");
2758 void __init xen_hvm_init_mmu_ops(void)
2760 if (is_pagetable_dying_supported())
2761 pv_mmu_ops.exit_mmap = xen_hvm_exit_mmap;
2762 #ifdef CONFIG_PROC_VMCORE
2763 register_oldmem_pfn_is_ram(&xen_oldmem_pfn_is_ram);
2768 #define REMAP_BATCH_SIZE 16
2774 struct mmu_update *mmu_update;
2777 static int remap_area_mfn_pte_fn(pte_t *ptep, pgtable_t token,
2778 unsigned long addr, void *data)
2780 struct remap_data *rmd = data;
2781 pte_t pte = pte_mkspecial(mfn_pte(*rmd->mfn, rmd->prot));
2783 /* If we have a contiguous range, just update the mfn itself,
2784 else update pointer to be "next mfn". */
2785 if (rmd->contiguous)
2790 rmd->mmu_update->ptr = virt_to_machine(ptep).maddr;
2791 rmd->mmu_update->val = pte_val_ma(pte);
2797 static int do_remap_gfn(struct vm_area_struct *vma,
2799 xen_pfn_t *gfn, int nr,
2800 int *err_ptr, pgprot_t prot,
2802 struct page **pages)
2805 struct remap_data rmd;
2806 struct mmu_update mmu_update[REMAP_BATCH_SIZE];
2807 unsigned long range;
2810 BUG_ON(!((vma->vm_flags & (VM_PFNMAP | VM_IO)) == (VM_PFNMAP | VM_IO)));
2812 if (xen_feature(XENFEAT_auto_translated_physmap)) {
2813 #ifdef CONFIG_XEN_PVH
2814 /* We need to update the local page tables and the xen HAP */
2815 return xen_xlate_remap_gfn_array(vma, addr, gfn, nr, err_ptr,
2816 prot, domid, pages);
2824 /* We use the err_ptr to indicate if there we are doing a contiguous
2825 * mapping or a discontigious mapping. */
2826 rmd.contiguous = !err_ptr;
2831 int batch = min(REMAP_BATCH_SIZE, nr);
2832 int batch_left = batch;
2833 range = (unsigned long)batch << PAGE_SHIFT;
2835 rmd.mmu_update = mmu_update;
2836 err = apply_to_page_range(vma->vm_mm, addr, range,
2837 remap_area_mfn_pte_fn, &rmd);
2841 /* We record the error for each page that gives an error, but
2842 * continue mapping until the whole set is done */
2846 err = HYPERVISOR_mmu_update(&mmu_update[index],
2847 batch_left, &done, domid);
2850 * @err_ptr may be the same buffer as @gfn, so
2851 * only clear it after each chunk of @gfn is
2855 for (i = index; i < index + done; i++)
2862 done++; /* Skip failed frame. */
2867 } while (batch_left);
2877 xen_flush_tlb_all();
2879 return err < 0 ? err : mapped;
2882 int xen_remap_domain_gfn_range(struct vm_area_struct *vma,
2884 xen_pfn_t gfn, int nr,
2885 pgprot_t prot, unsigned domid,
2886 struct page **pages)
2888 return do_remap_gfn(vma, addr, &gfn, nr, NULL, prot, domid, pages);
2890 EXPORT_SYMBOL_GPL(xen_remap_domain_gfn_range);
2892 int xen_remap_domain_gfn_array(struct vm_area_struct *vma,
2894 xen_pfn_t *gfn, int nr,
2895 int *err_ptr, pgprot_t prot,
2896 unsigned domid, struct page **pages)
2898 /* We BUG_ON because it's a programmer error to pass a NULL err_ptr,
2899 * and the consequences later is quite hard to detect what the actual
2900 * cause of "wrong memory was mapped in".
2902 BUG_ON(err_ptr == NULL);
2903 return do_remap_gfn(vma, addr, gfn, nr, err_ptr, prot, domid, pages);
2905 EXPORT_SYMBOL_GPL(xen_remap_domain_gfn_array);
2908 /* Returns: 0 success */
2909 int xen_unmap_domain_gfn_range(struct vm_area_struct *vma,
2910 int numpgs, struct page **pages)
2912 if (!pages || !xen_feature(XENFEAT_auto_translated_physmap))
2915 #ifdef CONFIG_XEN_PVH
2916 return xen_xlate_unmap_gfn_range(vma, numpgs, pages);
2921 EXPORT_SYMBOL_GPL(xen_unmap_domain_gfn_range);