// SPDX-License-Identifier: GPL-2.0 /* * This is a module to test the HMM (Heterogeneous Memory Management) * mirror and zone device private memory migration APIs of the kernel. * Userspace programs can register with the driver to mirror their own address * space and can use the device to read/write any valid virtual address. */ #include <linux/init.h> #include <linux/fs.h> #include <linux/mm.h> #include <linux/module.h> #include <linux/kernel.h> #include <linux/cdev.h> #include <linux/device.h> #include <linux/mutex.h> #include <linux/rwsem.h> #include <linux/sched.h> #include <linux/slab.h> #include <linux/highmem.h> #include <linux/delay.h> #include <linux/pagemap.h> #include <linux/hmm.h> #include <linux/vmalloc.h> #include <linux/swap.h> #include <linux/swapops.h> #include <linux/sched/mm.h> #include <linux/platform_device.h> #include "test_hmm_uapi.h" #define DMIRROR_NDEVICES 2 #define DMIRROR_RANGE_FAULT_TIMEOUT 1000 #define DEVMEM_CHUNK_SIZE (256 * 1024 * 1024U) #define DEVMEM_CHUNKS_RESERVE 16 static const struct dev_pagemap_ops dmirror_devmem_ops; static const struct mmu_interval_notifier_ops dmirror_min_ops; static dev_t dmirror_dev; static struct page *dmirror_zero_page; struct dmirror_device; struct dmirror_bounce { void *ptr; unsigned long size; unsigned long addr; unsigned long cpages; }; #define DPT_XA_TAG_WRITE 3UL /* * Data structure to track address ranges and register for mmu interval * notifier updates. */ struct dmirror_interval { struct mmu_interval_notifier notifier; struct dmirror *dmirror; }; /* * Data attached to the open device file. * Note that it might be shared after a fork(). */ struct dmirror { struct dmirror_device *mdevice; struct xarray pt; struct mmu_interval_notifier notifier; struct mutex mutex; }; /* * ZONE_DEVICE pages for migration and simulating device memory. */ struct dmirror_chunk { struct dev_pagemap pagemap; struct dmirror_device *mdevice; }; /* * Per device data. */ struct dmirror_device { struct cdev cdevice; struct hmm_devmem *devmem; unsigned int devmem_capacity; unsigned int devmem_count; struct dmirror_chunk **devmem_chunks; struct mutex devmem_lock; /* protects the above */ unsigned long calloc; unsigned long cfree; struct page *free_pages; spinlock_t lock; /* protects the above */ }; static struct dmirror_device dmirror_devices[DMIRROR_NDEVICES]; static int dmirror_bounce_init(struct dmirror_bounce *bounce, unsigned long addr, unsigned long size) { bounce->addr = addr; bounce->size = size; bounce->cpages = 0; bounce->ptr = vmalloc(size); if (!bounce->ptr) return -ENOMEM; return 0; } static void dmirror_bounce_fini(struct dmirror_bounce *bounce) { vfree(bounce->ptr); } static int dmirror_fops_open(struct inode *inode, struct file *filp) { struct cdev *cdev = inode->i_cdev; struct dmirror *dmirror; int ret; /* Mirror this process address space */ dmirror = kzalloc(sizeof(*dmirror), GFP_KERNEL); if (dmirror == NULL) return -ENOMEM; dmirror->mdevice = container_of(cdev, struct dmirror_device, cdevice); mutex_init(&dmirror->mutex); xa_init(&dmirror->pt); ret = mmu_interval_notifier_insert(&dmirror->notifier, current->mm, 0, ULONG_MAX & PAGE_MASK, &dmirror_min_ops); if (ret) { kfree(dmirror); return ret; } filp->private_data = dmirror; return 0; } static int dmirror_fops_release(struct inode *inode, struct file *filp) { struct dmirror *dmirror = filp->private_data; mmu_interval_notifier_remove(&dmirror->notifier); xa_destroy(&dmirror->pt); kfree(dmirror); return 0; } static struct dmirror_device *dmirror_page_to_device(struct page *page) { return container_of(page->pgmap, struct dmirror_chunk, pagemap)->mdevice; } static int dmirror_do_fault(struct dmirror *dmirror, struct hmm_range *range) { unsigned long *pfns = range->hmm_pfns; unsigned long pfn; for (pfn = (range->start >> PAGE_SHIFT); pfn < (range->end >> PAGE_SHIFT); pfn++, pfns++) { struct page *page; void *entry; /* * Since we asked for hmm_range_fault() to populate pages, * it shouldn't return an error entry on success. */ WARN_ON(*pfns & HMM_PFN_ERROR); WARN_ON(!(*pfns & HMM_PFN_VALID)); page = hmm_pfn_to_page(*pfns); WARN_ON(!page); entry = page; if (*pfns & HMM_PFN_WRITE) entry = xa_tag_pointer(entry, DPT_XA_TAG_WRITE); else if (WARN_ON(range->default_flags & HMM_PFN_WRITE)) return -EFAULT; entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC); if (xa_is_err(entry)) return xa_err(entry); } return 0; } static void dmirror_do_update(struct dmirror *dmirror, unsigned long start, unsigned long end) { unsigned long pfn; void *entry; /* * The XArray doesn't hold references to pages since it relies on * the mmu notifier to clear page pointers when they become stale. * Therefore, it is OK to just clear the entry. */ xa_for_each_range(&dmirror->pt, pfn, entry, start >> PAGE_SHIFT, end >> PAGE_SHIFT) xa_erase(&dmirror->pt, pfn); } static bool dmirror_interval_invalidate(struct mmu_interval_notifier *mni, const struct mmu_notifier_range *range, unsigned long cur_seq) { struct dmirror *dmirror = container_of(mni, struct dmirror, notifier); if (mmu_notifier_range_blockable(range)) mutex_lock(&dmirror->mutex); else if (!mutex_trylock(&dmirror->mutex)) return false; mmu_interval_set_seq(mni, cur_seq); dmirror_do_update(dmirror, range->start, range->end); mutex_unlock(&dmirror->mutex); return true; } static const struct mmu_interval_notifier_ops dmirror_min_ops = { .invalidate = dmirror_interval_invalidate, }; static int dmirror_range_fault(struct dmirror *dmirror, struct hmm_range *range) { struct mm_struct *mm = dmirror->notifier.mm; unsigned long timeout = jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); int ret; while (true) { if (time_after(jiffies, timeout)) { ret = -EBUSY; goto out; } range->notifier_seq = mmu_interval_read_begin(range->notifier); mmap_read_lock(mm); ret = hmm_range_fault(range); mmap_read_unlock(mm); if (ret) { if (ret == -EBUSY) continue; goto out; } mutex_lock(&dmirror->mutex); if (mmu_interval_read_retry(range->notifier, range->notifier_seq)) { mutex_unlock(&dmirror->mutex); continue; } break; } ret = dmirror_do_fault(dmirror, range); mutex_unlock(&dmirror->mutex); out: return ret; } static int dmirror_fault(struct dmirror *dmirror, unsigned long start, unsigned long end, bool write) { struct mm_struct *mm = dmirror->notifier.mm; unsigned long addr; unsigned long pfns[64]; struct hmm_range range = { .notifier = &dmirror->notifier, .hmm_pfns = pfns, .pfn_flags_mask = 0, .default_flags = HMM_PFN_REQ_FAULT | (write ? HMM_PFN_REQ_WRITE : 0), .dev_private_owner = dmirror->mdevice, }; int ret = 0; /* Since the mm is for the mirrored process, get a reference first. */ if (!mmget_not_zero(mm)) return 0; for (addr = start; addr < end; addr = range.end) { range.start = addr; range.end = min(addr + (ARRAY_SIZE(pfns) << PAGE_SHIFT), end); ret = dmirror_range_fault(dmirror, &range); if (ret) break; } mmput(mm); return ret; } static int dmirror_do_read(struct dmirror *dmirror, unsigned long start, unsigned long end, struct dmirror_bounce *bounce) { unsigned long pfn; void *ptr; ptr = bounce->ptr + ((start - bounce->addr) & PAGE_MASK); for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) { void *entry; struct page *page; void *tmp; entry = xa_load(&dmirror->pt, pfn); page = xa_untag_pointer(entry); if (!page) return -ENOENT; tmp = kmap(page); memcpy(ptr, tmp, PAGE_SIZE); kunmap(page); ptr += PAGE_SIZE; bounce->cpages++; } return 0; } static int dmirror_read(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd) { struct dmirror_bounce bounce; unsigned long start, end; unsigned long size = cmd->npages << PAGE_SHIFT; int ret; start = cmd->addr; end = start + size; if (end < start) return -EINVAL; ret = dmirror_bounce_init(&bounce, start, size); if (ret) return ret; while (1) { mutex_lock(&dmirror->mutex); ret = dmirror_do_read(dmirror, start, end, &bounce); mutex_unlock(&dmirror->mutex); if (ret != -ENOENT) break; start = cmd->addr + (bounce.cpages << PAGE_SHIFT); ret = dmirror_fault(dmirror, start, end, false); if (ret) break; cmd->faults++; } if (ret == 0) { if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr, bounce.size)) ret = -EFAULT; } cmd->cpages = bounce.cpages; dmirror_bounce_fini(&bounce); return ret; } static int dmirror_do_write(struct dmirror *dmirror, unsigned long start, unsigned long end, struct dmirror_bounce *bounce) { unsigned long pfn; void *ptr; ptr = bounce->ptr + ((start - bounce->addr) & PAGE_MASK); for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) { void *entry; struct page *page; void *tmp; entry = xa_load(&dmirror->pt, pfn); page = xa_untag_pointer(entry); if (!page || xa_pointer_tag(entry) != DPT_XA_TAG_WRITE) return -ENOENT; tmp = kmap(page); memcpy(tmp, ptr, PAGE_SIZE); kunmap(page); ptr += PAGE_SIZE; bounce->cpages++; } return 0; } static int dmirror_write(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd) { struct dmirror_bounce bounce; unsigned long start, end; unsigned long size = cmd->npages << PAGE_SHIFT; int ret; start = cmd->addr; end = start + size; if (end < start) return -EINVAL; ret = dmirror_bounce_init(&bounce, start, size); if (ret) return ret; if (copy_from_user(bounce.ptr, u64_to_user_ptr(cmd->ptr), bounce.size)) { ret = -EFAULT; goto fini; } while (1) { mutex_lock(&dmirror->mutex); ret = dmirror_do_write(dmirror, start, end, &bounce); mutex_unlock(&dmirror->mutex); if (ret != -ENOENT) break; start = cmd->addr + (bounce.cpages << PAGE_SHIFT); ret = dmirror_fault(dmirror, start, end, true); if (ret) break; cmd->faults++; } fini: cmd->cpages = bounce.cpages; dmirror_bounce_fini(&bounce); return ret; } static bool dmirror_allocate_chunk(struct dmirror_device *mdevice, struct page **ppage) { struct dmirror_chunk *devmem; struct resource *res; unsigned long pfn; unsigned long pfn_first; unsigned long pfn_last; void *ptr; mutex_lock(&mdevice->devmem_lock); if (mdevice->devmem_count == mdevice->devmem_capacity) { struct dmirror_chunk **new_chunks; unsigned int new_capacity; new_capacity = mdevice->devmem_capacity + DEVMEM_CHUNKS_RESERVE; new_chunks = krealloc(mdevice->devmem_chunks, sizeof(new_chunks[0]) * new_capacity, GFP_KERNEL); if (!new_chunks) goto err; mdevice->devmem_capacity = new_capacity; mdevice->devmem_chunks = new_chunks; } res = request_free_mem_region(&iomem_resource, DEVMEM_CHUNK_SIZE, "hmm_dmirror"); if (IS_ERR(res)) goto err; devmem = kzalloc(sizeof(*devmem), GFP_KERNEL); if (!devmem) goto err_release; devmem->pagemap.type = MEMORY_DEVICE_PRIVATE; devmem->pagemap.res = *res; devmem->pagemap.ops = &dmirror_devmem_ops; devmem->pagemap.owner = mdevice; ptr = memremap_pages(&devmem->pagemap, numa_node_id()); if (IS_ERR(ptr)) goto err_free; devmem->mdevice = mdevice; pfn_first = devmem->pagemap.res.start >> PAGE_SHIFT; pfn_last = pfn_first + (resource_size(&devmem->pagemap.res) >> PAGE_SHIFT); mdevice->devmem_chunks[mdevice->devmem_count++] = devmem; mutex_unlock(&mdevice->devmem_lock); pr_info("added new %u MB chunk (total %u chunks, %u MB) PFNs [0x%lx 0x%lx)\n", DEVMEM_CHUNK_SIZE / (1024 * 1024), mdevice->devmem_count, mdevice->devmem_count * (DEVMEM_CHUNK_SIZE / (1024 * 1024)), pfn_first, pfn_last); spin_lock(&mdevice->lock); for (pfn = pfn_first; pfn < pfn_last; pfn++) { struct page *page = pfn_to_page(pfn); page->zone_device_data = mdevice->free_pages; mdevice->free_pages = page; } if (ppage) { *ppage = mdevice->free_pages; mdevice->free_pages = (*ppage)->zone_device_data; mdevice->calloc++; } spin_unlock(&mdevice->lock); return true; err_free: kfree(devmem); err_release: release_mem_region(devmem->pagemap.res.start, resource_size(&devmem->pagemap.res)); err: mutex_unlock(&mdevice->devmem_lock); return false; } static struct page *dmirror_devmem_alloc_page(struct dmirror_device *mdevice) { struct page *dpage = NULL; struct page *rpage; /* * This is a fake device so we alloc real system memory to store * our device memory. */ rpage = alloc_page(GFP_HIGHUSER); if (!rpage) return NULL; spin_lock(&mdevice->lock); if (mdevice->free_pages) { dpage = mdevice->free_pages; mdevice->free_pages = dpage->zone_device_data; mdevice->calloc++; spin_unlock(&mdevice->lock); } else { spin_unlock(&mdevice->lock); if (!dmirror_allocate_chunk(mdevice, &dpage)) goto error; } dpage->zone_device_data = rpage; get_page(dpage); lock_page(dpage); return dpage; error: __free_page(rpage); return NULL; } static void dmirror_migrate_alloc_and_copy(struct migrate_vma *args, struct dmirror *dmirror) { struct dmirror_device *mdevice = dmirror->mdevice; const unsigned long *src = args->src; unsigned long *dst = args->dst; unsigned long addr; for (addr = args->start; addr < args->end; addr += PAGE_SIZE, src++, dst++) { struct page *spage; struct page *dpage; struct page *rpage; if (!(*src & MIGRATE_PFN_MIGRATE)) continue; /* * Note that spage might be NULL which is OK since it is an * unallocated pte_none() or read-only zero page. */ spage = migrate_pfn_to_page(*src); /* * Don't migrate device private pages from our own driver or * others. For our own we would do a device private memory copy * not a migration and for others, we would need to fault the * other device's page into system memory first. */ if (spage && is_zone_device_page(spage)) continue; dpage = dmirror_devmem_alloc_page(mdevice); if (!dpage) continue; rpage = dpage->zone_device_data; if (spage) copy_highpage(rpage, spage); else clear_highpage(rpage); /* * Normally, a device would use the page->zone_device_data to * point to the mirror but here we use it to hold the page for * the simulated device memory and that page holds the pointer * to the mirror. */ rpage->zone_device_data = dmirror; *dst = migrate_pfn(page_to_pfn(dpage)) | MIGRATE_PFN_LOCKED; if ((*src & MIGRATE_PFN_WRITE) || (!spage && args->vma->vm_flags & VM_WRITE)) *dst |= MIGRATE_PFN_WRITE; } } static int dmirror_migrate_finalize_and_map(struct migrate_vma *args, struct dmirror *dmirror) { unsigned long start = args->start; unsigned long end = args->end; const unsigned long *src = args->src; const unsigned long *dst = args->dst; unsigned long pfn; /* Map the migrated pages into the device's page tables. */ mutex_lock(&dmirror->mutex); for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++, src++, dst++) { struct page *dpage; void *entry; if (!(*src & MIGRATE_PFN_MIGRATE)) continue; dpage = migrate_pfn_to_page(*dst); if (!dpage) continue; /* * Store the page that holds the data so the page table * doesn't have to deal with ZONE_DEVICE private pages. */ entry = dpage->zone_device_data; if (*dst & MIGRATE_PFN_WRITE) entry = xa_tag_pointer(entry, DPT_XA_TAG_WRITE); entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC); if (xa_is_err(entry)) { mutex_unlock(&dmirror->mutex); return xa_err(entry); } } mutex_unlock(&dmirror->mutex); return 0; } static int dmirror_migrate(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd) { unsigned long start, end, addr; unsigned long size = cmd->npages << PAGE_SHIFT; struct mm_struct *mm = dmirror->notifier.mm; struct vm_area_struct *vma; unsigned long src_pfns[64]; unsigned long dst_pfns[64]; struct dmirror_bounce bounce; struct migrate_vma args; unsigned long next; int ret; start = cmd->addr; end = start + size; if (end < start) return -EINVAL; /* Since the mm is for the mirrored process, get a reference first. */ if (!mmget_not_zero(mm)) return -EINVAL; mmap_read_lock(mm); for (addr = start; addr < end; addr = next) { vma = find_vma(mm, addr); if (!vma || addr < vma->vm_start || !(vma->vm_flags & VM_READ)) { ret = -EINVAL; goto out; } next = min(end, addr + (ARRAY_SIZE(src_pfns) << PAGE_SHIFT)); if (next > vma->vm_end) next = vma->vm_end; args.vma = vma; args.src = src_pfns; args.dst = dst_pfns; args.start = addr; args.end = next; args.src_owner = NULL; ret = migrate_vma_setup(&args); if (ret) goto out; dmirror_migrate_alloc_and_copy(&args, dmirror); migrate_vma_pages(&args); dmirror_migrate_finalize_and_map(&args, dmirror); migrate_vma_finalize(&args); } mmap_read_unlock(mm); mmput(mm); /* Return the migrated data for verification. */ ret = dmirror_bounce_init(&bounce, start, size); if (ret) return ret; mutex_lock(&dmirror->mutex); ret = dmirror_do_read(dmirror, start, end, &bounce); mutex_unlock(&dmirror->mutex); if (ret == 0) { if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr, bounce.size)) ret = -EFAULT; } cmd->cpages = bounce.cpages; dmirror_bounce_fini(&bounce); return ret; out: mmap_read_unlock(mm); mmput(mm); return ret; } static void dmirror_mkentry(struct dmirror *dmirror, struct hmm_range *range, unsigned char *perm, unsigned long entry) { struct page *page; if (entry & HMM_PFN_ERROR) { *perm = HMM_DMIRROR_PROT_ERROR; return; } if (!(entry & HMM_PFN_VALID)) { *perm = HMM_DMIRROR_PROT_NONE; return; } page = hmm_pfn_to_page(entry); if (is_device_private_page(page)) { /* Is the page migrated to this device or some other? */ if (dmirror->mdevice == dmirror_page_to_device(page)) *perm = HMM_DMIRROR_PROT_DEV_PRIVATE_LOCAL; else *perm = HMM_DMIRROR_PROT_DEV_PRIVATE_REMOTE; } else if (is_zero_pfn(page_to_pfn(page))) *perm = HMM_DMIRROR_PROT_ZERO; else *perm = HMM_DMIRROR_PROT_NONE; if (entry & HMM_PFN_WRITE) *perm |= HMM_DMIRROR_PROT_WRITE; else *perm |= HMM_DMIRROR_PROT_READ; } static bool dmirror_snapshot_invalidate(struct mmu_interval_notifier *mni, const struct mmu_notifier_range *range, unsigned long cur_seq) { struct dmirror_interval *dmi = container_of(mni, struct dmirror_interval, notifier); struct dmirror *dmirror = dmi->dmirror; if (mmu_notifier_range_blockable(range)) mutex_lock(&dmirror->mutex); else if (!mutex_trylock(&dmirror->mutex)) return false; /* * Snapshots only need to set the sequence number since any * invalidation in the interval invalidates the whole snapshot. */ mmu_interval_set_seq(mni, cur_seq); mutex_unlock(&dmirror->mutex); return true; } static const struct mmu_interval_notifier_ops dmirror_mrn_ops = { .invalidate = dmirror_snapshot_invalidate, }; static int dmirror_range_snapshot(struct dmirror *dmirror, struct hmm_range *range, unsigned char *perm) { struct mm_struct *mm = dmirror->notifier.mm; struct dmirror_interval notifier; unsigned long timeout = jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); unsigned long i; unsigned long n; int ret = 0; notifier.dmirror = dmirror; range->notifier = ¬ifier.notifier; ret = mmu_interval_notifier_insert(range->notifier, mm, range->start, range->end - range->start, &dmirror_mrn_ops); if (ret) return ret; while (true) { if (time_after(jiffies, timeout)) { ret = -EBUSY; goto out; } range->notifier_seq = mmu_interval_read_begin(range->notifier); mmap_read_lock(mm); ret = hmm_range_fault(range); mmap_read_unlock(mm); if (ret) { if (ret == -EBUSY) continue; goto out; } mutex_lock(&dmirror->mutex); if (mmu_interval_read_retry(range->notifier, range->notifier_seq)) { mutex_unlock(&dmirror->mutex); continue; } break; } n = (range->end - range->start) >> PAGE_SHIFT; for (i = 0; i < n; i++) dmirror_mkentry(dmirror, range, perm + i, range->hmm_pfns[i]); mutex_unlock(&dmirror->mutex); out: mmu_interval_notifier_remove(range->notifier); return ret; } static int dmirror_snapshot(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd) { struct mm_struct *mm = dmirror->notifier.mm; unsigned long start, end; unsigned long size = cmd->npages << PAGE_SHIFT; unsigned long addr; unsigned long next; unsigned long pfns[64]; unsigned char perm[64]; char __user *uptr; struct hmm_range range = { .hmm_pfns = pfns, .dev_private_owner = dmirror->mdevice, }; int ret = 0; start = cmd->addr; end = start + size; if (end < start) return -EINVAL; /* Since the mm is for the mirrored process, get a reference first. */ if (!mmget_not_zero(mm)) return -EINVAL; /* * Register a temporary notifier to detect invalidations even if it * overlaps with other mmu_interval_notifiers. */ uptr = u64_to_user_ptr(cmd->ptr); for (addr = start; addr < end; addr = next) { unsigned long n; next = min(addr + (ARRAY_SIZE(pfns) << PAGE_SHIFT), end); range.start = addr; range.end = next; ret = dmirror_range_snapshot(dmirror, &range, perm); if (ret) break; n = (range.end - range.start) >> PAGE_SHIFT; if (copy_to_user(uptr, perm, n)) { ret = -EFAULT; break; } cmd->cpages += n; uptr += n; } mmput(mm); return ret; } static long dmirror_fops_unlocked_ioctl(struct file *filp, unsigned int command, unsigned long arg) { void __user *uarg = (void __user *)arg; struct hmm_dmirror_cmd cmd; struct dmirror *dmirror; int ret; dmirror = filp->private_data; if (!dmirror) return -EINVAL; if (copy_from_user(&cmd, uarg, sizeof(cmd))) return -EFAULT; if (cmd.addr & ~PAGE_MASK) return -EINVAL; if (cmd.addr >= (cmd.addr + (cmd.npages << PAGE_SHIFT))) return -EINVAL; cmd.cpages = 0; cmd.faults = 0; switch (command) { case HMM_DMIRROR_READ: ret = dmirror_read(dmirror, &cmd); break; case HMM_DMIRROR_WRITE: ret = dmirror_write(dmirror, &cmd); break; case HMM_DMIRROR_MIGRATE: ret = dmirror_migrate(dmirror, &cmd); break; case HMM_DMIRROR_SNAPSHOT: ret = dmirror_snapshot(dmirror, &cmd); break; default: return -EINVAL; } if (ret) return ret; if (copy_to_user(uarg, &cmd, sizeof(cmd))) return -EFAULT; return 0; } static const struct file_operations dmirror_fops = { .open = dmirror_fops_open, .release = dmirror_fops_release, .unlocked_ioctl = dmirror_fops_unlocked_ioctl, .llseek = default_llseek, .owner = THIS_MODULE, }; static void dmirror_devmem_free(struct page *page) { struct page *rpage = page->zone_device_data; struct dmirror_device *mdevice; if (rpage) __free_page(rpage); mdevice = dmirror_page_to_device(page); spin_lock(&mdevice->lock); mdevice->cfree++; page->zone_device_data = mdevice->free_pages; mdevice->free_pages = page; spin_unlock(&mdevice->lock); } static vm_fault_t dmirror_devmem_fault_alloc_and_copy(struct migrate_vma *args, struct dmirror_device *mdevice) { const unsigned long *src = args->src; unsigned long *dst = args->dst; unsigned long start = args->start; unsigned long end = args->end; unsigned long addr; for (addr = start; addr < end; addr += PAGE_SIZE, src++, dst++) { struct page *dpage, *spage; spage = migrate_pfn_to_page(*src); if (!spage || !(*src & MIGRATE_PFN_MIGRATE)) continue; spage = spage->zone_device_data; dpage = alloc_page_vma(GFP_HIGHUSER_MOVABLE, args->vma, addr); if (!dpage) continue; lock_page(dpage); copy_highpage(dpage, spage); *dst = migrate_pfn(page_to_pfn(dpage)) | MIGRATE_PFN_LOCKED; if (*src & MIGRATE_PFN_WRITE) *dst |= MIGRATE_PFN_WRITE; } return 0; } static void dmirror_devmem_fault_finalize_and_map(struct migrate_vma *args, struct dmirror *dmirror) { /* Invalidate the device's page table mapping. */ mutex_lock(&dmirror->mutex); dmirror_do_update(dmirror, args->start, args->end); mutex_unlock(&dmirror->mutex); } static vm_fault_t dmirror_devmem_fault(struct vm_fault *vmf) { struct migrate_vma args; unsigned long src_pfns; unsigned long dst_pfns; struct page *rpage; struct dmirror *dmirror; vm_fault_t ret; /* * Normally, a device would use the page->zone_device_data to point to * the mirror but here we use it to hold the page for the simulated * device memory and that page holds the pointer to the mirror. */ rpage = vmf->page->zone_device_data; dmirror = rpage->zone_device_data; /* FIXME demonstrate how we can adjust migrate range */ args.vma = vmf->vma; args.start = vmf->address; args.end = args.start + PAGE_SIZE; args.src = &src_pfns; args.dst = &dst_pfns; args.src_owner = dmirror->mdevice; if (migrate_vma_setup(&args)) return VM_FAULT_SIGBUS; ret = dmirror_devmem_fault_alloc_and_copy(&args, dmirror->mdevice); if (ret) return ret; migrate_vma_pages(&args); dmirror_devmem_fault_finalize_and_map(&args, dmirror); migrate_vma_finalize(&args); return 0; } static const struct dev_pagemap_ops dmirror_devmem_ops = { .page_free = dmirror_devmem_free, .migrate_to_ram = dmirror_devmem_fault, }; static int dmirror_device_init(struct dmirror_device *mdevice, int id) { dev_t dev; int ret; dev = MKDEV(MAJOR(dmirror_dev), id); mutex_init(&mdevice->devmem_lock); spin_lock_init(&mdevice->lock); cdev_init(&mdevice->cdevice, &dmirror_fops); mdevice->cdevice.owner = THIS_MODULE; ret = cdev_add(&mdevice->cdevice, dev, 1); if (ret) return ret; /* Build a list of free ZONE_DEVICE private struct pages */ dmirror_allocate_chunk(mdevice, NULL); return 0; } static void dmirror_device_remove(struct dmirror_device *mdevice) { unsigned int i; if (mdevice->devmem_chunks) { for (i = 0; i < mdevice->devmem_count; i++) { struct dmirror_chunk *devmem = mdevice->devmem_chunks[i]; memunmap_pages(&devmem->pagemap); release_mem_region(devmem->pagemap.res.start, resource_size(&devmem->pagemap.res)); kfree(devmem); } kfree(mdevice->devmem_chunks); } cdev_del(&mdevice->cdevice); } static int __init hmm_dmirror_init(void) { int ret; int id; ret = alloc_chrdev_region(&dmirror_dev, 0, DMIRROR_NDEVICES, "HMM_DMIRROR"); if (ret) goto err_unreg; for (id = 0; id < DMIRROR_NDEVICES; id++) { ret = dmirror_device_init(dmirror_devices + id, id); if (ret) goto err_chrdev; } /* * Allocate a zero page to simulate a reserved page of device private * memory which is always zero. The zero_pfn page isn't used just to * make the code here simpler (i.e., we need a struct page for it). */ dmirror_zero_page = alloc_page(GFP_HIGHUSER | __GFP_ZERO); if (!dmirror_zero_page) { ret = -ENOMEM; goto err_chrdev; } pr_info("HMM test module loaded. This is only for testing HMM.\n"); return 0; err_chrdev: while (--id >= 0) dmirror_device_remove(dmirror_devices + id); unregister_chrdev_region(dmirror_dev, DMIRROR_NDEVICES); err_unreg: return ret; } static void __exit hmm_dmirror_exit(void) { int id; if (dmirror_zero_page) __free_page(dmirror_zero_page); for (id = 0; id < DMIRROR_NDEVICES; id++) dmirror_device_remove(dmirror_devices + id); unregister_chrdev_region(dmirror_dev, DMIRROR_NDEVICES); } module_init(hmm_dmirror_init); module_exit(hmm_dmirror_exit); MODULE_LICENSE("GPL");