diff options
Diffstat (limited to 'include')
| -rw-r--r-- | include/linux/hmm.h | 24 | ||||
| -rw-r--r-- | include/linux/migrate.h | 16 | ||||
| -rw-r--r-- | include/linux/mmu_notifier.h | 6 |
3 files changed, 40 insertions, 6 deletions
diff --git a/include/linux/hmm.h b/include/linux/hmm.h index f4a09ed223ac..866a0fa104c4 100644 --- a/include/linux/hmm.h +++ b/include/linux/hmm.h @@ -37,16 +37,17 @@ * will fail. Must be combined with HMM_PFN_REQ_FAULT. */ enum hmm_pfn_flags { - /* Output flags */ + /* Output fields and flags */ HMM_PFN_VALID = 1UL << (BITS_PER_LONG - 1), HMM_PFN_WRITE = 1UL << (BITS_PER_LONG - 2), HMM_PFN_ERROR = 1UL << (BITS_PER_LONG - 3), + HMM_PFN_ORDER_SHIFT = (BITS_PER_LONG - 8), /* Input flags */ HMM_PFN_REQ_FAULT = HMM_PFN_VALID, HMM_PFN_REQ_WRITE = HMM_PFN_WRITE, - HMM_PFN_FLAGS = HMM_PFN_VALID | HMM_PFN_WRITE | HMM_PFN_ERROR, + HMM_PFN_FLAGS = 0xFFUL << HMM_PFN_ORDER_SHIFT, }; /* @@ -62,6 +63,25 @@ static inline struct page *hmm_pfn_to_page(unsigned long hmm_pfn) } /* + * hmm_pfn_to_map_order() - return the CPU mapping size order + * + * This is optionally useful to optimize processing of the pfn result + * array. It indicates that the page starts at the order aligned VA and is + * 1<<order bytes long. Every pfn within an high order page will have the + * same pfn flags, both access protections and the map_order. The caller must + * be careful with edge cases as the start and end VA of the given page may + * extend past the range used with hmm_range_fault(). + * + * This must be called under the caller 'user_lock' after a successful + * mmu_interval_read_begin(). The caller must have tested for HMM_PFN_VALID + * already. + */ +static inline unsigned int hmm_pfn_to_map_order(unsigned long hmm_pfn) +{ + return (hmm_pfn >> HMM_PFN_ORDER_SHIFT) & 0x1F; +} + +/* * struct hmm_range - track invalidation lock on virtual address range * * @notifier: a mmu_interval_notifier that includes the start/end diff --git a/include/linux/migrate.h b/include/linux/migrate.h index 3e546cbf03dd..540998d9810b 100644 --- a/include/linux/migrate.h +++ b/include/linux/migrate.h @@ -180,6 +180,11 @@ static inline unsigned long migrate_pfn(unsigned long pfn) return (pfn << MIGRATE_PFN_SHIFT) | MIGRATE_PFN_VALID; } +enum migrate_vma_direction { + MIGRATE_VMA_SELECT_SYSTEM = 1 << 0, + MIGRATE_VMA_SELECT_DEVICE_PRIVATE = 1 << 1, +}; + struct migrate_vma { struct vm_area_struct *vma; /* @@ -199,11 +204,14 @@ struct migrate_vma { /* * Set to the owner value also stored in page->pgmap->owner for - * migrating out of device private memory. If set only device - * private pages with this owner are migrated. If not set - * device private pages are not migrated at all. + * migrating out of device private memory. The flags also need to + * be set to MIGRATE_VMA_SELECT_DEVICE_PRIVATE. + * The caller should always set this field when using mmu notifier + * callbacks to avoid device MMU invalidations for device private + * pages that are not being migrated. */ - void *src_owner; + void *pgmap_owner; + unsigned long flags; }; int migrate_vma_setup(struct migrate_vma *args); diff --git a/include/linux/mmu_notifier.h b/include/linux/mmu_notifier.h index fc68f3570e19..c6f0708195cd 100644 --- a/include/linux/mmu_notifier.h +++ b/include/linux/mmu_notifier.h @@ -38,6 +38,10 @@ struct mmu_interval_notifier; * * @MMU_NOTIFY_RELEASE: used during mmu_interval_notifier invalidate to signal * that the mm refcount is zero and the range is no longer accessible. + * + * @MMU_NOTIFY_MIGRATE: used during migrate_vma_collect() invalidate to signal + * a device driver to possibly ignore the invalidation if the + * migrate_pgmap_owner field matches the driver's device private pgmap owner. */ enum mmu_notifier_event { MMU_NOTIFY_UNMAP = 0, @@ -46,6 +50,7 @@ enum mmu_notifier_event { MMU_NOTIFY_PROTECTION_PAGE, MMU_NOTIFY_SOFT_DIRTY, MMU_NOTIFY_RELEASE, + MMU_NOTIFY_MIGRATE, }; #define MMU_NOTIFIER_RANGE_BLOCKABLE (1 << 0) @@ -264,6 +269,7 @@ struct mmu_notifier_range { unsigned long end; unsigned flags; enum mmu_notifier_event event; + void *migrate_pgmap_owner; }; static inline int mm_has_notifiers(struct mm_struct *mm) |