Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874// SPDX-License-Identifier: GPL-2.0/* * This is a module to test the HMM (Heterogeneous Memory Management) * mirror and zone device private memory migration APIs of the kernel. * Userspace programs can register with the driver to mirror their own address * space and can use the device to read/write any valid virtual address. */#include <linux/init.h>#include <linux/fs.h>#include <linux/mm.h>#include <linux/module.h>#include <linux/kernel.h>#include <linux/cdev.h>#include <linux/device.h>#include <linux/memremap.h>#include <linux/mutex.h>#include <linux/rwsem.h>#include <linux/sched.h>#include <linux/slab.h>#include <linux/highmem.h>#include <linux/delay.h>#include <linux/pagemap.h>#include <linux/hmm.h>#include <linux/vmalloc.h>#include <linux/swap.h>#include <linux/swapops.h>#include <linux/sched/mm.h>#include <linux/platform_device.h>#include <linux/rmap.h>#include <linux/mmu_notifier.h>#include <linux/migrate.h>
#include "test_hmm_uapi.h"
#define DMIRROR_NDEVICES 4#define DMIRROR_RANGE_FAULT_TIMEOUT 1000#define DEVMEM_CHUNK_SIZE (256 * 1024 * 1024U)#define DEVMEM_CHUNKS_RESERVE 16
/* * For device_private pages, dpage is just a dummy struct page * representing a piece of device memory. dmirror_devmem_alloc_page * allocates a real system memory page as backing storage to fake a * real device. zone_device_data points to that backing page. But * for device_coherent memory, the struct page represents real * physical CPU-accessible memory that we can use directly. */#define BACKING_PAGE(page) (is_device_private_page((page)) ? \ (page)->zone_device_data : (page))
static unsigned long spm_addr_dev0;module_param(spm_addr_dev0, long, 0644);MODULE_PARM_DESC(spm_addr_dev0, "Specify start address for SPM (special purpose memory) used for device 0. By setting this Coherent device type will be used. Make sure spm_addr_dev1 is set too. Minimum SPM size should be DEVMEM_CHUNK_SIZE.");
static unsigned long spm_addr_dev1;module_param(spm_addr_dev1, long, 0644);MODULE_PARM_DESC(spm_addr_dev1, "Specify start address for SPM (special purpose memory) used for device 1. By setting this Coherent device type will be used. Make sure spm_addr_dev0 is set too. Minimum SPM size should be DEVMEM_CHUNK_SIZE.");
static const struct dev_pagemap_ops dmirror_devmem_ops;static const struct mmu_interval_notifier_ops dmirror_min_ops;static dev_t dmirror_dev;
struct dmirror_device;
struct dmirror_bounce { void *ptr; unsigned long size; unsigned long addr; unsigned long cpages;};
#define DPT_XA_TAG_ATOMIC 1UL#define DPT_XA_TAG_WRITE 3UL
/* * Data structure to track address ranges and register for mmu interval * notifier updates. */struct dmirror_interval { struct mmu_interval_notifier notifier; struct dmirror *dmirror;};
/* * Data attached to the open device file. * Note that it might be shared after a fork(). */struct dmirror { struct dmirror_device *mdevice; struct xarray pt; struct mmu_interval_notifier notifier; struct mutex mutex; __u64 flags;};
/* * ZONE_DEVICE pages for migration and simulating device memory. */struct dmirror_chunk { struct dev_pagemap pagemap; struct dmirror_device *mdevice; bool remove;};
/* * Per device data. */struct dmirror_device { struct cdev cdevice; unsigned int zone_device_type; struct device device;
unsigned int devmem_capacity; unsigned int devmem_count; struct dmirror_chunk **devmem_chunks; struct mutex devmem_lock; /* protects the above */
unsigned long calloc; unsigned long cfree; struct page *free_pages; struct folio *free_folios; spinlock_t lock; /* protects the above */};
static struct dmirror_device dmirror_devices[DMIRROR_NDEVICES];
static int dmirror_bounce_init(struct dmirror_bounce *bounce, unsigned long addr, unsigned long size){ bounce->addr = addr; bounce->size = size; bounce->cpages = 0; bounce->ptr = vmalloc(size); if (!bounce->ptr) return -ENOMEM; return 0;}
static bool dmirror_is_private_zone(struct dmirror_device *mdevice){ return (mdevice->zone_device_type == HMM_DMIRROR_MEMORY_DEVICE_PRIVATE);}
static enum migrate_vma_directiondmirror_select_device(struct dmirror *dmirror){ return (dmirror->mdevice->zone_device_type == HMM_DMIRROR_MEMORY_DEVICE_PRIVATE) ? MIGRATE_VMA_SELECT_DEVICE_PRIVATE : MIGRATE_VMA_SELECT_DEVICE_COHERENT;}
static void dmirror_bounce_fini(struct dmirror_bounce *bounce){ vfree(bounce->ptr);}
static int dmirror_fops_open(struct inode *inode, struct file *filp){ struct cdev *cdev = inode->i_cdev; struct dmirror *dmirror; int ret;
/* Mirror this process address space */ dmirror = kzalloc_obj(*dmirror); if (dmirror == NULL) return -ENOMEM;
dmirror->mdevice = container_of(cdev, struct dmirror_device, cdevice); mutex_init(&dmirror->mutex); xa_init(&dmirror->pt);
ret = mmu_interval_notifier_insert(&dmirror->notifier, current->mm, 0, ULONG_MAX & PAGE_MASK, &dmirror_min_ops); if (ret) { kfree(dmirror); return ret; }
filp->private_data = dmirror; return 0;}
static void dmirror_device_evict_chunk(struct dmirror_chunk *chunk){ unsigned long start_pfn = chunk->pagemap.range.start >> PAGE_SHIFT; unsigned long end_pfn = chunk->pagemap.range.end >> PAGE_SHIFT; unsigned long npages = end_pfn - start_pfn + 1; unsigned long i; unsigned long *src_pfns; unsigned long *dst_pfns; unsigned int order = 0;
src_pfns = kvcalloc(npages, sizeof(*src_pfns), GFP_KERNEL | __GFP_NOFAIL); dst_pfns = kvcalloc(npages, sizeof(*dst_pfns), GFP_KERNEL | __GFP_NOFAIL);
migrate_device_range(src_pfns, start_pfn, npages); for (i = 0; i < npages; i++) { struct page *dpage, *spage;
spage = migrate_pfn_to_page(src_pfns[i]); if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE)) continue;
if (WARN_ON(!is_device_private_page(spage) && !is_device_coherent_page(spage))) continue;
order = folio_order(page_folio(spage)); spage = BACKING_PAGE(spage); if (src_pfns[i] & MIGRATE_PFN_COMPOUND) { dpage = folio_page(folio_alloc(GFP_HIGHUSER_MOVABLE, order), 0); } else { dpage = alloc_page(GFP_HIGHUSER_MOVABLE | __GFP_NOFAIL); order = 0; }
/* TODO Support splitting here */ lock_page(dpage); dst_pfns[i] = migrate_pfn(page_to_pfn(dpage)); if (src_pfns[i] & MIGRATE_PFN_WRITE) dst_pfns[i] |= MIGRATE_PFN_WRITE; if (order) dst_pfns[i] |= MIGRATE_PFN_COMPOUND; folio_copy(page_folio(dpage), page_folio(spage)); } migrate_device_pages(src_pfns, dst_pfns, npages); migrate_device_finalize(src_pfns, dst_pfns, npages); kvfree(src_pfns); kvfree(dst_pfns);}
static int dmirror_fops_release(struct inode *inode, struct file *filp){ struct dmirror *dmirror = filp->private_data; struct dmirror_device *mdevice = dmirror->mdevice; int i;
mmu_interval_notifier_remove(&dmirror->notifier);
if (mdevice->devmem_chunks) { for (i = 0; i < mdevice->devmem_count; i++) { struct dmirror_chunk *devmem = mdevice->devmem_chunks[i];
dmirror_device_evict_chunk(devmem); } }
xa_destroy(&dmirror->pt); kfree(dmirror); return 0;}
static struct dmirror_chunk *dmirror_page_to_chunk(struct page *page){ return container_of(page_pgmap(page), struct dmirror_chunk, pagemap);}
static struct dmirror_device *dmirror_page_to_device(struct page *page)
{ return dmirror_page_to_chunk(page)->mdevice;}
static int dmirror_do_fault(struct dmirror *dmirror, struct hmm_range *range){ unsigned long *pfns = range->hmm_pfns; unsigned long pfn;
for (pfn = (range->start >> PAGE_SHIFT); pfn < (range->end >> PAGE_SHIFT); pfn++, pfns++) { struct page *page; void *entry;
/* * Since we asked for hmm_range_fault() to populate pages, * it shouldn't return an error entry on success. */ WARN_ON(*pfns & HMM_PFN_ERROR); WARN_ON(!(*pfns & HMM_PFN_VALID));
page = hmm_pfn_to_page(*pfns); WARN_ON(!page);
entry = page; if (*pfns & HMM_PFN_WRITE) entry = xa_tag_pointer(entry, DPT_XA_TAG_WRITE); else if (WARN_ON(range->default_flags & HMM_PFN_WRITE)) return -EFAULT; entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC); if (xa_is_err(entry)) return xa_err(entry); }
return 0;}
static void dmirror_do_update(struct dmirror *dmirror, unsigned long start, unsigned long end){ unsigned long pfn; void *entry;
/* * The XArray doesn't hold references to pages since it relies on * the mmu notifier to clear page pointers when they become stale. * Therefore, it is OK to just clear the entry. */ xa_for_each_range(&dmirror->pt, pfn, entry, start >> PAGE_SHIFT, end >> PAGE_SHIFT) xa_erase(&dmirror->pt, pfn);}
static bool dmirror_interval_invalidate(struct mmu_interval_notifier *mni, const struct mmu_notifier_range *range, unsigned long cur_seq){ struct dmirror *dmirror = container_of(mni, struct dmirror, notifier);
/* * Ignore invalidation callbacks for device private pages since * the invalidation is handled as part of the migration process. */ if (range->event == MMU_NOTIFY_MIGRATE && range->owner == dmirror->mdevice) return true;
if (mmu_notifier_range_blockable(range)) mutex_lock(&dmirror->mutex); else if (!mutex_trylock(&dmirror->mutex)) return false;
mmu_interval_set_seq(mni, cur_seq); dmirror_do_update(dmirror, range->start, range->end);
mutex_unlock(&dmirror->mutex); return true;}
static const struct mmu_interval_notifier_ops dmirror_min_ops = { .invalidate = dmirror_interval_invalidate,};
static int dmirror_range_fault(struct dmirror *dmirror, struct hmm_range *range){ struct mm_struct *mm = dmirror->notifier.mm; unsigned long timeout = jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); int ret;
while (true) { if (time_after(jiffies, timeout)) { ret = -EBUSY; goto out; }
range->notifier_seq = mmu_interval_read_begin(range->notifier); mmap_read_lock(mm); ret = hmm_range_fault(range); mmap_read_unlock(mm); if (ret) { if (ret == -EBUSY) continue; goto out; }
mutex_lock(&dmirror->mutex); if (mmu_interval_read_retry(range->notifier, range->notifier_seq)) { mutex_unlock(&dmirror->mutex); continue; } break; }
ret = dmirror_do_fault(dmirror, range);
mutex_unlock(&dmirror->mutex);out: return ret;}
static int dmirror_fault(struct dmirror *dmirror, unsigned long start, unsigned long end, bool write){ struct mm_struct *mm = dmirror->notifier.mm; unsigned long addr; unsigned long pfns[32]; struct hmm_range range = { .notifier = &dmirror->notifier, .hmm_pfns = pfns, .pfn_flags_mask = 0, .default_flags = HMM_PFN_REQ_FAULT | (write ? HMM_PFN_REQ_WRITE : 0), .dev_private_owner = dmirror->mdevice, }; int ret = 0;
/* Since the mm is for the mirrored process, get a reference first. */ if (!mmget_not_zero(mm)) return 0;
for (addr = start; addr < end; addr = range.end) { range.start = addr; range.end = min(addr + (ARRAY_SIZE(pfns) << PAGE_SHIFT), end);
ret = dmirror_range_fault(dmirror, &range); if (ret) break; }
mmput(mm); return ret;}
static int dmirror_do_read(struct dmirror *dmirror, unsigned long start, unsigned long end, struct dmirror_bounce *bounce){ unsigned long pfn; void *ptr;
ptr = bounce->ptr + ((start - bounce->addr) & PAGE_MASK);
for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) { void *entry; struct page *page;
entry = xa_load(&dmirror->pt, pfn); page = xa_untag_pointer(entry); if (!page) return -ENOENT;
memcpy_from_page(ptr, page, 0, PAGE_SIZE);
ptr += PAGE_SIZE; bounce->cpages++; }
return 0;}
static int dmirror_read(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd){ struct dmirror_bounce bounce; unsigned long start, end; unsigned long size = cmd->npages << PAGE_SHIFT; int ret;
start = cmd->addr; end = start + size; if (end < start) return -EINVAL;
ret = dmirror_bounce_init(&bounce, start, size); if (ret) return ret;
while (1) { mutex_lock(&dmirror->mutex); ret = dmirror_do_read(dmirror, start, end, &bounce); mutex_unlock(&dmirror->mutex); if (ret != -ENOENT) break;
start = cmd->addr + (bounce.cpages << PAGE_SHIFT); ret = dmirror_fault(dmirror, start, end, false); if (ret) break; cmd->faults++; }
if (ret == 0) { if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr, bounce.size)) ret = -EFAULT; } cmd->cpages = bounce.cpages; dmirror_bounce_fini(&bounce); return ret;}
static int dmirror_do_write(struct dmirror *dmirror, unsigned long start, unsigned long end, struct dmirror_bounce *bounce){ unsigned long pfn; void *ptr;
ptr = bounce->ptr + ((start - bounce->addr) & PAGE_MASK);
for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) { void *entry; struct page *page;
entry = xa_load(&dmirror->pt, pfn); page = xa_untag_pointer(entry); if (!page || xa_pointer_tag(entry) != DPT_XA_TAG_WRITE) return -ENOENT;
memcpy_to_page(page, 0, ptr, PAGE_SIZE);
ptr += PAGE_SIZE; bounce->cpages++; }
return 0;}
static int dmirror_write(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd){ struct dmirror_bounce bounce; unsigned long start, end; unsigned long size = cmd->npages << PAGE_SHIFT; int ret;
start = cmd->addr; end = start + size; if (end < start) return -EINVAL;
ret = dmirror_bounce_init(&bounce, start, size); if (ret) return ret; if (copy_from_user(bounce.ptr, u64_to_user_ptr(cmd->ptr), bounce.size)) { ret = -EFAULT; goto fini; }
while (1) { mutex_lock(&dmirror->mutex); ret = dmirror_do_write(dmirror, start, end, &bounce); mutex_unlock(&dmirror->mutex); if (ret != -ENOENT) break;
start = cmd->addr + (bounce.cpages << PAGE_SHIFT); ret = dmirror_fault(dmirror, start, end, true); if (ret) break; cmd->faults++; }
fini: cmd->cpages = bounce.cpages; dmirror_bounce_fini(&bounce); return ret;}
static int dmirror_allocate_chunk(struct dmirror_device *mdevice, struct page **ppage, bool is_large){ struct dmirror_chunk *devmem; struct resource *res = NULL; unsigned long pfn; unsigned long pfn_first; unsigned long pfn_last; void *ptr; int ret = -ENOMEM;
devmem = kzalloc_obj(*devmem); if (!devmem) return ret;
switch (mdevice->zone_device_type) { case HMM_DMIRROR_MEMORY_DEVICE_PRIVATE: res = request_free_mem_region(&iomem_resource, DEVMEM_CHUNK_SIZE, "hmm_dmirror"); if (IS_ERR_OR_NULL(res)) goto err_devmem; devmem->pagemap.range.start = res->start; devmem->pagemap.range.end = res->end; devmem->pagemap.type = MEMORY_DEVICE_PRIVATE; break; case HMM_DMIRROR_MEMORY_DEVICE_COHERENT: devmem->pagemap.range.start = (MINOR(mdevice->device.devt) - 2) ? spm_addr_dev0 : spm_addr_dev1; devmem->pagemap.range.end = devmem->pagemap.range.start + DEVMEM_CHUNK_SIZE - 1; devmem->pagemap.type = MEMORY_DEVICE_COHERENT; break; default: ret = -EINVAL; goto err_devmem; }
devmem->pagemap.nr_range = 1; devmem->pagemap.ops = &dmirror_devmem_ops; devmem->pagemap.owner = mdevice;
mutex_lock(&mdevice->devmem_lock);
if (mdevice->devmem_count == mdevice->devmem_capacity) { struct dmirror_chunk **new_chunks; unsigned int new_capacity;
new_capacity = mdevice->devmem_capacity + DEVMEM_CHUNKS_RESERVE; new_chunks = krealloc(mdevice->devmem_chunks, sizeof(new_chunks[0]) * new_capacity, GFP_KERNEL); if (!new_chunks) goto err_release; mdevice->devmem_capacity = new_capacity; mdevice->devmem_chunks = new_chunks; } ptr = memremap_pages(&devmem->pagemap, numa_node_id()); if (IS_ERR_OR_NULL(ptr)) { if (ptr) ret = PTR_ERR(ptr); else ret = -EFAULT; goto err_release; }
devmem->mdevice = mdevice; pfn_first = devmem->pagemap.range.start >> PAGE_SHIFT; pfn_last = pfn_first + (range_len(&devmem->pagemap.range) >> PAGE_SHIFT); mdevice->devmem_chunks[mdevice->devmem_count++] = devmem;
mutex_unlock(&mdevice->devmem_lock);
pr_info("added new %u MB chunk (total %u chunks, %u MB) PFNs [0x%lx 0x%lx)\n", DEVMEM_CHUNK_SIZE / (1024 * 1024), mdevice->devmem_count, mdevice->devmem_count * (DEVMEM_CHUNK_SIZE / (1024 * 1024)), pfn_first, pfn_last);
spin_lock(&mdevice->lock); for (pfn = pfn_first; pfn < pfn_last; ) { struct page *page = pfn_to_page(pfn);
if (is_large && IS_ALIGNED(pfn, HPAGE_PMD_NR) && (pfn + HPAGE_PMD_NR <= pfn_last)) { page->zone_device_data = mdevice->free_folios; mdevice->free_folios = page_folio(page); pfn += HPAGE_PMD_NR; continue; }
page->zone_device_data = mdevice->free_pages; mdevice->free_pages = page; pfn++; }
ret = 0; if (ppage) { if (is_large) { if (!mdevice->free_folios) { ret = -ENOMEM; goto err_unlock; } *ppage = folio_page(mdevice->free_folios, 0); mdevice->free_folios = (*ppage)->zone_device_data; mdevice->calloc += HPAGE_PMD_NR; } else if (mdevice->free_pages) { *ppage = mdevice->free_pages; mdevice->free_pages = (*ppage)->zone_device_data; mdevice->calloc++; } else { ret = -ENOMEM; goto err_unlock; } }err_unlock: spin_unlock(&mdevice->lock);
return ret;
err_release: mutex_unlock(&mdevice->devmem_lock); if (res && devmem->pagemap.type == MEMORY_DEVICE_PRIVATE) release_mem_region(devmem->pagemap.range.start, range_len(&devmem->pagemap.range));err_devmem: kfree(devmem);
return ret;}
static struct page *dmirror_devmem_alloc_page(struct dmirror *dmirror, bool is_large){ struct page *dpage = NULL; struct page *rpage = NULL; unsigned int order = is_large ? HPAGE_PMD_ORDER : 0; struct dmirror_device *mdevice = dmirror->mdevice;
/* * For ZONE_DEVICE private type, this is a fake device so we allocate * real system memory to store our device memory. * For ZONE_DEVICE coherent type we use the actual dpage to store the * data and ignore rpage. */ if (dmirror_is_private_zone(mdevice)) { rpage = folio_page(folio_alloc(GFP_HIGHUSER, order), 0); if (!rpage) return NULL; } spin_lock(&mdevice->lock);
if (is_large && mdevice->free_folios) { dpage = folio_page(mdevice->free_folios, 0); mdevice->free_folios = dpage->zone_device_data; mdevice->calloc += 1 << order; spin_unlock(&mdevice->lock); } else if (!is_large && mdevice->free_pages) { dpage = mdevice->free_pages; mdevice->free_pages = dpage->zone_device_data; mdevice->calloc++; spin_unlock(&mdevice->lock); } else { spin_unlock(&mdevice->lock); if (dmirror_allocate_chunk(mdevice, &dpage, is_large)) goto error; }
zone_device_folio_init(page_folio(dpage), page_pgmap(folio_page(page_folio(dpage), 0)), order); dpage->zone_device_data = rpage; return dpage;
error: if (rpage) __free_pages(rpage, order); return NULL;}
static void dmirror_migrate_alloc_and_copy(struct migrate_vma *args, struct dmirror *dmirror){ const unsigned long *src = args->src; unsigned long *dst = args->dst; unsigned long addr;
for (addr = args->start; addr < args->end; ) { struct page *spage; struct page *dpage; struct page *rpage; bool is_large = *src & MIGRATE_PFN_COMPOUND; int write = (*src & MIGRATE_PFN_WRITE) ? MIGRATE_PFN_WRITE : 0; unsigned long nr = 1;
if (!(*src & MIGRATE_PFN_MIGRATE)) goto next;
/* * Note that spage might be NULL which is OK since it is an * unallocated pte_none() or read-only zero page. */ spage = migrate_pfn_to_page(*src); if (WARN(spage && is_zone_device_page(spage), "page already in device spage pfn: 0x%lx\n", page_to_pfn(spage))) goto next;
if (dmirror->flags & HMM_DMIRROR_FLAG_FAIL_ALLOC) { dmirror->flags &= ~HMM_DMIRROR_FLAG_FAIL_ALLOC; dpage = NULL; } else dpage = dmirror_devmem_alloc_page(dmirror, is_large);
if (!dpage) { struct folio *folio; unsigned long i; unsigned long spfn = *src >> MIGRATE_PFN_SHIFT; struct page *src_page;
if (!is_large) goto next;
if (!spage && is_large) { nr = HPAGE_PMD_NR; } else { folio = page_folio(spage); nr = folio_nr_pages(folio); }
for (i = 0; i < nr && addr < args->end; i++) { dpage = dmirror_devmem_alloc_page(dmirror, false); rpage = BACKING_PAGE(dpage); rpage->zone_device_data = dmirror;
*dst = migrate_pfn(page_to_pfn(dpage)) | write; src_page = pfn_to_page(spfn + i);
if (spage) copy_highpage(rpage, src_page); else clear_highpage(rpage); src++; dst++; addr += PAGE_SIZE; } continue; }
rpage = BACKING_PAGE(dpage);
/* * Normally, a device would use the page->zone_device_data to * point to the mirror but here we use it to hold the page for * the simulated device memory and that page holds the pointer * to the mirror. */ rpage->zone_device_data = dmirror;
pr_debug("migrating from sys to dev pfn src: 0x%lx pfn dst: 0x%lx\n", page_to_pfn(spage), page_to_pfn(dpage));
*dst = migrate_pfn(page_to_pfn(dpage)) | write;
if (is_large) { int i; struct folio *folio = page_folio(dpage); *dst |= MIGRATE_PFN_COMPOUND;
if (folio_test_large(folio)) { for (i = 0; i < folio_nr_pages(folio); i++) { struct page *dst_page = pfn_to_page(page_to_pfn(rpage) + i); struct page *src_page = pfn_to_page(page_to_pfn(spage) + i);
if (spage) copy_highpage(dst_page, src_page); else clear_highpage(dst_page); src++; dst++; addr += PAGE_SIZE; } continue; } }
if (spage) copy_highpage(rpage, spage); else clear_highpage(rpage);
next: src++; dst++; addr += PAGE_SIZE; }}
static int dmirror_check_atomic(struct dmirror *dmirror, unsigned long start, unsigned long end){ unsigned long pfn;
for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) { void *entry;
entry = xa_load(&dmirror->pt, pfn); if (xa_pointer_tag(entry) == DPT_XA_TAG_ATOMIC) return -EPERM; }
return 0;}
static int dmirror_atomic_map(unsigned long addr, struct page *page, struct dmirror *dmirror){ void *entry;
/* Map the migrated pages into the device's page tables. */ mutex_lock(&dmirror->mutex);
entry = xa_tag_pointer(page, DPT_XA_TAG_ATOMIC); entry = xa_store(&dmirror->pt, addr >> PAGE_SHIFT, entry, GFP_ATOMIC); if (xa_is_err(entry)) { mutex_unlock(&dmirror->mutex); return xa_err(entry); }
mutex_unlock(&dmirror->mutex); return 0;}
static int dmirror_migrate_finalize_and_map(struct migrate_vma *args, struct dmirror *dmirror){ unsigned long start = args->start; unsigned long end = args->end; const unsigned long *src = args->src; const unsigned long *dst = args->dst; unsigned long pfn; const unsigned long start_pfn = start >> PAGE_SHIFT; const unsigned long end_pfn = end >> PAGE_SHIFT;
/* Map the migrated pages into the device's page tables. */ mutex_lock(&dmirror->mutex);
for (pfn = start_pfn; pfn < end_pfn; pfn++, src++, dst++) { struct page *dpage; void *entry; int nr, i; struct page *rpage;
if (!(*src & MIGRATE_PFN_MIGRATE)) continue;
dpage = migrate_pfn_to_page(*dst); if (!dpage) continue;
if (*dst & MIGRATE_PFN_COMPOUND) nr = folio_nr_pages(page_folio(dpage)); else nr = 1;
WARN_ON_ONCE(end_pfn < start_pfn + nr);
rpage = BACKING_PAGE(dpage); VM_WARN_ON(folio_nr_pages(page_folio(rpage)) != nr);
for (i = 0; i < nr; i++) { entry = folio_page(page_folio(rpage), i); if (*dst & MIGRATE_PFN_WRITE) entry = xa_tag_pointer(entry, DPT_XA_TAG_WRITE); entry = xa_store(&dmirror->pt, pfn + i, entry, GFP_ATOMIC); if (xa_is_err(entry)) { mutex_unlock(&dmirror->mutex); return xa_err(entry); } } }
mutex_unlock(&dmirror->mutex); return 0;}
static int dmirror_exclusive(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd){ unsigned long start, end, addr; unsigned long size = cmd->npages << PAGE_SHIFT; struct mm_struct *mm = dmirror->notifier.mm; struct dmirror_bounce bounce; int ret = 0;
start = cmd->addr; end = start + size; if (end < start) return -EINVAL;
/* Since the mm is for the mirrored process, get a reference first. */ if (!mmget_not_zero(mm)) return -EINVAL;
mmap_read_lock(mm); for (addr = start; !ret && addr < end; addr += PAGE_SIZE) { struct folio *folio; struct page *page;
page = make_device_exclusive(mm, addr, NULL, &folio); if (IS_ERR(page)) { ret = PTR_ERR(page); break; }
ret = dmirror_atomic_map(addr, page, dmirror); folio_unlock(folio); folio_put(folio); } mmap_read_unlock(mm); mmput(mm);
if (ret) return ret;
/* Return the migrated data for verification. */ ret = dmirror_bounce_init(&bounce, start, size); if (ret) return ret; mutex_lock(&dmirror->mutex); ret = dmirror_do_read(dmirror, start, end, &bounce); mutex_unlock(&dmirror->mutex); if (ret == 0) { if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr, bounce.size)) ret = -EFAULT; }
cmd->cpages = bounce.cpages; dmirror_bounce_fini(&bounce); return ret;}
static vm_fault_t dmirror_devmem_fault_alloc_and_copy(struct migrate_vma *args, struct dmirror *dmirror){ const unsigned long *src = args->src; unsigned long *dst = args->dst; unsigned long start = args->start; unsigned long end = args->end; unsigned long addr; unsigned int order = 0; int i;
for (addr = start; addr < end; ) { struct page *dpage, *spage;
spage = migrate_pfn_to_page(*src); if (!spage || !(*src & MIGRATE_PFN_MIGRATE)) { addr += PAGE_SIZE; goto next; }
if (WARN_ON(!is_device_private_page(spage) && !is_device_coherent_page(spage))) { addr += PAGE_SIZE; goto next; }
spage = BACKING_PAGE(spage); order = folio_order(page_folio(spage)); if (order) *dst = MIGRATE_PFN_COMPOUND; if (*src & MIGRATE_PFN_WRITE) *dst |= MIGRATE_PFN_WRITE;
if (dmirror->flags & HMM_DMIRROR_FLAG_FAIL_ALLOC) { dmirror->flags &= ~HMM_DMIRROR_FLAG_FAIL_ALLOC; *dst &= ~MIGRATE_PFN_COMPOUND; dpage = NULL; } else if (order) { dpage = folio_page(vma_alloc_folio(GFP_HIGHUSER_MOVABLE, order, args->vma, addr), 0); } else { dpage = alloc_page_vma(GFP_HIGHUSER_MOVABLE, args->vma, addr); }
if (!dpage && !order) return VM_FAULT_OOM;
pr_debug("migrating from sys to dev pfn src: 0x%lx pfn dst: 0x%lx\n", page_to_pfn(spage), page_to_pfn(dpage));
if (dpage) { lock_page(dpage); *dst |= migrate_pfn(page_to_pfn(dpage)); }
for (i = 0; i < (1 << order); i++) { struct page *src_page; struct page *dst_page;
/* Try with smaller pages if large allocation fails */ if (!dpage && order) { dpage = alloc_page_vma(GFP_HIGHUSER_MOVABLE, args->vma, addr); if (!dpage) { /* Unlock and free pages already allocated. */ while (i > 0) { struct page *fpage;
fpage = migrate_pfn_to_page(dst[--i]); unlock_page(fpage); __free_page(fpage); } /* Clear remaining dst entries to avoid * migrate_vma_pages/finalize() using * uninitialized values. */ while (i < (1 << order)) { dst[i] = 0; i++; } return VM_FAULT_OOM; } lock_page(dpage); dst[i] = migrate_pfn(page_to_pfn(dpage)); dst_page = pfn_to_page(page_to_pfn(dpage)); dpage = NULL; /* For the next iteration */ } else { dst_page = pfn_to_page(page_to_pfn(dpage) + i); }
src_page = pfn_to_page(page_to_pfn(spage) + i);
xa_erase(&dmirror->pt, addr >> PAGE_SHIFT); addr += PAGE_SIZE; copy_highpage(dst_page, src_page); }next: src += 1 << order; dst += 1 << order; } return 0;}
static unsigned longdmirror_successful_migrated_pages(struct migrate_vma *migrate){ unsigned long cpages = 0; unsigned long i;
for (i = 0; i < migrate->npages; i++) { if (migrate->src[i] & MIGRATE_PFN_VALID && migrate->src[i] & MIGRATE_PFN_MIGRATE) cpages++; } return cpages;}
static int dmirror_migrate_to_system(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd){ unsigned long start, end, addr; unsigned long size = cmd->npages << PAGE_SHIFT; struct mm_struct *mm = dmirror->notifier.mm; struct vm_area_struct *vma; struct migrate_vma args = { 0 }; unsigned long next; int ret; unsigned long *src_pfns; unsigned long *dst_pfns;
start = cmd->addr; end = start + size; if (end < start) return -EINVAL;
/* Since the mm is for the mirrored process, get a reference first. */ if (!mmget_not_zero(mm)) return -EINVAL;
src_pfns = kvcalloc(PTRS_PER_PTE, sizeof(*src_pfns), GFP_KERNEL | __GFP_NOFAIL); dst_pfns = kvcalloc(PTRS_PER_PTE, sizeof(*dst_pfns), GFP_KERNEL | __GFP_NOFAIL);
cmd->cpages = 0; mmap_read_lock(mm); for (addr = start; addr < end; addr = next) { vma = vma_lookup(mm, addr); if (!vma || !(vma->vm_flags & VM_READ)) { ret = -EINVAL; goto out; } next = min(end, addr + (PTRS_PER_PTE << PAGE_SHIFT)); if (next > vma->vm_end) next = vma->vm_end;
args.vma = vma; args.src = src_pfns; args.dst = dst_pfns; args.start = addr; args.end = next; args.pgmap_owner = dmirror->mdevice; args.flags = dmirror_select_device(dmirror) | MIGRATE_VMA_SELECT_COMPOUND;
ret = migrate_vma_setup(&args); if (ret) goto out;
pr_debug("Migrating from device mem to sys mem\n"); if (dmirror_devmem_fault_alloc_and_copy(&args, dmirror)) { migrate_vma_finalize(&args); ret = -ENOMEM; goto out; }
migrate_vma_pages(&args); cmd->cpages += dmirror_successful_migrated_pages(&args); migrate_vma_finalize(&args); }out: mmap_read_unlock(mm); mmput(mm); kvfree(src_pfns); kvfree(dst_pfns);
return ret;}
static int dmirror_migrate_to_device(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd){ unsigned long start, end, addr; unsigned long size = cmd->npages << PAGE_SHIFT; struct mm_struct *mm = dmirror->notifier.mm; struct vm_area_struct *vma; struct dmirror_bounce bounce; struct migrate_vma args = { 0 }; unsigned long next; int ret; unsigned long *src_pfns = NULL; unsigned long *dst_pfns = NULL;
start = cmd->addr; end = start + size; if (end < start) return -EINVAL;
/* Since the mm is for the mirrored process, get a reference first. */ if (!mmget_not_zero(mm)) return -EINVAL;
ret = -ENOMEM; src_pfns = kvcalloc(PTRS_PER_PTE, sizeof(*src_pfns), GFP_KERNEL | __GFP_NOFAIL); if (!src_pfns) goto free_mem;
dst_pfns = kvcalloc(PTRS_PER_PTE, sizeof(*dst_pfns), GFP_KERNEL | __GFP_NOFAIL); if (!dst_pfns) goto free_mem;
ret = 0; mmap_read_lock(mm); for (addr = start; addr < end; addr = next) { vma = vma_lookup(mm, addr); if (!vma || !(vma->vm_flags & VM_READ)) { ret = -EINVAL; goto out; } next = min(end, addr + (PTRS_PER_PTE << PAGE_SHIFT)); if (next > vma->vm_end) next = vma->vm_end;
args.vma = vma; args.src = src_pfns; args.dst = dst_pfns; args.start = addr; args.end = next; args.pgmap_owner = dmirror->mdevice; args.flags = MIGRATE_VMA_SELECT_SYSTEM | MIGRATE_VMA_SELECT_COMPOUND; ret = migrate_vma_setup(&args); if (ret) goto out;
pr_debug("Migrating from sys mem to device mem\n"); dmirror_migrate_alloc_and_copy(&args, dmirror); migrate_vma_pages(&args); dmirror_migrate_finalize_and_map(&args, dmirror); migrate_vma_finalize(&args); } mmap_read_unlock(mm); mmput(mm);
/* * Return the migrated data for verification. * Only for pages in device zone */ ret = dmirror_bounce_init(&bounce, start, size); if (ret) goto free_mem; mutex_lock(&dmirror->mutex); ret = dmirror_do_read(dmirror, start, end, &bounce); mutex_unlock(&dmirror->mutex); if (ret == 0) { if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr, bounce.size)) ret = -EFAULT; } cmd->cpages = bounce.cpages; dmirror_bounce_fini(&bounce); goto free_mem;
out: mmap_read_unlock(mm); mmput(mm);free_mem: kvfree(src_pfns); kvfree(dst_pfns); return ret;}
static void dmirror_mkentry(struct dmirror *dmirror, struct hmm_range *range, unsigned char *perm, unsigned long entry){ struct page *page;
if (entry & HMM_PFN_ERROR) { *perm = HMM_DMIRROR_PROT_ERROR; return; } if (!(entry & HMM_PFN_VALID)) { *perm = HMM_DMIRROR_PROT_NONE; return; }
page = hmm_pfn_to_page(entry); if (is_device_private_page(page)) { /* Is the page migrated to this device or some other? */ if (dmirror->mdevice == dmirror_page_to_device(page)) *perm = HMM_DMIRROR_PROT_DEV_PRIVATE_LOCAL; else *perm = HMM_DMIRROR_PROT_DEV_PRIVATE_REMOTE; } else if (is_device_coherent_page(page)) { /* Is the page migrated to this device or some other? */ if (dmirror->mdevice == dmirror_page_to_device(page)) *perm = HMM_DMIRROR_PROT_DEV_COHERENT_LOCAL; else *perm = HMM_DMIRROR_PROT_DEV_COHERENT_REMOTE; } else if (is_zero_pfn(page_to_pfn(page))) *perm = HMM_DMIRROR_PROT_ZERO; else *perm = HMM_DMIRROR_PROT_NONE; if (entry & HMM_PFN_WRITE) *perm |= HMM_DMIRROR_PROT_WRITE; else *perm |= HMM_DMIRROR_PROT_READ; if (hmm_pfn_to_map_order(entry) + PAGE_SHIFT == PMD_SHIFT) *perm |= HMM_DMIRROR_PROT_PMD; else if (hmm_pfn_to_map_order(entry) + PAGE_SHIFT == PUD_SHIFT) *perm |= HMM_DMIRROR_PROT_PUD;}
static bool dmirror_snapshot_invalidate(struct mmu_interval_notifier *mni, const struct mmu_notifier_range *range, unsigned long cur_seq){ struct dmirror_interval *dmi = container_of(mni, struct dmirror_interval, notifier); struct dmirror *dmirror = dmi->dmirror;
if (mmu_notifier_range_blockable(range)) mutex_lock(&dmirror->mutex); else if (!mutex_trylock(&dmirror->mutex)) return false;
/* * Snapshots only need to set the sequence number since any * invalidation in the interval invalidates the whole snapshot. */ mmu_interval_set_seq(mni, cur_seq);
mutex_unlock(&dmirror->mutex); return true;}
static const struct mmu_interval_notifier_ops dmirror_mrn_ops = { .invalidate = dmirror_snapshot_invalidate,};
static int dmirror_range_snapshot(struct dmirror *dmirror, struct hmm_range *range, unsigned char *perm){ struct mm_struct *mm = dmirror->notifier.mm; struct dmirror_interval notifier; unsigned long timeout = jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); unsigned long i; unsigned long n; int ret = 0;
notifier.dmirror = dmirror; range->notifier = ¬ifier.notifier;
ret = mmu_interval_notifier_insert(range->notifier, mm, range->start, range->end - range->start, &dmirror_mrn_ops); if (ret) return ret;
while (true) { if (time_after(jiffies, timeout)) { ret = -EBUSY; goto out; }
range->notifier_seq = mmu_interval_read_begin(range->notifier);
mmap_read_lock(mm); ret = hmm_range_fault(range); mmap_read_unlock(mm); if (ret) { if (ret == -EBUSY) continue; goto out; }
mutex_lock(&dmirror->mutex); if (mmu_interval_read_retry(range->notifier, range->notifier_seq)) { mutex_unlock(&dmirror->mutex); continue; } break; }
n = (range->end - range->start) >> PAGE_SHIFT; for (i = 0; i < n; i++) dmirror_mkentry(dmirror, range, perm + i, range->hmm_pfns[i]);
mutex_unlock(&dmirror->mutex);out: mmu_interval_notifier_remove(range->notifier); return ret;}
static int dmirror_snapshot(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd){ struct mm_struct *mm = dmirror->notifier.mm; unsigned long start, end; unsigned long size = cmd->npages << PAGE_SHIFT; unsigned long addr; unsigned long next; unsigned long pfns[32]; unsigned char perm[32]; char __user *uptr; struct hmm_range range = { .hmm_pfns = pfns, .dev_private_owner = dmirror->mdevice, }; int ret = 0;
start = cmd->addr; end = start + size; if (end < start) return -EINVAL;
/* Since the mm is for the mirrored process, get a reference first. */ if (!mmget_not_zero(mm)) return -EINVAL;
/* * Register a temporary notifier to detect invalidations even if it * overlaps with other mmu_interval_notifiers. */ uptr = u64_to_user_ptr(cmd->ptr); for (addr = start; addr < end; addr = next) { unsigned long n;
next = min(addr + (ARRAY_SIZE(pfns) << PAGE_SHIFT), end); range.start = addr; range.end = next;
ret = dmirror_range_snapshot(dmirror, &range, perm); if (ret) break;
n = (range.end - range.start) >> PAGE_SHIFT; if (copy_to_user(uptr, perm, n)) { ret = -EFAULT; break; }
cmd->cpages += n; uptr += n; } mmput(mm);
return ret;}
/* Removes free pages from the free list so they can't be re-allocated */static void dmirror_remove_free_pages(struct dmirror_chunk *devmem){ struct dmirror_device *mdevice = devmem->mdevice; struct page *page; struct folio *folio;
for (folio = mdevice->free_folios; folio; folio = folio_zone_device_data(folio)) if (dmirror_page_to_chunk(folio_page(folio, 0)) == devmem) mdevice->free_folios = folio_zone_device_data(folio); for (page = mdevice->free_pages; page; page = page->zone_device_data) if (dmirror_page_to_chunk(page) == devmem) mdevice->free_pages = page->zone_device_data;}
static void dmirror_device_remove_chunks(struct dmirror_device *mdevice){ unsigned int i;
mutex_lock(&mdevice->devmem_lock); if (mdevice->devmem_chunks) { for (i = 0; i < mdevice->devmem_count; i++) { struct dmirror_chunk *devmem = mdevice->devmem_chunks[i];
spin_lock(&mdevice->lock); devmem->remove = true; dmirror_remove_free_pages(devmem); spin_unlock(&mdevice->lock);
dmirror_device_evict_chunk(devmem); memunmap_pages(&devmem->pagemap); if (devmem->pagemap.type == MEMORY_DEVICE_PRIVATE) release_mem_region(devmem->pagemap.range.start, range_len(&devmem->pagemap.range)); kfree(devmem); } mdevice->devmem_count = 0; mdevice->devmem_capacity = 0; mdevice->free_pages = NULL; mdevice->free_folios = NULL; kfree(mdevice->devmem_chunks); mdevice->devmem_chunks = NULL; } mutex_unlock(&mdevice->devmem_lock);}
static long dmirror_fops_unlocked_ioctl(struct file *filp, unsigned int command, unsigned long arg){ void __user *uarg = (void __user *)arg; struct hmm_dmirror_cmd cmd; struct dmirror *dmirror; int ret;
dmirror = filp->private_data; if (!dmirror) return -EINVAL;
if (copy_from_user(&cmd, uarg, sizeof(cmd))) return -EFAULT;
if (cmd.addr & ~PAGE_MASK) return -EINVAL; if (cmd.addr >= (cmd.addr + (cmd.npages << PAGE_SHIFT))) return -EINVAL;
cmd.cpages = 0; cmd.faults = 0;
switch (command) { case HMM_DMIRROR_READ: ret = dmirror_read(dmirror, &cmd); break;
case HMM_DMIRROR_WRITE: ret = dmirror_write(dmirror, &cmd); break;
case HMM_DMIRROR_MIGRATE_TO_DEV: ret = dmirror_migrate_to_device(dmirror, &cmd); break;
case HMM_DMIRROR_MIGRATE_TO_SYS: ret = dmirror_migrate_to_system(dmirror, &cmd); break;
case HMM_DMIRROR_EXCLUSIVE: ret = dmirror_exclusive(dmirror, &cmd); break;
case HMM_DMIRROR_CHECK_EXCLUSIVE: ret = dmirror_check_atomic(dmirror, cmd.addr, cmd.addr + (cmd.npages << PAGE_SHIFT)); break;
case HMM_DMIRROR_SNAPSHOT: ret = dmirror_snapshot(dmirror, &cmd); break;
case HMM_DMIRROR_RELEASE: dmirror_device_remove_chunks(dmirror->mdevice); ret = 0; break; case HMM_DMIRROR_FLAGS: dmirror->flags = cmd.npages; ret = 0; break;
default: return -EINVAL; } if (ret) return ret;
if (copy_to_user(uarg, &cmd, sizeof(cmd))) return -EFAULT;
return 0;}
static int dmirror_fops_mmap(struct file *file, struct vm_area_struct *vma){ unsigned long addr;
for (addr = vma->vm_start; addr < vma->vm_end; addr += PAGE_SIZE) { struct page *page; int ret;
page = alloc_page(GFP_KERNEL | __GFP_ZERO); if (!page) return -ENOMEM;
ret = vm_insert_page(vma, addr, page); if (ret) { __free_page(page); return ret; } put_page(page); }
return 0;}
static const struct file_operations dmirror_fops = { .open = dmirror_fops_open, .release = dmirror_fops_release, .mmap = dmirror_fops_mmap, .unlocked_ioctl = dmirror_fops_unlocked_ioctl, .llseek = default_llseek, .owner = THIS_MODULE,};
static void dmirror_devmem_free(struct folio *folio){ struct page *page = &folio->page; struct page *rpage = BACKING_PAGE(page); struct dmirror_device *mdevice; struct folio *rfolio = page_folio(rpage); unsigned int order = folio_order(rfolio);
if (rpage != page) { if (order) __free_pages(rpage, order); else __free_page(rpage); rpage = NULL; }
mdevice = dmirror_page_to_device(page); spin_lock(&mdevice->lock);
/* Return page to our allocator if not freeing the chunk */ if (!dmirror_page_to_chunk(page)->remove) { mdevice->cfree += 1 << order; if (order) { page->zone_device_data = mdevice->free_folios; mdevice->free_folios = page_folio(page); } else { page->zone_device_data = mdevice->free_pages; mdevice->free_pages = page; } } spin_unlock(&mdevice->lock);}
static vm_fault_t dmirror_devmem_fault(struct vm_fault *vmf){ struct migrate_vma args = { 0 }; struct page *rpage; struct dmirror *dmirror; vm_fault_t ret = 0; unsigned int order, nr;
/* * Normally, a device would use the page->zone_device_data to point to * the mirror but here we use it to hold the page for the simulated * device memory and that page holds the pointer to the mirror. */ rpage = folio_zone_device_data(page_folio(vmf->page)); dmirror = rpage->zone_device_data;
/* FIXME demonstrate how we can adjust migrate range */ order = folio_order(page_folio(vmf->page)); nr = 1 << order;
/* * When folios are partially mapped, we can't rely on the folio * order of vmf->page as the folio might not be fully split yet */ if (vmf->pte) { order = 0; nr = 1; }
/* * Consider a per-cpu cache of src and dst pfns, but with * large number of cpus that might not scale well. */ args.start = ALIGN_DOWN(vmf->address, (PAGE_SIZE << order)); args.vma = vmf->vma; args.end = args.start + (PAGE_SIZE << order);
nr = (args.end - args.start) >> PAGE_SHIFT; args.src = kcalloc(nr, sizeof(unsigned long), GFP_KERNEL); args.dst = kcalloc(nr, sizeof(unsigned long), GFP_KERNEL); args.pgmap_owner = dmirror->mdevice; args.flags = dmirror_select_device(dmirror); args.fault_page = vmf->page;
if (!args.src || !args.dst) { ret = VM_FAULT_OOM; goto err; }
if (order) args.flags |= MIGRATE_VMA_SELECT_COMPOUND;
/* * In practice migrate_vma_setup() should never fail unless the * test is wrong as it just tests some static VMA properties. */ if (migrate_vma_setup(&args)) { ret = VM_FAULT_SIGBUS; goto err; }
ret = dmirror_devmem_fault_alloc_and_copy(&args, dmirror); if (ret) { migrate_vma_finalize(&args); goto err; } migrate_vma_pages(&args); /* * No device finalize step is needed since * dmirror_devmem_fault_alloc_and_copy() will have already * invalidated the device page table. */ migrate_vma_finalize(&args);err: kfree(args.src); kfree(args.dst); return ret;}
static void dmirror_devmem_folio_split(struct folio *head, struct folio *tail){ struct page *rpage = BACKING_PAGE(folio_page(head, 0)); struct page *rpage_tail; struct folio *rfolio; unsigned long offset = 0;
if (!rpage) { tail->page.zone_device_data = NULL; return; }
rfolio = page_folio(rpage);
if (tail == NULL) { folio_reset_order(rfolio); rfolio->mapping = NULL; folio_set_count(rfolio, 1); return; }
offset = folio_pfn(tail) - folio_pfn(head);
rpage_tail = folio_page(rfolio, offset); tail->page.zone_device_data = rpage_tail; rpage_tail->zone_device_data = rpage->zone_device_data; clear_compound_head(rpage_tail); rpage_tail->mapping = NULL;
folio_page(tail, 0)->mapping = folio_page(head, 0)->mapping; tail->pgmap = head->pgmap; folio_set_count(page_folio(rpage_tail), 1);}
static const struct dev_pagemap_ops dmirror_devmem_ops = { .folio_free = dmirror_devmem_free, .migrate_to_ram = dmirror_devmem_fault, .folio_split = dmirror_devmem_folio_split,};
static void dmirror_device_release(struct device *dev){ struct dmirror_device *mdevice = container_of(dev, struct dmirror_device, device);
dmirror_device_remove_chunks(mdevice);}
static int dmirror_device_init(struct dmirror_device *mdevice, int id){ dev_t dev; int ret;
dev = MKDEV(MAJOR(dmirror_dev), id); mutex_init(&mdevice->devmem_lock); spin_lock_init(&mdevice->lock);
cdev_init(&mdevice->cdevice, &dmirror_fops); mdevice->cdevice.owner = THIS_MODULE; mdevice->device.release = dmirror_device_release;
device_initialize(&mdevice->device); mdevice->device.devt = dev;
ret = dev_set_name(&mdevice->device, "hmm_dmirror%u", id); if (ret) goto put_device;
/* Build a list of free ZONE_DEVICE struct pages */ ret = dmirror_allocate_chunk(mdevice, NULL, false); if (ret) goto put_device;
ret = cdev_device_add(&mdevice->cdevice, &mdevice->device); if (ret) goto put_device;
return 0;
put_device: put_device(&mdevice->device); return ret;}
static void dmirror_device_remove(struct dmirror_device *mdevice){ cdev_device_del(&mdevice->cdevice, &mdevice->device); put_device(&mdevice->device);}
static int __init hmm_dmirror_init(void){ int ret; int id = 0; int ndevices = 0;
ret = alloc_chrdev_region(&dmirror_dev, 0, DMIRROR_NDEVICES, "HMM_DMIRROR"); if (ret) goto err_unreg;
memset(dmirror_devices, 0, DMIRROR_NDEVICES * sizeof(dmirror_devices[0])); dmirror_devices[ndevices++].zone_device_type = HMM_DMIRROR_MEMORY_DEVICE_PRIVATE; dmirror_devices[ndevices++].zone_device_type = HMM_DMIRROR_MEMORY_DEVICE_PRIVATE; if (spm_addr_dev0 && spm_addr_dev1) { dmirror_devices[ndevices++].zone_device_type = HMM_DMIRROR_MEMORY_DEVICE_COHERENT; dmirror_devices[ndevices++].zone_device_type = HMM_DMIRROR_MEMORY_DEVICE_COHERENT; } for (id = 0; id < ndevices; id++) { ret = dmirror_device_init(dmirror_devices + id, id); if (ret) goto err_chrdev; }
pr_info("HMM test module loaded. This is only for testing HMM.\n"); return 0;
err_chrdev: while (--id >= 0) dmirror_device_remove(dmirror_devices + id); unregister_chrdev_region(dmirror_dev, DMIRROR_NDEVICES);err_unreg: return ret;}
static void __exit hmm_dmirror_exit(void){ int id;
for (id = 0; id < DMIRROR_NDEVICES; id++) if (dmirror_devices[id].zone_device_type) dmirror_device_remove(dmirror_devices + id); unregister_chrdev_region(dmirror_dev, DMIRROR_NDEVICES);}
module_init(hmm_dmirror_init);module_exit(hmm_dmirror_exit);MODULE_DESCRIPTION("HMM (Heterogeneous Memory Management) test module");MODULE_LICENSE("GPL");