Skip to content

Commit 1fe058f

Browse files
bibrakbibrak
committed
feature: multi gpu SVM madvise
Implements madvise SVM allocated memory for multi-gpu Signed-off-by: Chandio, Bibrak Qamar <[email protected]>
1 parent f31341f commit 1fe058f

File tree

17 files changed

+71
-47
lines changed

17 files changed

+71
-47
lines changed

‎level_zero/core/source/cmdlist/cmdlist_hw.inl‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1355,10 +1355,19 @@ ze_result_t CommandListCoreFamily<gfxCoreFamily>::executeMemAdvise(ze_device_han
13551355
ze_memory_advice_t advice) {
13561356

13571357
auto driverHandle = device->getDriverHandle();
1358+
auto callingNEODevice = device->getNEODevice();
13581359
auto allocData = driverHandle->getSvmAllocsManager()->getSVMAlloc(ptr);
13591360

13601361
if (!allocData) {
1361-
if (device->getNEODevice()->areSharedSystemAllocationsAllowed()) {
1362+
if (callingNEODevice->areSharedSystemAllocationsAllowed()) {
1363+
1364+
DeviceImp *targetDeviceImp = static_cast<DeviceImp *>((L0::Device::fromHandle(hDevice)));
1365+
auto targetNEODevice = targetDeviceImp->getNEODevice();
1366+
1367+
if (!targetNEODevice->areSharedSystemAllocationsAllowed()) {
1368+
return ZE_RESULT_ERROR_INVALID_ARGUMENT;
1369+
}
1370+
13621371
NEO::MemAdvise memAdviseOp = NEO::MemAdvise::invalidAdvise;
13631372

13641373
switch (advice) {
@@ -1384,10 +1393,9 @@ ze_result_t CommandListCoreFamily<gfxCoreFamily>::executeMemAdvise(ze_device_han
13841393
return ZE_RESULT_SUCCESS;
13851394
}
13861395

1387-
DeviceImp *deviceImp = static_cast<DeviceImp *>((L0::Device::fromHandle(hDevice)));
13881396
auto unifiedMemoryManager = driverHandle->getSvmAllocsManager();
13891397

1390-
unifiedMemoryManager->sharedSystemMemAdvise(*deviceImp->getNEODevice(), memAdviseOp, ptr, size);
1398+
unifiedMemoryManager->sharedSystemMemAdvise(*callingNEODevice, *targetNEODevice, memAdviseOp, ptr, size);
13911399

13921400
return ZE_RESULT_SUCCESS;
13931401
} else {

‎shared/source/memory_manager/memory_manager.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -287,7 +287,7 @@ class MemoryManager {
287287
virtual AllocationStatus registerLocalMemAlloc(GraphicsAllocation *allocation, uint32_t rootDeviceIndex);
288288

289289
virtual bool setMemAdvise(GraphicsAllocation *gfxAllocation, MemAdviseFlags flags, uint32_t rootDeviceIndex) { return true; }
290-
virtual bool setSharedSystemMemAdvise(const void *ptr, const size_t size, MemAdvise memAdviseOp, SubDeviceIdsVec &subDeviceIds, uint32_t rootDeviceIndex) { return true; }
290+
virtual bool setSharedSystemMemAdvise(const void *ptr, const size_t size, MemAdvise memAdviseOp, SubDeviceIdsVec &subDeviceIds, uint32_t callingRootDeviceIndex, uint32_t targetRootDeviceIndex) { return true; }
291291
virtual bool setMemPrefetch(GraphicsAllocation *gfxAllocation, SubDeviceIdsVec &subDeviceIds, uint32_t rootDeviceIndex) { return true; }
292292
virtual bool prefetchSharedSystemAlloc(const void *ptr, const size_t size, SubDeviceIdsVec &subDeviceIds, uint32_t rootDeviceIndex) { return true; }
293293
virtual bool setAtomicAccess(GraphicsAllocation *gfxAllocation, size_t size, AtomicAccessMode mode, uint32_t rootDeviceIndex) { return true; }

‎shared/source/memory_manager/unified_memory_manager.cpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1141,12 +1141,12 @@ static NEO::SubDeviceIdsVec getSubDeviceIds(CommandStreamReceiver &csr) {
11411141
return subDeviceIds;
11421142
};
11431143

1144-
void SVMAllocsManager::sharedSystemMemAdvise(Device &device, MemAdvise memAdviseOp, const void *ptr, const size_t size) {
1144+
void SVMAllocsManager::sharedSystemMemAdvise(Device &callingDevice, Device &targetDevice, MemAdvise memAdviseOp, const void *ptr, const size_t size) {
11451145

11461146
// All vm_ids on a single device for shared system USM allocation
1147-
auto subDeviceIds = NEO::SubDevice::getSubDeviceIdsFromDevice(device);
1147+
auto subDeviceIds = NEO::SubDevice::getSubDeviceIdsFromDevice(targetDevice);
11481148

1149-
memoryManager->setSharedSystemMemAdvise(ptr, size, memAdviseOp, subDeviceIds, device.getRootDeviceIndex());
1149+
memoryManager->setSharedSystemMemAdvise(ptr, size, memAdviseOp, subDeviceIds, callingDevice.getRootDeviceIndex(), targetDevice.getRootDeviceIndex());
11501150
}
11511151

11521152
void SVMAllocsManager::prefetchMemory(Device &device, CommandStreamReceiver &commandStreamReceiver, const void *ptr, const size_t size) {

‎shared/source/memory_manager/unified_memory_manager.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -304,7 +304,7 @@ class SVMAllocsManager {
304304
std::atomic<uint32_t> allocationsCounter = 0;
305305
MOCKABLE_VIRTUAL void makeIndirectAllocationsResident(CommandStreamReceiver &commandStreamReceiver, TaskCountType taskCount);
306306
void prepareIndirectAllocationForDestruction(SvmAllocationData *allocationData, bool isNonBlockingFree);
307-
void sharedSystemMemAdvise(Device &device, MemAdvise memAdviseOp, const void *ptr, const size_t size);
307+
void sharedSystemMemAdvise(Device &callingDevice, Device &targetDevice, MemAdvise memAdviseOp, const void *ptr, const size_t size);
308308
MOCKABLE_VIRTUAL void prefetchMemory(Device &device, CommandStreamReceiver &commandStreamReceiver, const void *ptr, const size_t size);
309309
void prefetchSVMAllocs(Device &device, CommandStreamReceiver &commandStreamReceiver);
310310
void sharedSystemAtomicAccess(Device &device, AtomicAccessMode mode, const void *ptr, const size_t size);

‎shared/source/os_interface/linux/drm_memory_manager.cpp‎

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -308,22 +308,26 @@ bool DrmMemoryManager::setMemAdvise(GraphicsAllocation *gfxAllocation, MemAdvise
308308
return drmAllocation->setMemAdvise(&this->getDrm(rootDeviceIndex), flags);
309309
}
310310

311-
bool DrmMemoryManager::setSharedSystemMemAdvise(const void *ptr, const size_t size, MemAdvise memAdviseOp, SubDeviceIdsVec &subDeviceIds, uint32_t rootDeviceIndex) {
311+
bool DrmMemoryManager::setSharedSystemMemAdvise(const void *ptr, const size_t size, MemAdvise memAdviseOp, SubDeviceIdsVec &subDeviceIds, uint32_t callingRootDeviceIndex, uint32_t targetRootDeviceIndex) {
312312

313-
auto &drm = this->getDrm(rootDeviceIndex);
314-
auto ioctlHelper = drm.getIoctlHelper();
313+
auto &targetDeviceDrm = this->getDrm(targetRootDeviceIndex);
314+
auto targetDeviceIoctlHelper = targetDeviceDrm.getIoctlHelper();
315+
auto targetDeviceFd = targetDeviceDrm.getFileDescriptor();
315316

316-
uint32_t attribute = ioctlHelper->getPreferredLocationAdvise();
317-
uint64_t param = ioctlHelper->getPreferredLocationArgs(memAdviseOp);
317+
uint32_t attribute = targetDeviceIoctlHelper->getPreferredLocationAdvise();
318+
uint64_t param = targetDeviceIoctlHelper->getPreferredLocationArgs(targetDeviceFd, memAdviseOp);
318319

319320
// Apply the shared system USM IOCTL to all the VMs of the device
320321
std::vector<uint32_t> vmIds;
321322
vmIds.reserve(subDeviceIds.size());
322323
for (auto subDeviceId : subDeviceIds) {
323-
vmIds.push_back(drm.getVirtualMemoryAddressSpace(subDeviceId));
324+
vmIds.push_back(targetDeviceDrm.getVirtualMemoryAddressSpace(subDeviceId));
324325
}
325326

326-
auto result = ioctlHelper->setVmSharedSystemMemAdvise(reinterpret_cast<uint64_t>(ptr), size, attribute, param, vmIds);
327+
auto &callingDeviceDrm = this->getDrm(callingRootDeviceIndex);
328+
auto callingDeviceIoctlHelper = callingDeviceDrm.getIoctlHelper();
329+
330+
auto result = callingDeviceIoctlHelper->setVmSharedSystemMemAdvise(reinterpret_cast<uint64_t>(ptr), size, attribute, param, vmIds);
327331

328332
return result;
329333
}

‎shared/source/os_interface/linux/drm_memory_manager.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ class DrmMemoryManager : public MemoryManager {
9090
bool isKmdMigrationAvailable(uint32_t rootDeviceIndex) override;
9191

9292
bool setMemAdvise(GraphicsAllocation *gfxAllocation, MemAdviseFlags flags, uint32_t rootDeviceIndex) override;
93-
bool setSharedSystemMemAdvise(const void *ptr, const size_t size, MemAdvise memAdviseOp, SubDeviceIdsVec &subDeviceIds, uint32_t rootDeviceIndex) override;
93+
bool setSharedSystemMemAdvise(const void *ptr, const size_t size, MemAdvise memAdviseOp, SubDeviceIdsVec &subDeviceIds, uint32_t callingRootDeviceIndex, uint32_t targetRootDeviceIndex) override;
9494
bool setMemPrefetch(GraphicsAllocation *gfxAllocation, SubDeviceIdsVec &subDeviceIds, uint32_t rootDeviceIndex) override;
9595
bool prefetchSharedSystemAlloc(const void *ptr, const size_t size, SubDeviceIdsVec &subDeviceIds, uint32_t rootDeviceIndex) override;
9696
bool setAtomicAccess(GraphicsAllocation *gfxAllocation, size_t size, AtomicAccessMode mode, uint32_t rootDeviceIndex) override;

‎shared/source/os_interface/linux/ioctl_helper.h‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -129,7 +129,7 @@ class IoctlHelper {
129129
bool userInterrupt, uint32_t externalInterruptId, GraphicsAllocation *allocForInterruptWait) = 0;
130130
virtual uint32_t getAtomicAdvise(bool isNonAtomic) = 0;
131131
virtual uint32_t getAtomicAccess(AtomicAccessMode mode) = 0;
132-
virtual uint64_t getPreferredLocationArgs(MemAdvise memAdviseOp) = 0;
132+
virtual uint64_t getPreferredLocationArgs(int deviceFd, MemAdvise memAdviseOp) = 0;
133133
virtual uint32_t getPreferredLocationAdvise() = 0;
134134
virtual std::optional<MemoryClassInstance> getPreferredLocationRegion(PreferredLocation memoryLocation, uint32_t memoryInstance) = 0;
135135
virtual bool setVmBoAdvise(int32_t handle, uint32_t attribute, void *region) = 0;
@@ -328,7 +328,7 @@ class IoctlHelperUpstream : public IoctlHelperI915 {
328328
bool userInterrupt, uint32_t externalInterruptId, GraphicsAllocation *allocForInterruptWait) override;
329329
uint32_t getAtomicAdvise(bool isNonAtomic) override;
330330
uint32_t getAtomicAccess(AtomicAccessMode mode) override;
331-
uint64_t getPreferredLocationArgs(MemAdvise memAdviseOp) override;
331+
uint64_t getPreferredLocationArgs(int deviceFd, MemAdvise memAdviseOp) override;
332332
uint32_t getPreferredLocationAdvise() override;
333333
std::optional<MemoryClassInstance> getPreferredLocationRegion(PreferredLocation memoryLocation, uint32_t memoryInstance) override;
334334
bool setVmBoAdvise(int32_t handle, uint32_t attribute, void *region) override;
@@ -404,7 +404,7 @@ class IoctlHelperPrelim20 : public IoctlHelperI915 {
404404
bool userInterrupt, uint32_t externalInterruptId, GraphicsAllocation *allocForInterruptWait) override;
405405
uint32_t getAtomicAdvise(bool isNonAtomic) override;
406406
uint32_t getAtomicAccess(AtomicAccessMode mode) override;
407-
uint64_t getPreferredLocationArgs(MemAdvise memAdviseOp) override;
407+
uint64_t getPreferredLocationArgs(int deviceFd, MemAdvise memAdviseOp) override;
408408
uint32_t getPreferredLocationAdvise() override;
409409
std::optional<MemoryClassInstance> getPreferredLocationRegion(PreferredLocation memoryLocation, uint32_t memoryInstance) override;
410410
bool setVmBoAdvise(int32_t handle, uint32_t attribute, void *region) override;

‎shared/source/os_interface/linux/ioctl_helper_prelim.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -345,7 +345,7 @@ uint32_t IoctlHelperPrelim20::getAtomicAdvise(bool isNonAtomic) {
345345
return isNonAtomic ? PRELIM_I915_VM_ADVISE_ATOMIC_NONE : PRELIM_I915_VM_ADVISE_ATOMIC_SYSTEM;
346346
}
347347

348-
uint64_t IoctlHelperPrelim20::getPreferredLocationArgs(MemAdvise memAdviseOp) {
348+
uint64_t IoctlHelperPrelim20::getPreferredLocationArgs(int deviceFd, MemAdvise memAdviseOp) {
349349
return 0;
350350
}
351351

‎shared/source/os_interface/linux/ioctl_helper_upstream.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -140,7 +140,7 @@ uint32_t IoctlHelperUpstream::getAtomicAccess(AtomicAccessMode mode) {
140140
return 0;
141141
}
142142

143-
uint64_t IoctlHelperUpstream::getPreferredLocationArgs(MemAdvise memAdviseOp) {
143+
uint64_t IoctlHelperUpstream::getPreferredLocationArgs(int deviceFd, MemAdvise memAdviseOp) {
144144
return 0;
145145
}
146146

‎shared/source/os_interface/linux/xe/ioctl_helper_xe.cpp‎

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -794,24 +794,30 @@ uint32_t IoctlHelperXe::getAtomicAccess(AtomicAccessMode mode) {
794794
return retVal;
795795
}
796796

797-
uint64_t IoctlHelperXe::getPreferredLocationArgs(MemAdvise memAdviseOp) {
797+
uint64_t IoctlHelperXe::getPreferredLocationArgs(int deviceFd, MemAdvise memAdviseOp) {
798798
xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__);
799799
uint64_t param = 0;
800800

801801
switch (memAdviseOp) {
802-
case MemAdvise::setPreferredLocation:
803802
case MemAdvise::clearPreferredLocation:
804-
805803
case MemAdvise::clearSystemMemoryPreferredLocation: {
806804
// Assumes that the default location is Device VRAM.
807805
const auto preferredLocation = static_cast<uint64_t>(getDrmParamValue(DrmParam::memoryAdviseLocationDevice));
808806
const auto policy = static_cast<uint64_t>(getDrmParamValue(DrmParam::memoryAdviseMigrationPolicyAllPages));
809-
param = (preferredLocation << 32) | policy;
807+
const auto regionInstance = static_cast<uint64_t>(0);
808+
param = (preferredLocation << 32) | (policy << 16) | regionInstance;
809+
} break;
810+
case MemAdvise::setPreferredLocation: {
811+
const auto preferredLocation = static_cast<uint64_t>(deviceFd);
812+
const auto policy = static_cast<uint64_t>(getDrmParamValue(DrmParam::memoryAdviseMigrationPolicyAllPages));
813+
const auto regionInstance = static_cast<uint64_t>(1);
814+
param = (preferredLocation << 32) | (policy << 16) | regionInstance;
810815
} break;
811816
case MemAdvise::setSystemMemoryPreferredLocation: {
812817
const auto preferredLocation = static_cast<uint64_t>(getDrmParamValue(DrmParam::memoryAdviseLocationSystem));
813818
const auto policy = static_cast<uint64_t>(getDrmParamValue(DrmParam::memoryAdviseMigrationPolicySystemPages));
814-
param = (preferredLocation << 32) | policy;
819+
const auto regionInstance = static_cast<uint64_t>(0);
820+
param = (preferredLocation << 32) | (policy << 16) | regionInstance;
815821
} break;
816822
default:
817823
xeLog(" Invalid advise operation %s\n", __FUNCTION__);
@@ -837,10 +843,12 @@ bool IoctlHelperXe::setVmBoAdvise(int32_t handle, uint32_t attribute, void *regi
837843
inline void setMemoryAdvisePreferredLocationParam(drm_xe_madvise &vmAdvise, const uint64_t param) {
838844

839845
uint32_t devmemFd = static_cast<uint32_t>(param >> 32);
840-
uint16_t migrationPolicy = static_cast<uint16_t>(param & 0xFFFF);
846+
uint16_t migrationPolicy = static_cast<uint16_t>((param >> 16) & 0xFFFF);
847+
uint16_t regionInstance = static_cast<uint16_t>(param & 0xFFFF);
841848

842849
vmAdvise.preferred_mem_loc.devmem_fd = devmemFd;
843850
vmAdvise.preferred_mem_loc.migration_policy = migrationPolicy;
851+
vmAdvise.preferred_mem_loc.region_instance = regionInstance;
844852
}
845853

846854
inline void setMemoryAdviseAtomicParam(drm_xe_madvise &vmAdvise, const uint64_t param) {

0 commit comments

Comments
 (0)