From 9f960da72b25054163cf555e622dcdc3b8ccc488 Mon Sep 17 00:00:00 2001 From: Pavel Tatashin Date: Tue, 16 Jul 2019 16:30:35 -0700 Subject: [PATCH] device-dax: "Hotremove" persistent memory that is used like normal RAM MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit It is now allowed to use persistent memory like a regular RAM, but currently there is no way to remove this memory until machine is rebooted. This work expands the functionality to also allows hotremoving previously hotplugged persistent memory, and recover the device for use for other purposes. To hotremove persistent memory, the management software must first offline all memory blocks of dax region, and than unbind it from device-dax/kmem driver. So, operations should look like this: echo offline > /sys/devices/system/memory/memoryN/state ... echo dax0.0 > /sys/bus/dax/drivers/kmem/unbind Note: if unbind is done without offlining memory beforehand, it won't be possible to do dax0.0 hotremove, and dax's memory is going to be part of System RAM until reboot. Link: http://lkml.kernel.org/r/20190517215438.6487-4-pasha.tatashin@soleen.com Signed-off-by: Pavel Tatashin Reviewed-by: David Hildenbrand Cc: James Morris Cc: Sasha Levin Cc: Michal Hocko Cc: Dave Hansen Cc: Dan Williams Cc: Keith Busch Cc: Vishal Verma Cc: Dave Jiang Cc: Ross Zwisler Cc: Tom Lendacky Cc: Huang Ying Cc: Fengguang Wu Cc: Borislav Petkov Cc: Bjorn Helgaas Cc: Yaowei Bai Cc: Takashi Iwai Cc: Jérôme Glisse Cc: Dave Hansen Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/dax/dax-private.h | 2 ++ drivers/dax/kmem.c | 41 +++++++++++++++++++++++++++++++++++---- 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/drivers/dax/dax-private.h b/drivers/dax/dax-private.h index c915889d1769..6ccca3b890d6 100644 --- a/drivers/dax/dax-private.h +++ b/drivers/dax/dax-private.h @@ -43,6 +43,7 @@ struct dax_region { * @target_node: effective numa node if dev_dax memory range is onlined * @dev - device core * @pgmap - pgmap for memmap setup / lifetime (driver owned) + * @dax_mem_res: physical address range of hotadded DAX memory */ struct dev_dax { struct dax_region *region; @@ -50,6 +51,7 @@ struct dev_dax { int target_node; struct device dev; struct dev_pagemap pgmap; + struct resource *dax_kmem_res; }; static inline struct dev_dax *to_dev_dax(struct device *dev) diff --git a/drivers/dax/kmem.c b/drivers/dax/kmem.c index 4c0131857133..3d0a7e702c94 100644 --- a/drivers/dax/kmem.c +++ b/drivers/dax/kmem.c @@ -71,21 +71,54 @@ int dev_dax_kmem_probe(struct device *dev) kfree(new_res); return rc; } + dev_dax->dax_kmem_res = new_res; return 0; } +#ifdef CONFIG_MEMORY_HOTREMOVE +static int dev_dax_kmem_remove(struct device *dev) +{ + struct dev_dax *dev_dax = to_dev_dax(dev); + struct resource *res = dev_dax->dax_kmem_res; + resource_size_t kmem_start = res->start; + resource_size_t kmem_size = resource_size(res); + int rc; + + /* + * We have one shot for removing memory, if some memory blocks were not + * offline prior to calling this function remove_memory() will fail, and + * there is no way to hotremove this memory until reboot because device + * unbind will succeed even if we return failure. + */ + rc = remove_memory(dev_dax->target_node, kmem_start, kmem_size); + if (rc) { + dev_err(dev, + "DAX region %pR cannot be hotremoved until the next reboot\n", + res); + return rc; + } + + /* Release and free dax resources */ + release_resource(res); + kfree(res); + dev_dax->dax_kmem_res = NULL; + + return 0; +} +#else static int dev_dax_kmem_remove(struct device *dev) { /* - * Purposely leak the request_mem_region() for the device-dax - * range and return '0' to ->remove() attempts. The removal of - * the device from the driver always succeeds, but the region - * is permanently pinned as reserved by the unreleased + * Without hotremove purposely leak the request_mem_region() for the + * device-dax range and return '0' to ->remove() attempts. The removal + * of the device from the driver always succeeds, but the region is + * permanently pinned as reserved by the unreleased * request_mem_region(). */ return 0; } +#endif /* CONFIG_MEMORY_HOTREMOVE */ static struct dax_device_driver device_dax_kmem_driver = { .drv = { -- 2.39.5