Home Home > GIT Browse > stable-xen
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorDenis Kirjanov <dkirjanov@suse.com>2019-07-31 15:15:48 +0300
committerDenis Kirjanov <dkirjanov@suse.com>2019-07-31 15:15:48 +0300
commitf81a69a442e2cbac261ad8dda0a76b26d8ee014f (patch)
treebb451054e5ac0d7e16a8cde6424ff89907d8b93b
parentcbd2da5045b37903d2795b805e4d5b2cdcec3fd1 (diff)
parentb40c0eb4bccf1e14d418565c21ecb0ad4364bba4 (diff)
Merge 'users/mhocko/SLE12-SP5/for-next' into SLE12-SP5
Pull dax and mm fixes from Michal Hocko Conflicts: blacklist.conf patches.arch/powerpc-tm-Fix-oops-on-sigreturn-on-systems-without-.patch series.conf suse-commit: 2f3e5bacedbc4ef6df7a52916dd6fa6ef48ccbe6
-rw-r--r--Documentation/ABI/obsolete/sysfs-class-dax22
-rw-r--r--arch/powerpc/platforms/pseries/papr_scm.c1
-rw-r--r--drivers/acpi/nfit/core.c8
-rw-r--r--drivers/acpi/numa.c1
-rw-r--r--drivers/base/memory.c1
-rw-r--r--drivers/dax/Kconfig28
-rw-r--r--drivers/dax/Makefile6
-rw-r--r--drivers/dax/bus.c503
-rw-r--r--drivers/dax/bus.h61
-rw-r--r--drivers/dax/dax-private.h34
-rw-r--r--drivers/dax/dax.h18
-rw-r--r--drivers/dax/device-dax.h25
-rw-r--r--drivers/dax/device.c362
-rw-r--r--drivers/dax/kmem.c111
-rw-r--r--drivers/dax/pmem.c161
-rw-r--r--drivers/dax/pmem/Makefile7
-rw-r--r--drivers/dax/pmem/compat.c73
-rw-r--r--drivers/dax/pmem/core.c71
-rw-r--r--drivers/dax/pmem/pmem.c40
-rw-r--r--drivers/dax/super.c41
-rw-r--r--drivers/edac/i10nm_base.c10
-rw-r--r--drivers/edac/skx_base.c2
-rw-r--r--drivers/edac/skx_common.c4
-rw-r--r--drivers/edac/skx_common.h2
-rw-r--r--drivers/nvdimm/e820.c1
-rw-r--r--drivers/nvdimm/nd.h2
-rw-r--r--drivers/nvdimm/of_pmem.c1
-rw-r--r--drivers/nvdimm/pmem.c13
-rw-r--r--drivers/nvdimm/region_devs.c1
-rw-r--r--include/linux/acpi.h5
-rw-r--r--include/linux/libnvdimm.h1
-rw-r--r--include/linux/memremap.h2
-rw-r--r--kernel/memremap.c39
-rw-r--r--kernel/resource.c18
-rw-r--r--mm/memory_hotplug.c32
-rw-r--r--tools/testing/nvdimm/Kbuild7
-rw-r--r--tools/testing/nvdimm/dax-dev.c16
-rw-r--r--tools/testing/nvdimm/test/iomap.c15
38 files changed, 1162 insertions, 583 deletions
diff --git a/Documentation/ABI/obsolete/sysfs-class-dax b/Documentation/ABI/obsolete/sysfs-class-dax
new file mode 100644
index 000000000000..2cb9fc5e8bd1
--- /dev/null
+++ b/Documentation/ABI/obsolete/sysfs-class-dax
@@ -0,0 +1,22 @@
+What: /sys/class/dax/
+Date: May, 2016
+KernelVersion: v4.7
+Contact: linux-nvdimm@lists.01.org
+Description: Device DAX is the device-centric analogue of Filesystem
+ DAX (CONFIG_FS_DAX). It allows memory ranges to be
+ allocated and mapped without need of an intervening file
+ system. Device DAX is strict, precise and predictable.
+ Specifically this interface:
+
+ 1/ Guarantees fault granularity with respect to a given
+ page size (pte, pmd, or pud) set at configuration time.
+
+ 2/ Enforces deterministic behavior by being strict about
+ what fault scenarios are supported.
+
+ The /sys/class/dax/ interface enumerates all the
+ device-dax instances in the system. The ABI is
+ deprecated and will be removed after 2020. It is
+ replaced with the DAX bus interface /sys/bus/dax/ where
+ device-dax instances can be found under
+ /sys/bus/dax/devices/
diff --git a/arch/powerpc/platforms/pseries/papr_scm.c b/arch/powerpc/platforms/pseries/papr_scm.c
index 32f1e3a33650..7c5e10c063a0 100644
--- a/arch/powerpc/platforms/pseries/papr_scm.c
+++ b/arch/powerpc/platforms/pseries/papr_scm.c
@@ -259,6 +259,7 @@ static int papr_scm_nvdimm_init(struct papr_scm_priv *p)
memset(&ndr_desc, 0, sizeof(ndr_desc));
ndr_desc.attr_groups = region_attr_groups;
ndr_desc.numa_node = dev_to_node(&p->pdev->dev);
+ ndr_desc.target_node = ndr_desc.numa_node;
ndr_desc.res = &p->res;
ndr_desc.of_node = p->dn;
ndr_desc.provider_data = p;
diff --git a/drivers/acpi/nfit/core.c b/drivers/acpi/nfit/core.c
index 5c39057721c1..2098f6c2fafc 100644
--- a/drivers/acpi/nfit/core.c
+++ b/drivers/acpi/nfit/core.c
@@ -2934,11 +2934,15 @@ static int acpi_nfit_register_region(struct acpi_nfit_desc *acpi_desc,
ndr_desc->res = &res;
ndr_desc->provider_data = nfit_spa;
ndr_desc->attr_groups = acpi_nfit_region_attribute_groups;
- if (spa->flags & ACPI_NFIT_PROXIMITY_VALID)
+ if (spa->flags & ACPI_NFIT_PROXIMITY_VALID) {
ndr_desc->numa_node = acpi_map_pxm_to_online_node(
spa->proximity_domain);
- else
+ ndr_desc->target_node = acpi_map_pxm_to_node(
+ spa->proximity_domain);
+ } else {
ndr_desc->numa_node = NUMA_NO_NODE;
+ ndr_desc->target_node = NUMA_NO_NODE;
+ }
/*
* Persistence domain bits are hierarchical, if
diff --git a/drivers/acpi/numa.c b/drivers/acpi/numa.c
index 90269e62efda..6572214f0f6d 100644
--- a/drivers/acpi/numa.c
+++ b/drivers/acpi/numa.c
@@ -85,6 +85,7 @@ int acpi_map_pxm_to_node(int pxm)
return node;
}
+EXPORT_SYMBOL(acpi_map_pxm_to_node);
/**
* acpi_map_pxm_to_online_node - Map proximity ID to online node
diff --git a/drivers/base/memory.c b/drivers/base/memory.c
index dc8bf1853930..d1e355549498 100644
--- a/drivers/base/memory.c
+++ b/drivers/base/memory.c
@@ -87,6 +87,7 @@ unsigned long __weak memory_block_size_bytes(void)
{
return MIN_MEMORY_BLOCK_SIZE;
}
+EXPORT_SYMBOL_GPL(memory_block_size_bytes);
static unsigned long get_memory_block_size(void)
{
diff --git a/drivers/dax/Kconfig b/drivers/dax/Kconfig
index e0700bf4893a..5ef624fe3934 100644
--- a/drivers/dax/Kconfig
+++ b/drivers/dax/Kconfig
@@ -23,12 +23,38 @@ config DEV_DAX
config DEV_DAX_PMEM
tristate "PMEM DAX: direct access to persistent memory"
depends on LIBNVDIMM && NVDIMM_DAX && DEV_DAX
+ depends on m # until we can kill DEV_DAX_PMEM_COMPAT
default DEV_DAX
help
Support raw access to persistent memory. Note that this
driver consumes memory ranges allocated and exported by the
libnvdimm sub-system.
- Say Y if unsure
+ Say M if unsure
+
+config DEV_DAX_KMEM
+ tristate "KMEM DAX: volatile-use of persistent memory"
+ default DEV_DAX
+ depends on DEV_DAX
+ depends on MEMORY_HOTPLUG # for add_memory() and friends
+ help
+ Support access to persistent memory as if it were RAM. This
+ allows easier use of persistent memory by unmodified
+ applications.
+
+ To use this feature, a DAX device must be unbound from the
+ device_dax driver (PMEM DAX) and bound to this kmem driver
+ on each boot.
+
+ Say N if unsure.
+
+config DEV_DAX_PMEM_COMPAT
+ tristate "PMEM DAX: support the deprecated /sys/class/dax interface"
+ depends on DEV_DAX_PMEM
+ default DEV_DAX_PMEM
+ help
+ Older versions of the libdaxctl library expect to find all
+ device-dax instances under /sys/class/dax. If libdaxctl in
+ your distribution is older than v58 say M, otherwise say N.
endif
diff --git a/drivers/dax/Makefile b/drivers/dax/Makefile
index dc7422530462..da64a0a4a057 100644
--- a/drivers/dax/Makefile
+++ b/drivers/dax/Makefile
@@ -1,7 +1,9 @@
obj-$(CONFIG_DAX) += dax.o
obj-$(CONFIG_DEV_DAX) += device_dax.o
-obj-$(CONFIG_DEV_DAX_PMEM) += dax_pmem.o
+obj-$(CONFIG_DEV_DAX_KMEM) += kmem.o
dax-y := super.o
-dax_pmem-y := pmem.o
+dax-y += bus.o
device_dax-y := device.o
+
+obj-y += pmem/
diff --git a/drivers/dax/bus.c b/drivers/dax/bus.c
new file mode 100644
index 000000000000..2109cfe80219
--- /dev/null
+++ b/drivers/dax/bus.c
@@ -0,0 +1,503 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 2017-2018 Intel Corporation. All rights reserved. */
+#include <linux/memremap.h>
+#include <linux/device.h>
+#include <linux/mutex.h>
+#include <linux/list.h>
+#include <linux/slab.h>
+#include <linux/dax.h>
+#include "dax-private.h"
+#include "bus.h"
+
+static struct class *dax_class;
+
+static DEFINE_MUTEX(dax_bus_lock);
+
+#define DAX_NAME_LEN 30
+struct dax_id {
+ struct list_head list;
+ char dev_name[DAX_NAME_LEN];
+};
+
+static int dax_bus_uevent(struct device *dev, struct kobj_uevent_env *env)
+{
+ /*
+ * We only ever expect to handle device-dax instances, i.e. the
+ * @type argument to MODULE_ALIAS_DAX_DEVICE() is always zero
+ */
+ return add_uevent_var(env, "MODALIAS=" DAX_DEVICE_MODALIAS_FMT, 0);
+}
+
+static struct dax_device_driver *to_dax_drv(struct device_driver *drv)
+{
+ return container_of(drv, struct dax_device_driver, drv);
+}
+
+static struct dax_id *__dax_match_id(struct dax_device_driver *dax_drv,
+ const char *dev_name)
+{
+ struct dax_id *dax_id;
+
+ lockdep_assert_held(&dax_bus_lock);
+
+ list_for_each_entry(dax_id, &dax_drv->ids, list)
+ if (sysfs_streq(dax_id->dev_name, dev_name))
+ return dax_id;
+ return NULL;
+}
+
+static int dax_match_id(struct dax_device_driver *dax_drv, struct device *dev)
+{
+ int match;
+
+ mutex_lock(&dax_bus_lock);
+ match = !!__dax_match_id(dax_drv, dev_name(dev));
+ mutex_unlock(&dax_bus_lock);
+
+ return match;
+}
+
+enum id_action {
+ ID_REMOVE,
+ ID_ADD,
+};
+
+static ssize_t do_id_store(struct device_driver *drv, const char *buf,
+ size_t count, enum id_action action)
+{
+ struct dax_device_driver *dax_drv = to_dax_drv(drv);
+ unsigned int region_id, id;
+ char devname[DAX_NAME_LEN];
+ struct dax_id *dax_id;
+ ssize_t rc = count;
+ int fields;
+
+ fields = sscanf(buf, "dax%d.%d", &region_id, &id);
+ if (fields != 2)
+ return -EINVAL;
+ sprintf(devname, "dax%d.%d", region_id, id);
+ if (!sysfs_streq(buf, devname))
+ return -EINVAL;
+
+ mutex_lock(&dax_bus_lock);
+ dax_id = __dax_match_id(dax_drv, buf);
+ if (!dax_id) {
+ if (action == ID_ADD) {
+ dax_id = kzalloc(sizeof(*dax_id), GFP_KERNEL);
+ if (dax_id) {
+ strncpy(dax_id->dev_name, buf, DAX_NAME_LEN);
+ list_add(&dax_id->list, &dax_drv->ids);
+ } else
+ rc = -ENOMEM;
+ } else
+ /* nothing to remove */;
+ } else if (action == ID_REMOVE) {
+ list_del(&dax_id->list);
+ kfree(dax_id);
+ } else
+ /* dax_id already added */;
+ mutex_unlock(&dax_bus_lock);
+
+ if (rc < 0)
+ return rc;
+ if (action == ID_ADD)
+ rc = driver_attach(drv);
+ if (rc)
+ return rc;
+ return count;
+}
+
+static ssize_t new_id_store(struct device_driver *drv, const char *buf,
+ size_t count)
+{
+ return do_id_store(drv, buf, count, ID_ADD);
+}
+static DRIVER_ATTR_WO(new_id);
+
+static ssize_t remove_id_store(struct device_driver *drv, const char *buf,
+ size_t count)
+{
+ return do_id_store(drv, buf, count, ID_REMOVE);
+}
+static DRIVER_ATTR_WO(remove_id);
+
+static struct attribute *dax_drv_attrs[] = {
+ &driver_attr_new_id.attr,
+ &driver_attr_remove_id.attr,
+ NULL,
+};
+ATTRIBUTE_GROUPS(dax_drv);
+
+static int dax_bus_match(struct device *dev, struct device_driver *drv);
+
+static struct bus_type dax_bus_type = {
+ .name = "dax",
+ .uevent = dax_bus_uevent,
+ .match = dax_bus_match,
+ .drv_groups = dax_drv_groups,
+};
+
+static int dax_bus_match(struct device *dev, struct device_driver *drv)
+{
+ struct dax_device_driver *dax_drv = to_dax_drv(drv);
+
+ /*
+ * All but the 'device-dax' driver, which has 'match_always'
+ * set, requires an exact id match.
+ */
+ if (dax_drv->match_always)
+ return 1;
+
+ return dax_match_id(dax_drv, dev);
+}
+
+/*
+ * Rely on the fact that drvdata is set before the attributes are
+ * registered, and that the attributes are unregistered before drvdata
+ * is cleared to assume that drvdata is always valid.
+ */
+static ssize_t id_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct dax_region *dax_region = dev_get_drvdata(dev);
+
+ return sprintf(buf, "%d\n", dax_region->id);
+}
+static DEVICE_ATTR_RO(id);
+
+static ssize_t region_size_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct dax_region *dax_region = dev_get_drvdata(dev);
+
+ return sprintf(buf, "%llu\n", (unsigned long long)
+ resource_size(&dax_region->res));
+}
+static struct device_attribute dev_attr_region_size = __ATTR(size, 0444,
+ region_size_show, NULL);
+
+static ssize_t align_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct dax_region *dax_region = dev_get_drvdata(dev);
+
+ return sprintf(buf, "%u\n", dax_region->align);
+}
+static DEVICE_ATTR_RO(align);
+
+static struct attribute *dax_region_attributes[] = {
+ &dev_attr_region_size.attr,
+ &dev_attr_align.attr,
+ &dev_attr_id.attr,
+ NULL,
+};
+
+static const struct attribute_group dax_region_attribute_group = {
+ .name = "dax_region",
+ .attrs = dax_region_attributes,
+};
+
+static const struct attribute_group *dax_region_attribute_groups[] = {
+ &dax_region_attribute_group,
+ NULL,
+};
+
+static void dax_region_free(struct kref *kref)
+{
+ struct dax_region *dax_region;
+
+ dax_region = container_of(kref, struct dax_region, kref);
+ kfree(dax_region);
+}
+
+void dax_region_put(struct dax_region *dax_region)
+{
+ kref_put(&dax_region->kref, dax_region_free);
+}
+EXPORT_SYMBOL_GPL(dax_region_put);
+
+static void dax_region_unregister(void *region)
+{
+ struct dax_region *dax_region = region;
+
+ sysfs_remove_groups(&dax_region->dev->kobj,
+ dax_region_attribute_groups);
+ dax_region_put(dax_region);
+}
+
+struct dax_region *alloc_dax_region(struct device *parent, int region_id,
+ struct resource *res, int target_node, unsigned int align,
+ unsigned long pfn_flags)
+{
+ struct dax_region *dax_region;
+
+ /*
+ * The DAX core assumes that it can store its private data in
+ * parent->driver_data. This WARN is a reminder / safeguard for
+ * developers of device-dax drivers.
+ */
+ if (dev_get_drvdata(parent)) {
+ dev_WARN(parent, "dax core failed to setup private data\n");
+ return NULL;
+ }
+
+ if (!IS_ALIGNED(res->start, align)
+ || !IS_ALIGNED(resource_size(res), align))
+ return NULL;
+
+ dax_region = kzalloc(sizeof(*dax_region), GFP_KERNEL);
+ if (!dax_region)
+ return NULL;
+
+ dev_set_drvdata(parent, dax_region);
+ memcpy(&dax_region->res, res, sizeof(*res));
+ dax_region->pfn_flags = pfn_flags;
+ kref_init(&dax_region->kref);
+ dax_region->id = region_id;
+ dax_region->align = align;
+ dax_region->dev = parent;
+ dax_region->target_node = target_node;
+ if (sysfs_create_groups(&parent->kobj, dax_region_attribute_groups)) {
+ kfree(dax_region);
+ return NULL;
+ }
+
+ kref_get(&dax_region->kref);
+ if (devm_add_action_or_reset(parent, dax_region_unregister, dax_region))
+ return NULL;
+ return dax_region;
+}
+EXPORT_SYMBOL_GPL(alloc_dax_region);
+
+static ssize_t size_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct dev_dax *dev_dax = to_dev_dax(dev);
+ unsigned long long size = resource_size(&dev_dax->region->res);
+
+ return sprintf(buf, "%llu\n", size);
+}
+static DEVICE_ATTR_RO(size);
+
+static int dev_dax_target_node(struct dev_dax *dev_dax)
+{
+ struct dax_region *dax_region = dev_dax->region;
+
+ return dax_region->target_node;
+}
+
+static ssize_t target_node_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct dev_dax *dev_dax = to_dev_dax(dev);
+
+ return sprintf(buf, "%d\n", dev_dax_target_node(dev_dax));
+}
+static DEVICE_ATTR_RO(target_node);
+
+static ssize_t modalias_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ /*
+ * We only ever expect to handle device-dax instances, i.e. the
+ * @type argument to MODULE_ALIAS_DAX_DEVICE() is always zero
+ */
+ return sprintf(buf, DAX_DEVICE_MODALIAS_FMT "\n", 0);
+}
+static DEVICE_ATTR_RO(modalias);
+
+static umode_t dev_dax_visible(struct kobject *kobj, struct attribute *a, int n)
+{
+ struct device *dev = container_of(kobj, struct device, kobj);
+ struct dev_dax *dev_dax = to_dev_dax(dev);
+
+ if (a == &dev_attr_target_node.attr && dev_dax_target_node(dev_dax) < 0)
+ return 0;
+ return a->mode;
+}
+
+static struct attribute *dev_dax_attributes[] = {
+ &dev_attr_modalias.attr,
+ &dev_attr_size.attr,
+ &dev_attr_target_node.attr,
+ NULL,
+};
+
+static const struct attribute_group dev_dax_attribute_group = {
+ .attrs = dev_dax_attributes,
+ .is_visible = dev_dax_visible,
+};
+
+static const struct attribute_group *dax_attribute_groups[] = {
+ &dev_dax_attribute_group,
+ NULL,
+};
+
+void kill_dev_dax(struct dev_dax *dev_dax)
+{
+ struct dax_device *dax_dev = dev_dax->dax_dev;
+ struct inode *inode = dax_inode(dax_dev);
+
+ kill_dax(dax_dev);
+ unmap_mapping_range(inode->i_mapping, 0, 0, 1);
+}
+EXPORT_SYMBOL_GPL(kill_dev_dax);
+
+static void dev_dax_release(struct device *dev)
+{
+ struct dev_dax *dev_dax = to_dev_dax(dev);
+ struct dax_region *dax_region = dev_dax->region;
+ struct dax_device *dax_dev = dev_dax->dax_dev;
+
+ dax_region_put(dax_region);
+ put_dax(dax_dev);
+ kfree(dev_dax);
+}
+
+static void unregister_dev_dax(void *dev)
+{
+ struct dev_dax *dev_dax = to_dev_dax(dev);
+
+ dev_dbg(dev, "%s\n", __func__);
+
+ kill_dev_dax(dev_dax);
+ device_del(dev);
+ put_device(dev);
+}
+
+struct dev_dax *__devm_create_dev_dax(struct dax_region *dax_region, int id,
+ struct dev_pagemap *pgmap, enum dev_dax_subsys subsys)
+{
+ struct device *parent = dax_region->dev;
+ struct dax_device *dax_dev;
+ struct dev_dax *dev_dax;
+ struct inode *inode;
+ struct device *dev;
+ int rc = -ENOMEM;
+
+ if (id < 0)
+ return ERR_PTR(-EINVAL);
+
+ dev_dax = kzalloc(sizeof(*dev_dax), GFP_KERNEL);
+ if (!dev_dax)
+ return ERR_PTR(-ENOMEM);
+
+ memcpy(&dev_dax->pgmap, pgmap, sizeof(*pgmap));
+
+ /*
+ * No 'host' or dax_operations since there is no access to this
+ * device outside of mmap of the resulting character device.
+ */
+ dax_dev = alloc_dax(dev_dax, NULL, NULL);
+ if (!dax_dev)
+ goto err;
+
+ /* a device_dax instance is dead while the driver is not attached */
+ kill_dax(dax_dev);
+
+ /* from here on we're committed to teardown via dax_dev_release() */
+ dev = &dev_dax->dev;
+ device_initialize(dev);
+
+ dev_dax->dax_dev = dax_dev;
+ dev_dax->region = dax_region;
+ dev_dax->target_node = dax_region->target_node;
+ kref_get(&dax_region->kref);
+
+ inode = dax_inode(dax_dev);
+ dev->devt = inode->i_rdev;
+ if (subsys == DEV_DAX_BUS)
+ dev->bus = &dax_bus_type;
+ else
+ dev->class = dax_class;
+ dev->parent = parent;
+ dev->groups = dax_attribute_groups;
+ dev->release = dev_dax_release;
+ dev_set_name(dev, "dax%d.%d", dax_region->id, id);
+
+ rc = device_add(dev);
+ if (rc) {
+ kill_dev_dax(dev_dax);
+ put_device(dev);
+ return ERR_PTR(rc);
+ }
+
+ rc = devm_add_action_or_reset(dax_region->dev, unregister_dev_dax, dev);
+ if (rc)
+ return ERR_PTR(rc);
+
+ return dev_dax;
+
+ err:
+ kfree(dev_dax);
+
+ return ERR_PTR(rc);
+}
+EXPORT_SYMBOL_GPL(__devm_create_dev_dax);
+
+static int match_always_count;
+
+int __dax_driver_register(struct dax_device_driver *dax_drv,
+ struct module *module, const char *mod_name)
+{
+ struct device_driver *drv = &dax_drv->drv;
+ int rc = 0;
+
+ INIT_LIST_HEAD(&dax_drv->ids);
+ drv->owner = module;
+ drv->name = mod_name;
+ drv->mod_name = mod_name;
+ drv->bus = &dax_bus_type;
+
+ /* there can only be one default driver */
+ mutex_lock(&dax_bus_lock);
+ match_always_count += dax_drv->match_always;
+ if (match_always_count > 1) {
+ match_always_count--;
+ WARN_ON(1);
+ rc = -EINVAL;
+ }
+ mutex_unlock(&dax_bus_lock);
+ if (rc)
+ return rc;
+ return driver_register(drv);
+}
+EXPORT_SYMBOL_GPL(__dax_driver_register);
+
+void dax_driver_unregister(struct dax_device_driver *dax_drv)
+{
+ struct device_driver *drv = &dax_drv->drv;
+ struct dax_id *dax_id, *_id;
+
+ mutex_lock(&dax_bus_lock);
+ match_always_count -= dax_drv->match_always;
+ list_for_each_entry_safe(dax_id, _id, &dax_drv->ids, list) {
+ list_del(&dax_id->list);
+ kfree(dax_id);
+ }
+ mutex_unlock(&dax_bus_lock);
+ driver_unregister(drv);
+}
+EXPORT_SYMBOL_GPL(dax_driver_unregister);
+
+int __init dax_bus_init(void)
+{
+ int rc;
+
+ if (IS_ENABLED(CONFIG_DEV_DAX_PMEM_COMPAT)) {
+ dax_class = class_create(THIS_MODULE, "dax");
+ if (IS_ERR(dax_class))
+ return PTR_ERR(dax_class);
+ }
+
+ rc = bus_register(&dax_bus_type);
+ if (rc)
+ class_destroy(dax_class);
+ return rc;
+}
+
+void __exit dax_bus_exit(void)
+{
+ bus_unregister(&dax_bus_type);
+ class_destroy(dax_class);
+}
diff --git a/drivers/dax/bus.h b/drivers/dax/bus.h
new file mode 100644
index 000000000000..8619e3299943
--- /dev/null
+++ b/drivers/dax/bus.h
@@ -0,0 +1,61 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 2016 - 2018 Intel Corporation. All rights reserved. */
+#ifndef __DAX_BUS_H__
+#define __DAX_BUS_H__
+#include <linux/device.h>
+
+struct dev_dax;
+struct resource;
+struct dax_device;
+struct dax_region;
+void dax_region_put(struct dax_region *dax_region);
+struct dax_region *alloc_dax_region(struct device *parent, int region_id,
+ struct resource *res, int target_node, unsigned int align,
+ unsigned long flags);
+
+enum dev_dax_subsys {
+ DEV_DAX_BUS,
+ DEV_DAX_CLASS,
+};
+
+struct dev_dax *__devm_create_dev_dax(struct dax_region *dax_region, int id,
+ struct dev_pagemap *pgmap, enum dev_dax_subsys subsys);
+
+static inline struct dev_dax *devm_create_dev_dax(struct dax_region *dax_region,
+ int id, struct dev_pagemap *pgmap)
+{
+ return __devm_create_dev_dax(dax_region, id, pgmap, DEV_DAX_BUS);
+}
+
+/* to be deleted when DEV_DAX_CLASS is removed */
+struct dev_dax *__dax_pmem_probe(struct device *dev, enum dev_dax_subsys subsys);
+
+struct dax_device_driver {
+ struct device_driver drv;
+ struct list_head ids;
+ int match_always;
+};
+
+int __dax_driver_register(struct dax_device_driver *dax_drv,
+ struct module *module, const char *mod_name);
+#define dax_driver_register(driver) \
+ __dax_driver_register(driver, THIS_MODULE, KBUILD_MODNAME)
+void dax_driver_unregister(struct dax_device_driver *dax_drv);
+void kill_dev_dax(struct dev_dax *dev_dax);
+
+#if IS_ENABLED(CONFIG_DEV_DAX_PMEM_COMPAT)
+int dev_dax_probe(struct device *dev);
+#endif
+
+/*
+ * While run_dax() is potentially a generic operation that could be
+ * defined in include/linux/dax.h we don't want to grow any users
+ * outside of drivers/dax/
+ */
+void run_dax(struct dax_device *dax_dev);
+
+#define MODULE_ALIAS_DAX_DEVICE(type) \
+ MODULE_ALIAS("dax:t" __stringify(type) "*")
+#define DAX_DEVICE_MODALIAS_FMT "dax:t%d"
+
+#endif /* __DAX_BUS_H__ */
diff --git a/drivers/dax/dax-private.h b/drivers/dax/dax-private.h
index b6fc4f04636d..a45612148ca0 100644
--- a/drivers/dax/dax-private.h
+++ b/drivers/dax/dax-private.h
@@ -16,10 +16,17 @@
#include <linux/device.h>
#include <linux/cdev.h>
+/* private routines between core files */
+struct dax_device;
+struct dax_device *inode_dax(struct inode *inode);
+struct inode *dax_inode(struct dax_device *dax_dev);
+int dax_bus_init(void);
+void dax_bus_exit(void);
+
/**
* struct dax_region - mapping infrastructure for dax devices
* @id: kernel-wide unique region for a memory range
- * @base: linear address corresponding to @res
+ * @target_node: effective numa node if this memory range is onlined
* @kref: to pin while other agents have a need to do lookups
* @dev: parent device backing this region
* @align: allocation and mapping alignment for child dax devices
@@ -28,8 +35,7 @@
*/
struct dax_region {
int id;
- struct ida ida;
- void *base;
+ int target_node;
struct kref kref;
struct device *dev;
unsigned int align;
@@ -38,20 +44,28 @@ struct dax_region {
};
/**
- * struct dev_dax - instance data for a subdivision of a dax region
+ * struct dev_dax - instance data for a subdivision of a dax region, and
+ * data while the device is activated in the driver.
* @region - parent region
* @dax_dev - core dax functionality
+ * @target_node: effective numa node if dev_dax memory range is onlined
* @dev - device core
- * @id - child id in the region
- * @num_resources - number of physical address extents in this device
- * @res - array of physical address ranges
+ * @pgmap - pgmap for memmap setup / lifetime (driver owned)
+ * @ref: pgmap reference count (driver owned)
+ * @cmp: @ref final put completion (driver owned)
*/
struct dev_dax {
struct dax_region *region;
struct dax_device *dax_dev;
+ int target_node;
struct device dev;
- int id;
- int num_resources;
- struct resource res[0];
+ struct dev_pagemap pgmap;
+ struct percpu_ref ref;
+ struct completion cmp;
};
+
+static inline struct dev_dax *to_dev_dax(struct device *dev)
+{
+ return container_of(dev, struct dev_dax, dev);
+}
#endif
diff --git a/drivers/dax/dax.h b/drivers/dax/dax.h
deleted file mode 100644
index f9e5feea742c..000000000000
--- a/drivers/dax/dax.h
+++ /dev/null
@@ -1,18 +0,0 @@
-/*
- * Copyright(c) 2016 - 2017 Intel Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * General Public License for more details.
- */
-#ifndef __DAX_H__
-#define __DAX_H__
-struct dax_device;
-struct dax_device *inode_dax(struct inode *inode);
-struct inode *dax_inode(struct dax_device *dax_dev);
-#endif /* __DAX_H__ */
diff --git a/drivers/dax/device-dax.h b/drivers/dax/device-dax.h
deleted file mode 100644
index 688b051750bd..000000000000
--- a/drivers/dax/device-dax.h
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Copyright(c) 2016 Intel Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * General Public License for more details.
- */
-#ifndef __DEVICE_DAX_H__
-#define __DEVICE_DAX_H__
-struct device;
-struct dev_dax;
-struct resource;
-struct dax_region;
-void dax_region_put(struct dax_region *dax_region);
-struct dax_region *alloc_dax_region(struct device *parent,
- int region_id, struct resource *res, unsigned int align,
- void *addr, unsigned long flags);
-struct dev_dax *devm_create_dev_dax(struct dax_region *dax_region,
- int id, struct resource *res, int count);
-#endif /* __DEVICE_DAX_H__ */
diff --git a/drivers/dax/device.c b/drivers/dax/device.c
index fc0e03583bff..70052a50b5b6 100644
--- a/drivers/dax/device.c
+++ b/drivers/dax/device.c
@@ -1,15 +1,6 @@
-/*
- * Copyright(c) 2016 - 2017 Intel Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * General Public License for more details.
- */
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 2016-2018 Intel Corporation. All rights reserved. */
+#include <linux/memremap.h>
#include <linux/pagemap.h>
#include <linux/module.h>
#include <linux/device.h>
@@ -21,161 +12,39 @@
#include <linux/mm.h>
#include <linux/mman.h>
#include "dax-private.h"
-#include "dax.h"
+#include "bus.h"
-static struct class *dax_class;
-
-/*
- * Rely on the fact that drvdata is set before the attributes are
- * registered, and that the attributes are unregistered before drvdata
- * is cleared to assume that drvdata is always valid.
- */
-static ssize_t id_show(struct device *dev,
- struct device_attribute *attr, char *buf)
-{
- struct dax_region *dax_region = dev_get_drvdata(dev);
-
- return sprintf(buf, "%d\n", dax_region->id);
-}
-static DEVICE_ATTR_RO(id);
-
-static ssize_t region_size_show(struct device *dev,
- struct device_attribute *attr, char *buf)
+static struct dev_dax *ref_to_dev_dax(struct percpu_ref *ref)
{
- struct dax_region *dax_region = dev_get_drvdata(dev);
-
- return sprintf(buf, "%llu\n", (unsigned long long)
- resource_size(&dax_region->res));
+ return container_of(ref, struct dev_dax, ref);
}
-static struct device_attribute dev_attr_region_size = __ATTR(size, 0444,
- region_size_show, NULL);
-static ssize_t align_show(struct device *dev,
- struct device_attribute *attr, char *buf)
+static void dev_dax_percpu_release(struct percpu_ref *ref)
{
- struct dax_region *dax_region = dev_get_drvdata(dev);
+ struct dev_dax *dev_dax = ref_to_dev_dax(ref);
- return sprintf(buf, "%u\n", dax_region->align);
-}
-static DEVICE_ATTR_RO(align);
-
-static struct attribute *dax_region_attributes[] = {
- &dev_attr_region_size.attr,
- &dev_attr_align.attr,
- &dev_attr_id.attr,
- NULL,
-};
-
-static const struct attribute_group dax_region_attribute_group = {
- .name = "dax_region",
- .attrs = dax_region_attributes,
-};
-
-static const struct attribute_group *dax_region_attribute_groups[] = {
- &dax_region_attribute_group,
- NULL,
-};
-
-static void dax_region_free(struct kref *kref)
-{
- struct dax_region *dax_region;
-
- dax_region = container_of(kref, struct dax_region, kref);
- kfree(dax_region);
-}
-
-void dax_region_put(struct dax_region *dax_region)
-{
- kref_put(&dax_region->kref, dax_region_free);
-}
-EXPORT_SYMBOL_GPL(dax_region_put);
-
-static void dax_region_unregister(void *region)
-{
- struct dax_region *dax_region = region;
-
- sysfs_remove_groups(&dax_region->dev->kobj,
- dax_region_attribute_groups);
- dax_region_put(dax_region);
+ dev_dbg(&dev_dax->dev, "%s\n", __func__);
+ complete(&dev_dax->cmp);
}
-struct dax_region *alloc_dax_region(struct device *parent, int region_id,
- struct resource *res, unsigned int align, void *addr,
- unsigned long pfn_flags)
+static void dev_dax_percpu_exit(void *data)
{
- struct dax_region *dax_region;
-
- /*
- * The DAX core assumes that it can store its private data in
- * parent->driver_data. This WARN is a reminder / safeguard for
- * developers of device-dax drivers.
- */
- if (dev_get_drvdata(parent)) {
- dev_WARN(parent, "dax core failed to setup private data\n");
- return NULL;
- }
-
- if (!IS_ALIGNED(res->start, align)
- || !IS_ALIGNED(resource_size(res), align))
- return NULL;
-
- dax_region = kzalloc(sizeof(*dax_region), GFP_KERNEL);
- if (!dax_region)
- return NULL;
-
- dev_set_drvdata(parent, dax_region);
- memcpy(&dax_region->res, res, sizeof(*res));
- dax_region->pfn_flags = pfn_flags;
- kref_init(&dax_region->kref);
- dax_region->id = region_id;
- ida_init(&dax_region->ida);
- dax_region->align = align;
- dax_region->dev = parent;
- dax_region->base = addr;
- if (sysfs_create_groups(&parent->kobj, dax_region_attribute_groups)) {
- kfree(dax_region);
- return NULL;
- }
-
- kref_get(&dax_region->kref);
- if (devm_add_action_or_reset(parent, dax_region_unregister, dax_region))
- return NULL;
- return dax_region;
-}
-EXPORT_SYMBOL_GPL(alloc_dax_region);
+ struct percpu_ref *ref = data;
+ struct dev_dax *dev_dax = ref_to_dev_dax(ref);
-static struct dev_dax *to_dev_dax(struct device *dev)
-{
- return container_of(dev, struct dev_dax, dev);
+ dev_dbg(&dev_dax->dev, "%s\n", __func__);
+ wait_for_completion(&dev_dax->cmp);
+ percpu_ref_exit(ref);
}
-static ssize_t size_show(struct device *dev,
- struct device_attribute *attr, char *buf)
+static void dev_dax_percpu_kill(struct percpu_ref *data)
{
- struct dev_dax *dev_dax = to_dev_dax(dev);
- unsigned long long size = 0;
- int i;
-
- for (i = 0; i < dev_dax->num_resources; i++)
- size += resource_size(&dev_dax->res[i]);
+ struct percpu_ref *ref = data;
+ struct dev_dax *dev_dax = ref_to_dev_dax(ref);
- return sprintf(buf, "%llu\n", size);
+ dev_dbg(&dev_dax->dev, "%s\n", __func__);
+ percpu_ref_kill(ref);
}
-static DEVICE_ATTR_RO(size);
-
-static struct attribute *dev_dax_attributes[] = {
- &dev_attr_size.attr,
- NULL,
-};
-
-static const struct attribute_group dev_dax_attribute_group = {
- .attrs = dev_dax_attributes,
-};
-
-static const struct attribute_group *dax_attribute_groups[] = {
- &dev_dax_attribute_group,
- NULL,
-};
static int check_vma(struct dev_dax *dev_dax, struct vm_area_struct *vma,
const char *func)
@@ -226,21 +95,11 @@ static int check_vma(struct dev_dax *dev_dax, struct vm_area_struct *vma,
__weak phys_addr_t dax_pgoff_to_phys(struct dev_dax *dev_dax, pgoff_t pgoff,
unsigned long size)
{
- struct resource *res;
- /* gcc-4.6.3-nolibc for i386 complains that this is uninitialized */
- phys_addr_t uninitialized_var(phys);
- int i;
-
- for (i = 0; i < dev_dax->num_resources; i++) {
- res = &dev_dax->res[i];
- phys = pgoff * PAGE_SIZE + res->start;
- if (phys >= res->start && phys <= res->end)
- break;
- pgoff -= PHYS_PFN(resource_size(res));
- }
+ struct resource *res = &dev_dax->region->res;
+ phys_addr_t phys;
- if (i < dev_dax->num_resources) {
- res = &dev_dax->res[i];
+ phys = pgoff * PAGE_SIZE + res->start;
+ if (phys >= res->start && phys <= res->end) {
if (phys + size - 1 <= res->end)
return phys;
}
@@ -587,152 +446,101 @@ static const struct file_operations dax_fops = {
.mmap_supported_flags = MAP_SYNC,
};
-static void dev_dax_release(struct device *dev)
+static void dev_dax_cdev_del(void *cdev)
{
- struct dev_dax *dev_dax = to_dev_dax(dev);
- struct dax_region *dax_region = dev_dax->region;
- struct dax_device *dax_dev = dev_dax->dax_dev;
-
- if (dev_dax->id >= 0)
- ida_simple_remove(&dax_region->ida, dev_dax->id);
- dax_region_put(dax_region);
- put_dax(dax_dev);
- kfree(dev_dax);
+ cdev_del(cdev);
}
-static void kill_dev_dax(struct dev_dax *dev_dax)
+static void dev_dax_kill(void *dev_dax)
{
- struct dax_device *dax_dev = dev_dax->dax_dev;
- struct inode *inode = dax_inode(dax_dev);
-
- kill_dax(dax_dev);
- unmap_mapping_range(inode->i_mapping, 0, 0, 1);
+ kill_dev_dax(dev_dax);
}
-static void unregister_dev_dax(void *dev)
+int dev_dax_probe(struct device *dev)
{
struct dev_dax *dev_dax = to_dev_dax(dev);
struct dax_device *dax_dev = dev_dax->dax_dev;
- struct inode *inode = dax_inode(dax_dev);
- struct cdev *cdev = inode->i_cdev;
-
- dev_dbg(dev, "%s\n", __func__);
-
- kill_dev_dax(dev_dax);
- cdev_device_del(cdev, dev);
- put_device(dev);
-}
-
-struct dev_dax *devm_create_dev_dax(struct dax_region *dax_region,
- int id, struct resource *res, int count)
-{
- struct device *parent = dax_region->dev;
- struct dax_device *dax_dev;
- struct dev_dax *dev_dax;
+ struct resource *res = &dev_dax->region->res;
struct inode *inode;
- struct device *dev;
struct cdev *cdev;
- int rc, i;
-
- if (!count)
- return ERR_PTR(-EINVAL);
-
- dev_dax = kzalloc(sizeof(*dev_dax) + sizeof(*res) * count, GFP_KERNEL);
- if (!dev_dax)
- return ERR_PTR(-ENOMEM);
-
- for (i = 0; i < count; i++) {
- if (!IS_ALIGNED(res[i].start, dax_region->align)
- || !IS_ALIGNED(resource_size(&res[i]),
- dax_region->align)) {
- rc = -EINVAL;
- break;
- }
- dev_dax->res[i].start = res[i].start;
- dev_dax->res[i].end = res[i].end;
+ void *addr;
+ int rc;
+
+ /* 1:1 map region resource range to device-dax instance range */
+ if (!devm_request_mem_region(dev, res->start, resource_size(res),
+ dev_name(dev))) {
+ dev_warn(dev, "could not reserve region %pR\n", res);
+ return -EBUSY;
}
- if (i < count)
- goto err_id;
+ init_completion(&dev_dax->cmp);
+ rc = percpu_ref_init(&dev_dax->ref, dev_dax_percpu_release, 0,
+ GFP_KERNEL);
+ if (rc)
+ return rc;
- if (id < 0) {
- id = ida_simple_get(&dax_region->ida, 0, 0, GFP_KERNEL);
- dev_dax->id = id;
- if (id < 0) {
- rc = id;
- goto err_id;
- }
- } else {
- /* region provider owns @id lifetime */
- dev_dax->id = -1;
- }
+ rc = devm_add_action_or_reset(dev, dev_dax_percpu_exit, &dev_dax->ref);
+ if (rc)
+ return rc;
- /*
- * No 'host' or dax_operations since there is no access to this
- * device outside of mmap of the resulting character device.
- */
- dax_dev = alloc_dax(dev_dax, NULL, NULL);
- if (!dax_dev) {
- rc = -ENOMEM;
- goto err_dax;
+ dev_dax->pgmap.ref = &dev_dax->ref;
+ dev_dax->pgmap.kill = dev_dax_percpu_kill;
+ addr = devm_memremap_pages(dev, &dev_dax->pgmap);
+ if (IS_ERR(addr)) {
+ devm_remove_action(dev, dev_dax_percpu_exit, &dev_dax->ref);
+ percpu_ref_exit(&dev_dax->ref);
+ return PTR_ERR(addr);
}
- /* from here on we're committed to teardown via dax_dev_release() */
- dev = &dev_dax->dev;
- device_initialize(dev);
-
inode = dax_inode(dax_dev);
cdev = inode->i_cdev;
cdev_init(cdev, &dax_fops);
- cdev->owner = parent->driver->owner;
-
- dev_dax->num_resources = count;
- dev_dax->dax_dev = dax_dev;
- dev_dax->region = dax_region;
- kref_get(&dax_region->kref);
-
- dev->devt = inode->i_rdev;
- dev->class = dax_class;
- dev->parent = parent;
- dev->groups = dax_attribute_groups;
- dev->release = dev_dax_release;
- dev_set_name(dev, "dax%d.%d", dax_region->id, id);
-
- rc = cdev_device_add(cdev, dev);
- if (rc) {
- kill_dev_dax(dev_dax);
- put_device(dev);
- return ERR_PTR(rc);
- }
- rc = devm_add_action_or_reset(dax_region->dev, unregister_dev_dax, dev);
+ if (dev->class) {
+ /* for the CONFIG_DEV_DAX_PMEM_COMPAT case */
+ cdev->owner = dev->parent->driver->owner;
+ } else
+ cdev->owner = dev->driver->owner;
+ cdev_set_parent(cdev, &dev->kobj);
+ rc = cdev_add(cdev, dev->devt, 1);
if (rc)
- return ERR_PTR(rc);
+ return rc;
- return dev_dax;
+ rc = devm_add_action_or_reset(dev, dev_dax_cdev_del, cdev);
+ if (rc)
+ return rc;
- err_dax:
- if (dev_dax->id >= 0)
- ida_simple_remove(&dax_region->ida, dev_dax->id);
- err_id:
- kfree(dev_dax);
+ run_dax(dax_dev);
+ return devm_add_action_or_reset(dev, dev_dax_kill, dev_dax);
+}
+EXPORT_SYMBOL_GPL(dev_dax_probe);
- return ERR_PTR(rc);
+static int dev_dax_remove(struct device *dev)
+{
+ /* all probe actions are unwound by devm */
+ return 0;
}
-EXPORT_SYMBOL_GPL(devm_create_dev_dax);
+
+static struct dax_device_driver device_dax_driver = {
+ .drv = {
+ .probe = dev_dax_probe,
+ .remove = dev_dax_remove,
+ },
+ .match_always = 1,
+};
static int __init dax_init(void)
{
- dax_class = class_create(THIS_MODULE, "dax");
- return PTR_ERR_OR_ZERO(dax_class);
+ return dax_driver_register(&device_dax_driver);
}
static void __exit dax_exit(void)
{
- class_destroy(dax_class);
+ dax_driver_unregister(&device_dax_driver);
}
MODULE_AUTHOR("Intel Corporation");
MODULE_LICENSE("GPL v2");
-subsys_initcall(dax_init);
+module_init(dax_init);
module_exit(dax_exit);
+MODULE_ALIAS_DAX_DEVICE(0);
diff --git a/drivers/dax/kmem.c b/drivers/dax/kmem.c
new file mode 100644
index 000000000000..4c0131857133
--- /dev/null
+++ b/drivers/dax/kmem.c
@@ -0,0 +1,111 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 2016-2019 Intel Corporation. All rights reserved. */
+#include <linux/memremap.h>
+#include <linux/pagemap.h>
+#include <linux/memory.h>
+#include <linux/module.h>
+#include <linux/device.h>
+#include <linux/pfn_t.h>
+#include <linux/slab.h>
+#include <linux/dax.h>
+#include <linux/fs.h>
+#include <linux/mm.h>
+#include <linux/mman.h>
+#include "dax-private.h"
+#include "bus.h"
+
+int dev_dax_kmem_probe(struct device *dev)
+{
+ struct dev_dax *dev_dax = to_dev_dax(dev);
+ struct resource *res = &dev_dax->region->res;
+ resource_size_t kmem_start;
+ resource_size_t kmem_size;
+ resource_size_t kmem_end;
+ struct resource *new_res;
+ int numa_node;
+ int rc;
+
+ /*
+ * Ensure good NUMA information for the persistent memory.
+ * Without this check, there is a risk that slow memory
+ * could be mixed in a node with faster memory, causing
+ * unavoidable performance issues.
+ */
+ numa_node = dev_dax->target_node;
+ if (numa_node < 0) {
+ dev_warn(dev, "rejecting DAX region %pR with invalid node: %d\n",
+ res, numa_node);
+ return -EINVAL;
+ }
+
+ /* Hotplug starting at the beginning of the next block: */
+ kmem_start = ALIGN(res->start, memory_block_size_bytes());
+
+ kmem_size = resource_size(res);
+ /* Adjust the size down to compensate for moving up kmem_start: */
+ kmem_size -= kmem_start - res->start;
+ /* Align the size down to cover only complete blocks: */
+ kmem_size &= ~(memory_block_size_bytes() - 1);
+ kmem_end = kmem_start + kmem_size;
+
+ /* Region is permanently reserved. Hot-remove not yet implemented. */
+ new_res = request_mem_region(kmem_start, kmem_size, dev_name(dev));
+ if (!new_res) {
+ dev_warn(dev, "could not reserve region [%pa-%pa]\n",
+ &kmem_start, &kmem_end);
+ return -EBUSY;
+ }
+
+ /*
+ * Set flags appropriate for System RAM. Leave ..._BUSY clear
+ * so that add_memory() can add a child resource. Do not
+ * inherit flags from the parent since it may set new flags
+ * unknown to us that will break add_memory() below.
+ */
+ new_res->flags = IORESOURCE_SYSTEM_RAM;
+ new_res->name = dev_name(dev);
+
+ rc = add_memory(numa_node, new_res->start, resource_size(new_res));
+ if (rc) {
+ release_resource(new_res);
+ kfree(new_res);
+ return rc;
+ }
+
+ return 0;
+}
+
+static int dev_dax_kmem_remove(struct device *dev)
+{
+ /*
+ * Purposely leak the request_mem_region() for the device-dax
+ * range and return '0' to ->remove() attempts. The removal of
+ * the device from the driver always succeeds, but the region
+ * is permanently pinned as reserved by the unreleased
+ * request_mem_region().
+ */
+ return 0;
+}
+
+static struct dax_device_driver device_dax_kmem_driver = {
+ .drv = {
+ .probe = dev_dax_kmem_probe,
+ .remove = dev_dax_kmem_remove,
+ },
+};
+
+static int __init dax_kmem_init(void)
+{
+ return dax_driver_register(&device_dax_kmem_driver);
+}
+
+static void __exit dax_kmem_exit(void)
+{
+ dax_driver_unregister(&device_dax_kmem_driver);
+}
+
+MODULE_AUTHOR("Intel Corporation");
+MODULE_LICENSE("GPL v2");
+module_init(dax_kmem_init);
+module_exit(dax_kmem_exit);
+MODULE_ALIAS_DAX_DEVICE(0);
diff --git a/drivers/dax/pmem.c b/drivers/dax/pmem.c
deleted file mode 100644
index 7b7feb6880d1..000000000000
--- a/drivers/dax/pmem.c
+++ /dev/null
@@ -1,161 +0,0 @@
-/*
- * Copyright(c) 2016 Intel Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * General Public License for more details.
- */
-#include <linux/percpu-refcount.h>
-#include <linux/memremap.h>
-#include <linux/module.h>
-#include <linux/pfn_t.h>
-#include "../nvdimm/pfn.h"
-#include "../nvdimm/nd.h"
-#include "device-dax.h"
-
-struct dax_pmem {
- struct device *dev;
- struct percpu_ref ref;
- struct dev_pagemap pgmap;
- struct completion cmp;
-};
-
-static struct dax_pmem *to_dax_pmem(struct percpu_ref *ref)
-{
- return container_of(ref, struct dax_pmem, ref);
-}
-
-static void dax_pmem_percpu_release(struct percpu_ref *ref)
-{
- struct dax_pmem *dax_pmem = to_dax_pmem(ref);
-
- dev_dbg(dax_pmem->dev, "%s\n", __func__);
- complete(&dax_pmem->cmp);
-}
-
-static void dax_pmem_percpu_exit(void *data)
-{
- struct percpu_ref *ref = data;
- struct dax_pmem *dax_pmem = to_dax_pmem(ref);
-
- dev_dbg(dax_pmem->dev, "%s\n", __func__);
- wait_for_completion(&dax_pmem->cmp);
- percpu_ref_exit(ref);
-}
-
-static void dax_pmem_percpu_kill(void *data)
-{
- struct percpu_ref *ref = data;
- struct dax_pmem *dax_pmem = to_dax_pmem(ref);
-
- dev_dbg(dax_pmem->dev, "%s\n", __func__);
- percpu_ref_kill(ref);
-}
-
-static int dax_pmem_probe(struct device *dev)
-{
- void *addr;
- struct resource res;
- int rc, id, region_id;
- struct nd_pfn_sb *pfn_sb;
- struct dev_dax *dev_dax;
- struct dax_pmem *dax_pmem;
- struct nd_namespace_io *nsio;
- struct dax_region *dax_region;
- struct nd_namespace_common *ndns;
- struct nd_dax *nd_dax = to_nd_dax(dev);
- struct nd_pfn *nd_pfn = &nd_dax->nd_pfn;
-
- ndns = nvdimm_namespace_common_probe(dev);
- if (IS_ERR(ndns))
- return PTR_ERR(ndns);
- nsio = to_nd_namespace_io(&ndns->dev);
-
- dax_pmem = devm_kzalloc(dev, sizeof(*dax_pmem), GFP_KERNEL);
- if (!dax_pmem)
- return -ENOMEM;
-
- /* parse the 'pfn' info block via ->rw_bytes */
- rc = devm_nsio_enable(dev, nsio);
- if (rc)
- return rc;
- rc = nvdimm_setup_pfn(nd_pfn, &dax_pmem->pgmap);
- if (rc)
- return rc;
- devm_nsio_disable(dev, nsio);
-
- pfn_sb = nd_pfn->pfn_sb;
-
- if (!devm_request_mem_region(dev, nsio->res.start,
- resource_size(&nsio->res),
- dev_name(&ndns->dev))) {
- dev_warn(dev, "could not reserve region %pR\n", &nsio->res);
- return -EBUSY;
- }
-
- dax_pmem->dev = dev;
- init_completion(&dax_pmem->cmp);
- rc = percpu_ref_init(&dax_pmem->ref, dax_pmem_percpu_release, 0,
- GFP_KERNEL);
- if (rc)
- return rc;
-
- rc = devm_add_action(dev, dax_pmem_percpu_exit, &dax_pmem->ref);
- if (rc) {
- percpu_ref_exit(&dax_pmem->ref);
- return rc;
- }
-
- dax_pmem->pgmap.ref = &dax_pmem->ref;
- addr = devm_memremap_pages(dev, &dax_pmem->pgmap);
- if (IS_ERR(addr)) {
- devm_remove_action(dev, dax_pmem_percpu_exit, &dax_pmem->ref);
- percpu_ref_exit(&dax_pmem->ref);
- return PTR_ERR(addr);
- }
-
- rc = devm_add_action_or_reset(dev, dax_pmem_percpu_kill,
- &dax_pmem->ref);
- if (rc)
- return rc;
-
- /* adjust the dax_region resource to the start of data */
- memcpy(&res, &dax_pmem->pgmap.res, sizeof(res));
- res.start += le64_to_cpu(pfn_sb->dataoff);
-
- rc = sscanf(dev_name(&ndns->dev), "namespace%d.%d", &region_id, &id);
- if (rc != 2)
- return -EINVAL;
-
- dax_region = alloc_dax_region(dev, region_id, &res,
- le32_to_cpu(pfn_sb->align), addr, PFN_DEV|PFN_MAP);
- if (!dax_region)
- return -ENOMEM;
-
- /* TODO: support for subdividing a dax region... */
- dev_dax = devm_create_dev_dax(dax_region, id, &res, 1);
-
- /* child dev_dax instances now own the lifetime of the dax_region */
- dax_region_put(dax_region);
-
- return PTR_ERR_OR_ZERO(dev_dax);
-}
-
-static struct nd_device_driver dax_pmem_driver = {
- .probe = dax_pmem_probe,
- .drv = {
- .name = "dax_pmem",
- },
- .type = ND_DRIVER_DAX_PMEM,
-};
-
-module_nd_driver(dax_pmem_driver);
-
-MODULE_LICENSE("GPL v2");
-MODULE_AUTHOR("Intel Corporation");
-MODULE_ALIAS_ND_DEVICE(ND_DEVICE_DAX_PMEM);
diff --git a/drivers/dax/pmem/Makefile b/drivers/dax/pmem/Makefile
new file mode 100644
index 000000000000..e2e79bd3fdcf
--- /dev/null
+++ b/drivers/dax/pmem/Makefile
@@ -0,0 +1,7 @@
+obj-$(CONFIG_DEV_DAX_PMEM) += dax_pmem.o
+obj-$(CONFIG_DEV_DAX_PMEM) += dax_pmem_core.o
+obj-$(CONFIG_DEV_DAX_PMEM_COMPAT) += dax_pmem_compat.o
+
+dax_pmem-y := pmem.o
+dax_pmem_core-y := core.o
+dax_pmem_compat-y := compat.o
diff --git a/drivers/dax/pmem/compat.c b/drivers/dax/pmem/compat.c
new file mode 100644
index 000000000000..d7b15e6f30c5
--- /dev/null
+++ b/drivers/dax/pmem/compat.c
@@ -0,0 +1,73 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 2016 - 2018 Intel Corporation. All rights reserved. */
+#include <linux/percpu-refcount.h>
+#include <linux/memremap.h>
+#include <linux/module.h>
+#include <linux/pfn_t.h>
+#include <linux/nd.h>
+#include "../bus.h"
+
+/* we need the private definitions to implement compat suport */
+#include "../dax-private.h"
+
+static int dax_pmem_compat_probe(struct device *dev)
+{
+ struct dev_dax *dev_dax = __dax_pmem_probe(dev, DEV_DAX_CLASS);
+ int rc;
+
+ if (IS_ERR(dev_dax))
+ return PTR_ERR(dev_dax);
+
+ if (!devres_open_group(&dev_dax->dev, dev_dax, GFP_KERNEL))
+ return -ENOMEM;
+
+ device_lock(&dev_dax->dev);
+ rc = dev_dax_probe(&dev_dax->dev);
+ device_unlock(&dev_dax->dev);
+
+ devres_close_group(&dev_dax->dev, dev_dax);
+ if (rc)
+ devres_release_group(&dev_dax->dev, dev_dax);
+
+ return rc;
+}
+
+static int dax_pmem_compat_release(struct device *dev, void *data)
+{
+ device_lock(dev);
+ devres_release_group(dev, to_dev_dax(dev));
+ device_unlock(dev);
+
+ return 0;
+}
+
+static int dax_pmem_compat_remove(struct device *dev)
+{
+ device_for_each_child(dev, NULL, dax_pmem_compat_release);
+ return 0;
+}
+
+static struct nd_device_driver dax_pmem_compat_driver = {
+ .probe = dax_pmem_compat_probe,
+ .remove = dax_pmem_compat_remove,
+ .drv = {
+ .name = "dax_pmem_compat",
+ },
+ .type = ND_DRIVER_DAX_PMEM,
+};
+
+static int __init dax_pmem_compat_init(void)
+{
+ return nd_driver_register(&dax_pmem_compat_driver);
+}
+module_init(dax_pmem_compat_init);
+
+static void __exit dax_pmem_compat_exit(void)
+{
+ driver_unregister(&dax_pmem_compat_driver.drv);
+}
+module_exit(dax_pmem_compat_exit);
+
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Intel Corporation");
+MODULE_ALIAS_ND_DEVICE(ND_DEVICE_DAX_PMEM);
diff --git a/drivers/dax/pmem/core.c b/drivers/dax/pmem/core.c
new file mode 100644
index 000000000000..f71019ce0647
--- /dev/null
+++ b/drivers/dax/pmem/core.c
@@ -0,0 +1,71 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 2016 - 2018 Intel Corporation. All rights reserved. */
+#include <linux/memremap.h>
+#include <linux/module.h>
+#include <linux/pfn_t.h>
+#include "../../nvdimm/pfn.h"
+#include "../../nvdimm/nd.h"
+#include "../bus.h"
+
+struct dev_dax *__dax_pmem_probe(struct device *dev, enum dev_dax_subsys subsys)
+{
+ struct resource res;
+ int rc, id, region_id;
+ resource_size_t offset;
+ struct nd_pfn_sb *pfn_sb;
+ struct dev_dax *dev_dax;
+ struct nd_namespace_io *nsio;
+ struct dax_region *dax_region;
+ struct dev_pagemap pgmap = { 0 };
+ struct nd_namespace_common *ndns;
+ struct nd_dax *nd_dax = to_nd_dax(dev);
+ struct nd_pfn *nd_pfn = &nd_dax->nd_pfn;
+ struct nd_region *nd_region = to_nd_region(dev->parent);
+
+ ndns = nvdimm_namespace_common_probe(dev);
+ if (IS_ERR(ndns))
+ return ERR_CAST(ndns);
+ nsio = to_nd_namespace_io(&ndns->dev);
+
+ /* parse the 'pfn' info block via ->rw_bytes */
+ rc = devm_nsio_enable(dev, nsio);
+ if (rc)
+ return ERR_PTR(rc);
+ rc = nvdimm_setup_pfn(nd_pfn, &pgmap);
+ if (rc)
+ return ERR_PTR(rc);
+ devm_nsio_disable(dev, nsio);
+
+ /* reserve the metadata area, device-dax will reserve the data */
+ pfn_sb = nd_pfn->pfn_sb;
+ offset = le64_to_cpu(pfn_sb->dataoff);
+ if (!devm_request_mem_region(dev, nsio->res.start, offset,
+ dev_name(&ndns->dev))) {
+ dev_warn(dev, "could not reserve metadata\n");
+ return ERR_PTR(-EBUSY);
+ }
+
+ rc = sscanf(dev_name(&ndns->dev), "namespace%d.%d", &region_id, &id);
+ if (rc != 2)
+ return ERR_PTR(-EINVAL);
+
+ /* adjust the dax_region resource to the start of data */
+ memcpy(&res, &pgmap.res, sizeof(res));
+ res.start += offset;
+ dax_region = alloc_dax_region(dev, region_id, &res,
+ nd_region->target_node, le32_to_cpu(pfn_sb->align),
+ PFN_DEV|PFN_MAP);
+ if (!dax_region)
+ return ERR_PTR(-ENOMEM);
+
+ dev_dax = __devm_create_dev_dax(dax_region, id, &pgmap, subsys);
+
+ /* child dev_dax instances now own the lifetime of the dax_region */
+ dax_region_put(dax_region);
+
+ return dev_dax;
+}
+EXPORT_SYMBOL_GPL(__dax_pmem_probe);
+
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Intel Corporation");
diff --git a/drivers/dax/pmem/pmem.c b/drivers/dax/pmem/pmem.c
new file mode 100644
index 000000000000..0ae4238a0ef8
--- /dev/null
+++ b/drivers/dax/pmem/pmem.c
@@ -0,0 +1,40 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 2016 - 2018 Intel Corporation. All rights reserved. */
+#include <linux/percpu-refcount.h>
+#include <linux/memremap.h>
+#include <linux/module.h>
+#include <linux/pfn_t.h>
+#include <linux/nd.h>
+#include "../bus.h"
+
+static int dax_pmem_probe(struct device *dev)
+{
+ return PTR_ERR_OR_ZERO(__dax_pmem_probe(dev, DEV_DAX_BUS));
+}
+
+static struct nd_device_driver dax_pmem_driver = {
+ .probe = dax_pmem_probe,
+ .drv = {
+ .name = "dax_pmem",
+ },
+ .type = ND_DRIVER_DAX_PMEM,
+};
+
+static int __init dax_pmem_init(void)
+{
+ return nd_driver_register(&dax_pmem_driver);
+}
+module_init(dax_pmem_init);
+
+static void __exit dax_pmem_exit(void)
+{
+ driver_unregister(&dax_pmem_driver.drv);
+}
+module_exit(dax_pmem_exit);
+
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Intel Corporation");
+#if !IS_ENABLED(CONFIG_DEV_DAX_PMEM_COMPAT)
+/* For compat builds, don't load this module by default */
+MODULE_ALIAS_ND_DEVICE(ND_DEVICE_DAX_PMEM);
+#endif
diff --git a/drivers/dax/super.c b/drivers/dax/super.c
index be8e984240ab..144a63aa8f18 100644
--- a/drivers/dax/super.c
+++ b/drivers/dax/super.c
@@ -22,6 +22,7 @@
#include <linux/uio.h>
#include <linux/dax.h>
#include <linux/fs.h>
+#include "dax-private.h"
static dev_t dax_devt;
DEFINE_STATIC_SRCU(dax_srcu);
@@ -366,11 +367,15 @@ void kill_dax(struct dax_device *dax_dev)
spin_lock(&dax_host_lock);
hlist_del_init(&dax_dev->list);
spin_unlock(&dax_host_lock);
-
- dax_dev->private = NULL;
}
EXPORT_SYMBOL_GPL(kill_dax);
+void run_dax(struct dax_device *dax_dev)
+{
+ set_bit(DAXDEV_ALIVE, &dax_dev->flags);
+}
+EXPORT_SYMBOL_GPL(run_dax);
+
static struct inode *dax_alloc_inode(struct super_block *sb)
{
struct dax_device *dax_dev;
@@ -585,6 +590,8 @@ EXPORT_SYMBOL_GPL(dax_inode);
void *dax_get_private(struct dax_device *dax_dev)
{
+ if (!test_bit(DAXDEV_ALIVE, &dax_dev->flags))
+ return NULL;
return dax_dev->private;
}
EXPORT_SYMBOL_GPL(dax_get_private);
@@ -598,7 +605,7 @@ static void init_once(void *_dax_dev)
inode_init_once(inode);
}
-static int __dax_fs_init(void)
+static int dax_fs_init(void)
{
int rc;
@@ -630,35 +637,45 @@ static int __dax_fs_init(void)
return rc;
}
-static void __dax_fs_exit(void)
+static void dax_fs_exit(void)
{
kern_unmount(dax_mnt);
unregister_filesystem(&dax_fs_type);
kmem_cache_destroy(dax_cache);
}
-static int __init dax_fs_init(void)
+static int __init dax_core_init(void)
{
int rc;
- rc = __dax_fs_init();
+ rc = dax_fs_init();
if (rc)
return rc;
rc = alloc_chrdev_region(&dax_devt, 0, MINORMASK+1, "dax");
if (rc)
- __dax_fs_exit();
- return rc;
+ goto err_chrdev;
+
+ rc = dax_bus_init();
+ if (rc)
+ goto err_bus;
+ return 0;
+
+err_bus:
+ unregister_chrdev_region(dax_devt, MINORMASK+1);
+err_chrdev:
+ dax_fs_exit();
+ return 0;
}
-static void __exit dax_fs_exit(void)
+static void __exit dax_core_exit(void)
{
unregister_chrdev_region(dax_devt, MINORMASK+1);
ida_destroy(&dax_minor_ida);
- __dax_fs_exit();
+ dax_fs_exit();
}
MODULE_AUTHOR("Intel Corporation");
MODULE_LICENSE("GPL v2");
-subsys_initcall(dax_fs_init);
-module_exit(dax_fs_exit);
+subsys_initcall(dax_core_init);
+module_exit(dax_core_exit);
diff --git a/drivers/edac/i10nm_base.c b/drivers/edac/i10nm_base.c
index 6f06aec4877c..83392f2841de 100644
--- a/drivers/edac/i10nm_base.c
+++ b/drivers/edac/i10nm_base.c
@@ -124,6 +124,8 @@ static int i10nm_get_all_munits(void)
static const struct x86_cpu_id i10nm_cpuids[] = {
{ X86_VENDOR_INTEL, 6, INTEL_FAM6_ATOM_TREMONT_X, 0, 0 },
+ { X86_VENDOR_INTEL, 6, INTEL_FAM6_ICELAKE_X, 0, 0 },
+ { X86_VENDOR_INTEL, 6, INTEL_FAM6_ICELAKE_XEON_D, 0, 0 },
{ }
};
MODULE_DEVICE_TABLE(x86cpu, i10nm_cpuids);
@@ -166,9 +168,9 @@ static int i10nm_get_dimm_config(struct mem_ctl_info *mci)
ndimms += skx_get_nvdimm_info(dimm, imc, i, j,
EDAC_MOD_STR);
}
- if (ndimms && !i10nm_check_ecc(imc, 0)) {
- i10nm_printk(KERN_ERR, "ECC is disabled on imc %d\n",
- imc->mc);
+ if (ndimms && !i10nm_check_ecc(imc, i)) {
+ i10nm_printk(KERN_ERR, "ECC is disabled on imc %d channel %d\n",
+ imc->mc, i);
return -ENODEV;
}
}
@@ -265,7 +267,7 @@ static int __init i10nm_init(void)
goto fail;
list_for_each_entry(d, i10nm_edac_list, list) {
- rc = skx_get_src_id(d, &src_id);
+ rc = skx_get_src_id(d, 0xf8, &src_id);
if (rc < 0)
goto fail;
diff --git a/drivers/edac/skx_base.c b/drivers/edac/skx_base.c
index a5c8fa3a249a..0fcf3785e8f3 100644
--- a/drivers/edac/skx_base.c
+++ b/drivers/edac/skx_base.c
@@ -639,7 +639,7 @@ static int __init skx_init(void)
}
list_for_each_entry(d, skx_edac_list, list) {
- rc = skx_get_src_id(d, &src_id);
+ rc = skx_get_src_id(d, 0xf0, &src_id);
if (rc < 0)
goto fail;
rc = skx_get_node_id(d, &node_id);
diff --git a/drivers/edac/skx_common.c b/drivers/edac/skx_common.c
index b0dddcfa9baa..d8ff63d91b86 100644
--- a/drivers/edac/skx_common.c
+++ b/drivers/edac/skx_common.c
@@ -136,11 +136,11 @@ void skx_set_decode(skx_decode_f decode)
skx_decode = decode;
}
-int skx_get_src_id(struct skx_dev *d, u8 *id)
+int skx_get_src_id(struct skx_dev *d, int off, u8 *id)
{
u32 reg;
- if (pci_read_config_dword(d->util_all, 0xf0, &reg)) {
+ if (pci_read_config_dword(d->util_all, off, &reg)) {
skx_printk(KERN_ERR, "Failed to read src id\n");
return -ENODEV;
}
diff --git a/drivers/edac/skx_common.h b/drivers/edac/skx_common.h
index d18fa98669af..08cc971a50ea 100644
--- a/drivers/edac/skx_common.h
+++ b/drivers/edac/skx_common.h
@@ -118,7 +118,7 @@ int __init skx_adxl_get(void);
void __exit skx_adxl_put(void);
void skx_set_decode(skx_decode_f decode);
-int skx_get_src_id(struct skx_dev *d, u8 *id);
+int skx_get_src_id(struct skx_dev *d, int off, u8 *id);
int skx_get_node_id(struct skx_dev *d, u8 *id);
int skx_get_all_bus_mappings(unsigned int did, int off, enum type,
diff --git a/drivers/nvdimm/e820.c b/drivers/nvdimm/e820.c
index 521eaf53a52a..36be9b619187 100644
--- a/drivers/nvdimm/e820.c
+++ b/drivers/nvdimm/e820.c
@@ -47,6 +47,7 @@ static int e820_register_one(struct resource *res, void *data)
ndr_desc.res = res;
ndr_desc.attr_groups = e820_pmem_region_attribute_groups;
ndr_desc.numa_node = e820_range_to_nid(res->start);
+ ndr_desc.target_node = ndr_desc.numa_node;
set_bit(ND_REGION_PAGEMAP, &ndr_desc.flags);
if (!nvdimm_pmem_region_create(nvdimm_bus, &ndr_desc))
return -ENXIO;
diff --git a/drivers/nvdimm/nd.h b/drivers/nvdimm/nd.h
index a3023a5017ea..104471bc1277 100644
--- a/drivers/nvdimm/nd.h
+++ b/drivers/nvdimm/nd.h
@@ -157,7 +157,7 @@ struct nd_region {
u16 ndr_mappings;
u64 ndr_size;
u64 ndr_start;
- int id, num_lanes, ro, numa_node;
+ int id, num_lanes, ro, numa_node, target_node;
void *provider_data;
struct kernfs_node *bb_state;
struct badblocks bb;
diff --git a/drivers/nvdimm/of_pmem.c b/drivers/nvdimm/of_pmem.c
index 0a701837dfc0..ecaaa27438e2 100644
--- a/drivers/nvdimm/of_pmem.c
+++ b/drivers/nvdimm/of_pmem.c
@@ -68,6 +68,7 @@ static int of_pmem_region_probe(struct platform_device *pdev)
memset(&ndr_desc, 0, sizeof(ndr_desc));
ndr_desc.attr_groups = region_attr_groups;
ndr_desc.numa_node = dev_to_node(&pdev->dev);
+ ndr_desc.target_node = ndr_desc.numa_node;
ndr_desc.res = &pdev->resource[i];
ndr_desc.of_node = np;
set_bit(ND_REGION_PAGEMAP, &ndr_desc.flags);
diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c
index 5e5a85c0c677..5991188efa28 100644
--- a/drivers/nvdimm/pmem.c
+++ b/drivers/nvdimm/pmem.c
@@ -309,8 +309,11 @@ static void pmem_release_queue(void *q)
blk_cleanup_queue(q);
}
-static void pmem_freeze_queue(void *q)
+static void pmem_freeze_queue(struct percpu_ref *ref)
{
+ struct request_queue *q;
+
+ q = container_of(ref, typeof(*q), q_usage_counter);
blk_freeze_queue_start(q);
}
@@ -402,6 +405,7 @@ static int pmem_attach_disk(struct device *dev,
pmem->pfn_flags = PFN_DEV;
pmem->pgmap.ref = &q->q_usage_counter;
+ pmem->pgmap.kill = pmem_freeze_queue;
if (is_nd_pfn(dev)) {
if (setup_pagemap_fsdax(dev, &pmem->pgmap))
return -ENOMEM;
@@ -427,13 +431,6 @@ static int pmem_attach_disk(struct device *dev,
memcpy(&bb_res, &nsio->res, sizeof(bb_res));
}
- /*
- * At release time the queue must be frozen before
- * devm_memremap_pages is unwound
- */
- if (devm_add_action_or_reset(dev, pmem_freeze_queue, q))
- return -ENOMEM;
-
if (IS_ERR(addr))
return PTR_ERR(addr);
pmem->virt_addr = addr;
diff --git a/drivers/nvdimm/region_devs.c b/drivers/nvdimm/region_devs.c
index 23eb317d0c42..cccd1fbfbe39 100644
--- a/drivers/nvdimm/region_devs.c
+++ b/drivers/nvdimm/region_devs.c
@@ -1065,6 +1065,7 @@ static struct nd_region *nd_region_create(struct nvdimm_bus *nvdimm_bus,
nd_region->flags = ndr_desc->flags;
nd_region->ro = ro;
nd_region->numa_node = ndr_desc->numa_node;
+ nd_region->target_node = ndr_desc->target_node;
ida_init(&nd_region->ns_ida);
ida_init(&nd_region->btt_ida);
ida_init(&nd_region->pfn_ida);
diff --git a/include/linux/acpi.h b/include/linux/acpi.h
index a7116b084d11..fd05a8712184 100644
--- a/include/linux/acpi.h
+++ b/include/linux/acpi.h
@@ -393,12 +393,17 @@ extern bool acpi_osi_is_win8(void);
#ifdef CONFIG_ACPI_NUMA
int acpi_map_pxm_to_online_node(int pxm);
+int acpi_map_pxm_to_node(int pxm);
int acpi_get_node(acpi_handle handle);
#else
static inline int acpi_map_pxm_to_online_node(int pxm)
{
return 0;
}
+static inline int acpi_map_pxm_to_node(int pxm)
+{
+ return 0;
+}
static inline int acpi_get_node(acpi_handle handle)
{
return 0;
diff --git a/include/linux/libnvdimm.h b/include/linux/libnvdimm.h
index ad609617aeb8..4b2b157c8fbe 100644
--- a/include/linux/libnvdimm.h
+++ b/include/linux/libnvdimm.h
@@ -128,6 +128,7 @@ struct nd_region_desc {
void *provider_data;
int num_lanes;
int numa_node;
+ int target_node;
unsigned long flags;
struct device_node *of_node;
};
diff --git a/include/linux/memremap.h b/include/linux/memremap.h
index 8b83e71354b4..a4c1b14e86f6 100644
--- a/include/linux/memremap.h
+++ b/include/linux/memremap.h
@@ -110,6 +110,7 @@ typedef void (*dev_page_free_t)(struct page *page, void *data);
* @altmap: pre-allocated/reserved memory for vmemmap allocations
* @res: physical address range covered by @ref
* @ref: reference count that pins the devm_memremap_pages() mapping
+ * @kill: callback to transition @ref to the dead state
* @dev: host device of the mapping for debug
* @data: private data pointer for page_free()
* @type: memory type: see MEMORY_* in memory_hotplug.h
@@ -121,6 +122,7 @@ struct dev_pagemap {
bool altmap_valid;
struct resource res;
struct percpu_ref *ref;
+ void (*kill)(struct percpu_ref *ref);
struct device *dev;
void *data;
enum memory_type type;
diff --git a/kernel/memremap.c b/kernel/memremap.c
index 97baac4acdf9..a1892e20eb3c 100644
--- a/kernel/memremap.c
+++ b/kernel/memremap.c
@@ -294,14 +294,10 @@ static void devm_memremap_pages_release(void *data)
resource_size_t align_start, align_size;
unsigned long pfn;
+ pgmap->kill(pgmap->ref);
for_each_device_pfn(pfn, pgmap)
put_page(pfn_to_page(pfn));
- if (percpu_ref_tryget_live(pgmap->ref)) {
- dev_WARN(dev, "%s: page mapping is still live!\n", __func__);
- percpu_ref_put(pgmap->ref);
- }
-
/* pages are dead and unused, undo the arch mapping */
align_start = res->start & ~(SECTION_SIZE - 1);
align_size = ALIGN(res->start + resource_size(res), SECTION_SIZE)
@@ -321,7 +317,7 @@ static void devm_memremap_pages_release(void *data)
/**
* devm_memremap_pages - remap and provide memmap backing for the given resource
* @dev: hosting device for @res
- * @pgmap: pointer to a struct dev_pgmap
+ * @pgmap: pointer to a struct dev_pagemap
*
* Notes:
* 1/ At a minimum the res, ref and type members of @pgmap must be initialized
@@ -330,11 +326,8 @@ static void devm_memremap_pages_release(void *data)
* 2/ The altmap field may optionally be initialized, in which case altmap_valid
* must be set to true
*
- * 3/ pgmap.ref must be 'live' on entry and 'dead' before devm_memunmap_pages()
- * time (or devm release event). The expected order of events is that ref has
- * been through percpu_ref_kill() before devm_memremap_pages_release(). The
- * wait for the completion of all references being dropped and
- * percpu_ref_exit() must occur after devm_memremap_pages_release().
+ * 3/ pgmap->ref must be 'live' on entry and will be killed at
+ * devm_memremap_pages_release() time, or if this routine fails.
*
* 4/ res is expected to be a host memory range that could feasibly be
* treated as a "System RAM" range, i.e. not a device mmio range, but
@@ -350,24 +343,22 @@ void *devm_memremap_pages(struct device *dev, struct dev_pagemap *pgmap)
int error, nid, is_ram;
struct resource *res = &pgmap->res;
+ if (!pgmap->ref || !pgmap->kill)
+ return ERR_PTR(-EINVAL);
+
align_start = res->start & ~(SECTION_SIZE - 1);
align_size = ALIGN(res->start + resource_size(res), SECTION_SIZE)
- align_start;
is_ram = region_intersects(align_start, align_size,
IORESOURCE_SYSTEM_RAM, IORES_DESC_NONE);
- if (is_ram == REGION_MIXED) {
- WARN_ONCE(1, "%s attempted on mixed region %pr\n",
- __func__, res);
- return ERR_PTR(-ENXIO);
+ if (is_ram != REGION_DISJOINT) {
+ WARN_ONCE(1, "%s attempted on %s region %pr\n", __func__,
+ is_ram == REGION_MIXED ? "mixed" : "ram", res);
+ error = -ENXIO;
+ goto err_radix;
}
- if (is_ram == REGION_INTERSECTS)
- return __va(res->start);
-
- if (!pgmap->ref)
- return ERR_PTR(-EINVAL);
-
pgmap->dev = dev;
mutex_lock(&pgmap_lock);
@@ -414,7 +405,10 @@ void *devm_memremap_pages(struct device *dev, struct dev_pagemap *pgmap)
align_size >> PAGE_SHIFT, pgmap);
percpu_ref_get_many(pgmap->ref, pfn_end(pgmap) - pfn_first(pgmap));
- devm_add_action(dev, devm_memremap_pages_release, pgmap);
+ error = devm_add_action_or_reset(dev, devm_memremap_pages_release,
+ pgmap);
+ if (error)
+ return ERR_PTR(error);
return __va(res->start);
@@ -423,6 +417,7 @@ void *devm_memremap_pages(struct device *dev, struct dev_pagemap *pgmap)
err_pfn_remap:
err_radix:
pgmap_radix_release(res, pgoff);
+ pgmap->kill(pgmap->ref);
return ERR_PTR(error);
}
EXPORT_SYMBOL(devm_memremap_pages);
diff --git a/kernel/resource.c b/kernel/resource.c
index d6352bb6ecaf..d6b26cc0474d 100644
--- a/kernel/resource.c
+++ b/kernel/resource.c
@@ -408,7 +408,7 @@ static int __walk_iomem_res_desc(struct resource *res, unsigned long desc,
int (*func)(struct resource *, void *))
{
u64 orig_end = res->end;
- int ret = -1;
+ int ret = -EINVAL;
while ((res->start < res->end) &&
!find_next_iomem_res(res, desc, first_level_children_only)) {
@@ -493,6 +493,9 @@ int walk_mem_res(u64 start, u64 end, void *arg,
* This function calls the @func callback against all memory ranges of type
* System RAM which are marked as IORESOURCE_SYSTEM_RAM and IORESOUCE_BUSY.
* It is to be used only for System RAM.
+ *
+ * This will find System RAM ranges that are children of top-level resources
+ * in addition to top-level System RAM resources.
*/
int walk_system_ram_range(unsigned long start_pfn, unsigned long nr_pages,
void *arg, int (*func)(unsigned long, unsigned long, void *))
@@ -500,14 +503,14 @@ int walk_system_ram_range(unsigned long start_pfn, unsigned long nr_pages,
struct resource res;
unsigned long pfn, end_pfn;
u64 orig_end;
- int ret = -1;
+ int ret = -EINVAL;
res.start = (u64) start_pfn << PAGE_SHIFT;
res.end = ((u64)(start_pfn + nr_pages) << PAGE_SHIFT) - 1;
res.flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY;
orig_end = res.end;
while ((res.start < res.end) &&
- (find_next_iomem_res(&res, IORES_DESC_NONE, true) >= 0)) {
+ (find_next_iomem_res(&res, IORES_DESC_NONE, false) >= 0)) {
pfn = (res.start + PAGE_SIZE - 1) >> PAGE_SHIFT;
end_pfn = (res.end + 1) >> PAGE_SHIFT;
if (end_pfn > pfn)
@@ -1170,6 +1173,15 @@ struct resource * __request_region(struct resource *parent,
conflict = __request_resource(parent, res);
if (!conflict)
break;
+ /*
+ * mm/hmm.c reserves physical addresses which then
+ * become unavailable to other users. Conflicts are
+ * not expected. Warn to aid debugging if encountered.
+ */
+ if (conflict->desc == IORES_DESC_DEVICE_PRIVATE_MEMORY) {
+ pr_warn("Unaddressable device %s %pR conflicts with %pR",
+ conflict->name, conflict, res);
+ }
if (conflict != parent) {
if (!(conflict->flags & IORESOURCE_BUSY)) {
parent = conflict;
diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c
index 2fa1d1c822e5..7907729c6270 100644
--- a/mm/memory_hotplug.c
+++ b/mm/memory_hotplug.c
@@ -159,28 +159,24 @@ u64 max_mem_size = U64_MAX;
/* add this memory to iomem resource */
static struct resource *register_memory_resource(u64 start, u64 size)
{
- struct resource *res, *conflict;
+ struct resource *res;
+ unsigned long flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY;
+ char *resource_name = "System RAM";
if (start + size > max_mem_size)
return ERR_PTR(-E2BIG);
- res = kzalloc(sizeof(struct resource), GFP_KERNEL);
- if (!res)
- return ERR_PTR(-ENOMEM);
-
- res->name = "System RAM";
- res->start = start;
- res->end = start + size - 1;
- res->flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY;
- conflict = request_resource_conflict(&iomem_resource, res);
- if (conflict) {
- if (conflict->desc == IORES_DESC_DEVICE_PRIVATE_MEMORY) {
- pr_debug("Device unaddressable memory block "
- "memory hotplug at %#010llx !\n",
- (unsigned long long)start);
- }
- pr_debug("System RAM resource %pR cannot be added\n", res);
- kfree(res);
+ /*
+ * Request ownership of the new memory range. This might be
+ * a child of an existing resource that was present but
+ * not marked as busy.
+ */
+ res = __request_region(&iomem_resource, start, size,
+ resource_name, flags);
+
+ if (!res) {
+ pr_debug("Unable to reserve System RAM region: %016llx->%016llx\n",
+ start, start + size);
return ERR_PTR(-EEXIST);
}
return res;
diff --git a/tools/testing/nvdimm/Kbuild b/tools/testing/nvdimm/Kbuild
index 26b4de02841e..d948263f5fe7 100644
--- a/tools/testing/nvdimm/Kbuild
+++ b/tools/testing/nvdimm/Kbuild
@@ -34,6 +34,8 @@ obj-$(CONFIG_DAX) += dax.o
endif
obj-$(CONFIG_DEV_DAX) += device_dax.o
obj-$(CONFIG_DEV_DAX_PMEM) += dax_pmem.o
+obj-$(CONFIG_DEV_DAX_PMEM) += dax_pmem_core.o
+obj-$(CONFIG_DEV_DAX_PMEM_COMPAT) += dax_pmem_compat.o
nfit-y := $(ACPI_SRC)/core.o
nfit-y += $(ACPI_SRC)/intel.o
@@ -56,6 +58,7 @@ nd_e820-y := $(NVDIMM_SRC)/e820.o
nd_e820-y += config_check.o
dax-y := $(DAX_SRC)/super.o
+dax-y += $(DAX_SRC)/bus.o
dax-y += config_check.o
device_dax-y := $(DAX_SRC)/device.o
@@ -63,7 +66,9 @@ device_dax-y += dax-dev.o
device_dax-y += device_dax_test.o
device_dax-y += config_check.o
-dax_pmem-y := $(DAX_SRC)/pmem.o
+dax_pmem-y := $(DAX_SRC)/pmem/pmem.o
+dax_pmem_core-y := $(DAX_SRC)/pmem/core.o
+dax_pmem_compat-y := $(DAX_SRC)/pmem/compat.o
dax_pmem-y += config_check.o
libnvdimm-y := $(NVDIMM_SRC)/core.o
diff --git a/tools/testing/nvdimm/dax-dev.c b/tools/testing/nvdimm/dax-dev.c
index 36ee3d8797c3..f36e708265b8 100644
--- a/tools/testing/nvdimm/dax-dev.c
+++ b/tools/testing/nvdimm/dax-dev.c
@@ -17,20 +17,11 @@
phys_addr_t dax_pgoff_to_phys(struct dev_dax *dev_dax, pgoff_t pgoff,
unsigned long size)
{
- struct resource *res;
+ struct resource *res = &dev_dax->region->res;
phys_addr_t addr;
- int i;
- for (i = 0; i < dev_dax->num_resources; i++) {
- res = &dev_dax->res[i];
- addr = pgoff * PAGE_SIZE + res->start;
- if (addr >= res->start && addr <= res->end)
- break;
- pgoff -= PHYS_PFN(resource_size(res));
- }
-
- if (i < dev_dax->num_resources) {
- res = &dev_dax->res[i];
+ addr = pgoff * PAGE_SIZE + res->start;
+ if (addr >= res->start && addr <= res->end) {
if (addr + size - 1 <= res->end) {
if (get_nfit_res(addr)) {
struct page *page;
@@ -44,6 +35,5 @@ phys_addr_t dax_pgoff_to_phys(struct dev_dax *dev_dax, pgoff_t pgoff,
return addr;
}
}
-
return -1;
}
diff --git a/tools/testing/nvdimm/test/iomap.c b/tools/testing/nvdimm/test/iomap.c
index ff9d3a5825e1..c94054f20f39 100644
--- a/tools/testing/nvdimm/test/iomap.c
+++ b/tools/testing/nvdimm/test/iomap.c
@@ -104,13 +104,26 @@ void *__wrap_devm_memremap(struct device *dev, resource_size_t offset,
}
EXPORT_SYMBOL(__wrap_devm_memremap);
+static void nfit_test_kill(void *_pgmap)
+{
+ struct dev_pagemap *pgmap = _pgmap;
+
+ pgmap->kill(pgmap->ref);
+}
+
void *__wrap_devm_memremap_pages(struct device *dev, struct dev_pagemap *pgmap)
{
resource_size_t offset = pgmap->res.start;
struct nfit_test_resource *nfit_res = get_nfit_res(offset);
- if (nfit_res)
+ if (nfit_res) {
+ int rc;
+
+ rc = devm_add_action_or_reset(dev, nfit_test_kill, pgmap);
+ if (rc)
+ return ERR_PTR(rc);
return nfit_res->buf + offset - nfit_res->res.start;
+ }
return devm_memremap_pages(dev, pgmap);
}
EXPORT_SYMBOL(__wrap_devm_memremap_pages);