From cc2a4878700b2467f36e03f581a0a877ae6a568d Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Thu, 26 May 2022 12:15:25 -0700 Subject: cxl/mem: Add a debugfs version of 'iomem' for DPA, 'dpamem' Dump the device-physical-address map for a CXL expander in /proc/iomem style format. E.g.: cat /sys/kernel/debug/cxl/mem1/dpamem 00000000-0fffffff : ram 10000000-1fffffff : pmem Reviewed-by: Jonathan Cameron Link: https://lore.kernel.org/r/165603885318.551046.8308248564880066726.stgit@dwillia2-xfh Signed-off-by: Dan Williams --- drivers/cxl/mem.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) (limited to 'drivers/cxl/mem.c') diff --git a/drivers/cxl/mem.c b/drivers/cxl/mem.c index a979d0b484d5..7513bea55145 100644 --- a/drivers/cxl/mem.c +++ b/drivers/cxl/mem.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only /* Copyright(c) 2022 Intel Corporation. All rights reserved. */ +#include #include #include #include @@ -56,10 +57,26 @@ static void enable_suspend(void *data) cxl_mem_active_dec(); } +static void remove_debugfs(void *dentry) +{ + debugfs_remove_recursive(dentry); +} + +static int cxl_mem_dpa_show(struct seq_file *file, void *data) +{ + struct device *dev = file->private; + struct cxl_memdev *cxlmd = to_cxl_memdev(dev); + + cxl_dpa_debug(file, cxlmd->cxlds); + + return 0; +} + static int cxl_mem_probe(struct device *dev) { struct cxl_memdev *cxlmd = to_cxl_memdev(dev); struct cxl_port *parent_port; + struct dentry *dentry; int rc; /* @@ -73,6 +90,12 @@ static int cxl_mem_probe(struct device *dev) if (work_pending(&cxlmd->detach_work)) return -EBUSY; + dentry = cxl_debugfs_create_dir(dev_name(dev)); + debugfs_create_devm_seqfile(dev, "dpamem", dentry, cxl_mem_dpa_show); + rc = devm_add_action_or_reset(dev, remove_debugfs, dentry); + if (rc) + return rc; + rc = devm_cxl_enumerate_ports(cxlmd); if (rc) return rc; -- cgit v1.2.3 From 1b58b4cac6fc6fab55f34f74087594125fc60b84 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Fri, 27 May 2022 10:57:01 -0700 Subject: cxl/port: Record parent dport when adding ports At the time that cxl_port instances are being created, cache the dport from the parent port that points to this new child port. This will be useful for region provisioning when walking the tree to calculate decoder targets, and saves rewalking the dport list after the fact to build this information. Reviewed-by: Jonathan Cameron Link: https://lore.kernel.org/r/20220624041950.559155-1-dan.j.williams@intel.com Signed-off-by: Dan Williams --- drivers/cxl/acpi.c | 3 +-- drivers/cxl/core/port.c | 27 +++++++++++++++------------ drivers/cxl/cxl.h | 7 +++++-- drivers/cxl/mem.c | 10 ++++++---- 4 files changed, 27 insertions(+), 20 deletions(-) (limited to 'drivers/cxl/mem.c') diff --git a/drivers/cxl/acpi.c b/drivers/cxl/acpi.c index 8f021241699f..64004eb672d0 100644 --- a/drivers/cxl/acpi.c +++ b/drivers/cxl/acpi.c @@ -211,8 +211,7 @@ static int add_host_bridge_uport(struct device *match, void *arg) if (rc) return rc; - port = devm_cxl_add_port(host, match, dport->component_reg_phys, - root_port); + port = devm_cxl_add_port(host, match, dport->component_reg_phys, dport); if (IS_ERR(port)) return PTR_ERR(port); dev_dbg(host, "%s: add: %s\n", dev_name(match), dev_name(&port->dev)); diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index 43b827b22bb2..0de6c1355393 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -526,7 +526,7 @@ static struct lock_class_key cxl_port_key; static struct cxl_port *cxl_port_alloc(struct device *uport, resource_size_t component_reg_phys, - struct cxl_port *parent_port) + struct cxl_dport *parent_dport) { struct cxl_port *port; struct device *dev; @@ -549,11 +549,13 @@ static struct cxl_port *cxl_port_alloc(struct device *uport, * description. */ dev = &port->dev; - if (parent_port) { + if (parent_dport) { + struct cxl_port *parent_port = parent_dport->port; struct cxl_port *iter; dev->parent = &parent_port->dev; port->depth = parent_port->depth + 1; + port->parent_dport = parent_dport; /* * walk to the host bridge, or the first ancestor that knows @@ -595,24 +597,24 @@ err: * @host: host device for devm operations * @uport: "physical" device implementing this upstream port * @component_reg_phys: (optional) for configurable cxl_port instances - * @parent_port: next hop up in the CXL memory decode hierarchy + * @parent_dport: next hop up in the CXL memory decode hierarchy */ struct cxl_port *devm_cxl_add_port(struct device *host, struct device *uport, resource_size_t component_reg_phys, - struct cxl_port *parent_port) + struct cxl_dport *parent_dport) { struct cxl_port *port; struct device *dev; int rc; - port = cxl_port_alloc(uport, component_reg_phys, parent_port); + port = cxl_port_alloc(uport, component_reg_phys, parent_dport); if (IS_ERR(port)) return port; dev = &port->dev; if (is_cxl_memdev(uport)) rc = dev_set_name(dev, "endpoint%d", port->id); - else if (parent_port) + else if (parent_dport) rc = dev_set_name(dev, "port%d", port->id); else rc = dev_set_name(dev, "root%d", port->id); @@ -1014,7 +1016,7 @@ static void delete_endpoint(void *data) struct cxl_port *parent_port; struct device *parent; - parent_port = cxl_mem_find_port(cxlmd); + parent_port = cxl_mem_find_port(cxlmd, NULL); if (!parent_port) goto out; parent = &parent_port->dev; @@ -1149,8 +1151,8 @@ static int add_port_attach_ep(struct cxl_memdev *cxlmd, { struct device *dparent = grandparent(dport_dev); struct cxl_port *port, *parent_port = NULL; + struct cxl_dport *dport, *parent_dport; resource_size_t component_reg_phys; - struct cxl_dport *dport; int rc; if (!dparent) { @@ -1164,7 +1166,7 @@ static int add_port_attach_ep(struct cxl_memdev *cxlmd, return -ENXIO; } - parent_port = find_cxl_port(dparent, NULL); + parent_port = find_cxl_port(dparent, &parent_dport); if (!parent_port) { /* iterate to create this parent_port */ return -EAGAIN; @@ -1183,7 +1185,7 @@ static int add_port_attach_ep(struct cxl_memdev *cxlmd, if (!port) { component_reg_phys = find_component_registers(uport_dev); port = devm_cxl_add_port(&parent_port->dev, uport_dev, - component_reg_phys, parent_port); + component_reg_phys, parent_dport); /* retry find to pick up the new dport information */ if (!IS_ERR(port)) port = find_cxl_port_at(parent_port, dport_dev, &dport); @@ -1290,9 +1292,10 @@ retry: } EXPORT_SYMBOL_NS_GPL(devm_cxl_enumerate_ports, CXL); -struct cxl_port *cxl_mem_find_port(struct cxl_memdev *cxlmd) +struct cxl_port *cxl_mem_find_port(struct cxl_memdev *cxlmd, + struct cxl_dport **dport) { - return find_cxl_port(grandparent(&cxlmd->dev), NULL); + return find_cxl_port(grandparent(&cxlmd->dev), dport); } EXPORT_SYMBOL_NS_GPL(cxl_mem_find_port, CXL); diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index 326283f1e328..4bdc47600f9d 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -332,6 +332,7 @@ struct cxl_nvdimm { * @id: id for port device-name * @dports: cxl_dport instances referenced by decoders * @endpoints: cxl_ep instances, endpoints that are a descendant of this port + * @parent_dport: dport that points to this port in the parent * @decoder_ida: allocator for decoder ids * @hdm_end: track last allocated HDM decoder instance for allocation ordering * @component_reg_phys: component register capability base address (optional) @@ -347,6 +348,7 @@ struct cxl_port { int id; struct list_head dports; struct list_head endpoints; + struct cxl_dport *parent_dport; struct ida decoder_ida; int hdm_end; resource_size_t component_reg_phys; @@ -406,11 +408,12 @@ int devm_cxl_register_pci_bus(struct device *host, struct device *uport, struct pci_bus *cxl_port_to_pci_bus(struct cxl_port *port); struct cxl_port *devm_cxl_add_port(struct device *host, struct device *uport, resource_size_t component_reg_phys, - struct cxl_port *parent_port); + struct cxl_dport *parent_dport); struct cxl_port *find_cxl_root(struct device *dev); int devm_cxl_enumerate_ports(struct cxl_memdev *cxlmd); int cxl_bus_rescan(void); -struct cxl_port *cxl_mem_find_port(struct cxl_memdev *cxlmd); +struct cxl_port *cxl_mem_find_port(struct cxl_memdev *cxlmd, + struct cxl_dport **dport); bool schedule_cxl_memdev_detach(struct cxl_memdev *cxlmd); struct cxl_dport *devm_cxl_add_dport(struct cxl_port *port, diff --git a/drivers/cxl/mem.c b/drivers/cxl/mem.c index 7513bea55145..2786d3402c9e 100644 --- a/drivers/cxl/mem.c +++ b/drivers/cxl/mem.c @@ -26,14 +26,15 @@ */ static int create_endpoint(struct cxl_memdev *cxlmd, - struct cxl_port *parent_port) + struct cxl_dport *parent_dport) { + struct cxl_port *parent_port = parent_dport->port; struct cxl_dev_state *cxlds = cxlmd->cxlds; struct cxl_port *endpoint; int rc; endpoint = devm_cxl_add_port(&parent_port->dev, &cxlmd->dev, - cxlds->component_reg_phys, parent_port); + cxlds->component_reg_phys, parent_dport); if (IS_ERR(endpoint)) return PTR_ERR(endpoint); @@ -76,6 +77,7 @@ static int cxl_mem_probe(struct device *dev) { struct cxl_memdev *cxlmd = to_cxl_memdev(dev); struct cxl_port *parent_port; + struct cxl_dport *dport; struct dentry *dentry; int rc; @@ -100,7 +102,7 @@ static int cxl_mem_probe(struct device *dev) if (rc) return rc; - parent_port = cxl_mem_find_port(cxlmd); + parent_port = cxl_mem_find_port(cxlmd, &dport); if (!parent_port) { dev_err(dev, "CXL port topology not found\n"); return -ENXIO; @@ -114,7 +116,7 @@ static int cxl_mem_probe(struct device *dev) goto unlock; } - rc = create_endpoint(cxlmd, parent_port); + rc = create_endpoint(cxlmd, dport); unlock: device_unlock(&parent_port->dev); put_device(&parent_port->dev); -- cgit v1.2.3 From 7f8faf96a2fb562833db73595640329ca8da7b1d Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Tue, 7 Jun 2022 10:35:39 -0700 Subject: cxl/mem: Enumerate port targets before adding endpoints The port scanning algorithm in devm_cxl_enumerate_ports() walks up the topology and adds cxl_port objects starting from the root down to the endpoint. When those ports are initially created they know all their dports, but they do not know the downstream cxl_port instance that represents the next descendant in the topology. Rework create_endpoint() into devm_cxl_add_endpoint() that enumerates the downstream cxl_port topology into each port's 'struct cxl_ep' record for each endpoint it that the port is an ancestor. Reviewed-by: Jonathan Cameron Link: https://lore.kernel.org/r/20220624041950.559155-7-dan.j.williams@intel.com Signed-off-by: Dan Williams --- drivers/cxl/core/port.c | 41 +++++++++++++++++++++++++++++++++++++++++ drivers/cxl/cxl.h | 5 +++++ drivers/cxl/mem.c | 30 +----------------------------- 3 files changed, 47 insertions(+), 29 deletions(-) (limited to 'drivers/cxl/mem.c') diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index 719563d85ce1..60c637a826a2 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -1087,6 +1087,47 @@ static struct cxl_ep *cxl_ep_load(struct cxl_port *port, return xa_load(&port->endpoints, (unsigned long)&cxlmd->dev); } +int devm_cxl_add_endpoint(struct cxl_memdev *cxlmd, + struct cxl_dport *parent_dport) +{ + struct cxl_port *parent_port = parent_dport->port; + struct cxl_dev_state *cxlds = cxlmd->cxlds; + struct cxl_port *endpoint, *iter, *down; + int rc; + + /* + * Now that the path to the root is established record all the + * intervening ports in the chain. + */ + for (iter = parent_port, down = NULL; !is_cxl_root(iter); + down = iter, iter = to_cxl_port(iter->dev.parent)) { + struct cxl_ep *ep; + + ep = cxl_ep_load(iter, cxlmd); + ep->next = down; + } + + endpoint = devm_cxl_add_port(&parent_port->dev, &cxlmd->dev, + cxlds->component_reg_phys, parent_dport); + if (IS_ERR(endpoint)) + return PTR_ERR(endpoint); + + dev_dbg(&cxlmd->dev, "add: %s\n", dev_name(&endpoint->dev)); + + rc = cxl_endpoint_autoremove(cxlmd, endpoint); + if (rc) + return rc; + + if (!endpoint->dev.driver) { + dev_err(&cxlmd->dev, "%s failed probe\n", + dev_name(&endpoint->dev)); + return -ENXIO; + } + + return 0; +} +EXPORT_SYMBOL_NS_GPL(devm_cxl_add_endpoint, CXL); + static void cxl_detach_ep(void *data) { struct cxl_memdev *cxlmd = data; diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index ac950fb45dd2..c3c62568d287 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -385,10 +385,13 @@ struct cxl_dport { * struct cxl_ep - track an endpoint's interest in a port * @ep: device that hosts a generic CXL endpoint (expander or accelerator) * @dport: which dport routes to this endpoint on @port + * @next: cxl switch port across the link attached to @dport NULL if + * attached to an endpoint */ struct cxl_ep { struct device *ep; struct cxl_dport *dport; + struct cxl_port *next; }; /* @@ -411,6 +414,8 @@ struct pci_bus *cxl_port_to_pci_bus(struct cxl_port *port); struct cxl_port *devm_cxl_add_port(struct device *host, struct device *uport, resource_size_t component_reg_phys, struct cxl_dport *parent_dport); +int devm_cxl_add_endpoint(struct cxl_memdev *cxlmd, + struct cxl_dport *parent_dport); struct cxl_port *find_cxl_root(struct device *dev); int devm_cxl_enumerate_ports(struct cxl_memdev *cxlmd); int cxl_bus_rescan(void); diff --git a/drivers/cxl/mem.c b/drivers/cxl/mem.c index 2786d3402c9e..64ccf053d32c 100644 --- a/drivers/cxl/mem.c +++ b/drivers/cxl/mem.c @@ -25,34 +25,6 @@ * in higher level operations. */ -static int create_endpoint(struct cxl_memdev *cxlmd, - struct cxl_dport *parent_dport) -{ - struct cxl_port *parent_port = parent_dport->port; - struct cxl_dev_state *cxlds = cxlmd->cxlds; - struct cxl_port *endpoint; - int rc; - - endpoint = devm_cxl_add_port(&parent_port->dev, &cxlmd->dev, - cxlds->component_reg_phys, parent_dport); - if (IS_ERR(endpoint)) - return PTR_ERR(endpoint); - - dev_dbg(&cxlmd->dev, "add: %s\n", dev_name(&endpoint->dev)); - - rc = cxl_endpoint_autoremove(cxlmd, endpoint); - if (rc) - return rc; - - if (!endpoint->dev.driver) { - dev_err(&cxlmd->dev, "%s failed probe\n", - dev_name(&endpoint->dev)); - return -ENXIO; - } - - return 0; -} - static void enable_suspend(void *data) { cxl_mem_active_dec(); @@ -116,7 +88,7 @@ static int cxl_mem_probe(struct device *dev) goto unlock; } - rc = create_endpoint(cxlmd, dport); + rc = devm_cxl_add_endpoint(cxlmd, dport); unlock: device_unlock(&parent_port->dev); put_device(&parent_port->dev); -- cgit v1.2.3