Web lists-archives.com

[PATCH 4/8] irqchip/gic-v3: add ability to save/restore GIC/ITS state




Some platforms power off GIC logic in S3, so we need to save/restore
state. This adds a DT-binding to save/restore the GICD/GICR/GITS
states using the new CPU_PM_SYSTEM_ENTER/EXIT CPU PM states.

Change-Id: I1fb2117296373fa67397fdd4a8960077b241462e
Signed-off-by: Derek Basehore <dbasehore@xxxxxxxxxxxx>
Signed-off-by: Brian Norris <briannorris@xxxxxxxxxxxx>
---
 drivers/irqchip/irq-gic-v3-its.c   |  51 ++++++
 drivers/irqchip/irq-gic-v3.c       | 333 +++++++++++++++++++++++++++++++++++--
 include/linux/irqchip/arm-gic-v3.h |  17 ++
 3 files changed, 391 insertions(+), 10 deletions(-)

diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c
index 06f025fd5726..5cb808e3d0bf 100644
--- a/drivers/irqchip/irq-gic-v3-its.c
+++ b/drivers/irqchip/irq-gic-v3-its.c
@@ -85,6 +85,16 @@ struct its_baser {
 
 struct its_device;
 
+/*
+ * Saved ITS state - this is where saved state for the ITS is stored
+ * when it's disabled during system suspend.
+ */
+struct its_ctx {
+	u64			cbaser;
+	u64			cwriter;
+	u32			ctlr;
+};
+
 /*
  * The ITS structure - contains most of the infrastructure, with the
  * top-level MSI domain, the command queue, the collections, and the
@@ -101,6 +111,7 @@ struct its_node {
 	struct its_collection	*collections;
 	struct fwnode_handle	*fwnode_handle;
 	u64			(*get_msi_base)(struct its_device *its_dev);
+	struct its_ctx		its_ctx;
 	struct list_head	its_device_list;
 	u64			flags;
 	unsigned long		list_nr;
@@ -3042,6 +3053,46 @@ static void its_enable_quirks(struct its_node *its)
 	gic_enable_quirks(iidr, its_quirks, its);
 }
 
+void its_save_disable(void)
+{
+	struct its_node *its;
+
+	spin_lock(&its_lock);
+	list_for_each_entry(its, &its_nodes, entry) {
+		struct its_ctx *ctx = &its->its_ctx;
+		void __iomem *base = its->base;
+		int i;
+
+		ctx->ctlr = readl_relaxed(base + GITS_CTLR);
+		its_force_quiescent(base);
+		ctx->cbaser = gits_read_cbaser(base + GITS_CBASER);
+		ctx->cwriter = readq_relaxed(base + GITS_CWRITER);
+		for (i = 0; i < ARRAY_SIZE(its->tables); i++)
+			its->tables[i].val = its_read_baser(its, &its->tables[i]);
+	}
+	spin_unlock(&its_lock);
+}
+
+void its_restore_enable(void)
+{
+	struct its_node *its;
+
+	spin_lock(&its_lock);
+	list_for_each_entry(its, &its_nodes, entry) {
+		struct its_ctx *ctx = &its->its_ctx;
+		void __iomem *base = its->base;
+		int i;
+
+		gits_write_cbaser(ctx->cbaser, base + GITS_CBASER);
+		gits_write_cwriter(ctx->cwriter, base + GITS_CWRITER);
+		for (i = 0; i < ARRAY_SIZE(its->tables); i++)
+			its_write_baser(its, &its->tables[i],
+					its->tables[i].val);
+		writel_relaxed(ctx->ctlr, base + GITS_CTLR);
+	}
+	spin_unlock(&its_lock);
+}
+
 static int its_init_domain(struct fwnode_handle *handle, struct its_node *its)
 {
 	struct irq_domain *inner_domain;
diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c
index 9a7a15049903..95d37fb6f458 100644
--- a/drivers/irqchip/irq-gic-v3.c
+++ b/drivers/irqchip/irq-gic-v3.c
@@ -47,6 +47,36 @@ struct redist_region {
 	bool			single_redist;
 };
 
+struct gic_dist_ctx {
+	u64			*irouter;
+	u32			*igroupr;
+	u32			*isenabler;
+	u32			*ispendr;
+	u32			*isactiver;
+	u32			*ipriorityr;
+	u32			*icfgr;
+	u32			*igrpmodr;
+	u32			*nsacr;
+	u32			ctlr;
+};
+
+struct gic_redist_ctx {
+	u64			propbaser;
+	u64			pendbaser;
+	u32			ipriorityr[8];
+	u32			ctlr;
+	u32			igroupr0;
+	u32			isenabler0;
+	u32			ispendr0;
+	u32			isactiver0;
+	u32			icfgr0;
+	u32			icfgr1;
+	u32			igrpmodr0;
+	u32			nsacr;
+};
+
+#define GICD_NR_REGS(reg, nr_irq)	((nr_irq) >> GICD_## reg ##_SHIFT)
+
 struct gic_chip_data {
 	struct fwnode_handle	*fwnode;
 	void __iomem		*dist_base;
@@ -58,6 +88,8 @@ struct gic_chip_data {
 	bool			has_rss;
 	unsigned int		irq_nr;
 	struct partition_desc	*ppi_descs[16];
+	struct gic_dist_ctx	*gicd_ctx;
+	struct gic_redist_ctx	*gicr_ctx;
 };
 
 static struct gic_chip_data gic_data __read_mostly;
@@ -133,13 +165,13 @@ static u64 __maybe_unused gic_read_iar(void)
 }
 #endif
 
-static void gic_enable_redist(bool enable)
+static void gic_enable_redist_base(int cpu, bool enable)
 {
 	void __iomem *rbase;
 	u32 count = 1000000;	/* 1s! */
 	u32 val;
 
-	rbase = gic_data_rdist_rd_base();
+	rbase = per_cpu_ptr(gic_data.rdists.rdist, cpu)->rd_base;
 
 	val = readl_relaxed(rbase + GICR_WAKER);
 	if (enable)
@@ -167,6 +199,11 @@ static void gic_enable_redist(bool enable)
 				   enable ? "wakeup" : "sleep");
 }
 
+static void gic_enable_redist(bool enable)
+{
+	gic_enable_redist_base(smp_processor_id(), enable);
+}
+
 /*
  * Routines to disable, enable, EOI and route interrupts
  */
@@ -757,6 +794,155 @@ static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
 #define gic_smp_init()		do { } while(0)
 #endif
 
+static void gic_redist_save(int cpu)
+{
+	struct gic_redist_ctx *ctx = &gic_data.gicr_ctx[cpu];
+	void __iomem *base = per_cpu_ptr(gic_data.rdists.rdist, cpu)->rd_base;
+
+	gic_do_wait_for_rwp(base);
+	ctx->ctlr = readl_relaxed(base + GICR_CTLR);
+	ctx->propbaser = readq_relaxed(base + GICR_PROPBASER);
+	ctx->pendbaser = readq_relaxed(base + GICR_PENDBASER);
+
+	base += GICR_SGI_BASE_OFFSET;
+	ctx->igroupr0 = readl_relaxed(base + GICR_IGROUPR0);
+	ctx->isenabler0 = readl_relaxed(base + GICR_ISENABLER0);
+	ctx->ispendr0 = readl_relaxed(base + GICR_ISPENDR0);
+	ctx->isactiver0 = readl_relaxed(base + GICR_ISACTIVER0);
+	__ioread32_copy(ctx->ipriorityr, base + GICR_IPRIORITYR0, 8);
+	ctx->icfgr0 = readl_relaxed(base + GICR_ICFGR0);
+	ctx->icfgr1 = readl_relaxed(base + GICR_ICFGR0 + sizeof(u32));
+	ctx->igrpmodr0 = readl_relaxed(base + GICR_IGRPMODR0);
+	ctx->nsacr = readl_relaxed(base + GICR_NSACR);
+}
+
+static void gic_dist_save(void)
+{
+	struct gic_dist_ctx *ctx = gic_data.gicd_ctx;
+	void __iomem *base = gic_data.dist_base;
+	int irq_nr = gic_data.irq_nr;
+
+	__ioread64_copy(ctx->irouter, base + GICD_IROUTER,
+			GICD_NR_REGS(IROUTER, irq_nr));
+	__ioread32_copy(ctx->igroupr, base + GICD_IGROUPR,
+			GICD_NR_REGS(IGROUPR, irq_nr));
+	__ioread32_copy(ctx->isenabler, base + GICD_ISENABLER,
+			GICD_NR_REGS(ISENABLER, irq_nr));
+	__ioread32_copy(ctx->ispendr, base + GICD_ISPENDR,
+			GICD_NR_REGS(ISPENDR, irq_nr));
+	__ioread32_copy(ctx->isactiver, base + GICD_ISACTIVER,
+			GICD_NR_REGS(ISACTIVER, irq_nr));
+	__ioread32_copy(ctx->icfgr, base + GICD_ICFGR,
+			GICD_NR_REGS(ICFGR, irq_nr));
+	__ioread32_copy(ctx->igrpmodr, base + GICD_IGRPMODR,
+			GICD_NR_REGS(IGRPMODR, irq_nr));
+	__ioread32_copy(ctx->nsacr, base + GICD_NSACR,
+			GICD_NR_REGS(NSACR, irq_nr));
+	ctx->ctlr = readl_relaxed(base + GICD_CTLR);
+}
+
+static int gic_suspend(void)
+{
+	void __iomem *rbase;
+	int cpu;
+
+	if (!gic_data.gicd_ctx || !gic_data.gicr_ctx)
+		return 0;
+
+	for_each_possible_cpu(cpu) {
+		/*
+		 * The current processor will have the redistributor disabled in
+		 * the next step.
+		 */
+		if (cpu == smp_processor_id())
+			continue;
+
+		/* Each redistributor must be disabled to save state. */
+		rbase = per_cpu_ptr(gic_data.rdists.rdist, cpu)->rd_base;
+		if (!(readl_relaxed(rbase + GICR_WAKER) &
+		     GICR_WAKER_ProcessorSleep)) {
+			pr_err("Redistributor for CPU: %d enabled \n", cpu);
+			return -EPERM;
+		}
+	}
+
+	/* Disable the redist in case it wasn't in CPU_PM_ENTER. */
+	gic_enable_redist(false);
+	for_each_possible_cpu(cpu)
+		gic_redist_save(cpu);
+
+	its_save_disable();
+	gic_dist_save();
+
+	return 0;
+}
+
+static void gic_rdist_restore(int cpu)
+{
+	struct gic_redist_ctx *ctx = &gic_data.gicr_ctx[cpu];
+	void __iomem *base = per_cpu_ptr(gic_data.rdists.rdist, cpu)->rd_base;
+	void __iomem *sgi_base = base + GICR_SGI_BASE_OFFSET;
+
+	writel_relaxed(ctx->ctlr & ~(GICR_CTLR_ENABLE_LPIS), base + GICR_CTLR);
+	gic_do_wait_for_rwp(base);
+	writeq_relaxed(ctx->propbaser, base + GICR_PROPBASER);
+	writeq_relaxed(ctx->pendbaser, base + GICR_PENDBASER);
+
+	writel_relaxed(ctx->igroupr0, sgi_base + GICR_IGROUPR0);
+	writel_relaxed(ctx->isenabler0, sgi_base + GICR_ISENABLER0);
+	writel_relaxed(ctx->ispendr0, sgi_base + GICR_ISPENDR0);
+	writel_relaxed(ctx->isactiver0, sgi_base + GICR_ISACTIVER0);
+	__iowrite32_copy(sgi_base + GICR_IPRIORITYR0, ctx->ipriorityr, 8);
+	writel_relaxed(ctx->icfgr0, sgi_base + GICR_ICFGR0);
+	writel_relaxed(ctx->icfgr1, sgi_base + GICR_ICFGR0 + sizeof(u32));
+	writel_relaxed(ctx->igrpmodr0, sgi_base + GICR_IGRPMODR0);
+	writel_relaxed(ctx->nsacr, sgi_base + GICR_NSACR);
+	writel_relaxed(ctx->ctlr, sgi_base + GICR_CTLR);
+	gic_do_wait_for_rwp(base);
+}
+
+static void gic_dist_restore(void)
+{
+	struct gic_dist_ctx *ctx = gic_data.gicd_ctx;
+	void __iomem *base = gic_data.dist_base;
+	int irq_nr = gic_data.irq_nr;
+
+	gic_dist_wait_for_rwp();
+	__iowrite64_copy(base + GICD_IROUTER, ctx->irouter,
+			 GICD_NR_REGS(IROUTER, irq_nr));
+	__iowrite32_copy(base + GICD_IGROUPR, ctx->igroupr,
+			 GICD_NR_REGS(IGROUPR, irq_nr));
+	__iowrite32_copy(base + GICD_ISENABLER, ctx->isenabler,
+			 GICD_NR_REGS(ISENABLER, irq_nr));
+	__iowrite32_copy(base + GICD_ISPENDR, ctx->ispendr,
+			 GICD_NR_REGS(ISPENDR, irq_nr));
+	__iowrite32_copy(base + GICD_ISACTIVER, ctx->isactiver,
+			 GICD_NR_REGS(ISACTIVER, irq_nr));
+	__iowrite32_copy(base + GICD_ICFGR, ctx->icfgr,
+			 GICD_NR_REGS(ICFGR, irq_nr));
+	__iowrite32_copy(base + GICD_IGRPMODR, ctx->igrpmodr,
+			 GICD_NR_REGS(IGRPMODR, irq_nr));
+	__iowrite32_copy(base + GICD_NSACR, ctx->nsacr,
+			 GICD_NR_REGS(NSACR, irq_nr));
+	writel_relaxed(ctx->ctlr, base + GICD_CTLR);
+	gic_dist_wait_for_rwp();
+}
+
+static void gic_resume(void)
+{
+	int cpu;
+
+	if (!gic_data.gicd_ctx || !gic_data.gicr_ctx)
+		return;
+
+	gic_dist_restore();
+	its_restore_enable();
+	for_each_possible_cpu(cpu)
+		gic_rdist_restore(cpu);
+
+	gic_enable_redist(true);
+}
+
 #ifdef CONFIG_CPU_PM
 /* Check whether it's single security state view */
 static bool gic_dist_security_disabled(void)
@@ -767,13 +953,24 @@ static bool gic_dist_security_disabled(void)
 static int gic_cpu_pm_notifier(struct notifier_block *self,
 			       unsigned long cmd, void *v)
 {
-	if (cmd == CPU_PM_EXIT) {
+	switch (cmd) {
+	case CPU_PM_EXIT:
 		if (gic_dist_security_disabled())
 			gic_enable_redist(true);
 		gic_cpu_sys_reg_init();
-	} else if (cmd == CPU_PM_ENTER && gic_dist_security_disabled()) {
-		gic_write_grpen1(0);
-		gic_enable_redist(false);
+		break;
+	case CPU_PM_ENTER:
+		if (gic_dist_security_disabled()) {
+			gic_write_grpen1(0);
+			gic_enable_redist(false);
+		}
+		break;
+	case CPU_SYSTEM_PM_EXIT:
+		gic_resume();
+		break;
+	case CPU_SYSTEM_PM_ENTER:
+		gic_suspend();
+		break;
 	}
 	return NOTIFY_OK;
 }
@@ -991,11 +1188,99 @@ static const struct irq_domain_ops partition_domain_ops = {
 	.select = gic_irq_domain_select,
 };
 
+static int gic_populate_ctx(struct gic_dist_ctx *ctx, int irqs)
+{
+	int err;
+
+	ctx->irouter = kcalloc(GICD_NR_REGS(IROUTER, irqs),
+			       sizeof(*ctx->irouter), GFP_KERNEL);
+	if (IS_ERR(ctx->irouter))
+		return PTR_ERR(ctx->irouter);
+
+	ctx->igroupr = kcalloc(GICD_NR_REGS(IGROUPR, irqs),
+			       sizeof(*ctx->igroupr), GFP_KERNEL);
+	if (IS_ERR(ctx->igroupr)) {
+		err = PTR_ERR(ctx->igroupr);
+		goto out_irouter;
+	}
+
+	ctx->isenabler = kcalloc(GICD_NR_REGS(ISENABLER, irqs),
+				 sizeof(*ctx->isenabler), GFP_KERNEL);
+	if (IS_ERR(ctx->isenabler)) {
+		err = PTR_ERR(ctx->isenabler);
+		goto out_igroupr;
+	}
+
+	ctx->ispendr = kcalloc(GICD_NR_REGS(ISPENDR, irqs),
+			       sizeof(*ctx->ispendr), GFP_KERNEL);
+	if (IS_ERR(ctx->ispendr)) {
+		err = PTR_ERR(ctx->ispendr);
+		goto out_isenabler;
+	}
+
+	ctx->isactiver = kcalloc(GICD_NR_REGS(ISACTIVER, irqs),
+				 sizeof(*ctx->isactiver), GFP_KERNEL);
+	if (IS_ERR(ctx->isactiver)) {
+		err = PTR_ERR(ctx->isactiver);
+		goto out_ispendr;
+	}
+
+	ctx->ipriorityr = kcalloc(GICD_NR_REGS(IPRIORITYR, irqs),
+				  sizeof(*ctx->ipriorityr), GFP_KERNEL);
+	if (IS_ERR(ctx->ipriorityr)) {
+		err = PTR_ERR(ctx->ipriorityr);
+		goto out_isactiver;
+	}
+
+	ctx->icfgr = kcalloc(GICD_NR_REGS(ICFGR, irqs), sizeof(*ctx->icfgr),
+			     GFP_KERNEL);
+	if (IS_ERR(ctx->icfgr)) {
+		err = PTR_ERR(ctx->icfgr);
+		goto out_ipriorityr;
+	}
+
+	ctx->igrpmodr = kcalloc(GICD_NR_REGS(IGRPMODR, irqs),
+				sizeof(*ctx->igrpmodr), GFP_KERNEL);
+	if (IS_ERR(ctx->igrpmodr)) {
+		err = PTR_ERR(ctx->igrpmodr);
+		goto out_icfgr;
+	}
+
+	ctx->nsacr = kcalloc(GICD_NR_REGS(NSACR, irqs), sizeof(*ctx->nsacr),
+			     GFP_KERNEL);
+	if (IS_ERR(ctx->nsacr)) {
+		err = PTR_ERR(ctx->nsacr);
+		goto out_igrpmodr;
+	}
+
+	return 0;
+
+out_irouter:
+	kfree(ctx->irouter);
+out_igroupr:
+	kfree(ctx->igroupr);
+out_isenabler:
+	kfree(ctx->isenabler);
+out_ispendr:
+	kfree(ctx->ispendr);
+out_isactiver:
+	kfree(ctx->isactiver);
+out_ipriorityr:
+	kfree(ctx->ipriorityr);
+out_icfgr:
+	kfree(ctx->icfgr);
+out_igrpmodr:
+	kfree(ctx->igrpmodr);
+	return err;
+}
+
 static int __init gic_init_bases(void __iomem *dist_base,
 				 struct redist_region *rdist_regs,
 				 u32 nr_redist_regions,
 				 u64 redist_stride,
-				 struct fwnode_handle *handle)
+				 struct fwnode_handle *handle,
+				 struct gic_dist_ctx *gicd_ctx,
+				 struct gic_redist_ctx *gicr_ctx)
 {
 	u32 typer;
 	int gic_irqs;
@@ -1012,6 +1297,8 @@ static int __init gic_init_bases(void __iomem *dist_base,
 	gic_data.redist_regions = rdist_regs;
 	gic_data.nr_redist_regions = nr_redist_regions;
 	gic_data.redist_stride = redist_stride;
+	gic_data.gicd_ctx = gicd_ctx;
+	gic_data.gicr_ctx = gicr_ctx;
 
 	/*
 	 * Find out how many interrupts are supported.
@@ -1035,6 +1322,12 @@ static int __init gic_init_bases(void __iomem *dist_base,
 		goto out_free;
 	}
 
+	if (gicd_ctx) {
+		err = gic_populate_ctx(gicd_ctx, gic_irqs);
+		if (err)
+			goto out_free;
+	}
+
 	gic_data.has_rss = !!(typer & GICD_TYPER_RSS);
 	pr_info("Distributor has %sRange Selector support\n",
 		gic_data.has_rss ? "" : "no ");
@@ -1190,6 +1483,8 @@ static int __init gic_of_init(struct device_node *node, struct device_node *pare
 {
 	void __iomem *dist_base;
 	struct redist_region *rdist_regs;
+	struct gic_dist_ctx *gicd_ctx = NULL;
+	struct gic_redist_ctx *gicr_ctx = NULL;
 	u64 redist_stride;
 	u32 nr_redist_regions;
 	int err, i;
@@ -1232,17 +1527,35 @@ static int __init gic_of_init(struct device_node *node, struct device_node *pare
 	if (of_property_read_u64(node, "redistributor-stride", &redist_stride))
 		redist_stride = 0;
 
+	if (of_property_read_bool(node, "save-suspend-state")) {
+		gicd_ctx = kzalloc(sizeof(*gicd_ctx), GFP_KERNEL);
+		if (IS_ERR(gicd_ctx)) {
+			err = PTR_ERR(gicd_ctx);
+			goto out_unmap_rdist;
+		}
+
+		gicr_ctx = kcalloc(num_possible_cpus(), sizeof(*gicr_ctx),
+				   GFP_KERNEL);
+		if (IS_ERR(gicr_ctx)) {
+			err = PTR_ERR(gicr_ctx);
+			goto out_free_gicd_ctx;
+		}
+	}
+
 	err = gic_init_bases(dist_base, rdist_regs, nr_redist_regions,
-			     redist_stride, &node->fwnode);
+			     redist_stride, &node->fwnode, gicd_ctx, gicr_ctx);
 	if (err)
-		goto out_unmap_rdist;
+		goto out_free_gicr_ctx;
 
 	gic_populate_ppi_partitions(node);
 
 	if (static_key_true(&supports_deactivate))
 		gic_of_setup_kvm_info(node);
 	return 0;
-
+out_free_gicr_ctx:
+	kfree(gicr_ctx);
+out_free_gicd_ctx:
+	kfree(gicd_ctx);
 out_unmap_rdist:
 	for (i = 0; i < nr_redist_regions; i++)
 		if (rdist_regs[i].redist_base)
diff --git a/include/linux/irqchip/arm-gic-v3.h b/include/linux/irqchip/arm-gic-v3.h
index c00c4c33e432..f086987e3cb4 100644
--- a/include/linux/irqchip/arm-gic-v3.h
+++ b/include/linux/irqchip/arm-gic-v3.h
@@ -46,6 +46,19 @@
 #define GICD_IDREGS			0xFFD0
 #define GICD_PIDR2			0xFFE8
 
+#define GICD_IGROUPR_SHIFT		5
+#define GICD_ISENABLER_SHIFT		5
+#define GICD_ICENABLER_SHIFT		GICD_ISENABLER_SHIFT
+#define GICD_ISPENDR_SHIFT		5
+#define GICD_ICPENDR_SHIFT		GICD_ISPENDR_SHIFT
+#define GICD_ISACTIVER_SHIFT		5
+#define GICD_ICACTIVER_SHIFT		GICD_ISACTIVER_SHIFT
+#define GICD_IPRIORITYR_SHIFT		2
+#define GICD_ICFGR_SHIFT		4
+#define GICD_IGRPMODR_SHIFT		5
+#define GICD_NSACR_SHIFT		4
+#define GICD_IROUTER_SHIFT		0
+
 /*
  * Those registers are actually from GICv2, but the spec demands that they
  * are implemented as RES0 if ARE is 1 (which we do in KVM's emulated GICv3).
@@ -188,6 +201,8 @@
 
 #define GICR_PENDBASER_PTZ				BIT_ULL(62)
 
+#define GICR_SGI_BASE_OFFSET		(1U << 16)
+
 /*
  * Re-Distributor registers, offsets from SGI_base
  */
@@ -581,6 +596,8 @@ struct rdists {
 
 struct irq_domain;
 struct fwnode_handle;
+void its_save_disable(void);
+void its_restore_enable(void);
 int its_cpu_init(void);
 int its_init(struct fwnode_handle *handle, struct rdists *rdists,
 	     struct irq_domain *domain);
-- 
2.16.0.rc1.238.g530d649a79-goog