[PATCH v10 0/4] dtc: Dynamic DT support

[PATCH v10 0/4] dtc: Dynamic DT support

[Pantelis Antoniou]

This patchset adds Dynamic DT support in the DTC compiler
as used in a number of boards like the beaglebone/rpi/chip and others.

The first patch passes a boot_info argument instead of the root node
to the check methods.

The second patch documents the internals of overlay generation, while
the third one adds dynamic object/overlay support proper.

The last patch simply adds a few overlay tests verifying operation.

This patchset is against DTC mainline and is also available for a pull
request from https://github.com/pantoniou/dtc/tree/overlays

Regards

-- Pantelis

Changes since v9:
* Reversed -M switch to by default use new DTBO magic value.
* Removed global versionflags in the parser by using inherited
attributes.
* build_node instead of malloc at add_orphan_node().
* Do not use escape for path copy
* Do not generate /plugin/ when generating a dts file even when
the plugin flag is set..

Changes since v8:
* Removed extra member of boot_info in each node; passing boot_info
parameter to the check methods instead.
* Reworked yacc syntax that supports both old and new plugin syntax
* Added handling for new magic number (enabled by 'M' switch).
* Dropped dtbo/asmo formats.
* Added overlay testsuite.
* Addressed last version maintainer comments.

Changes since v7:
* Dropped xasprintf & backward compatibility patch
* Rebased against dgibson's overlay branch
* Minor doc wording fixes.

Changes since v6:
* Introduced xasprintf
* Added append_to_property and used it
* Changed some die()'s to assert
* Reordered node generation to respect sort
* Addressed remaining maintainer changes from v6

Changes since v5:
* Rebase to latest dtc version.
* Addressed all the maintainer requested changes from v5
* Added new magic value for dynamic objects and new format

Changes since v4:
* Rebase to latest dtc version.
* Completely redesigned the generation of resolution data.
Now instead of being generated as part of blob generation
they are created in the live tree.
* Consequently the patchset is much smaller.
* Added -A auto-label alias generation option.
* Addressed maintainer comments.
* Added syntactic sugar for overlays in the form of .dtsi
* Added /dts-v1/ /plugin/ preferred plugin form and deprecate
the previous form (although still works for backward compatibility)

Changes since v3:
* Rebase to latest dtc version.

Changes since v2:
* Split single patch to a patchset.
* Updated to dtc mainline.
* Changed __local_fixups__ format
* Clean up for better legibility.

Pantelis Antoniou (4):
  checks: Pass boot_info instead of root node
  dtc: Document the dynamic plugin internals
  dtc: Plugin and fixup support
  tests: Add overlay tests

 Documentation/dt-object-internal.txt | 318 +++++++++++++++++++++++++++++++++++
 Documentation/manual.txt             |  25 ++-
 checks.c                             |  86 +++++-----
 dtc-lexer.l                          |   5 +
 dtc-parser.y                         |  50 +++++-
 dtc.c                                |  39 ++++-
 dtc.h                                |  20 ++-
 fdtdump.c                            |   2 +-
 flattree.c                           |  17 +-
 fstree.c                             |   2 +-
 libfdt/fdt.c                         |   2 +-
 libfdt/fdt.h                         |   3 +-
 livetree.c                           | 225 ++++++++++++++++++++++++-
 tests/mangle-layout.c                |   7 +-
 tests/overlay_overlay_dtc.dts        |  76 +--------
 tests/overlay_overlay_dtc.dtsi       |  83 +++++++++
 tests/overlay_overlay_new_dtc.dts    |  11 ++
 tests/overlay_overlay_simple.dts     |  12 ++
 tests/run_tests.sh                   |  20 +++
 19 files changed, 862 insertions(+), 141 deletions(-)
 create mode 100644 Documentation/dt-object-internal.txt
 create mode 100644 tests/overlay_overlay_dtc.dtsi
 create mode 100644 tests/overlay_overlay_new_dtc.dts
 create mode 100644 tests/overlay_overlay_simple.dts

-- 
2.1.4

[PATCH v10 1/4] checks: Pass boot_info instead of root node

[Pantelis Antoniou]

As preparation for overlay support we need to pass the boot info
parameter instead of the root node to each check method.

The root node can be retrieved by accessing boot info's dt member.

No other functional changes are made.

Signed-off-by: Pantelis Antoniou <pantelis.antoniou-OWPKS81ov/FWk0Htik3J/w@public.gmane.org>
---
 checks.c | 78 ++++++++++++++++++++++++++++++++++------------------------------
 1 file changed, 42 insertions(+), 36 deletions(-)

diff --git a/checks.c b/checks.c
index 0381c98..2bd27a4 100644
--- a/checks.c
+++ b/checks.c
@@ -40,7 +40,7 @@ enum checkstatus {
 
 struct check;
 
-typedef void (*check_fn)(struct check *c, struct node *dt, struct node *node);
+typedef void (*check_fn)(struct check *c, struct boot_info *bi, struct node *node);
 
 struct check {
 	const char *name;
@@ -97,20 +97,21 @@ static inline void check_msg(struct check *c, const char *fmt, ...)
 		check_msg((c), __VA_ARGS__); \
 	} while (0)
 
-static void check_nodes_props(struct check *c, struct node *dt, struct node *node)
+static void check_nodes_props(struct check *c, struct boot_info *bi, struct node *node)
 {
 	struct node *child;
 
 	TRACE(c, "%s", node->fullpath);
 	if (c->fn)
-		c->fn(c, dt, node);
+		c->fn(c, bi, node);
 
 	for_each_child(node, child)
-		check_nodes_props(c, dt, child);
+		check_nodes_props(c, bi, child);
 }
 
-static bool run_check(struct check *c, struct node *dt)
+static bool run_check(struct check *c, struct boot_info *bi)
 {
+	struct node *dt = bi->dt;
 	bool error = false;
 	int i;
 
@@ -123,7 +124,7 @@ static bool run_check(struct check *c, struct node *dt)
 
 	for (i = 0; i < c->num_prereqs; i++) {
 		struct check *prq = c->prereq[i];
-		error = error || run_check(prq, dt);
+		error = error || run_check(prq, bi);
 		if (prq->status != PASSED) {
 			c->status = PREREQ;
 			check_msg(c, "Failed prerequisite '%s'",
@@ -134,7 +135,7 @@ static bool run_check(struct check *c, struct node *dt)
 	if (c->status != UNCHECKED)
 		goto out;
 
-	check_nodes_props(c, dt, dt);
+	check_nodes_props(c, bi, dt);
 
 	if (c->status == UNCHECKED)
 		c->status = PASSED;
@@ -153,14 +154,14 @@ out:
  */
 
 /* A check which always fails, for testing purposes only */
-static inline void check_always_fail(struct check *c, struct node *dt,
+static inline void check_always_fail(struct check *c, struct boot_info *bi,
 				     struct node *node)
 {
 	FAIL(c, "always_fail check");
 }
 CHECK(always_fail, check_always_fail, NULL);
 
-static void check_is_string(struct check *c, struct node *root,
+static void check_is_string(struct check *c, struct boot_info *bi,
 			    struct node *node)
 {
 	struct property *prop;
@@ -179,7 +180,7 @@ static void check_is_string(struct check *c, struct node *root,
 #define ERROR_IF_NOT_STRING(nm, propname) \
 	ERROR(nm, check_is_string, (propname))
 
-static void check_is_cell(struct check *c, struct node *root,
+static void check_is_cell(struct check *c, struct boot_info *bi,
 			  struct node *node)
 {
 	struct property *prop;
@@ -202,7 +203,7 @@ static void check_is_cell(struct check *c, struct node *root,
  * Structural check functions
  */
 
-static void check_duplicate_node_names(struct check *c, struct node *dt,
+static void check_duplicate_node_names(struct check *c, struct boot_info *bi,
 				       struct node *node)
 {
 	struct node *child, *child2;
@@ -217,7 +218,7 @@ static void check_duplicate_node_names(struct check *c, struct node *dt,
 }
 ERROR(duplicate_node_names, check_duplicate_node_names, NULL);
 
-static void check_duplicate_property_names(struct check *c, struct node *dt,
+static void check_duplicate_property_names(struct check *c, struct boot_info *bi,
 					   struct node *node)
 {
 	struct property *prop, *prop2;
@@ -239,7 +240,7 @@ ERROR(duplicate_property_names, check_duplicate_property_names, NULL);
 #define DIGITS		"0123456789"
 #define PROPNODECHARS	LOWERCASE UPPERCASE DIGITS ",._+*#?-"
 
-static void check_node_name_chars(struct check *c, struct node *dt,
+static void check_node_name_chars(struct check *c, struct boot_info *bi,
 				  struct node *node)
 {
 	int n = strspn(node->name, c->data);
@@ -250,7 +251,7 @@ static void check_node_name_chars(struct check *c, struct node *dt,
 }
 ERROR(node_name_chars, check_node_name_chars, PROPNODECHARS "@");
 
-static void check_node_name_format(struct check *c, struct node *dt,
+static void check_node_name_format(struct check *c, struct boot_info *bi,
 				   struct node *node)
 {
 	if (strchr(get_unitname(node), '@'))
@@ -259,8 +260,8 @@ static void check_node_name_format(struct check *c, struct node *dt,
 }
 ERROR(node_name_format, check_node_name_format, NULL, &node_name_chars);
 
-static void check_unit_address_vs_reg(struct check *c, struct node *dt,
-			     struct node *node)
+static void check_unit_address_vs_reg(struct check *c, struct boot_info *bi,
+				      struct node *node)
 {
 	const char *unitname = get_unitname(node);
 	struct property *prop = get_property(node, "reg");
@@ -283,7 +284,7 @@ static void check_unit_address_vs_reg(struct check *c, struct node *dt,
 }
 WARNING(unit_address_vs_reg, check_unit_address_vs_reg, NULL);
 
-static void check_property_name_chars(struct check *c, struct node *dt,
+static void check_property_name_chars(struct check *c, struct boot_info *bi,
 				      struct node *node)
 {
 	struct property *prop;
@@ -305,10 +306,11 @@ ERROR(property_name_chars, check_property_name_chars, PROPNODECHARS);
 	((prop) ? (prop)->name : ""), \
 	((prop) ? "' in " : ""), (node)->fullpath
 
-static void check_duplicate_label(struct check *c, struct node *dt,
+static void check_duplicate_label(struct check *c, struct boot_info *bi,
 				  const char *label, struct node *node,
 				  struct property *prop, struct marker *mark)
 {
+	struct node *dt = bi->dt;
 	struct node *othernode = NULL;
 	struct property *otherprop = NULL;
 	struct marker *othermark = NULL;
@@ -331,30 +333,31 @@ static void check_duplicate_label(struct check *c, struct node *dt,
 		     DESCLABEL_ARGS(othernode, otherprop, othermark));
 }
 
-static void check_duplicate_label_node(struct check *c, struct node *dt,
+static void check_duplicate_label_node(struct check *c, struct boot_info *bi,
 				       struct node *node)
 {
 	struct label *l;
 	struct property *prop;
 
 	for_each_label(node->labels, l)
-		check_duplicate_label(c, dt, l->label, node, NULL, NULL);
+		check_duplicate_label(c, bi, l->label, node, NULL, NULL);
 
 	for_each_property(node, prop) {
 		struct marker *m = prop->val.markers;
 
 		for_each_label(prop->labels, l)
-			check_duplicate_label(c, dt, l->label, node, prop, NULL);
+			check_duplicate_label(c, bi, l->label, node, prop, NULL);
 
 		for_each_marker_of_type(m, LABEL)
-			check_duplicate_label(c, dt, m->ref, node, prop, m);
+			check_duplicate_label(c, bi, m->ref, node, prop, m);
 	}
 }
 ERROR(duplicate_label, check_duplicate_label_node, NULL);
 
-static cell_t check_phandle_prop(struct check *c, struct node *root,
+static cell_t check_phandle_prop(struct check *c, struct boot_info *bi,
 				 struct node *node, const char *propname)
 {
+	struct node *root = bi->dt;
 	struct property *prop;
 	struct marker *m;
 	cell_t phandle;
@@ -398,18 +401,19 @@ static cell_t check_phandle_prop(struct check *c, struct node *root,
 	return phandle;
 }
 
-static void check_explicit_phandles(struct check *c, struct node *root,
+static void check_explicit_phandles(struct check *c, struct boot_info *bi,
 				    struct node *node)
 {
+	struct node *root = bi->dt;
 	struct node *other;
 	cell_t phandle, linux_phandle;
 
 	/* Nothing should have assigned phandles yet */
 	assert(!node->phandle);
 
-	phandle = check_phandle_prop(c, root, node, "phandle");
+	phandle = check_phandle_prop(c, bi, node, "phandle");
 
-	linux_phandle = check_phandle_prop(c, root, node, "linux,phandle");
+	linux_phandle = check_phandle_prop(c, bi, node, "linux,phandle");
 
 	if (!phandle && !linux_phandle)
 		/* No valid phandles; nothing further to check */
@@ -433,7 +437,7 @@ static void check_explicit_phandles(struct check *c, struct node *root,
 }
 ERROR(explicit_phandles, check_explicit_phandles, NULL);
 
-static void check_name_properties(struct check *c, struct node *root,
+static void check_name_properties(struct check *c, struct boot_info *bi,
 				  struct node *node)
 {
 	struct property **pp, *prop = NULL;
@@ -467,9 +471,10 @@ ERROR(name_properties, check_name_properties, NULL, &name_is_string);
  * Reference fixup functions
  */
 
-static void fixup_phandle_references(struct check *c, struct node *dt,
+static void fixup_phandle_references(struct check *c, struct boot_info *bi,
 				     struct node *node)
 {
+	struct node *dt = bi->dt;
 	struct property *prop;
 
 	for_each_property(node, prop) {
@@ -495,9 +500,10 @@ static void fixup_phandle_references(struct check *c, struct node *dt,
 ERROR(phandle_references, fixup_phandle_references, NULL,
       &duplicate_node_names, &explicit_phandles);
 
-static void fixup_path_references(struct check *c, struct node *dt,
+static void fixup_path_references(struct check *c, struct boot_info *bi,
 				  struct node *node)
 {
+	struct node *dt = bi->dt;
 	struct property *prop;
 
 	for_each_property(node, prop) {
@@ -534,7 +540,7 @@ WARNING_IF_NOT_STRING(device_type_is_string, "device_type");
 WARNING_IF_NOT_STRING(model_is_string, "model");
 WARNING_IF_NOT_STRING(status_is_string, "status");
 
-static void fixup_addr_size_cells(struct check *c, struct node *dt,
+static void fixup_addr_size_cells(struct check *c, struct boot_info *bi,
 				  struct node *node)
 {
 	struct property *prop;
@@ -558,7 +564,7 @@ WARNING(addr_size_cells, fixup_addr_size_cells, NULL,
 #define node_size_cells(n) \
 	(((n)->size_cells == -1) ? 1 : (n)->size_cells)
 
-static void check_reg_format(struct check *c, struct node *dt,
+static void check_reg_format(struct check *c, struct boot_info *bi,
 			     struct node *node)
 {
 	struct property *prop;
@@ -587,7 +593,7 @@ static void check_reg_format(struct check *c, struct node *dt,
 }
 WARNING(reg_format, check_reg_format, NULL, &addr_size_cells);
 
-static void check_ranges_format(struct check *c, struct node *dt,
+static void check_ranges_format(struct check *c, struct boot_info *bi,
 				struct node *node)
 {
 	struct property *prop;
@@ -631,7 +637,7 @@ WARNING(ranges_format, check_ranges_format, NULL, &addr_size_cells);
 /*
  * Style checks
  */
-static void check_avoid_default_addr_size(struct check *c, struct node *dt,
+static void check_avoid_default_addr_size(struct check *c, struct boot_info *bi,
 					  struct node *node)
 {
 	struct property *reg, *ranges;
@@ -657,9 +663,10 @@ WARNING(avoid_default_addr_size, check_avoid_default_addr_size, NULL,
 	&addr_size_cells);
 
 static void check_obsolete_chosen_interrupt_controller(struct check *c,
-						       struct node *dt,
+						       struct boot_info *bi,
 						       struct node *node)
 {
+	struct node *dt = bi->dt;
 	struct node *chosen;
 	struct property *prop;
 
@@ -765,7 +772,6 @@ void parse_checks_option(bool warn, bool error, const char *arg)
 
 void process_checks(bool force, struct boot_info *bi)
 {
-	struct node *dt = bi->dt;
 	int i;
 	int error = 0;
 
@@ -773,7 +779,7 @@ void process_checks(bool force, struct boot_info *bi)
 		struct check *c = check_table[i];
 
 		if (c->warn || c->error)
-			error = error || run_check(c, dt);
+			error = error || run_check(c, bi);
 	}
 
 	if (error) {
-- 
2.1.4

--
To unsubscribe from this list: send the line "unsubscribe devicetree" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v10 1/4] checks: Pass boot_info instead of root node

[David Gibson]

[-- Attachment #1: Type: text/plain, Size: 12923 bytes --]

On Fri, Nov 25, 2016 at 02:32:08PM +0200, Pantelis Antoniou wrote:
> As preparation for overlay support we need to pass the boot info
> parameter instead of the root node to each check method.
> 
> The root node can be retrieved by accessing boot info's dt member.
> 
> No other functional changes are made.
> 
> Signed-off-by: Pantelis Antoniou <pantelis.antoniou-OWPKS81ov/FWk0Htik3J/w@public.gmane.org>

I've applied this preliminary patch to the master branch.  I think
I'll also apply (an updated version of) my patch to rename 'boot_info'
to something a bit less silly.  But I'll hold off for now to avoid
creating extra conflicts with your patches.

> ---
>  checks.c | 78 ++++++++++++++++++++++++++++++++++------------------------------
>  1 file changed, 42 insertions(+), 36 deletions(-)
> 
> diff --git a/checks.c b/checks.c
> index 0381c98..2bd27a4 100644
> --- a/checks.c
> +++ b/checks.c
> @@ -40,7 +40,7 @@ enum checkstatus {
>  
>  struct check;
>  
> -typedef void (*check_fn)(struct check *c, struct node *dt, struct node *node);
> +typedef void (*check_fn)(struct check *c, struct boot_info *bi, struct node *node);
>  
>  struct check {
>  	const char *name;
> @@ -97,20 +97,21 @@ static inline void check_msg(struct check *c, const char *fmt, ...)
>  		check_msg((c), __VA_ARGS__); \
>  	} while (0)
>  
> -static void check_nodes_props(struct check *c, struct node *dt, struct node *node)
> +static void check_nodes_props(struct check *c, struct boot_info *bi, struct node *node)
>  {
>  	struct node *child;
>  
>  	TRACE(c, "%s", node->fullpath);
>  	if (c->fn)
> -		c->fn(c, dt, node);
> +		c->fn(c, bi, node);
>  
>  	for_each_child(node, child)
> -		check_nodes_props(c, dt, child);
> +		check_nodes_props(c, bi, child);
>  }
>  
> -static bool run_check(struct check *c, struct node *dt)
> +static bool run_check(struct check *c, struct boot_info *bi)
>  {
> +	struct node *dt = bi->dt;
>  	bool error = false;
>  	int i;
>  
> @@ -123,7 +124,7 @@ static bool run_check(struct check *c, struct node *dt)
>  
>  	for (i = 0; i < c->num_prereqs; i++) {
>  		struct check *prq = c->prereq[i];
> -		error = error || run_check(prq, dt);
> +		error = error || run_check(prq, bi);
>  		if (prq->status != PASSED) {
>  			c->status = PREREQ;
>  			check_msg(c, "Failed prerequisite '%s'",
> @@ -134,7 +135,7 @@ static bool run_check(struct check *c, struct node *dt)
>  	if (c->status != UNCHECKED)
>  		goto out;
>  
> -	check_nodes_props(c, dt, dt);
> +	check_nodes_props(c, bi, dt);
>  
>  	if (c->status == UNCHECKED)
>  		c->status = PASSED;
> @@ -153,14 +154,14 @@ out:
>   */
>  
>  /* A check which always fails, for testing purposes only */
> -static inline void check_always_fail(struct check *c, struct node *dt,
> +static inline void check_always_fail(struct check *c, struct boot_info *bi,
>  				     struct node *node)
>  {
>  	FAIL(c, "always_fail check");
>  }
>  CHECK(always_fail, check_always_fail, NULL);
>  
> -static void check_is_string(struct check *c, struct node *root,
> +static void check_is_string(struct check *c, struct boot_info *bi,
>  			    struct node *node)
>  {
>  	struct property *prop;
> @@ -179,7 +180,7 @@ static void check_is_string(struct check *c, struct node *root,
>  #define ERROR_IF_NOT_STRING(nm, propname) \
>  	ERROR(nm, check_is_string, (propname))
>  
> -static void check_is_cell(struct check *c, struct node *root,
> +static void check_is_cell(struct check *c, struct boot_info *bi,
>  			  struct node *node)
>  {
>  	struct property *prop;
> @@ -202,7 +203,7 @@ static void check_is_cell(struct check *c, struct node *root,
>   * Structural check functions
>   */
>  
> -static void check_duplicate_node_names(struct check *c, struct node *dt,
> +static void check_duplicate_node_names(struct check *c, struct boot_info *bi,
>  				       struct node *node)
>  {
>  	struct node *child, *child2;
> @@ -217,7 +218,7 @@ static void check_duplicate_node_names(struct check *c, struct node *dt,
>  }
>  ERROR(duplicate_node_names, check_duplicate_node_names, NULL);
>  
> -static void check_duplicate_property_names(struct check *c, struct node *dt,
> +static void check_duplicate_property_names(struct check *c, struct boot_info *bi,
>  					   struct node *node)
>  {
>  	struct property *prop, *prop2;
> @@ -239,7 +240,7 @@ ERROR(duplicate_property_names, check_duplicate_property_names, NULL);
>  #define DIGITS		"0123456789"
>  #define PROPNODECHARS	LOWERCASE UPPERCASE DIGITS ",._+*#?-"
>  
> -static void check_node_name_chars(struct check *c, struct node *dt,
> +static void check_node_name_chars(struct check *c, struct boot_info *bi,
>  				  struct node *node)
>  {
>  	int n = strspn(node->name, c->data);
> @@ -250,7 +251,7 @@ static void check_node_name_chars(struct check *c, struct node *dt,
>  }
>  ERROR(node_name_chars, check_node_name_chars, PROPNODECHARS "@");
>  
> -static void check_node_name_format(struct check *c, struct node *dt,
> +static void check_node_name_format(struct check *c, struct boot_info *bi,
>  				   struct node *node)
>  {
>  	if (strchr(get_unitname(node), '@'))
> @@ -259,8 +260,8 @@ static void check_node_name_format(struct check *c, struct node *dt,
>  }
>  ERROR(node_name_format, check_node_name_format, NULL, &node_name_chars);
>  
> -static void check_unit_address_vs_reg(struct check *c, struct node *dt,
> -			     struct node *node)
> +static void check_unit_address_vs_reg(struct check *c, struct boot_info *bi,
> +				      struct node *node)
>  {
>  	const char *unitname = get_unitname(node);
>  	struct property *prop = get_property(node, "reg");
> @@ -283,7 +284,7 @@ static void check_unit_address_vs_reg(struct check *c, struct node *dt,
>  }
>  WARNING(unit_address_vs_reg, check_unit_address_vs_reg, NULL);
>  
> -static void check_property_name_chars(struct check *c, struct node *dt,
> +static void check_property_name_chars(struct check *c, struct boot_info *bi,
>  				      struct node *node)
>  {
>  	struct property *prop;
> @@ -305,10 +306,11 @@ ERROR(property_name_chars, check_property_name_chars, PROPNODECHARS);
>  	((prop) ? (prop)->name : ""), \
>  	((prop) ? "' in " : ""), (node)->fullpath
>  
> -static void check_duplicate_label(struct check *c, struct node *dt,
> +static void check_duplicate_label(struct check *c, struct boot_info *bi,
>  				  const char *label, struct node *node,
>  				  struct property *prop, struct marker *mark)
>  {
> +	struct node *dt = bi->dt;
>  	struct node *othernode = NULL;
>  	struct property *otherprop = NULL;
>  	struct marker *othermark = NULL;
> @@ -331,30 +333,31 @@ static void check_duplicate_label(struct check *c, struct node *dt,
>  		     DESCLABEL_ARGS(othernode, otherprop, othermark));
>  }
>  
> -static void check_duplicate_label_node(struct check *c, struct node *dt,
> +static void check_duplicate_label_node(struct check *c, struct boot_info *bi,
>  				       struct node *node)
>  {
>  	struct label *l;
>  	struct property *prop;
>  
>  	for_each_label(node->labels, l)
> -		check_duplicate_label(c, dt, l->label, node, NULL, NULL);
> +		check_duplicate_label(c, bi, l->label, node, NULL, NULL);
>  
>  	for_each_property(node, prop) {
>  		struct marker *m = prop->val.markers;
>  
>  		for_each_label(prop->labels, l)
> -			check_duplicate_label(c, dt, l->label, node, prop, NULL);
> +			check_duplicate_label(c, bi, l->label, node, prop, NULL);
>  
>  		for_each_marker_of_type(m, LABEL)
> -			check_duplicate_label(c, dt, m->ref, node, prop, m);
> +			check_duplicate_label(c, bi, m->ref, node, prop, m);
>  	}
>  }
>  ERROR(duplicate_label, check_duplicate_label_node, NULL);
>  
> -static cell_t check_phandle_prop(struct check *c, struct node *root,
> +static cell_t check_phandle_prop(struct check *c, struct boot_info *bi,
>  				 struct node *node, const char *propname)
>  {
> +	struct node *root = bi->dt;
>  	struct property *prop;
>  	struct marker *m;
>  	cell_t phandle;
> @@ -398,18 +401,19 @@ static cell_t check_phandle_prop(struct check *c, struct node *root,
>  	return phandle;
>  }
>  
> -static void check_explicit_phandles(struct check *c, struct node *root,
> +static void check_explicit_phandles(struct check *c, struct boot_info *bi,
>  				    struct node *node)
>  {
> +	struct node *root = bi->dt;
>  	struct node *other;
>  	cell_t phandle, linux_phandle;
>  
>  	/* Nothing should have assigned phandles yet */
>  	assert(!node->phandle);
>  
> -	phandle = check_phandle_prop(c, root, node, "phandle");
> +	phandle = check_phandle_prop(c, bi, node, "phandle");
>  
> -	linux_phandle = check_phandle_prop(c, root, node, "linux,phandle");
> +	linux_phandle = check_phandle_prop(c, bi, node, "linux,phandle");
>  
>  	if (!phandle && !linux_phandle)
>  		/* No valid phandles; nothing further to check */
> @@ -433,7 +437,7 @@ static void check_explicit_phandles(struct check *c, struct node *root,
>  }
>  ERROR(explicit_phandles, check_explicit_phandles, NULL);
>  
> -static void check_name_properties(struct check *c, struct node *root,
> +static void check_name_properties(struct check *c, struct boot_info *bi,
>  				  struct node *node)
>  {
>  	struct property **pp, *prop = NULL;
> @@ -467,9 +471,10 @@ ERROR(name_properties, check_name_properties, NULL, &name_is_string);
>   * Reference fixup functions
>   */
>  
> -static void fixup_phandle_references(struct check *c, struct node *dt,
> +static void fixup_phandle_references(struct check *c, struct boot_info *bi,
>  				     struct node *node)
>  {
> +	struct node *dt = bi->dt;
>  	struct property *prop;
>  
>  	for_each_property(node, prop) {
> @@ -495,9 +500,10 @@ static void fixup_phandle_references(struct check *c, struct node *dt,
>  ERROR(phandle_references, fixup_phandle_references, NULL,
>        &duplicate_node_names, &explicit_phandles);
>  
> -static void fixup_path_references(struct check *c, struct node *dt,
> +static void fixup_path_references(struct check *c, struct boot_info *bi,
>  				  struct node *node)
>  {
> +	struct node *dt = bi->dt;
>  	struct property *prop;
>  
>  	for_each_property(node, prop) {
> @@ -534,7 +540,7 @@ WARNING_IF_NOT_STRING(device_type_is_string, "device_type");
>  WARNING_IF_NOT_STRING(model_is_string, "model");
>  WARNING_IF_NOT_STRING(status_is_string, "status");
>  
> -static void fixup_addr_size_cells(struct check *c, struct node *dt,
> +static void fixup_addr_size_cells(struct check *c, struct boot_info *bi,
>  				  struct node *node)
>  {
>  	struct property *prop;
> @@ -558,7 +564,7 @@ WARNING(addr_size_cells, fixup_addr_size_cells, NULL,
>  #define node_size_cells(n) \
>  	(((n)->size_cells == -1) ? 1 : (n)->size_cells)
>  
> -static void check_reg_format(struct check *c, struct node *dt,
> +static void check_reg_format(struct check *c, struct boot_info *bi,
>  			     struct node *node)
>  {
>  	struct property *prop;
> @@ -587,7 +593,7 @@ static void check_reg_format(struct check *c, struct node *dt,
>  }
>  WARNING(reg_format, check_reg_format, NULL, &addr_size_cells);
>  
> -static void check_ranges_format(struct check *c, struct node *dt,
> +static void check_ranges_format(struct check *c, struct boot_info *bi,
>  				struct node *node)
>  {
>  	struct property *prop;
> @@ -631,7 +637,7 @@ WARNING(ranges_format, check_ranges_format, NULL, &addr_size_cells);
>  /*
>   * Style checks
>   */
> -static void check_avoid_default_addr_size(struct check *c, struct node *dt,
> +static void check_avoid_default_addr_size(struct check *c, struct boot_info *bi,
>  					  struct node *node)
>  {
>  	struct property *reg, *ranges;
> @@ -657,9 +663,10 @@ WARNING(avoid_default_addr_size, check_avoid_default_addr_size, NULL,
>  	&addr_size_cells);
>  
>  static void check_obsolete_chosen_interrupt_controller(struct check *c,
> -						       struct node *dt,
> +						       struct boot_info *bi,
>  						       struct node *node)
>  {
> +	struct node *dt = bi->dt;
>  	struct node *chosen;
>  	struct property *prop;
>  
> @@ -765,7 +772,6 @@ void parse_checks_option(bool warn, bool error, const char *arg)
>  
>  void process_checks(bool force, struct boot_info *bi)
>  {
> -	struct node *dt = bi->dt;
>  	int i;
>  	int error = 0;
>  
> @@ -773,7 +779,7 @@ void process_checks(bool force, struct boot_info *bi)
>  		struct check *c = check_table[i];
>  
>  		if (c->warn || c->error)
> -			error = error || run_check(c, dt);
> +			error = error || run_check(c, bi);
>  	}
>  
>  	if (error) {

-- 
David Gibson			| I'll have my music baroque, and my code
david AT gibson.dropbear.id.au	| minimalist, thank you.  NOT _the_ _other_
				| _way_ _around_!
http://www.ozlabs.org/~dgibson

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 819 bytes --]

[PATCH v10 2/4] dtc: Document the dynamic plugin internals

[Pantelis Antoniou]

Provides the document explaining the internal mechanics of
plugins and options.

Signed-off-by: Pantelis Antoniou <pantelis.antoniou-OWPKS81ov/FWk0Htik3J/w@public.gmane.org>
---
 Documentation/dt-object-internal.txt | 318 +++++++++++++++++++++++++++++++++++
 1 file changed, 318 insertions(+)
 create mode 100644 Documentation/dt-object-internal.txt

diff --git a/Documentation/dt-object-internal.txt b/Documentation/dt-object-internal.txt
new file mode 100644
index 0000000..d5b841e
--- /dev/null
+++ b/Documentation/dt-object-internal.txt
@@ -0,0 +1,318 @@
+Device Tree Dynamic Object format internals
+-------------------------------------------
+
+The Device Tree for most platforms is a static representation of
+the hardware capabilities. This is insufficient for many platforms
+that need to dynamically insert device tree fragments to the
+running kernel's live tree.
+
+This document explains the the device tree object format and the
+modifications made to the device tree compiler, which make it possible.
+
+1. Simplified Problem Definition
+--------------------------------
+
+Assume we have a platform which boots using following simplified device tree.
+
+---- foo.dts -----------------------------------------------------------------
+	/* FOO platform */
+	/ {
+		compatible = "corp,foo";
+
+		/* shared resources */
+		res: res {
+		};
+
+		/* On chip peripherals */
+		ocp: ocp {
+			/* peripherals that are always instantiated */
+			peripheral1 { ... };
+		};
+	};
+---- foo.dts -----------------------------------------------------------------
+
+We have a number of peripherals that after probing (using some undefined method)
+should result in different device tree configuration.
+
+We cannot boot with this static tree because due to the configuration of the
+foo platform there exist multiple conficting peripherals DT fragments.
+
+So for the bar peripheral we would have this:
+
+---- foo+bar.dts -------------------------------------------------------------
+	/* FOO platform + bar peripheral */
+	/ {
+		compatible = "corp,foo";
+
+		/* shared resources */
+		res: res {
+		};
+
+		/* On chip peripherals */
+		ocp: ocp {
+			/* peripherals that are always instantiated */
+			peripheral1 { ... };
+
+			/* bar peripheral */
+			bar {
+				compatible = "corp,bar";
+				... /* various properties and child nodes */
+			};
+		};
+	};
+---- foo+bar.dts -------------------------------------------------------------
+
+While for the baz peripheral we would have this:
+
+---- foo+baz.dts -------------------------------------------------------------
+	/* FOO platform + baz peripheral */
+	/ {
+		compatible = "corp,foo";
+
+		/* shared resources */
+		res: res {
+			/* baz resources */
+			baz_res: res_baz { ... };
+		};
+
+		/* On chip peripherals */
+		ocp: ocp {
+			/* peripherals that are always instantiated */
+			peripheral1 { ... };
+
+			/* baz peripheral */
+			baz {
+				compatible = "corp,baz";
+				/* reference to another point in the tree */
+				ref-to-res = <&baz_res>;
+				... /* various properties and child nodes */
+			};
+		};
+	};
+---- foo+baz.dts -------------------------------------------------------------
+
+We note that the baz case is more complicated, since the baz peripheral needs to
+reference another node in the DT tree.
+
+2. Device Tree Object Format Requirements
+-----------------------------------------
+
+Since the device tree is used for booting a number of very different hardware
+platforms it is imperative that we tread very carefully.
+
+2.a) No changes to the Device Tree binary format for the base tree. We cannot
+modify the tree format at all and all the information we require should be
+encoded using device tree itself. We can add nodes that can be safely ignored
+by both bootloaders and the kernel. The plugin dtb's are optionally tagged
+with a different magic number in the header but otherwise they too are simple
+blobs.
+
+2.b) Changes to the DTS source format should be absolutely minimal, and should
+only be needed for the DT fragment definitions, and not the base boot DT.
+
+2.c) An explicit option should be used to instruct DTC to generate the required
+information needed for object resolution. Platforms that don't use the
+dynamic object format can safely ignore it.
+
+2.d) Finally, DT syntax changes should be kept to a minimum. It should be
+possible to express everything using the existing DT syntax.
+
+3. Implementation
+-----------------
+
+The basic unit of addressing in Device Tree is the phandle. Turns out it's
+relatively simple to extend the way phandles are generated and referenced
+so that it's possible to dynamically convert symbolic references (labels)
+to phandle values. This is a valid assumption as long as the author uses
+reference syntax and does not assign phandle values manually (which might
+be a problem with decompiled source files).
+
+We can roughly divide the operation into two steps.
+
+3.a) Compilation of the base board DTS file using the '-@' option
+generates a valid DT blob with an added __symbols__ node at the root node,
+containing a list of all nodes that are marked with a label.
+
+Using the foo.dts file above the following node will be generated;
+
+$ dtc -@ -O dtb -o foo.dtb -b 0 foo.dts
+$ fdtdump foo.dtb
+...
+/ {
+	...
+	res {
+		...
+		phandle = <0x00000001>;
+		...
+	};
+	ocp {
+		...
+		phandle = <0x00000002>;
+		...
+	};
+	__symbols__ {
+		res="/res";
+		ocp="/ocp";
+	};
+};
+
+Notice that all the nodes that had a label have been recorded, and that
+phandles have been generated for them.
+
+This blob can be used to boot the board normally, the __symbols__ node will
+be safely ignored both by the bootloader and the kernel (the only loss will
+be a few bytes of memory and disk space).
+
+3.b) The Device Tree fragments must be compiled with the same option but they
+must also have a tag (/plugin/) that allows undefined references to nodes
+that are not present at compilation time to be recorded so that the runtime
+loader can fix them.
+
+So the bar peripheral's DTS format would be of the form:
+
+/dts-v1/ /plugin/;	/* allow undefined references and record them */
+/ {
+	....	/* various properties for loader use; i.e. part id etc. */
+	fragment@0 {
+		target = <&ocp>;
+		__overlay__ {
+			/* bar peripheral */
+			bar {
+				compatible = "corp,bar";
+				... /* various properties and child nodes */
+			}
+		};
+	};
+};
+
+Note that there's a target property that specifies the location where the
+contents of the overlay node will be placed, and it references the node
+in the foo.dts file.
+
+$ dtc -@ -O dtb -o bar.dtbo -b 0 bar.dts
+$ fdtdump bar.dtbo
+...
+/ {
+	... /* properties */
+	fragment@0 {
+		target = <0xffffffff>;
+		__overlay__ {
+			bar {
+				compatible = "corp,bar";
+				... /* various properties and child nodes */
+			}
+		};
+	};
+	__fixups__ {
+	    ocp = "/fragment@0:target:0";
+	};
+};
+
+No __symbols__ has been generated (no label in bar.dts).
+Note that the target's ocp label is undefined, so the phandle handle
+value is filled with the illegal value '0xffffffff', while a __fixups__
+node has been generated, which marks the location in the tree where
+the label lookup should store the runtime phandle value of the ocp node.
+
+The format of the __fixups__ node entry is
+
+	<label> = "<local-full-path>:<property-name>:<offset>";
+
+<label> 		Is the label we're referring
+<local-full-path>	Is the full path of the node the reference is
+<property-name>		Is the name of the property containing the
+			reference
+<offset>		The offset (in bytes) of where the property's
+			phandle value is located.
+
+Doing the same with the baz peripheral's DTS format is a little bit more
+involved, since baz contains references to local labels which require
+local fixups.
+
+/dts-v1/ /plugin/;	/* allow undefined label references and record them */
+/ {
+	....	/* various properties for loader use; i.e. part id etc. */
+	fragment@0 {
+		target = <&res>;
+		__overlay__ {
+			/* baz resources */
+			baz_res: res_baz { ... };
+		};
+	};
+	fragment@1 {
+		target = <&ocp>;
+		__overlay__ {
+			/* baz peripheral */
+			baz {
+				compatible = "corp,baz";
+				/* reference to another point in the tree */
+				ref-to-res = <&baz_res>;
+				... /* various properties and child nodes */
+			}
+		};
+	};
+};
+
+Note that &bar_res reference.
+
+$ dtc -@ -O dtb -o baz.dtbo -b 0 baz.dts
+$ fdtdump baz.dtbo
+...
+/ {
+	... /* properties */
+	fragment@0 {
+		target = <0xffffffff>;
+		__overlay__ {
+			res_baz {
+				....
+				phandle = <0x00000001>;
+			};
+		};
+	};
+	fragment@1 {
+		target = <0xffffffff>;
+		__overlay__ {
+			baz {
+				compatible = "corp,baz";
+				... /* various properties and child nodes */
+				ref-to-res = <0x00000001>;
+			}
+		};
+	};
+	__fixups__ {
+		res = "/fragment@0:target:0";
+		ocp = "/fragment@1:target:0";
+	};
+	__local_fixups__ {
+		fragment@1 {
+			__overlay__ {
+				baz {
+					ref-to-res = <0>;
+				};
+			};
+		};
+	};
+};
+
+This is similar to the bar case, but the reference of a local label by the
+baz node generates a __local_fixups__ entry that records the place that the
+local reference is being made. No matter how phandles are allocated from dtc
+the run time loader must apply an offset to each phandle in every dynamic
+DT object loaded. The __local_fixups__ node records the place of every
+local reference so that the loader can apply the offset.
+
+There is an alternative syntax to the expanded form for overlays with phandle
+targets which makes the format similar to the one using in .dtsi include files.
+
+So for the &ocp target example above one can simply write:
+
+/dts-v1/ /plugin/;
+&ocp {
+	/* bar peripheral */
+	bar {
+		compatible = "corp,bar";
+		... /* various properties and child nodes */
+	}
+};
+
+The resulting dtb object is identical.
-- 
2.1.4

Re: [PATCH v10 2/4] dtc: Document the dynamic plugin internals

[Stephen Boyd]

Quoting Pantelis Antoniou (2016-11-25 04:32:09)
> diff --git a/Documentation/dt-object-internal.txt b/Documentation/dt-object-internal.txt
> new file mode 100644
> index 0000000..d5b841e
> --- /dev/null
> +++ b/Documentation/dt-object-internal.txt
> @@ -0,0 +1,318 @@
> +Device Tree Dynamic Object format internals
> +-------------------------------------------
> +
> +The Device Tree for most platforms is a static representation of
> +the hardware capabilities. This is insufficient for many platforms

s/many//

> +that need to dynamically insert device tree fragments to the

that need to dynamically insert device tree fragments into the

Also, should device tree be capitalized here?

> +running kernel's live tree.

Drop "running kernel's" as it's implicit with "live tree"?

> +
> +This document explains the the device tree object format and the

s/the//

> +modifications made to the device tree compiler, which make it possible.
> +
> +1. Simplified Problem Definition
> +--------------------------------
> +
> +Assume we have a platform which boots using following simplified device tree.
> +
> +---- foo.dts -----------------------------------------------------------------
> +       /* FOO platform */
> +       / {
> +               compatible = "corp,foo";
> +
> +               /* shared resources */
> +               res: res {
> +               };
> +
> +               /* On chip peripherals */
> +               ocp: ocp {
> +                       /* peripherals that are always instantiated */
> +                       peripheral1 { ... };
> +               };
> +       };
> +---- foo.dts -----------------------------------------------------------------
> +
> +We have a number of peripherals that after probing (using some undefined method)
> +should result in different device tree configuration.
> +
> +We cannot boot with this static tree because due to the configuration of the
> +foo platform there exist multiple conficting peripherals DT fragments.
> +
> +So for the bar peripheral we would have this:
> +
> +---- foo+bar.dts -------------------------------------------------------------
> +       /* FOO platform + bar peripheral */
> +       / {
> +               compatible = "corp,foo";
> +
> +               /* shared resources */
> +               res: res {
> +               };
> +
> +               /* On chip peripherals */
> +               ocp: ocp {
> +                       /* peripherals that are always instantiated */
> +                       peripheral1 { ... };
> +
> +                       /* bar peripheral */
> +                       bar {
> +                               compatible = "corp,bar";
> +                               ... /* various properties and child nodes */
> +                       };
> +               };
> +       };
> +---- foo+bar.dts -------------------------------------------------------------
> +
> +While for the baz peripheral we would have this:
> +
> +---- foo+baz.dts -------------------------------------------------------------
> +       /* FOO platform + baz peripheral */
> +       / {
> +               compatible = "corp,foo";
> +
> +               /* shared resources */
> +               res: res {
> +                       /* baz resources */
> +                       baz_res: res_baz { ... };
> +               };
> +
> +               /* On chip peripherals */
> +               ocp: ocp {
> +                       /* peripherals that are always instantiated */
> +                       peripheral1 { ... };
> +
> +                       /* baz peripheral */
> +                       baz {
> +                               compatible = "corp,baz";
> +                               /* reference to another point in the tree */
> +                               ref-to-res = <&baz_res>;
> +                               ... /* various properties and child nodes */
> +                       };
> +               };
> +       };
> +---- foo+baz.dts -------------------------------------------------------------
> +
> +We note that the baz case is more complicated, since the baz peripheral needs to
> +reference another node in the DT tree.
> +
> +2. Device Tree Object Format Requirements
> +-----------------------------------------
> +
> +Since the device tree is used for booting a number of very different hardware
> +platforms it is imperative that we tread very carefully.
> +
> +2.a) No changes to the Device Tree binary format for the base tree. We cannot
> +modify the tree format at all and all the information we require should be
> +encoded using device tree itself. We can add nodes that can be safely ignored
> +by both bootloaders and the kernel. The plugin dtb's are optionally tagged

s/dtb's/dtbs/

> +with a different magic number in the header but otherwise they too are simple
> +blobs.

but otherwise they're simple blobs.

> +
> +2.b) Changes to the DTS source format should be absolutely minimal, and should
> +only be needed for the DT fragment definitions, and not the base boot DT.
> +
> +2.c) An explicit option should be used to instruct DTC to generate the required
> +information needed for object resolution. Platforms that don't use the
> +dynamic object format can safely ignore it.

Why? We can't figure that out based on the /plugin/ label within the dts
file? And shouldn't we always generate a __symbols__ node in the base
dtb?

> +
> +2.d) Finally, DT syntax changes should be kept to a minimum. It should be
> +possible to express everything using the existing DT syntax.
> +
> +3. Implementation
> +-----------------
> +
> +The basic unit of addressing in Device Tree is the phandle. Turns out it's
> +relatively simple to extend the way phandles are generated and referenced
> +so that it's possible to dynamically convert symbolic references (labels)
> +to phandle values. This is a valid assumption as long as the author uses
> +reference syntax and does not assign phandle values manually (which might
> +be a problem with decompiled source files).
> +
> +We can roughly divide the operation into two steps.
> +
> +3.a) Compilation of the base board DTS file using the '-@' option
> +generates a valid DT blob with an added __symbols__ node at the root node,
> +containing a list of all nodes that are marked with a label.
> +
> +Using the foo.dts file above the following node will be generated;
> +
> +$ dtc -@ -O dtb -o foo.dtb -b 0 foo.dts
> +$ fdtdump foo.dtb
> +...
> +/ {
> +       ...
> +       res {
> +               ...
> +               phandle = <0x00000001>;
> +               ...
> +       };
> +       ocp {
> +               ...
> +               phandle = <0x00000002>;
> +               ...
> +       };
> +       __symbols__ {
> +               res="/res";
> +               ocp="/ocp";
> +       };
> +};
> +
> +Notice that all the nodes that had a label have been recorded, and that
> +phandles have been generated for them.
> +
> +This blob can be used to boot the board normally, the __symbols__ node will
> +be safely ignored both by the bootloader and the kernel (the only loss will
> +be a few bytes of memory and disk space).

This never really mentions why we need to generate a symbols node.
Perhaps we should say something like "we generate a __symbols__ node to
record nodes that had labels in the base tree so they can be matched up
with the fragments which reference the same labels"? Or something like
that.

I also wonder why it's even necessary. Couldn't we require overlays to
be compiled with the original dts files? Then we could encode the full
path of nodes referenced in the overlay into the overlay dtb itself.

> +
> +3.b) The Device Tree fragments must be compiled with the same option but they
> +must also have a tag (/plugin/) that allows undefined references to nodes
> +that are not present at compilation time to be recorded so that the runtime
> +loader can fix them.
> +
> +So the bar peripheral's DTS format would be of the form:
> +
> +/dts-v1/ /plugin/;     /* allow undefined references and record them */
> +/ {
> +       ....    /* various properties for loader use; i.e. part id etc. */
> +       fragment@0 {
> +               target = <&ocp>;
> +               __overlay__ {
> +                       /* bar peripheral */
> +                       bar {
> +                               compatible = "corp,bar";
> +                               ... /* various properties and child nodes */
> +                       }
> +               };
> +       };
> +};
> +
> +Note that there's a target property that specifies the location where the
> +contents of the overlay node will be placed, and it references the node
> +in the foo.dts file.
> +
> +$ dtc -@ -O dtb -o bar.dtbo -b 0 bar.dts
> +$ fdtdump bar.dtbo
> +...
> +/ {
> +       ... /* properties */
> +       fragment@0 {
> +               target = <0xffffffff>;
> +               __overlay__ {
> +                       bar {
> +                               compatible = "corp,bar";
> +                               ... /* various properties and child nodes */
> +                       }
> +               };
> +       };
> +       __fixups__ {
> +           ocp = "/fragment@0:target:0";
> +       };
> +};
> +
> +No __symbols__ has been generated (no label in bar.dts).

Add "node" after __symbols__ here?

> +Note that the target's ocp label is undefined, so the phandle handle

Drop handle after phandle?

> +value is filled with the illegal value '0xffffffff', while a __fixups__
> +node has been generated, which marks the location in the tree where
> +the label lookup should store the runtime phandle value of the ocp node.
> +
> +The format of the __fixups__ node entry is
> +
> +       <label> = "<local-full-path>:<property-name>:<offset>";
> +
> +<label>                Is the label we're referring
> +<local-full-path>      Is the full path of the node the reference is
> +<property-name>                Is the name of the property containing the

Weird alignment here.

> +                       reference
> +<offset>               The offset (in bytes) of where the property's
> +                       phandle value is located.

located within the property? Or "offset relative to the start of the
property in bytes where the phandle value is located"?

Is this a list? So multiple properties can be fixed up with the same
label? If so that isn't clear from this description.

> +
> +Doing the same with the baz peripheral's DTS format is a little bit more
> +involved, since baz contains references to local labels which require
> +local fixups.
> +
> +/dts-v1/ /plugin/;     /* allow undefined label references and record them */
> +/ {
> +       ....    /* various properties for loader use; i.e. part id etc. */
> +       fragment@0 {
> +               target = <&res>;
> +               __overlay__ {
> +                       /* baz resources */
> +                       baz_res: res_baz { ... };
> +               };
> +       };
> +       fragment@1 {
> +               target = <&ocp>;
> +               __overlay__ {
> +                       /* baz peripheral */
> +                       baz {
> +                               compatible = "corp,baz";
> +                               /* reference to another point in the tree */
> +                               ref-to-res = <&baz_res>;
> +                               ... /* various properties and child nodes */
> +                       }
> +               };
> +       };
> +};
> +
> +Note that &bar_res reference.
> +
> +$ dtc -@ -O dtb -o baz.dtbo -b 0 baz.dts
> +$ fdtdump baz.dtbo
> +...
> +/ {
> +       ... /* properties */
> +       fragment@0 {
> +               target = <0xffffffff>;
> +               __overlay__ {
> +                       res_baz {
> +                               ....
> +                               phandle = <0x00000001>;
> +                       };
> +               };
> +       };
> +       fragment@1 {
> +               target = <0xffffffff>;
> +               __overlay__ {
> +                       baz {
> +                               compatible = "corp,baz";
> +                               ... /* various properties and child nodes */
> +                               ref-to-res = <0x00000001>;
> +                       }
> +               };
> +       };
> +       __fixups__ {
> +               res = "/fragment@0:target:0";
> +               ocp = "/fragment@1:target:0";
> +       };
> +       __local_fixups__ {
> +               fragment@1 {
> +                       __overlay__ {
> +                               baz {
> +                                       ref-to-res = <0>;
> +                               };
> +                       };
> +               };
> +       };
> +};
> +
> +This is similar to the bar case, but the reference of a local label by the
> +baz node generates a __local_fixups__ entry that records the place that the
> +local reference is being made. No matter how phandles are allocated from dtc
> +the run time loader must apply an offset to each phandle in every dynamic
> +DT object loaded. The __local_fixups__ node records the place of every

records the offset relative to the start of the property of every local
reference within that property so that the loader...

> +local reference so that the loader can apply the offset.
> +
> +There is an alternative syntax to the expanded form for overlays with phandle
> +targets which makes the format similar to the one using in .dtsi include files.
> +
> +So for the &ocp target example above one can simply write:
> +
> +/dts-v1/ /plugin/;
> +&ocp {
> +       /* bar peripheral */
> +       bar {
> +               compatible = "corp,bar";
> +               ... /* various properties and child nodes */
> +       }
> +};
> +
> +The resulting dtb object is identical.

Re: [PATCH v10 2/4] dtc: Document the dynamic plugin internals

[Stephen Boyd]

Quoting Pantelis Antoniou (2016-11-25 04:32:09)
> diff --git a/Documentation/dt-object-internal.txt b/Documentation/dt-object-internal.txt
> new file mode 100644
> index 0000000..d5b841e
> --- /dev/null
> +++ b/Documentation/dt-object-internal.txt
> @@ -0,0 +1,318 @@
> +Device Tree Dynamic Object format internals
> +-------------------------------------------
> +
> +The Device Tree for most platforms is a static representation of
> +the hardware capabilities. This is insufficient for many platforms

s/many//

> +that need to dynamically insert device tree fragments to the

that need to dynamically insert device tree fragments into the

Also, should device tree be capitalized here?

> +running kernel's live tree.

Drop "running kernel's" as it's implicit with "live tree"?

> +
> +This document explains the the device tree object format and the

s/the//

> +modifications made to the device tree compiler, which make it possible.
> +
> +1. Simplified Problem Definition
> +--------------------------------
> +
> +Assume we have a platform which boots using following simplified device tree.
> +
> +---- foo.dts -----------------------------------------------------------------
> +       /* FOO platform */
> +       / {
> +               compatible = "corp,foo";
> +
> +               /* shared resources */
> +               res: res {
> +               };
> +
> +               /* On chip peripherals */
> +               ocp: ocp {
> +                       /* peripherals that are always instantiated */
> +                       peripheral1 { ... };
> +               };
> +       };
> +---- foo.dts -----------------------------------------------------------------
> +
> +We have a number of peripherals that after probing (using some undefined method)
> +should result in different device tree configuration.
> +
> +We cannot boot with this static tree because due to the configuration of the
> +foo platform there exist multiple conficting peripherals DT fragments.
> +
> +So for the bar peripheral we would have this:
> +
> +---- foo+bar.dts -------------------------------------------------------------
> +       /* FOO platform + bar peripheral */
> +       / {
> +               compatible = "corp,foo";
> +
> +               /* shared resources */
> +               res: res {
> +               };
> +
> +               /* On chip peripherals */
> +               ocp: ocp {
> +                       /* peripherals that are always instantiated */
> +                       peripheral1 { ... };
> +
> +                       /* bar peripheral */
> +                       bar {
> +                               compatible = "corp,bar";
> +                               ... /* various properties and child nodes */
> +                       };
> +               };
> +       };
> +---- foo+bar.dts -------------------------------------------------------------
> +
> +While for the baz peripheral we would have this:
> +
> +---- foo+baz.dts -------------------------------------------------------------
> +       /* FOO platform + baz peripheral */
> +       / {
> +               compatible = "corp,foo";
> +
> +               /* shared resources */
> +               res: res {
> +                       /* baz resources */
> +                       baz_res: res_baz { ... };
> +               };
> +
> +               /* On chip peripherals */
> +               ocp: ocp {
> +                       /* peripherals that are always instantiated */
> +                       peripheral1 { ... };
> +
> +                       /* baz peripheral */
> +                       baz {
> +                               compatible = "corp,baz";
> +                               /* reference to another point in the tree */
> +                               ref-to-res = <&baz_res>;
> +                               ... /* various properties and child nodes */
> +                       };
> +               };
> +       };
> +---- foo+baz.dts -------------------------------------------------------------
> +
> +We note that the baz case is more complicated, since the baz peripheral needs to
> +reference another node in the DT tree.
> +
> +2. Device Tree Object Format Requirements
> +-----------------------------------------
> +
> +Since the device tree is used for booting a number of very different hardware
> +platforms it is imperative that we tread very carefully.
> +
> +2.a) No changes to the Device Tree binary format for the base tree. We cannot
> +modify the tree format at all and all the information we require should be
> +encoded using device tree itself. We can add nodes that can be safely ignored
> +by both bootloaders and the kernel. The plugin dtb's are optionally tagged

s/dtb's/dtbs/

> +with a different magic number in the header but otherwise they too are simple
> +blobs.

but otherwise they're simple blobs.

> +
> +2.b) Changes to the DTS source format should be absolutely minimal, and should
> +only be needed for the DT fragment definitions, and not the base boot DT.
> +
> +2.c) An explicit option should be used to instruct DTC to generate the required
> +information needed for object resolution. Platforms that don't use the
> +dynamic object format can safely ignore it.

Why? We can't figure that out based on the /plugin/ label within the dts
file? And shouldn't we always generate a __symbols__ node in the base
dtb?

> +
> +2.d) Finally, DT syntax changes should be kept to a minimum. It should be
> +possible to express everything using the existing DT syntax.
> +
> +3. Implementation
> +-----------------
> +
> +The basic unit of addressing in Device Tree is the phandle. Turns out it's
> +relatively simple to extend the way phandles are generated and referenced
> +so that it's possible to dynamically convert symbolic references (labels)
> +to phandle values. This is a valid assumption as long as the author uses
> +reference syntax and does not assign phandle values manually (which might
> +be a problem with decompiled source files).
> +
> +We can roughly divide the operation into two steps.
> +
> +3.a) Compilation of the base board DTS file using the '-@' option
> +generates a valid DT blob with an added __symbols__ node at the root node,
> +containing a list of all nodes that are marked with a label.
> +
> +Using the foo.dts file above the following node will be generated;
> +
> +$ dtc -@ -O dtb -o foo.dtb -b 0 foo.dts
> +$ fdtdump foo.dtb
> +...
> +/ {
> +       ...
> +       res {
> +               ...
> +               phandle = <0x00000001>;
> +               ...
> +       };
> +       ocp {
> +               ...
> +               phandle = <0x00000002>;
> +               ...
> +       };
> +       __symbols__ {
> +               res="/res";
> +               ocp="/ocp";
> +       };
> +};
> +
> +Notice that all the nodes that had a label have been recorded, and that
> +phandles have been generated for them.
> +
> +This blob can be used to boot the board normally, the __symbols__ node will
> +be safely ignored both by the bootloader and the kernel (the only loss will
> +be a few bytes of memory and disk space).

This never really mentions why we need to generate a symbols node.
Perhaps we should say something like "we generate a __symbols__ node to
record nodes that had labels in the base tree so they can be matched up
with the fragments which reference the same labels"? Or something like
that.

I also wonder why it's even necessary. Couldn't we require overlays to
be compiled with the original dts files? Then we could encode the full
path of nodes referenced in the overlay into the overlay dtb itself.

> +
> +3.b) The Device Tree fragments must be compiled with the same option but they
> +must also have a tag (/plugin/) that allows undefined references to nodes
> +that are not present at compilation time to be recorded so that the runtime
> +loader can fix them.
> +
> +So the bar peripheral's DTS format would be of the form:
> +
> +/dts-v1/ /plugin/;     /* allow undefined references and record them */
> +/ {
> +       ....    /* various properties for loader use; i.e. part id etc. */
> +       fragment@0 {
> +               target = <&ocp>;
> +               __overlay__ {
> +                       /* bar peripheral */
> +                       bar {
> +                               compatible = "corp,bar";
> +                               ... /* various properties and child nodes */
> +                       }
> +               };
> +       };
> +};
> +
> +Note that there's a target property that specifies the location where the
> +contents of the overlay node will be placed, and it references the node
> +in the foo.dts file.
> +
> +$ dtc -@ -O dtb -o bar.dtbo -b 0 bar.dts
> +$ fdtdump bar.dtbo
> +...
> +/ {
> +       ... /* properties */
> +       fragment@0 {
> +               target = <0xffffffff>;
> +               __overlay__ {
> +                       bar {
> +                               compatible = "corp,bar";
> +                               ... /* various properties and child nodes */
> +                       }
> +               };
> +       };
> +       __fixups__ {
> +           ocp = "/fragment@0:target:0";
> +       };
> +};
> +
> +No __symbols__ has been generated (no label in bar.dts).

Add "node" after __symbols__ here?

> +Note that the target's ocp label is undefined, so the phandle handle

Drop handle after phandle?

> +value is filled with the illegal value '0xffffffff', while a __fixups__
> +node has been generated, which marks the location in the tree where
> +the label lookup should store the runtime phandle value of the ocp node.
> +
> +The format of the __fixups__ node entry is
> +
> +       <label> = "<local-full-path>:<property-name>:<offset>";
> +
> +<label>                Is the label we're referring
> +<local-full-path>      Is the full path of the node the reference is
> +<property-name>                Is the name of the property containing the

Weird alignment here.

> +                       reference
> +<offset>               The offset (in bytes) of where the property's
> +                       phandle value is located.

located within the property? Or "offset relative to the start of the
property in bytes where the phandle value is located"?

Is this a list? So multiple properties can be fixed up with the same
label? If so that isn't clear from this description.

> +
> +Doing the same with the baz peripheral's DTS format is a little bit more
> +involved, since baz contains references to local labels which require
> +local fixups.
> +
> +/dts-v1/ /plugin/;     /* allow undefined label references and record them */
> +/ {
> +       ....    /* various properties for loader use; i.e. part id etc. */
> +       fragment@0 {
> +               target = <&res>;
> +               __overlay__ {
> +                       /* baz resources */
> +                       baz_res: res_baz { ... };
> +               };
> +       };
> +       fragment@1 {
> +               target = <&ocp>;
> +               __overlay__ {
> +                       /* baz peripheral */
> +                       baz {
> +                               compatible = "corp,baz";
> +                               /* reference to another point in the tree */
> +                               ref-to-res = <&baz_res>;
> +                               ... /* various properties and child nodes */
> +                       }
> +               };
> +       };
> +};
> +
> +Note that &bar_res reference.
> +
> +$ dtc -@ -O dtb -o baz.dtbo -b 0 baz.dts
> +$ fdtdump baz.dtbo
> +...
> +/ {
> +       ... /* properties */
> +       fragment@0 {
> +               target = <0xffffffff>;
> +               __overlay__ {
> +                       res_baz {
> +                               ....
> +                               phandle = <0x00000001>;
> +                       };
> +               };
> +       };
> +       fragment@1 {
> +               target = <0xffffffff>;
> +               __overlay__ {
> +                       baz {
> +                               compatible = "corp,baz";
> +                               ... /* various properties and child nodes */
> +                               ref-to-res = <0x00000001>;
> +                       }
> +               };
> +       };
> +       __fixups__ {
> +               res = "/fragment@0:target:0";
> +               ocp = "/fragment@1:target:0";
> +       };
> +       __local_fixups__ {
> +               fragment@1 {
> +                       __overlay__ {
> +                               baz {
> +                                       ref-to-res = <0>;
> +                               };
> +                       };
> +               };
> +       };
> +};
> +
> +This is similar to the bar case, but the reference of a local label by the
> +baz node generates a __local_fixups__ entry that records the place that the
> +local reference is being made. No matter how phandles are allocated from dtc
> +the run time loader must apply an offset to each phandle in every dynamic
> +DT object loaded. The __local_fixups__ node records the place of every

records the offset relative to the start of the property of every local
reference within that property so that the loader...

> +local reference so that the loader can apply the offset.
> +
> +There is an alternative syntax to the expanded form for overlays with phandle
> +targets which makes the format similar to the one using in .dtsi include files.
> +
> +So for the &ocp target example above one can simply write:
> +
> +/dts-v1/ /plugin/;
> +&ocp {
> +       /* bar peripheral */
> +       bar {
> +               compatible = "corp,bar";
> +               ... /* various properties and child nodes */
> +       }
> +};
> +
> +The resulting dtb object is identical.

Re: [PATCH v10 2/4] dtc: Document the dynamic plugin internals

[Pantelis Antoniou]

Hi Stephen,

> On Nov 28, 2016, at 22:03 , Stephen Boyd <stephen.boyd-QSEj5FYQhm4dnm+yROfE0A@public.gmane.org> wrote:
> 
> Quoting Pantelis Antoniou (2016-11-25 04:32:09)
>> diff --git a/Documentation/dt-object-internal.txt b/Documentation/dt-object-internal.txt
>> new file mode 100644
>> index 0000000..d5b841e
>> --- /dev/null
>> +++ b/Documentation/dt-object-internal.txt
>> @@ -0,0 +1,318 @@
>> +Device Tree Dynamic Object format internals
>> +-------------------------------------------
>> +
>> +The Device Tree for most platforms is a static representation of
>> +the hardware capabilities. This is insufficient for many platforms
> 
> s/many//
> 
>> +that need to dynamically insert device tree fragments to the
> 
> that need to dynamically insert device tree fragments into the
> 
> Also, should device tree be capitalized here?
> 
>> +running kernel's live tree.
> 
> Drop "running kernel's" as it's implicit with "live tree"?
> 
>> +
>> +This document explains the the device tree object format and the
> 
> s/the//
> 
>> +modifications made to the device tree compiler, which make it possible.
>> +
>> +1. Simplified Problem Definition
>> +--------------------------------
>> +
>> +Assume we have a platform which boots using following simplified device tree.
>> +
>> +---- foo.dts -----------------------------------------------------------------
>> +       /* FOO platform */
>> +       / {
>> +               compatible = "corp,foo";
>> +
>> +               /* shared resources */
>> +               res: res {
>> +               };
>> +
>> +               /* On chip peripherals */
>> +               ocp: ocp {
>> +                       /* peripherals that are always instantiated */
>> +                       peripheral1 { ... };
>> +               };
>> +       };
>> +---- foo.dts -----------------------------------------------------------------
>> +
>> +We have a number of peripherals that after probing (using some undefined method)
>> +should result in different device tree configuration.
>> +
>> +We cannot boot with this static tree because due to the configuration of the
>> +foo platform there exist multiple conficting peripherals DT fragments.
>> +
>> +So for the bar peripheral we would have this:
>> +
>> +---- foo+bar.dts -------------------------------------------------------------
>> +       /* FOO platform + bar peripheral */
>> +       / {
>> +               compatible = "corp,foo";
>> +
>> +               /* shared resources */
>> +               res: res {
>> +               };
>> +
>> +               /* On chip peripherals */
>> +               ocp: ocp {
>> +                       /* peripherals that are always instantiated */
>> +                       peripheral1 { ... };
>> +
>> +                       /* bar peripheral */
>> +                       bar {
>> +                               compatible = "corp,bar";
>> +                               ... /* various properties and child nodes */
>> +                       };
>> +               };
>> +       };
>> +---- foo+bar.dts -------------------------------------------------------------
>> +
>> +While for the baz peripheral we would have this:
>> +
>> +---- foo+baz.dts -------------------------------------------------------------
>> +       /* FOO platform + baz peripheral */
>> +       / {
>> +               compatible = "corp,foo";
>> +
>> +               /* shared resources */
>> +               res: res {
>> +                       /* baz resources */
>> +                       baz_res: res_baz { ... };
>> +               };
>> +
>> +               /* On chip peripherals */
>> +               ocp: ocp {
>> +                       /* peripherals that are always instantiated */
>> +                       peripheral1 { ... };
>> +
>> +                       /* baz peripheral */
>> +                       baz {
>> +                               compatible = "corp,baz";
>> +                               /* reference to another point in the tree */
>> +                               ref-to-res = <&baz_res>;
>> +                               ... /* various properties and child nodes */
>> +                       };
>> +               };
>> +       };
>> +---- foo+baz.dts -------------------------------------------------------------
>> +
>> +We note that the baz case is more complicated, since the baz peripheral needs to
>> +reference another node in the DT tree.
>> +
>> +2. Device Tree Object Format Requirements
>> +-----------------------------------------
>> +
>> +Since the device tree is used for booting a number of very different hardware
>> +platforms it is imperative that we tread very carefully.
>> +
>> +2.a) No changes to the Device Tree binary format for the base tree. We cannot
>> +modify the tree format at all and all the information we require should be
>> +encoded using device tree itself. We can add nodes that can be safely ignored
>> +by both bootloaders and the kernel. The plugin dtb's are optionally tagged
> 
> s/dtb's/dtbs/
> 
>> +with a different magic number in the header but otherwise they too are simple
>> +blobs.
> 
> but otherwise they're simple blobs.
> 

OK on the spelling/grammar fixes above.

>> +
>> +2.b) Changes to the DTS source format should be absolutely minimal, and should
>> +only be needed for the DT fragment definitions, and not the base boot DT.
>> +
>> +2.c) An explicit option should be used to instruct DTC to generate the required
>> +information needed for object resolution. Platforms that don't use the
>> +dynamic object format can safely ignore it.
> 
> Why? We can't figure that out based on the /plugin/ label within the dts
> file? And shouldn't we always generate a __symbols__ node in the base
> dtb?
> 

Actually now we do. The last patchset does automatically generate those nodes
if a /plugin/ tag is encountered. For base dtbs I would suggest that generating
the symbols node automatically is what’s sane too, but unfortunately there are
some platforms out there that are having trouble with larger dtbs than what they
expect.

It is your call whether to enable it by default I guess.

>> +
>> +2.d) Finally, DT syntax changes should be kept to a minimum. It should be
>> +possible to express everything using the existing DT syntax.
>> +
>> +3. Implementation
>> +-----------------
>> +
>> +The basic unit of addressing in Device Tree is the phandle. Turns out it's
>> +relatively simple to extend the way phandles are generated and referenced
>> +so that it's possible to dynamically convert symbolic references (labels)
>> +to phandle values. This is a valid assumption as long as the author uses
>> +reference syntax and does not assign phandle values manually (which might
>> +be a problem with decompiled source files).
>> +
>> +We can roughly divide the operation into two steps.
>> +
>> +3.a) Compilation of the base board DTS file using the '-@' option
>> +generates a valid DT blob with an added __symbols__ node at the root node,
>> +containing a list of all nodes that are marked with a label.
>> +
>> +Using the foo.dts file above the following node will be generated;
>> +
>> +$ dtc -@ -O dtb -o foo.dtb -b 0 foo.dts
>> +$ fdtdump foo.dtb
>> +...
>> +/ {
>> +       ...
>> +       res {
>> +               ...
>> +               phandle = <0x00000001>;
>> +               ...
>> +       };
>> +       ocp {
>> +               ...
>> +               phandle = <0x00000002>;
>> +               ...
>> +       };
>> +       __symbols__ {
>> +               res="/res";
>> +               ocp="/ocp";
>> +       };
>> +};
>> +
>> +Notice that all the nodes that had a label have been recorded, and that
>> +phandles have been generated for them.
>> +
>> +This blob can be used to boot the board normally, the __symbols__ node will
>> +be safely ignored both by the bootloader and the kernel (the only loss will
>> +be a few bytes of memory and disk space).
> 
> This never really mentions why we need to generate a symbols node.
> Perhaps we should say something like "we generate a __symbols__ node to
> record nodes that had labels in the base tree so they can be matched up
> with the fragments which reference the same labels"? Or something like
> that.
> 

Hmm, yeah.

> I also wonder why it's even necessary. Couldn't we require overlays to
> be compiled with the original dts files? Then we could encode the full
> path of nodes referenced in the overlay into the overlay dtb itself.
> 

No, we can’t do that; the end-game of this is for overlays to be portable
for use in platforms having the same kind of connectors.


>> +
>> +3.b) The Device Tree fragments must be compiled with the same option but they
>> +must also have a tag (/plugin/) that allows undefined references to nodes
>> +that are not present at compilation time to be recorded so that the runtime
>> +loader can fix them.
>> +
>> +So the bar peripheral's DTS format would be of the form:
>> +
>> +/dts-v1/ /plugin/;     /* allow undefined references and record them */
>> +/ {
>> +       ....    /* various properties for loader use; i.e. part id etc. */
>> +       fragment@0 {
>> +               target = <&ocp>;
>> +               __overlay__ {
>> +                       /* bar peripheral */
>> +                       bar {
>> +                               compatible = "corp,bar";
>> +                               ... /* various properties and child nodes */
>> +                       }
>> +               };
>> +       };
>> +};
>> +
>> +Note that there's a target property that specifies the location where the
>> +contents of the overlay node will be placed, and it references the node
>> +in the foo.dts file.
>> +
>> +$ dtc -@ -O dtb -o bar.dtbo -b 0 bar.dts
>> +$ fdtdump bar.dtbo
>> +...
>> +/ {
>> +       ... /* properties */
>> +       fragment@0 {
>> +               target = <0xffffffff>;
>> +               __overlay__ {
>> +                       bar {
>> +                               compatible = "corp,bar";
>> +                               ... /* various properties and child nodes */
>> +                       }
>> +               };
>> +       };
>> +       __fixups__ {
>> +           ocp = "/fragment@0:target:0";
>> +       };
>> +};
>> +
>> +No __symbols__ has been generated (no label in bar.dts).
> 
> Add "node" after __symbols__ here?
> 
>> +Note that the target's ocp label is undefined, so the phandle handle
> 
> Drop handle after phandle?
> 
>> +value is filled with the illegal value '0xffffffff', while a __fixups__
>> +node has been generated, which marks the location in the tree where
>> +the label lookup should store the runtime phandle value of the ocp node.
>> +
>> +The format of the __fixups__ node entry is
>> +
>> +       <label> = "<local-full-path>:<property-name>:<offset>";
>> +
>> +<label>                Is the label we're referring
>> +<local-full-path>      Is the full path of the node the reference is
>> +<property-name>                Is the name of the property containing the
> 
> Weird alignment here.
> 
>> +                       reference
>> +<offset>               The offset (in bytes) of where the property's
>> +                       phandle value is located.
> 
> located within the property? Or "offset relative to the start of the
> property in bytes where the phandle value is located"?
> 
> Is this a list? So multiple properties can be fixed up with the same
> label? If so that isn't clear from this description.
> 
>> +
>> +Doing the same with the baz peripheral's DTS format is a little bit more
>> +involved, since baz contains references to local labels which require
>> +local fixups.
>> +
>> +/dts-v1/ /plugin/;     /* allow undefined label references and record them */
>> +/ {
>> +       ....    /* various properties for loader use; i.e. part id etc. */
>> +       fragment@0 {
>> +               target = <&res>;
>> +               __overlay__ {
>> +                       /* baz resources */
>> +                       baz_res: res_baz { ... };
>> +               };
>> +       };
>> +       fragment@1 {
>> +               target = <&ocp>;
>> +               __overlay__ {
>> +                       /* baz peripheral */
>> +                       baz {
>> +                               compatible = "corp,baz";
>> +                               /* reference to another point in the tree */
>> +                               ref-to-res = <&baz_res>;
>> +                               ... /* various properties and child nodes */
>> +                       }
>> +               };
>> +       };
>> +};
>> +
>> +Note that &bar_res reference.
>> +
>> +$ dtc -@ -O dtb -o baz.dtbo -b 0 baz.dts
>> +$ fdtdump baz.dtbo
>> +...
>> +/ {
>> +       ... /* properties */
>> +       fragment@0 {
>> +               target = <0xffffffff>;
>> +               __overlay__ {
>> +                       res_baz {
>> +                               ....
>> +                               phandle = <0x00000001>;
>> +                       };
>> +               };
>> +       };
>> +       fragment@1 {
>> +               target = <0xffffffff>;
>> +               __overlay__ {
>> +                       baz {
>> +                               compatible = "corp,baz";
>> +                               ... /* various properties and child nodes */
>> +                               ref-to-res = <0x00000001>;
>> +                       }
>> +               };
>> +       };
>> +       __fixups__ {
>> +               res = "/fragment@0:target:0";
>> +               ocp = "/fragment@1:target:0";
>> +       };
>> +       __local_fixups__ {
>> +               fragment@1 {
>> +                       __overlay__ {
>> +                               baz {
>> +                                       ref-to-res = <0>;
>> +                               };
>> +                       };
>> +               };
>> +       };
>> +};
>> +
>> +This is similar to the bar case, but the reference of a local label by the
>> +baz node generates a __local_fixups__ entry that records the place that the
>> +local reference is being made. No matter how phandles are allocated from dtc
>> +the run time loader must apply an offset to each phandle in every dynamic
>> +DT object loaded. The __local_fixups__ node records the place of every
> 
> records the offset relative to the start of the property of every local
> reference within that property so that the loader...
> 
>> +local reference so that the loader can apply the offset.
>> +
>> +There is an alternative syntax to the expanded form for overlays with phandle
>> +targets which makes the format similar to the one using in .dtsi include files.
>> +
>> +So for the &ocp target example above one can simply write:
>> +
>> +/dts-v1/ /plugin/;
>> +&ocp {
>> +       /* bar peripheral */
>> +       bar {
>> +               compatible = "corp,bar";
>> +               ... /* various properties and child nodes */
>> +       }
>> +};
>> +
>> +The resulting dtb object is identical.

Thanks for the review, I’ll submit a new doc patch shortly.

Regards

— Pantelis

Re: [PATCH v10 2/4] dtc: Document the dynamic plugin internals

[David Gibson]

[-- Attachment #1: Type: text/plain, Size: 9943 bytes --]

On Mon, Nov 28, 2016 at 10:29:05PM +0200, Pantelis Antoniou wrote:
> Hi Stephen,
> 
> > On Nov 28, 2016, at 22:03 , Stephen Boyd <stephen.boyd-QSEj5FYQhm4dnm+yROfE0A@public.gmane.org> wrote:
> > 
> > Quoting Pantelis Antoniou (2016-11-25 04:32:09)
> >> diff --git a/Documentation/dt-object-internal.txt b/Documentation/dt-object-internal.txt
> >> new file mode 100644
> >> index 0000000..d5b841e
> >> --- /dev/null
> >> +++ b/Documentation/dt-object-internal.txt
> >> @@ -0,0 +1,318 @@
> >> +Device Tree Dynamic Object format internals
> >> +-------------------------------------------
> >> +
> >> +The Device Tree for most platforms is a static representation of
> >> +the hardware capabilities. This is insufficient for many platforms
> > 
> > s/many//
> > 
> >> +that need to dynamically insert device tree fragments to the
> > 
> > that need to dynamically insert device tree fragments into the
> > 
> > Also, should device tree be capitalized here?
> > 
> >> +running kernel's live tree.
> > 
> > Drop "running kernel's" as it's implicit with "live tree"?
> > 
> >> +
> >> +This document explains the the device tree object format and the
> > 
> > s/the//
> > 
> >> +modifications made to the device tree compiler, which make it possible.
> >> +
> >> +1. Simplified Problem Definition
> >> +--------------------------------
> >> +
> >> +Assume we have a platform which boots using following simplified device tree.
> >> +
> >> +---- foo.dts -----------------------------------------------------------------
> >> +       /* FOO platform */
> >> +       / {
> >> +               compatible = "corp,foo";
> >> +
> >> +               /* shared resources */
> >> +               res: res {
> >> +               };
> >> +
> >> +               /* On chip peripherals */
> >> +               ocp: ocp {
> >> +                       /* peripherals that are always instantiated */
> >> +                       peripheral1 { ... };
> >> +               };
> >> +       };
> >> +---- foo.dts -----------------------------------------------------------------
> >> +
> >> +We have a number of peripherals that after probing (using some undefined method)
> >> +should result in different device tree configuration.
> >> +
> >> +We cannot boot with this static tree because due to the configuration of the
> >> +foo platform there exist multiple conficting peripherals DT fragments.
> >> +
> >> +So for the bar peripheral we would have this:
> >> +
> >> +---- foo+bar.dts -------------------------------------------------------------
> >> +       /* FOO platform + bar peripheral */
> >> +       / {
> >> +               compatible = "corp,foo";
> >> +
> >> +               /* shared resources */
> >> +               res: res {
> >> +               };
> >> +
> >> +               /* On chip peripherals */
> >> +               ocp: ocp {
> >> +                       /* peripherals that are always instantiated */
> >> +                       peripheral1 { ... };
> >> +
> >> +                       /* bar peripheral */
> >> +                       bar {
> >> +                               compatible = "corp,bar";
> >> +                               ... /* various properties and child nodes */
> >> +                       };
> >> +               };
> >> +       };
> >> +---- foo+bar.dts -------------------------------------------------------------
> >> +
> >> +While for the baz peripheral we would have this:
> >> +
> >> +---- foo+baz.dts -------------------------------------------------------------
> >> +       /* FOO platform + baz peripheral */
> >> +       / {
> >> +               compatible = "corp,foo";
> >> +
> >> +               /* shared resources */
> >> +               res: res {
> >> +                       /* baz resources */
> >> +                       baz_res: res_baz { ... };
> >> +               };
> >> +
> >> +               /* On chip peripherals */
> >> +               ocp: ocp {
> >> +                       /* peripherals that are always instantiated */
> >> +                       peripheral1 { ... };
> >> +
> >> +                       /* baz peripheral */
> >> +                       baz {
> >> +                               compatible = "corp,baz";
> >> +                               /* reference to another point in the tree */
> >> +                               ref-to-res = <&baz_res>;
> >> +                               ... /* various properties and child nodes */
> >> +                       };
> >> +               };
> >> +       };
> >> +---- foo+baz.dts -------------------------------------------------------------
> >> +
> >> +We note that the baz case is more complicated, since the baz peripheral needs to
> >> +reference another node in the DT tree.
> >> +
> >> +2. Device Tree Object Format Requirements
> >> +-----------------------------------------
> >> +
> >> +Since the device tree is used for booting a number of very different hardware
> >> +platforms it is imperative that we tread very carefully.
> >> +
> >> +2.a) No changes to the Device Tree binary format for the base tree. We cannot
> >> +modify the tree format at all and all the information we require should be
> >> +encoded using device tree itself. We can add nodes that can be safely ignored
> >> +by both bootloaders and the kernel. The plugin dtb's are optionally tagged
> > 
> > s/dtb's/dtbs/
> > 
> >> +with a different magic number in the header but otherwise they too are simple
> >> +blobs.
> > 
> > but otherwise they're simple blobs.
> > 
> 
> OK on the spelling/grammar fixes above.
> 
> >> +
> >> +2.b) Changes to the DTS source format should be absolutely minimal, and should
> >> +only be needed for the DT fragment definitions, and not the base boot DT.
> >> +
> >> +2.c) An explicit option should be used to instruct DTC to generate the required
> >> +information needed for object resolution. Platforms that don't use the
> >> +dynamic object format can safely ignore it.
> > 
> > Why? We can't figure that out based on the /plugin/ label within the dts
> > file? And shouldn't we always generate a __symbols__ node in the base
> > dtb?
> > 
> 
> Actually now we do. The last patchset does automatically generate those nodes
> if a /plugin/ tag is encountered. For base dtbs I would suggest that generating
> the symbols node automatically is what’s sane too, but unfortunately there are
> some platforms out there that are having trouble with larger dtbs than what they
> expect.
> 
> It is your call whether to enable it by default I guess.
> 
> >> +
> >> +2.d) Finally, DT syntax changes should be kept to a minimum. It should be
> >> +possible to express everything using the existing DT syntax.
> >> +
> >> +3. Implementation
> >> +-----------------
> >> +
> >> +The basic unit of addressing in Device Tree is the phandle. Turns out it's
> >> +relatively simple to extend the way phandles are generated and referenced
> >> +so that it's possible to dynamically convert symbolic references (labels)
> >> +to phandle values. This is a valid assumption as long as the author uses
> >> +reference syntax and does not assign phandle values manually (which might
> >> +be a problem with decompiled source files).
> >> +
> >> +We can roughly divide the operation into two steps.
> >> +
> >> +3.a) Compilation of the base board DTS file using the '-@' option
> >> +generates a valid DT blob with an added __symbols__ node at the root node,
> >> +containing a list of all nodes that are marked with a label.
> >> +
> >> +Using the foo.dts file above the following node will be generated;
> >> +
> >> +$ dtc -@ -O dtb -o foo.dtb -b 0 foo.dts
> >> +$ fdtdump foo.dtb
> >> +...
> >> +/ {
> >> +       ...
> >> +       res {
> >> +               ...
> >> +               phandle = <0x00000001>;
> >> +               ...
> >> +       };
> >> +       ocp {
> >> +               ...
> >> +               phandle = <0x00000002>;
> >> +               ...
> >> +       };
> >> +       __symbols__ {
> >> +               res="/res";
> >> +               ocp="/ocp";
> >> +       };
> >> +};
> >> +
> >> +Notice that all the nodes that had a label have been recorded, and that
> >> +phandles have been generated for them.
> >> +
> >> +This blob can be used to boot the board normally, the __symbols__ node will
> >> +be safely ignored both by the bootloader and the kernel (the only loss will
> >> +be a few bytes of memory and disk space).
> > 
> > This never really mentions why we need to generate a symbols node.
> > Perhaps we should say something like "we generate a __symbols__ node to
> > record nodes that had labels in the base tree so they can be matched up
> > with the fragments which reference the same labels"? Or something like
> > that.
> > 
> 
> Hmm, yeah.
> 
> > I also wonder why it's even necessary. Couldn't we require overlays to
> > be compiled with the original dts files? Then we could encode the full
> > path of nodes referenced in the overlay into the overlay dtb itself.
> > 
> 
> No, we can’t do that; the end-game of this is for overlays to be portable
> for use in platforms having the same kind of connectors.

That's kind of true, but actually I think we want to redesign the
connector format.  Obviously we'll take some stuff from the current
overlay format, but we can't be fully compatible with them, so we
should take the opportunity to remove some of the sillier design flaws
in the overlay format.

That said, even with their flaws and limitations, overlays in the
current format can sometimes be portable to multiple base trees.

-- 
David Gibson			| I'll have my music baroque, and my code
david AT gibson.dropbear.id.au	| minimalist, thank you.  NOT _the_ _other_
				| _way_ _around_!
http://www.ozlabs.org/~dgibson

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 819 bytes --]

Re: [PATCH v10 2/4] dtc: Document the dynamic plugin internals

[David Gibson]

[-- Attachment #1: Type: text/plain, Size: 15643 bytes --]

On Mon, Nov 28, 2016 at 12:03:50PM -0800, Stephen Boyd wrote:
> Quoting Pantelis Antoniou (2016-11-25 04:32:09)
> > diff --git a/Documentation/dt-object-internal.txt b/Documentation/dt-object-internal.txt
> > new file mode 100644
> > index 0000000..d5b841e
> > --- /dev/null
> > +++ b/Documentation/dt-object-internal.txt
> > @@ -0,0 +1,318 @@
> > +Device Tree Dynamic Object format internals
> > +-------------------------------------------
> > +
> > +The Device Tree for most platforms is a static representation of
> > +the hardware capabilities. This is insufficient for many platforms
> 
> s/many//
> 
> > +that need to dynamically insert device tree fragments to the
> 
> that need to dynamically insert device tree fragments into the
> 
> Also, should device tree be capitalized here?
> 
> > +running kernel's live tree.
> 
> Drop "running kernel's" as it's implicit with "live tree"?
> 
> > +
> > +This document explains the the device tree object format and the
> 
> s/the//
> 
> > +modifications made to the device tree compiler, which make it possible.
> > +
> > +1. Simplified Problem Definition
> > +--------------------------------
> > +
> > +Assume we have a platform which boots using following simplified device tree.
> > +
> > +---- foo.dts -----------------------------------------------------------------
> > +       /* FOO platform */
> > +       / {
> > +               compatible = "corp,foo";
> > +
> > +               /* shared resources */
> > +               res: res {
> > +               };
> > +
> > +               /* On chip peripherals */
> > +               ocp: ocp {
> > +                       /* peripherals that are always instantiated */
> > +                       peripheral1 { ... };
> > +               };
> > +       };
> > +---- foo.dts -----------------------------------------------------------------
> > +
> > +We have a number of peripherals that after probing (using some undefined method)
> > +should result in different device tree configuration.
> > +
> > +We cannot boot with this static tree because due to the configuration of the
> > +foo platform there exist multiple conficting peripherals DT fragments.
> > +
> > +So for the bar peripheral we would have this:
> > +
> > +---- foo+bar.dts -------------------------------------------------------------
> > +       /* FOO platform + bar peripheral */
> > +       / {
> > +               compatible = "corp,foo";
> > +
> > +               /* shared resources */
> > +               res: res {
> > +               };
> > +
> > +               /* On chip peripherals */
> > +               ocp: ocp {
> > +                       /* peripherals that are always instantiated */
> > +                       peripheral1 { ... };
> > +
> > +                       /* bar peripheral */
> > +                       bar {
> > +                               compatible = "corp,bar";
> > +                               ... /* various properties and child nodes */
> > +                       };
> > +               };
> > +       };
> > +---- foo+bar.dts -------------------------------------------------------------
> > +
> > +While for the baz peripheral we would have this:
> > +
> > +---- foo+baz.dts -------------------------------------------------------------
> > +       /* FOO platform + baz peripheral */
> > +       / {
> > +               compatible = "corp,foo";
> > +
> > +               /* shared resources */
> > +               res: res {
> > +                       /* baz resources */
> > +                       baz_res: res_baz { ... };
> > +               };
> > +
> > +               /* On chip peripherals */
> > +               ocp: ocp {
> > +                       /* peripherals that are always instantiated */
> > +                       peripheral1 { ... };
> > +
> > +                       /* baz peripheral */
> > +                       baz {
> > +                               compatible = "corp,baz";
> > +                               /* reference to another point in the tree */
> > +                               ref-to-res = <&baz_res>;
> > +                               ... /* various properties and child nodes */
> > +                       };
> > +               };
> > +       };
> > +---- foo+baz.dts -------------------------------------------------------------
> > +
> > +We note that the baz case is more complicated, since the baz peripheral needs to
> > +reference another node in the DT tree.
> > +
> > +2. Device Tree Object Format Requirements
> > +-----------------------------------------
> > +
> > +Since the device tree is used for booting a number of very different hardware
> > +platforms it is imperative that we tread very carefully.
> > +
> > +2.a) No changes to the Device Tree binary format for the base tree. We cannot
> > +modify the tree format at all and all the information we require should be
> > +encoded using device tree itself. We can add nodes that can be safely ignored
> > +by both bootloaders and the kernel. The plugin dtb's are optionally tagged
> 
> s/dtb's/dtbs/
> 
> > +with a different magic number in the header but otherwise they too are simple
> > +blobs.
> 
> but otherwise they're simple blobs.
> 
> > +
> > +2.b) Changes to the DTS source format should be absolutely minimal, and should
> > +only be needed for the DT fragment definitions, and not the base boot DT.
> > +
> > +2.c) An explicit option should be used to instruct DTC to generate the required
> > +information needed for object resolution. Platforms that don't use the
> > +dynamic object format can safely ignore it.
> 
> Why? We can't figure that out based on the /plugin/ label within the dts
> file? And shouldn't we always generate a __symbols__ node in the base
> dtb?

No, given it's a nonstandard extension on the basic device tree
contents, I don't think we should generate the symbol information by
default.  /plugin/ can let you determine whether to generate fixups,
but you need the symbols for the base tree.

> > +
> > +2.d) Finally, DT syntax changes should be kept to a minimum. It should be
> > +possible to express everything using the existing DT syntax.
> > +
> > +3. Implementation
> > +-----------------
> > +
> > +The basic unit of addressing in Device Tree is the phandle. Turns out it's
> > +relatively simple to extend the way phandles are generated and referenced
> > +so that it's possible to dynamically convert symbolic references (labels)
> > +to phandle values. This is a valid assumption as long as the author uses
> > +reference syntax and does not assign phandle values manually (which might
> > +be a problem with decompiled source files).
> > +
> > +We can roughly divide the operation into two steps.
> > +
> > +3.a) Compilation of the base board DTS file using the '-@' option
> > +generates a valid DT blob with an added __symbols__ node at the root node,
> > +containing a list of all nodes that are marked with a label.
> > +
> > +Using the foo.dts file above the following node will be generated;
> > +
> > +$ dtc -@ -O dtb -o foo.dtb -b 0 foo.dts
> > +$ fdtdump foo.dtb
> > +...
> > +/ {
> > +       ...
> > +       res {
> > +               ...
> > +               phandle = <0x00000001>;
> > +               ...
> > +       };
> > +       ocp {
> > +               ...
> > +               phandle = <0x00000002>;
> > +               ...
> > +       };
> > +       __symbols__ {
> > +               res="/res";
> > +               ocp="/ocp";
> > +       };
> > +};
> > +
> > +Notice that all the nodes that had a label have been recorded, and that
> > +phandles have been generated for them.
> > +
> > +This blob can be used to boot the board normally, the __symbols__ node will
> > +be safely ignored both by the bootloader and the kernel (the only loss will
> > +be a few bytes of memory and disk space).
> 
> This never really mentions why we need to generate a symbols node.
> Perhaps we should say something like "we generate a __symbols__ node to
> record nodes that had labels in the base tree so they can be matched up
> with the fragments which reference the same labels"? Or something like
> that.
> 
> I also wonder why it's even necessary. Couldn't we require overlays to
> be compiled with the original dts files? Then we could encode the full
> path of nodes referenced in the overlay into the overlay dtb itself.

That's one of many different design decisions that could have been
made, and might have been a better idea.  But the current design is
out in the wild now, flaws and all, so we do need to implement it.

> > +
> > +3.b) The Device Tree fragments must be compiled with the same option but they
> > +must also have a tag (/plugin/) that allows undefined references to nodes
> > +that are not present at compilation time to be recorded so that the runtime
> > +loader can fix them.
> > +
> > +So the bar peripheral's DTS format would be of the form:
> > +
> > +/dts-v1/ /plugin/;     /* allow undefined references and record them */
> > +/ {
> > +       ....    /* various properties for loader use; i.e. part id etc. */
> > +       fragment@0 {
> > +               target = <&ocp>;
> > +               __overlay__ {
> > +                       /* bar peripheral */
> > +                       bar {
> > +                               compatible = "corp,bar";
> > +                               ... /* various properties and child nodes */
> > +                       }
> > +               };
> > +       };
> > +};
> > +
> > +Note that there's a target property that specifies the location where the
> > +contents of the overlay node will be placed, and it references the node
> > +in the foo.dts file.
> > +
> > +$ dtc -@ -O dtb -o bar.dtbo -b 0 bar.dts
> > +$ fdtdump bar.dtbo
> > +...
> > +/ {
> > +       ... /* properties */
> > +       fragment@0 {
> > +               target = <0xffffffff>;
> > +               __overlay__ {
> > +                       bar {
> > +                               compatible = "corp,bar";
> > +                               ... /* various properties and child nodes */
> > +                       }
> > +               };
> > +       };
> > +       __fixups__ {
> > +           ocp = "/fragment@0:target:0";
> > +       };
> > +};
> > +
> > +No __symbols__ has been generated (no label in bar.dts).
> 
> Add "node" after __symbols__ here?
> 
> > +Note that the target's ocp label is undefined, so the phandle handle
> 
> Drop handle after phandle?
> 
> > +value is filled with the illegal value '0xffffffff', while a __fixups__
> > +node has been generated, which marks the location in the tree where
> > +the label lookup should store the runtime phandle value of the ocp node.
> > +
> > +The format of the __fixups__ node entry is
> > +
> > +       <label> = "<local-full-path>:<property-name>:<offset>";
> > +
> > +<label>                Is the label we're referring
> > +<local-full-path>      Is the full path of the node the reference is
> > +<property-name>                Is the name of the property containing the
> 
> Weird alignment here.
> 
> > +                       reference
> > +<offset>               The offset (in bytes) of where the property's
> > +                       phandle value is located.
> 
> located within the property? Or "offset relative to the start of the
> property in bytes where the phandle value is located"?
> 
> Is this a list? So multiple properties can be fixed up with the same
> label? If so that isn't clear from this description.
> 
> > +
> > +Doing the same with the baz peripheral's DTS format is a little bit more
> > +involved, since baz contains references to local labels which require
> > +local fixups.
> > +
> > +/dts-v1/ /plugin/;     /* allow undefined label references and record them */
> > +/ {
> > +       ....    /* various properties for loader use; i.e. part id etc. */
> > +       fragment@0 {
> > +               target = <&res>;
> > +               __overlay__ {
> > +                       /* baz resources */
> > +                       baz_res: res_baz { ... };
> > +               };
> > +       };
> > +       fragment@1 {
> > +               target = <&ocp>;
> > +               __overlay__ {
> > +                       /* baz peripheral */
> > +                       baz {
> > +                               compatible = "corp,baz";
> > +                               /* reference to another point in the tree */
> > +                               ref-to-res = <&baz_res>;
> > +                               ... /* various properties and child nodes */
> > +                       }
> > +               };
> > +       };
> > +};
> > +
> > +Note that &bar_res reference.
> > +
> > +$ dtc -@ -O dtb -o baz.dtbo -b 0 baz.dts
> > +$ fdtdump baz.dtbo
> > +...
> > +/ {
> > +       ... /* properties */
> > +       fragment@0 {
> > +               target = <0xffffffff>;
> > +               __overlay__ {
> > +                       res_baz {
> > +                               ....
> > +                               phandle = <0x00000001>;
> > +                       };
> > +               };
> > +       };
> > +       fragment@1 {
> > +               target = <0xffffffff>;
> > +               __overlay__ {
> > +                       baz {
> > +                               compatible = "corp,baz";
> > +                               ... /* various properties and child nodes */
> > +                               ref-to-res = <0x00000001>;
> > +                       }
> > +               };
> > +       };
> > +       __fixups__ {
> > +               res = "/fragment@0:target:0";
> > +               ocp = "/fragment@1:target:0";
> > +       };
> > +       __local_fixups__ {
> > +               fragment@1 {
> > +                       __overlay__ {
> > +                               baz {
> > +                                       ref-to-res = <0>;
> > +                               };
> > +                       };
> > +               };
> > +       };
> > +};
> > +
> > +This is similar to the bar case, but the reference of a local label by the
> > +baz node generates a __local_fixups__ entry that records the place that the
> > +local reference is being made. No matter how phandles are allocated from dtc
> > +the run time loader must apply an offset to each phandle in every dynamic
> > +DT object loaded. The __local_fixups__ node records the place of every
> 
> records the offset relative to the start of the property of every local
> reference within that property so that the loader...
> 
> > +local reference so that the loader can apply the offset.
> > +
> > +There is an alternative syntax to the expanded form for overlays with phandle
> > +targets which makes the format similar to the one using in .dtsi include files.
> > +
> > +So for the &ocp target example above one can simply write:
> > +
> > +/dts-v1/ /plugin/;
> > +&ocp {
> > +       /* bar peripheral */
> > +       bar {
> > +               compatible = "corp,bar";
> > +               ... /* various properties and child nodes */
> > +       }
> > +};
> > +
> > +The resulting dtb object is identical.

-- 
David Gibson			| I'll have my music baroque, and my code
david AT gibson.dropbear.id.au	| minimalist, thank you.  NOT _the_ _other_
				| _way_ _around_!
http://www.ozlabs.org/~dgibson

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 819 bytes --]

Re: [PATCH v10 2/4] dtc: Document the dynamic plugin internals

[Frank Rowand]

On 11/25/16 04:32, Pantelis Antoniou wrote:
> Provides the document explaining the internal mechanics of
> plugins and options.
> 
> Signed-off-by: Pantelis Antoniou <pantelis.antoniou-OWPKS81ov/FWk0Htik3J/w@public.gmane.org>
> ---
>  Documentation/dt-object-internal.txt | 318 +++++++++++++++++++++++++++++++++++
>  1 file changed, 318 insertions(+)
>  create mode 100644 Documentation/dt-object-internal.txt
> 
> diff --git a/Documentation/dt-object-internal.txt b/Documentation/dt-object-internal.txt
> new file mode 100644
> index 0000000..d5b841e
> --- /dev/null
> +++ b/Documentation/dt-object-internal.txt
> @@ -0,0 +1,318 @@
> +Device Tree Dynamic Object format internals
> +-------------------------------------------
> +
> +The Device Tree for most platforms is a static representation of
> +the hardware capabilities. This is insufficient for many platforms
> +that need to dynamically insert device tree fragments to the
> +running kernel's live tree.
> +
> +This document explains the the device tree object format and the
> +modifications made to the device tree compiler, which make it possible.
> +
> +1. Simplified Problem Definition
> +--------------------------------
> +
> +Assume we have a platform which boots using following simplified device tree.
> +
> +---- foo.dts -----------------------------------------------------------------
> +	/* FOO platform */
> +	/ {
> +		compatible = "corp,foo";
> +
> +		/* shared resources */
> +		res: res {
> +		};
> +
> +		/* On chip peripherals */
> +		ocp: ocp {
> +			/* peripherals that are always instantiated */
> +			peripheral1 { ... };
> +		};
> +	};
> +---- foo.dts -----------------------------------------------------------------
> +
> +We have a number of peripherals that after probing (using some undefined method)
> +should result in different device tree configuration.
> +
> +We cannot boot with this static tree because due to the configuration of the
> +foo platform there exist multiple conficting peripherals DT fragments.

                                     ^^^^^^^^^^  conflicting

I assume conflicting because, for instance, the different peripherals might
occupy the same address space, use the same interrupt, or use the same gpio.
Mentioning that would provide a fuller picture for the neophyte.

> +
> +So for the bar peripheral we would have this:
> +
> +---- foo+bar.dts -------------------------------------------------------------
> +	/* FOO platform + bar peripheral */
> +	/ {
> +		compatible = "corp,foo";
> +
> +		/* shared resources */
> +		res: res {
> +		};
> +
> +		/* On chip peripherals */
> +		ocp: ocp {
> +			/* peripherals that are always instantiated */
> +			peripheral1 { ... };
> +
> +			/* bar peripheral */
> +			bar {
> +				compatible = "corp,bar";
> +				... /* various properties and child nodes */
> +			};
> +		};
> +	};
> +---- foo+bar.dts -------------------------------------------------------------
> +
> +While for the baz peripheral we would have this:
> +
> +---- foo+baz.dts -------------------------------------------------------------
> +	/* FOO platform + baz peripheral */
> +	/ {
> +		compatible = "corp,foo";
> +
> +		/* shared resources */
> +		res: res {
> +			/* baz resources */
> +			baz_res: res_baz { ... };
> +		};
> +
> +		/* On chip peripherals */
> +		ocp: ocp {
> +			/* peripherals that are always instantiated */
> +			peripheral1 { ... };
> +
> +			/* baz peripheral */
> +			baz {
> +				compatible = "corp,baz";
> +				/* reference to another point in the tree */
> +				ref-to-res = <&baz_res>;
> +				... /* various properties and child nodes */
> +			};
> +		};
> +	};
> +---- foo+baz.dts -------------------------------------------------------------
> +
> +We note that the baz case is more complicated, since the baz peripheral needs to
> +reference another node in the DT tree.

I know that there are other situations that can justify overlays, so not
contesting the basic need with this comment.  But the above situation could
be handled in a much simpler fashion by setting the status property of each
of the conflicting devices to disabled, then after probing setting the status
to ok.  That method removes a lot of complexity.

A big driver for the concept of overlays was being able to describe different
add on boards at run time, instead of when the base dtb was created.  I think
we have agreed that moving to a connector model instead of a raw overlay is
the proper way to address add on boards.

Can you address how an overlay can be created that will work for a board
plugged into any of the identical sockets that is compatible with the
board?


> +
> +2. Device Tree Object Format Requirements
> +-----------------------------------------
> +
> +Since the device tree is used for booting a number of very different hardware
> +platforms it is imperative that we tread very carefully.
> +
> +2.a) No changes to the Device Tree binary format for the base tree. We cannot
> +modify the tree format at all and all the information we require should be
> +encoded using device tree itself. We can add nodes that can be safely ignored
> +by both bootloaders and the kernel. The plugin dtb's are optionally tagged
> +with a different magic number in the header but otherwise they too are simple
> +blobs.
> +
> +2.b) Changes to the DTS source format should be absolutely minimal, and should
> +only be needed for the DT fragment definitions, and not the base boot DT.
> +
> +2.c) An explicit option should be used to instruct DTC to generate the required
> +information needed for object resolution. Platforms that don't use the
> +dynamic object format can safely ignore it.
> +
> +2.d) Finally, DT syntax changes should be kept to a minimum. It should be
> +possible to express everything using the existing DT syntax.
> +
> +3. Implementation
> +-----------------
> +
> +The basic unit of addressing in Device Tree is the phandle. Turns out it's
> +relatively simple to extend the way phandles are generated and referenced
> +so that it's possible to dynamically convert symbolic references (labels)
> +to phandle values. This is a valid assumption as long as the author uses
> +reference syntax and does not assign phandle values manually (which might
> +be a problem with decompiled source files).
> +
> +We can roughly divide the operation into two steps.
> +
> +3.a) Compilation of the base board DTS file using the '-@' option
> +generates a valid DT blob with an added __symbols__ node at the root node,
> +containing a list of all nodes that are marked with a label.
> +
> +Using the foo.dts file above the following node will be generated;
> +
> +$ dtc -@ -O dtb -o foo.dtb -b 0 foo.dts
> +$ fdtdump foo.dtb
> +...
> +/ {
> +	...
> +	res {
> +		...
> +		phandle = <0x00000001>;
> +		...
> +	};
> +	ocp {
> +		...
> +		phandle = <0x00000002>;
> +		...
> +	};
> +	__symbols__ {
> +		res="/res";
> +		ocp="/ocp";
> +	};
> +};
> +
> +Notice that all the nodes that had a label have been recorded, and that
> +phandles have been generated for them.
> +
> +This blob can be used to boot the board normally, the __symbols__ node will
> +be safely ignored both by the bootloader and the kernel (the only loss will
> +be a few bytes of memory and disk space).
> +
> +3.b) The Device Tree fragments must be compiled with the same option but they
> +must also have a tag (/plugin/) that allows undefined references to nodes
> +that are not present at compilation time to be recorded so that the runtime
> +loader can fix them.
> +
> +So the bar peripheral's DTS format would be of the form:
> +
> +/dts-v1/ /plugin/;	/* allow undefined references and record them */
> +/ {
> +	....	/* various properties for loader use; i.e. part id etc. */
> +	fragment@0 {
> +		target = <&ocp>;
> +		__overlay__ {
> +			/* bar peripheral */
> +			bar {
> +				compatible = "corp,bar";
> +				... /* various properties and child nodes */
> +			}
> +		};
> +	};
> +};

The last version of your patches that I tested did not require specifying
the target property, the fragment node, and the __overlay__ node.  dtc
properly created all of those items automatically.  For example, I could
go to all of the trouble of creating those items in a dts like:

$ cat example_1_hand_coded.dts
/dts-v1/;
/plugin/;

/ {

	fragment@0 {
		target = <&am3353x_pinmux>;

		__overlay__ {

			i2c1_pins: pinmux_i2c1_pins {
				pinctrl-single,pins = <
					0x158 0x72
					0x15c 0x72
				>;
			};
		};
	};

	fragment@1 {
		target = <&i2c1>;

		__overlay__ {
			pinctrl-names = "default";
			pinctrl-0 = <&i2c1_pins>;
			clock-frequency = <400000>;
			status = "okay";

			at24@50 {
				compatible = "at,24c256";
				pagesize = <64>;
				reg = <0x50>;
			};
		};
	};
};


Or I could let dtc automagically create all the special features
(target, fragment, __overlay__) from an equivalent dts:

$ cat example_1.dts
/dts-v1/;
/plugin/;


		&am3353x_pinmux {
			i2c1_pins: pinmux_i2c1_pins {
				pinctrl-single,pins = <
					0x158 0x72
					0x15c 0x72
				>;
			};
		};

		&i2c1 {
			#address-cells = <1>;
			#size-cells = <0>;
			pinctrl-names = "default";
			pinctrl-0 = <&i2c1_pins>;
			clock-frequency = <400000>;
			status = "okay";

			at24@50 {
				compatible = "at,24c256";
				pagesize = <64>;
				reg = <0x50>;
			};
		};


I would much prefer that people never hand code the target, fragment, and
__overlay__ in a dts source file.  Exposing them at the source level adds
complexity, confusion, and an increased chance of creating an invalid
overlay dtb.

If possible, I would prefer target, fragment, and __overlay__ not be valid
input to dtc.  It would probably be difficult to prohibit target and fragment,
because however unlikely they are as property and node names, they are valid
dts syntax before adding the overlay enhancements to dtc.  However __overlay__
is not a valid node name without the overlay enhancements and could remain
invalid dts input.

I prefer that target, fragment, and __overlay__ be documented as a dtb to
target system API.  In this case, for the normal developer, they are
hidden in the binary dtb format and in the kernel (or boot loader)
overlay framework code.

I do recognize that if __overlay__ is not valid dtc input then it is not
possible to decompile an overlay into a dts containing __overlay__ and
then recompile that dts.  This could be resolved by a more complex
decompile that turned the overlay dtb back into the form of example_1.dts
above.

After reading to the end of this patch, I see that the simpler form of
.dts (like example_1.dts) is also noted as "an alternative syntax to
the expanded form for overlays".


> +
> +Note that there's a target property that specifies the location where the
> +contents of the overlay node will be placed, and it references the node
> +in the foo.dts file.
> +
> +$ dtc -@ -O dtb -o bar.dtbo -b 0 bar.dts
> +$ fdtdump bar.dtbo
> +...
> +/ {
> +	... /* properties */
> +	fragment@0 {
> +		target = <0xffffffff>;
> +		__overlay__ {
> +			bar {
> +				compatible = "corp,bar";
> +				... /* various properties and child nodes */
> +			}
> +		};
> +	};
> +	__fixups__ {
> +	    ocp = "/fragment@0:target:0";
> +	};
> +};
> +
> +No __symbols__ has been generated (no label in bar.dts).
> +Note that the target's ocp label is undefined, so the phandle handle
> +value is filled with the illegal value '0xffffffff', while a __fixups__
> +node has been generated, which marks the location in the tree where
> +the label lookup should store the runtime phandle value of the ocp node.
> +
> +The format of the __fixups__ node entry is
> +
> +	<label> = "<local-full-path>:<property-name>:<offset>";
> +
> +<label> 		Is the label we're referring
> +<local-full-path>	Is the full path of the node the reference is
> +<property-name>		Is the name of the property containing the
> +			reference
> +<offset>		The offset (in bytes) of where the property's
> +			phandle value is located.
> +
> +Doing the same with the baz peripheral's DTS format is a little bit more
> +involved, since baz contains references to local labels which require
> +local fixups.
> +
> +/dts-v1/ /plugin/;	/* allow undefined label references and record them */
> +/ {
> +	....	/* various properties for loader use; i.e. part id etc. */
> +	fragment@0 {
> +		target = <&res>;
> +		__overlay__ {
> +			/* baz resources */
> +			baz_res: res_baz { ... };
> +		};
> +	};
> +	fragment@1 {
> +		target = <&ocp>;
> +		__overlay__ {
> +			/* baz peripheral */
> +			baz {
> +				compatible = "corp,baz";
> +				/* reference to another point in the tree */
> +				ref-to-res = <&baz_res>;
> +				... /* various properties and child nodes */
> +			}
> +		};
> +	};
> +};
> +
> +Note that &bar_res reference.
> +
> +$ dtc -@ -O dtb -o baz.dtbo -b 0 baz.dts
> +$ fdtdump baz.dtbo
> +...
> +/ {
> +	... /* properties */
> +	fragment@0 {
> +		target = <0xffffffff>;
> +		__overlay__ {
> +			res_baz {
> +				....
> +				phandle = <0x00000001>;
> +			};
> +		};
> +	};
> +	fragment@1 {
> +		target = <0xffffffff>;
> +		__overlay__ {
> +			baz {
> +				compatible = "corp,baz";
> +				... /* various properties and child nodes */
> +				ref-to-res = <0x00000001>;
> +			}
> +		};
> +	};
> +	__fixups__ {
> +		res = "/fragment@0:target:0";
> +		ocp = "/fragment@1:target:0";
> +	};
> +	__local_fixups__ {
> +		fragment@1 {
> +			__overlay__ {
> +				baz {
> +					ref-to-res = <0>;
> +				};
> +			};
> +		};
> +	};
> +};
> +
> +This is similar to the bar case, but the reference of a local label by the
> +baz node generates a __local_fixups__ entry that records the place that the
> +local reference is being made. No matter how phandles are allocated from dtc
> +the run time loader must apply an offset to each phandle in every dynamic
> +DT object loaded. The __local_fixups__ node records the place of every
> +local reference so that the loader can apply the offset.
> +
> +There is an alternative syntax to the expanded form for overlays with phandle
> +targets which makes the format similar to the one using in .dtsi include files.
> +
> +So for the &ocp target example above one can simply write:
> +
> +/dts-v1/ /plugin/;
> +&ocp {
> +	/* bar peripheral */
> +	bar {
> +		compatible = "corp,bar";
> +		... /* various properties and child nodes */
> +	}
> +};
> +
> +The resulting dtb object is identical.
> 

Re: [PATCH v10 2/4] dtc: Document the dynamic plugin internals

[Pantelis Antoniou]

Hi Frank,

> On Nov 29, 2016, at 03:36 , Frank Rowand <frowand.list-Re5JQEeQqe8AvxtiuMwx3w@public.gmane.org> wrote:
> 
> On 11/25/16 04:32, Pantelis Antoniou wrote:
>> Provides the document explaining the internal mechanics of
>> plugins and options.
>> 
>> Signed-off-by: Pantelis Antoniou <pantelis.antoniou-OWPKS81ov/FWk0Htik3J/w@public.gmane.org>
>> ---
>> Documentation/dt-object-internal.txt | 318 +++++++++++++++++++++++++++++++++++
>> 1 file changed, 318 insertions(+)
>> create mode 100644 Documentation/dt-object-internal.txt
>> 
>> diff --git a/Documentation/dt-object-internal.txt b/Documentation/dt-object-internal.txt
>> new file mode 100644
>> index 0000000..d5b841e
>> --- /dev/null
>> +++ b/Documentation/dt-object-internal.txt
>> @@ -0,0 +1,318 @@
>> +Device Tree Dynamic Object format internals
>> +-------------------------------------------
>> +
>> +The Device Tree for most platforms is a static representation of
>> +the hardware capabilities. This is insufficient for many platforms
>> +that need to dynamically insert device tree fragments to the
>> +running kernel's live tree.
>> +
>> +This document explains the the device tree object format and the
>> +modifications made to the device tree compiler, which make it possible.
>> +
>> +1. Simplified Problem Definition
>> +--------------------------------
>> +
>> +Assume we have a platform which boots using following simplified device tree.
>> +
>> +---- foo.dts -----------------------------------------------------------------
>> +	/* FOO platform */
>> +	/ {
>> +		compatible = "corp,foo";
>> +
>> +		/* shared resources */
>> +		res: res {
>> +		};
>> +
>> +		/* On chip peripherals */
>> +		ocp: ocp {
>> +			/* peripherals that are always instantiated */
>> +			peripheral1 { ... };
>> +		};
>> +	};
>> +---- foo.dts -----------------------------------------------------------------
>> +
>> +We have a number of peripherals that after probing (using some undefined method)
>> +should result in different device tree configuration.
>> +
>> +We cannot boot with this static tree because due to the configuration of the
>> +foo platform there exist multiple conficting peripherals DT fragments.
> 
>                                     ^^^^^^^^^^  conflicting
> 
> I assume conflicting because, for instance, the different peripherals might
> occupy the same address space, use the same interrupt, or use the same gpio.
> Mentioning that would provide a fuller picture for the neophyte.
> 

Yes, thanks for bringing this to my attention. This document is heavy on the neophyte for sure.

>> +
>> +So for the bar peripheral we would have this:
>> +
>> +---- foo+bar.dts -------------------------------------------------------------
>> +	/* FOO platform + bar peripheral */
>> +	/ {
>> +		compatible = "corp,foo";
>> +
>> +		/* shared resources */
>> +		res: res {
>> +		};
>> +
>> +		/* On chip peripherals */
>> +		ocp: ocp {
>> +			/* peripherals that are always instantiated */
>> +			peripheral1 { ... };
>> +
>> +			/* bar peripheral */
>> +			bar {
>> +				compatible = "corp,bar";
>> +				... /* various properties and child nodes */
>> +			};
>> +		};
>> +	};
>> +---- foo+bar.dts -------------------------------------------------------------
>> +
>> +While for the baz peripheral we would have this:
>> +
>> +---- foo+baz.dts -------------------------------------------------------------
>> +	/* FOO platform + baz peripheral */
>> +	/ {
>> +		compatible = "corp,foo";
>> +
>> +		/* shared resources */
>> +		res: res {
>> +			/* baz resources */
>> +			baz_res: res_baz { ... };
>> +		};
>> +
>> +		/* On chip peripherals */
>> +		ocp: ocp {
>> +			/* peripherals that are always instantiated */
>> +			peripheral1 { ... };
>> +
>> +			/* baz peripheral */
>> +			baz {
>> +				compatible = "corp,baz";
>> +				/* reference to another point in the tree */
>> +				ref-to-res = <&baz_res>;
>> +				... /* various properties and child nodes */
>> +			};
>> +		};
>> +	};
>> +---- foo+baz.dts -------------------------------------------------------------
>> +
>> +We note that the baz case is more complicated, since the baz peripheral needs to
>> +reference another node in the DT tree.
> 
> I know that there are other situations that can justify overlays, so not
> contesting the basic need with this comment.  But the above situation could
> be handled in a much simpler fashion by setting the status property of each
> of the conflicting devices to disabled, then after probing setting the status
> to ok.  That method removes a lot of complexity.
> 
> A big driver for the concept of overlays was being able to describe different
> add on boards at run time, instead of when the base dtb was created.  I think
> we have agreed that moving to a connector model instead of a raw overlay is
> the proper way to address add on boards.
> 
> Can you address how an overlay can be created that will work for a board
> plugged into any of the identical sockets that is compatible with the
> board?
> 
> 

Yes, I will try to do so.

>> +
>> +2. Device Tree Object Format Requirements
>> +-----------------------------------------
>> +
>> +Since the device tree is used for booting a number of very different hardware
>> +platforms it is imperative that we tread very carefully.
>> +
>> +2.a) No changes to the Device Tree binary format for the base tree. We cannot
>> +modify the tree format at all and all the information we require should be
>> +encoded using device tree itself. We can add nodes that can be safely ignored
>> +by both bootloaders and the kernel. The plugin dtb's are optionally tagged
>> +with a different magic number in the header but otherwise they too are simple
>> +blobs.
>> +
>> +2.b) Changes to the DTS source format should be absolutely minimal, and should
>> +only be needed for the DT fragment definitions, and not the base boot DT.
>> +
>> +2.c) An explicit option should be used to instruct DTC to generate the required
>> +information needed for object resolution. Platforms that don't use the
>> +dynamic object format can safely ignore it.
>> +
>> +2.d) Finally, DT syntax changes should be kept to a minimum. It should be
>> +possible to express everything using the existing DT syntax.
>> +
>> +3. Implementation
>> +-----------------
>> +
>> +The basic unit of addressing in Device Tree is the phandle. Turns out it's
>> +relatively simple to extend the way phandles are generated and referenced
>> +so that it's possible to dynamically convert symbolic references (labels)
>> +to phandle values. This is a valid assumption as long as the author uses
>> +reference syntax and does not assign phandle values manually (which might
>> +be a problem with decompiled source files).
>> +
>> +We can roughly divide the operation into two steps.
>> +
>> +3.a) Compilation of the base board DTS file using the '-@' option
>> +generates a valid DT blob with an added __symbols__ node at the root node,
>> +containing a list of all nodes that are marked with a label.
>> +
>> +Using the foo.dts file above the following node will be generated;
>> +
>> +$ dtc -@ -O dtb -o foo.dtb -b 0 foo.dts
>> +$ fdtdump foo.dtb
>> +...
>> +/ {
>> +	...
>> +	res {
>> +		...
>> +		phandle = <0x00000001>;
>> +		...
>> +	};
>> +	ocp {
>> +		...
>> +		phandle = <0x00000002>;
>> +		...
>> +	};
>> +	__symbols__ {
>> +		res="/res";
>> +		ocp="/ocp";
>> +	};
>> +};
>> +
>> +Notice that all the nodes that had a label have been recorded, and that
>> +phandles have been generated for them.
>> +
>> +This blob can be used to boot the board normally, the __symbols__ node will
>> +be safely ignored both by the bootloader and the kernel (the only loss will
>> +be a few bytes of memory and disk space).
>> +
>> +3.b) The Device Tree fragments must be compiled with the same option but they
>> +must also have a tag (/plugin/) that allows undefined references to nodes
>> +that are not present at compilation time to be recorded so that the runtime
>> +loader can fix them.
>> +
>> +So the bar peripheral's DTS format would be of the form:
>> +
>> +/dts-v1/ /plugin/;	/* allow undefined references and record them */
>> +/ {
>> +	....	/* various properties for loader use; i.e. part id etc. */
>> +	fragment@0 {
>> +		target = <&ocp>;
>> +		__overlay__ {
>> +			/* bar peripheral */
>> +			bar {
>> +				compatible = "corp,bar";
>> +				... /* various properties and child nodes */
>> +			}
>> +		};
>> +	};
>> +};
> 
> The last version of your patches that I tested did not require specifying
> the target property, the fragment node, and the __overlay__ node.  dtc
> properly created all of those items automatically.  For example, I could
> go to all of the trouble of creating those items in a dts like:
> 
> $ cat example_1_hand_coded.dts
> /dts-v1/;
> /plugin/;
> 
> / {
> 
> 	fragment@0 {
> 		target = <&am3353x_pinmux>;
> 
> 		__overlay__ {
> 
> 			i2c1_pins: pinmux_i2c1_pins {
> 				pinctrl-single,pins = <
> 					0x158 0x72
> 					0x15c 0x72
> 				>;
> 			};
> 		};
> 	};
> 
> 	fragment@1 {
> 		target = <&i2c1>;
> 
> 		__overlay__ {
> 			pinctrl-names = "default";
> 			pinctrl-0 = <&i2c1_pins>;
> 			clock-frequency = <400000>;
> 			status = "okay";
> 
> 			at24@50 {
> 				compatible = "at,24c256";
> 				pagesize = <64>;
> 				reg = <0x50>;
> 			};
> 		};
> 	};
> };
> 
> 
> Or I could let dtc automagically create all the special features
> (target, fragment, __overlay__) from an equivalent dts:
> 
> $ cat example_1.dts
> /dts-v1/;
> /plugin/;
> 
> 
> 		&am3353x_pinmux {
> 			i2c1_pins: pinmux_i2c1_pins {
> 				pinctrl-single,pins = <
> 					0x158 0x72
> 					0x15c 0x72
> 				>;
> 			};
> 		};
> 
> 		&i2c1 {
> 			#address-cells = <1>;
> 			#size-cells = <0>;
> 			pinctrl-names = "default";
> 			pinctrl-0 = <&i2c1_pins>;
> 			clock-frequency = <400000>;
> 			status = "okay";
> 
> 			at24@50 {
> 				compatible = "at,24c256";
> 				pagesize = <64>;
> 				reg = <0x50>;
> 			};
> 		};
> 
> 
> I would much prefer that people never hand code the target, fragment, and
> __overlay__ in a dts source file.  Exposing them at the source level adds
> complexity, confusion, and an increased chance of creating an invalid
> overlay dtb.
> 
> If possible, I would prefer target, fragment, and __overlay__ not be valid
> input to dtc.  It would probably be difficult to prohibit target and fragment,
> because however unlikely they are as property and node names, they are valid
> dts syntax before adding the overlay enhancements to dtc.  However __overlay__
> is not a valid node name without the overlay enhancements and could remain
> invalid dts input.
> 
> I prefer that target, fragment, and __overlay__ be documented as a dtb to
> target system API.  In this case, for the normal developer, they are
> hidden in the binary dtb format and in the kernel (or boot loader)
> overlay framework code.
> 
> I do recognize that if __overlay__ is not valid dtc input then it is not
> possible to decompile an overlay into a dts containing __overlay__ and
> then recompile that dts.  This could be resolved by a more complex
> decompile that turned the overlay dtb back into the form of example_1.dts
> above.
> 
> After reading to the end of this patch, I see that the simpler form of
> .dts (like example_1.dts) is also noted as "an alternative syntax to
> the expanded form for overlays".
> 
> 

Phew.

Let me address all that.

When I started on this the main problem was that there was no support for applying
overlays in the kernel. The original patch series for dtc is meant to support the
encoding of the required information into device tree format.

The syntax of overlays like this '&foo { };’ is a new thing that can be subject to
change.

On the last patchset I’ve split it out so that it is clear.

Now, since we’ve settled on the internal encoding format (__overlays__, target, etc)
we can tackle the syntax cases and alternative target options.

So, yes we should forbid __overlay__ to be a valid node name eventually along with
a bunch of other syntax stuff.

Having come to mind, we should see what we need for the connector format to work.

>> +
>> +Note that there's a target property that specifies the location where the
>> +contents of the overlay node will be placed, and it references the node
>> +in the foo.dts file.
>> +
>> +$ dtc -@ -O dtb -o bar.dtbo -b 0 bar.dts
>> +$ fdtdump bar.dtbo
>> +...
>> +/ {
>> +	... /* properties */
>> +	fragment@0 {
>> +		target = <0xffffffff>;
>> +		__overlay__ {
>> +			bar {
>> +				compatible = "corp,bar";
>> +				... /* various properties and child nodes */
>> +			}
>> +		};
>> +	};
>> +	__fixups__ {
>> +	    ocp = "/fragment@0:target:0";
>> +	};
>> +};
>> +
>> +No __symbols__ has been generated (no label in bar.dts).
>> +Note that the target's ocp label is undefined, so the phandle handle
>> +value is filled with the illegal value '0xffffffff', while a __fixups__
>> +node has been generated, which marks the location in the tree where
>> +the label lookup should store the runtime phandle value of the ocp node.
>> +
>> +The format of the __fixups__ node entry is
>> +
>> +	<label> = "<local-full-path>:<property-name>:<offset>";
>> +
>> +<label> 		Is the label we're referring
>> +<local-full-path>	Is the full path of the node the reference is
>> +<property-name>		Is the name of the property containing the
>> +			reference
>> +<offset>		The offset (in bytes) of where the property's
>> +			phandle value is located.
>> +
>> +Doing the same with the baz peripheral's DTS format is a little bit more
>> +involved, since baz contains references to local labels which require
>> +local fixups.
>> +
>> +/dts-v1/ /plugin/;	/* allow undefined label references and record them */
>> +/ {
>> +	....	/* various properties for loader use; i.e. part id etc. */
>> +	fragment@0 {
>> +		target = <&res>;
>> +		__overlay__ {
>> +			/* baz resources */
>> +			baz_res: res_baz { ... };
>> +		};
>> +	};
>> +	fragment@1 {
>> +		target = <&ocp>;
>> +		__overlay__ {
>> +			/* baz peripheral */
>> +			baz {
>> +				compatible = "corp,baz";
>> +				/* reference to another point in the tree */
>> +				ref-to-res = <&baz_res>;
>> +				... /* various properties and child nodes */
>> +			}
>> +		};
>> +	};
>> +};
>> +
>> +Note that &bar_res reference.
>> +
>> +$ dtc -@ -O dtb -o baz.dtbo -b 0 baz.dts
>> +$ fdtdump baz.dtbo
>> +...
>> +/ {
>> +	... /* properties */
>> +	fragment@0 {
>> +		target = <0xffffffff>;
>> +		__overlay__ {
>> +			res_baz {
>> +				....
>> +				phandle = <0x00000001>;
>> +			};
>> +		};
>> +	};
>> +	fragment@1 {
>> +		target = <0xffffffff>;
>> +		__overlay__ {
>> +			baz {
>> +				compatible = "corp,baz";
>> +				... /* various properties and child nodes */
>> +				ref-to-res = <0x00000001>;
>> +			}
>> +		};
>> +	};
>> +	__fixups__ {
>> +		res = "/fragment@0:target:0";
>> +		ocp = "/fragment@1:target:0";
>> +	};
>> +	__local_fixups__ {
>> +		fragment@1 {
>> +			__overlay__ {
>> +				baz {
>> +					ref-to-res = <0>;
>> +				};
>> +			};
>> +		};
>> +	};
>> +};
>> +
>> +This is similar to the bar case, but the reference of a local label by the
>> +baz node generates a __local_fixups__ entry that records the place that the
>> +local reference is being made. No matter how phandles are allocated from dtc
>> +the run time loader must apply an offset to each phandle in every dynamic
>> +DT object loaded. The __local_fixups__ node records the place of every
>> +local reference so that the loader can apply the offset.
>> +
>> +There is an alternative syntax to the expanded form for overlays with phandle
>> +targets which makes the format similar to the one using in .dtsi include files.
>> +
>> +So for the &ocp target example above one can simply write:
>> +
>> +/dts-v1/ /plugin/;
>> +&ocp {
>> +	/* bar peripheral */
>> +	bar {
>> +		compatible = "corp,bar";
>> +		... /* various properties and child nodes */
>> +	}
>> +};
>> +
>> +The resulting dtb object is identical.
>> 
> 

Regards

— Pantelis

--
To unsubscribe from this list: send the line "unsubscribe devicetree" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v10 2/4] dtc: Document the dynamic plugin internals

[David Gibson]

[-- Attachment #1: Type: text/plain, Size: 18769 bytes --]

On Tue, Nov 29, 2016 at 01:21:40PM +0200, Pantelis Antoniou wrote:
> Hi Frank,
> 
> > On Nov 29, 2016, at 03:36 , Frank Rowand <frowand.list-Re5JQEeQqe8AvxtiuMwx3w@public.gmane.org> wrote:
> > 
> > On 11/25/16 04:32, Pantelis Antoniou wrote:
> >> Provides the document explaining the internal mechanics of
> >> plugins and options.
> >> 
> >> Signed-off-by: Pantelis Antoniou <pantelis.antoniou-OWPKS81ov/FWk0Htik3J/w@public.gmane.org>
> >> ---
> >> Documentation/dt-object-internal.txt | 318 +++++++++++++++++++++++++++++++++++
> >> 1 file changed, 318 insertions(+)
> >> create mode 100644 Documentation/dt-object-internal.txt
> >> 
> >> diff --git a/Documentation/dt-object-internal.txt b/Documentation/dt-object-internal.txt
> >> new file mode 100644
> >> index 0000000..d5b841e
> >> --- /dev/null
> >> +++ b/Documentation/dt-object-internal.txt
> >> @@ -0,0 +1,318 @@
> >> +Device Tree Dynamic Object format internals
> >> +-------------------------------------------
> >> +
> >> +The Device Tree for most platforms is a static representation of
> >> +the hardware capabilities. This is insufficient for many platforms
> >> +that need to dynamically insert device tree fragments to the
> >> +running kernel's live tree.
> >> +
> >> +This document explains the the device tree object format and the
> >> +modifications made to the device tree compiler, which make it possible.
> >> +
> >> +1. Simplified Problem Definition
> >> +--------------------------------
> >> +
> >> +Assume we have a platform which boots using following simplified device tree.
> >> +
> >> +---- foo.dts -----------------------------------------------------------------
> >> +	/* FOO platform */
> >> +	/ {
> >> +		compatible = "corp,foo";
> >> +
> >> +		/* shared resources */
> >> +		res: res {
> >> +		};
> >> +
> >> +		/* On chip peripherals */
> >> +		ocp: ocp {
> >> +			/* peripherals that are always instantiated */
> >> +			peripheral1 { ... };
> >> +		};
> >> +	};
> >> +---- foo.dts -----------------------------------------------------------------
> >> +
> >> +We have a number of peripherals that after probing (using some undefined method)
> >> +should result in different device tree configuration.
> >> +
> >> +We cannot boot with this static tree because due to the configuration of the
> >> +foo platform there exist multiple conficting peripherals DT fragments.
> > 
> >                                     ^^^^^^^^^^  conflicting
> > 
> > I assume conflicting because, for instance, the different peripherals might
> > occupy the same address space, use the same interrupt, or use the same gpio.
> > Mentioning that would provide a fuller picture for the neophyte.
> > 
> 
> Yes, thanks for bringing this to my attention. This document is heavy on the neophyte for sure.
> 
> >> +
> >> +So for the bar peripheral we would have this:
> >> +
> >> +---- foo+bar.dts -------------------------------------------------------------
> >> +	/* FOO platform + bar peripheral */
> >> +	/ {
> >> +		compatible = "corp,foo";
> >> +
> >> +		/* shared resources */
> >> +		res: res {
> >> +		};
> >> +
> >> +		/* On chip peripherals */
> >> +		ocp: ocp {
> >> +			/* peripherals that are always instantiated */
> >> +			peripheral1 { ... };
> >> +
> >> +			/* bar peripheral */
> >> +			bar {
> >> +				compatible = "corp,bar";
> >> +				... /* various properties and child nodes */
> >> +			};
> >> +		};
> >> +	};
> >> +---- foo+bar.dts -------------------------------------------------------------
> >> +
> >> +While for the baz peripheral we would have this:
> >> +
> >> +---- foo+baz.dts -------------------------------------------------------------
> >> +	/* FOO platform + baz peripheral */
> >> +	/ {
> >> +		compatible = "corp,foo";
> >> +
> >> +		/* shared resources */
> >> +		res: res {
> >> +			/* baz resources */
> >> +			baz_res: res_baz { ... };
> >> +		};
> >> +
> >> +		/* On chip peripherals */
> >> +		ocp: ocp {
> >> +			/* peripherals that are always instantiated */
> >> +			peripheral1 { ... };
> >> +
> >> +			/* baz peripheral */
> >> +			baz {
> >> +				compatible = "corp,baz";
> >> +				/* reference to another point in the tree */
> >> +				ref-to-res = <&baz_res>;
> >> +				... /* various properties and child nodes */
> >> +			};
> >> +		};
> >> +	};
> >> +---- foo+baz.dts -------------------------------------------------------------
> >> +
> >> +We note that the baz case is more complicated, since the baz peripheral needs to
> >> +reference another node in the DT tree.
> > 
> > I know that there are other situations that can justify overlays, so not
> > contesting the basic need with this comment.  But the above situation could
> > be handled in a much simpler fashion by setting the status property of each
> > of the conflicting devices to disabled, then after probing setting the status
> > to ok.  That method removes a lot of complexity.
> > 
> > A big driver for the concept of overlays was being able to describe different
> > add on boards at run time, instead of when the base dtb was created.  I think
> > we have agreed that moving to a connector model instead of a raw overlay is
> > the proper way to address add on boards.
> > 
> > Can you address how an overlay can be created that will work for a board
> > plugged into any of the identical sockets that is compatible with the
> > board?
> > 
> > 
> 
> Yes, I will try to do so.
> 
> >> +
> >> +2. Device Tree Object Format Requirements
> >> +-----------------------------------------
> >> +
> >> +Since the device tree is used for booting a number of very different hardware
> >> +platforms it is imperative that we tread very carefully.
> >> +
> >> +2.a) No changes to the Device Tree binary format for the base tree. We cannot
> >> +modify the tree format at all and all the information we require should be
> >> +encoded using device tree itself. We can add nodes that can be safely ignored
> >> +by both bootloaders and the kernel. The plugin dtb's are optionally tagged
> >> +with a different magic number in the header but otherwise they too are simple
> >> +blobs.
> >> +
> >> +2.b) Changes to the DTS source format should be absolutely minimal, and should
> >> +only be needed for the DT fragment definitions, and not the base boot DT.
> >> +
> >> +2.c) An explicit option should be used to instruct DTC to generate the required
> >> +information needed for object resolution. Platforms that don't use the
> >> +dynamic object format can safely ignore it.
> >> +
> >> +2.d) Finally, DT syntax changes should be kept to a minimum. It should be
> >> +possible to express everything using the existing DT syntax.
> >> +
> >> +3. Implementation
> >> +-----------------
> >> +
> >> +The basic unit of addressing in Device Tree is the phandle. Turns out it's
> >> +relatively simple to extend the way phandles are generated and referenced
> >> +so that it's possible to dynamically convert symbolic references (labels)
> >> +to phandle values. This is a valid assumption as long as the author uses
> >> +reference syntax and does not assign phandle values manually (which might
> >> +be a problem with decompiled source files).
> >> +
> >> +We can roughly divide the operation into two steps.
> >> +
> >> +3.a) Compilation of the base board DTS file using the '-@' option
> >> +generates a valid DT blob with an added __symbols__ node at the root node,
> >> +containing a list of all nodes that are marked with a label.
> >> +
> >> +Using the foo.dts file above the following node will be generated;
> >> +
> >> +$ dtc -@ -O dtb -o foo.dtb -b 0 foo.dts
> >> +$ fdtdump foo.dtb
> >> +...
> >> +/ {
> >> +	...
> >> +	res {
> >> +		...
> >> +		phandle = <0x00000001>;
> >> +		...
> >> +	};
> >> +	ocp {
> >> +		...
> >> +		phandle = <0x00000002>;
> >> +		...
> >> +	};
> >> +	__symbols__ {
> >> +		res="/res";
> >> +		ocp="/ocp";
> >> +	};
> >> +};
> >> +
> >> +Notice that all the nodes that had a label have been recorded, and that
> >> +phandles have been generated for them.
> >> +
> >> +This blob can be used to boot the board normally, the __symbols__ node will
> >> +be safely ignored both by the bootloader and the kernel (the only loss will
> >> +be a few bytes of memory and disk space).
> >> +
> >> +3.b) The Device Tree fragments must be compiled with the same option but they
> >> +must also have a tag (/plugin/) that allows undefined references to nodes
> >> +that are not present at compilation time to be recorded so that the runtime
> >> +loader can fix them.
> >> +
> >> +So the bar peripheral's DTS format would be of the form:
> >> +
> >> +/dts-v1/ /plugin/;	/* allow undefined references and record them */
> >> +/ {
> >> +	....	/* various properties for loader use; i.e. part id etc. */
> >> +	fragment@0 {
> >> +		target = <&ocp>;
> >> +		__overlay__ {
> >> +			/* bar peripheral */
> >> +			bar {
> >> +				compatible = "corp,bar";
> >> +				... /* various properties and child nodes */
> >> +			}
> >> +		};
> >> +	};
> >> +};
> > 
> > The last version of your patches that I tested did not require specifying
> > the target property, the fragment node, and the __overlay__ node.  dtc
> > properly created all of those items automatically.  For example, I could
> > go to all of the trouble of creating those items in a dts like:
> > 
> > $ cat example_1_hand_coded.dts
> > /dts-v1/;
> > /plugin/;
> > 
> > / {
> > 
> > 	fragment@0 {
> > 		target = <&am3353x_pinmux>;
> > 
> > 		__overlay__ {
> > 
> > 			i2c1_pins: pinmux_i2c1_pins {
> > 				pinctrl-single,pins = <
> > 					0x158 0x72
> > 					0x15c 0x72
> > 				>;
> > 			};
> > 		};
> > 	};
> > 
> > 	fragment@1 {
> > 		target = <&i2c1>;
> > 
> > 		__overlay__ {
> > 			pinctrl-names = "default";
> > 			pinctrl-0 = <&i2c1_pins>;
> > 			clock-frequency = <400000>;
> > 			status = "okay";
> > 
> > 			at24@50 {
> > 				compatible = "at,24c256";
> > 				pagesize = <64>;
> > 				reg = <0x50>;
> > 			};
> > 		};
> > 	};
> > };
> > 
> > 
> > Or I could let dtc automagically create all the special features
> > (target, fragment, __overlay__) from an equivalent dts:
> > 
> > $ cat example_1.dts
> > /dts-v1/;
> > /plugin/;
> > 
> > 
> > 		&am3353x_pinmux {
> > 			i2c1_pins: pinmux_i2c1_pins {
> > 				pinctrl-single,pins = <
> > 					0x158 0x72
> > 					0x15c 0x72
> > 				>;
> > 			};
> > 		};
> > 
> > 		&i2c1 {
> > 			#address-cells = <1>;
> > 			#size-cells = <0>;
> > 			pinctrl-names = "default";
> > 			pinctrl-0 = <&i2c1_pins>;
> > 			clock-frequency = <400000>;
> > 			status = "okay";
> > 
> > 			at24@50 {
> > 				compatible = "at,24c256";
> > 				pagesize = <64>;
> > 				reg = <0x50>;
> > 			};
> > 		};
> > 
> > 
> > I would much prefer that people never hand code the target, fragment, and
> > __overlay__ in a dts source file.  Exposing them at the source level adds
> > complexity, confusion, and an increased chance of creating an invalid
> > overlay dtb.
> > 
> > If possible, I would prefer target, fragment, and __overlay__ not be valid
> > input to dtc.  It would probably be difficult to prohibit target and fragment,
> > because however unlikely they are as property and node names, they are valid
> > dts syntax before adding the overlay enhancements to dtc.  However __overlay__
> > is not a valid node name without the overlay enhancements and could remain
> > invalid dts input.
> > 
> > I prefer that target, fragment, and __overlay__ be documented as a dtb to
> > target system API.  In this case, for the normal developer, they are
> > hidden in the binary dtb format and in the kernel (or boot loader)
> > overlay framework code.
> > 
> > I do recognize that if __overlay__ is not valid dtc input then it is not
> > possible to decompile an overlay into a dts containing __overlay__ and
> > then recompile that dts.  This could be resolved by a more complex
> > decompile that turned the overlay dtb back into the form of example_1.dts
> > above.
> > 
> > After reading to the end of this patch, I see that the simpler form of
> > .dts (like example_1.dts) is also noted as "an alternative syntax to
> > the expanded form for overlays".
> > 
> > 
> 
> Phew.
> 
> Let me address all that.
> 
> When I started on this the main problem was that there was no support for applying
> overlays in the kernel. The original patch series for dtc is meant to support the
> encoding of the required information into device tree format.
> 
> The syntax of overlays like this '&foo { };’ is a new thing that can be subject to
> change.

Well.. yes and no.  What I'm going to call "compile time overlays"
using that syntax have been around for ages (rather longer than
dynamic overlays).  The semantics you hve for runtime overlay
application are pretty much identical to those for compile time
overlays, except (duh) applied later.

That's why I want to unify the syntax between the two.  And, up to a
point, to unify the concepts as well.  This is why I want to treat
this as having dtc parse the source into a bundle of overlays which it
then decides whether it needs to apply immediately (compile time
overlay) or encode them into the dtbo format for the bootloader or
kernel to apply later (dynamic overlay).

> On the last patchset I’ve split it out so that it is clear.

Yeah, but you're splitting it based on the history, rather than what I
think is the conceptually clearer approach:  first, allow the overlay
(&ref { ... }) syntax to be either compile-time or dynamic.  second,
add in backwards compatiblity hacks for manually encoded dts files.

> Now, since we’ve settled on the internal encoding format (__overlays__, target, etc)
> we can tackle the syntax cases and alternative target options.

But that's not an internal encoding format, it's an _external_
encoding format.

> So, yes we should forbid __overlay__ to be a valid node name eventually along with
> a bunch of other syntax stuff.
> 
> Having come to mind, we should see what we need for the connector
> format to work.

No argument there.

> 
> >> +
> >> +Note that there's a target property that specifies the location where the
> >> +contents of the overlay node will be placed, and it references the node
> >> +in the foo.dts file.
> >> +
> >> +$ dtc -@ -O dtb -o bar.dtbo -b 0 bar.dts
> >> +$ fdtdump bar.dtbo
> >> +...
> >> +/ {
> >> +	... /* properties */
> >> +	fragment@0 {
> >> +		target = <0xffffffff>;
> >> +		__overlay__ {
> >> +			bar {
> >> +				compatible = "corp,bar";
> >> +				... /* various properties and child nodes */
> >> +			}
> >> +		};
> >> +	};
> >> +	__fixups__ {
> >> +	    ocp = "/fragment@0:target:0";
> >> +	};
> >> +};
> >> +
> >> +No __symbols__ has been generated (no label in bar.dts).
> >> +Note that the target's ocp label is undefined, so the phandle handle
> >> +value is filled with the illegal value '0xffffffff', while a __fixups__
> >> +node has been generated, which marks the location in the tree where
> >> +the label lookup should store the runtime phandle value of the ocp node.
> >> +
> >> +The format of the __fixups__ node entry is
> >> +
> >> +	<label> = "<local-full-path>:<property-name>:<offset>";
> >> +
> >> +<label> 		Is the label we're referring
> >> +<local-full-path>	Is the full path of the node the reference is
> >> +<property-name>		Is the name of the property containing the
> >> +			reference
> >> +<offset>		The offset (in bytes) of where the property's
> >> +			phandle value is located.
> >> +
> >> +Doing the same with the baz peripheral's DTS format is a little bit more
> >> +involved, since baz contains references to local labels which require
> >> +local fixups.
> >> +
> >> +/dts-v1/ /plugin/;	/* allow undefined label references and record them */
> >> +/ {
> >> +	....	/* various properties for loader use; i.e. part id etc. */
> >> +	fragment@0 {
> >> +		target = <&res>;
> >> +		__overlay__ {
> >> +			/* baz resources */
> >> +			baz_res: res_baz { ... };
> >> +		};
> >> +	};
> >> +	fragment@1 {
> >> +		target = <&ocp>;
> >> +		__overlay__ {
> >> +			/* baz peripheral */
> >> +			baz {
> >> +				compatible = "corp,baz";
> >> +				/* reference to another point in the tree */
> >> +				ref-to-res = <&baz_res>;
> >> +				... /* various properties and child nodes */
> >> +			}
> >> +		};
> >> +	};
> >> +};
> >> +
> >> +Note that &bar_res reference.
> >> +
> >> +$ dtc -@ -O dtb -o baz.dtbo -b 0 baz.dts
> >> +$ fdtdump baz.dtbo
> >> +...
> >> +/ {
> >> +	... /* properties */
> >> +	fragment@0 {
> >> +		target = <0xffffffff>;
> >> +		__overlay__ {
> >> +			res_baz {
> >> +				....
> >> +				phandle = <0x00000001>;
> >> +			};
> >> +		};
> >> +	};
> >> +	fragment@1 {
> >> +		target = <0xffffffff>;
> >> +		__overlay__ {
> >> +			baz {
> >> +				compatible = "corp,baz";
> >> +				... /* various properties and child nodes */
> >> +				ref-to-res = <0x00000001>;
> >> +			}
> >> +		};
> >> +	};
> >> +	__fixups__ {
> >> +		res = "/fragment@0:target:0";
> >> +		ocp = "/fragment@1:target:0";
> >> +	};
> >> +	__local_fixups__ {
> >> +		fragment@1 {
> >> +			__overlay__ {
> >> +				baz {
> >> +					ref-to-res = <0>;
> >> +				};
> >> +			};
> >> +		};
> >> +	};
> >> +};
> >> +
> >> +This is similar to the bar case, but the reference of a local label by the
> >> +baz node generates a __local_fixups__ entry that records the place that the
> >> +local reference is being made. No matter how phandles are allocated from dtc
> >> +the run time loader must apply an offset to each phandle in every dynamic
> >> +DT object loaded. The __local_fixups__ node records the place of every
> >> +local reference so that the loader can apply the offset.
> >> +
> >> +There is an alternative syntax to the expanded form for overlays with phandle
> >> +targets which makes the format similar to the one using in .dtsi include files.
> >> +
> >> +So for the &ocp target example above one can simply write:
> >> +
> >> +/dts-v1/ /plugin/;
> >> +&ocp {
> >> +	/* bar peripheral */
> >> +	bar {
> >> +		compatible = "corp,bar";
> >> +		... /* various properties and child nodes */
> >> +	}
> >> +};
> >> +
> >> +The resulting dtb object is identical.
> >> 
> > 
> 
> Regards
> 
> — Pantelis
> 

-- 
David Gibson			| I'll have my music baroque, and my code
david AT gibson.dropbear.id.au	| minimalist, thank you.  NOT _the_ _other_
				| _way_ _around_!
http://www.ozlabs.org/~dgibson

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 819 bytes --]

Re: [PATCH v10 2/4] dtc: Document the dynamic plugin internals

[Pantelis Antoniou]

Hi David,

> On Dec 2, 2016, at 05:25 , David Gibson <david-xT8FGy+AXnRB3Ne2BGzF6laj5H9X9Tb+@public.gmane.org> wrote:
> 
> On Tue, Nov 29, 2016 at 01:21:40PM +0200, Pantelis Antoniou wrote:
>> Hi Frank,
>> 
>>> On Nov 29, 2016, at 03:36 , Frank Rowand <frowand.list-Re5JQEeQqe8AvxtiuMwx3w@public.gmane.org> wrote:
>>> 
>>> On 11/25/16 04:32, Pantelis Antoniou wrote:
>>>> Provides the document explaining the internal mechanics of
>>>> plugins and options.
>>>> 
>>>> Signed-off-by: Pantelis Antoniou <pantelis.antoniou-OWPKS81ov/FWk0Htik3J/w@public.gmane.org>
>>>> ---
>>>> Documentation/dt-object-internal.txt | 318 +++++++++++++++++++++++++++++++++++
>>>> 1 file changed, 318 insertions(+)
>>>> create mode 100644 Documentation/dt-object-internal.txt
>>>> 
>>>> diff --git a/Documentation/dt-object-internal.txt b/Documentation/dt-object-internal.txt
>>>> new file mode 100644
>>>> index 0000000..d5b841e
>>>> --- /dev/null
>>>> +++ b/Documentation/dt-object-internal.txt
>>>> @@ -0,0 +1,318 @@
>>>> +Device Tree Dynamic Object format internals
>>>> +-------------------------------------------
>>>> +
>>>> +The Device Tree for most platforms is a static representation of
>>>> +the hardware capabilities. This is insufficient for many platforms
>>>> +that need to dynamically insert device tree fragments to the
>>>> +running kernel's live tree.
>>>> +
>>>> +This document explains the the device tree object format and the
>>>> +modifications made to the device tree compiler, which make it possible.
>>>> +
>>>> +1. Simplified Problem Definition
>>>> +--------------------------------
>>>> +
>>>> +Assume we have a platform which boots using following simplified device tree.
>>>> +
>>>> +---- foo.dts -----------------------------------------------------------------
>>>> +	/* FOO platform */
>>>> +	/ {
>>>> +		compatible = "corp,foo";
>>>> +
>>>> +		/* shared resources */
>>>> +		res: res {
>>>> +		};
>>>> +
>>>> +		/* On chip peripherals */
>>>> +		ocp: ocp {
>>>> +			/* peripherals that are always instantiated */
>>>> +			peripheral1 { ... };
>>>> +		};
>>>> +	};
>>>> +---- foo.dts -----------------------------------------------------------------
>>>> +
>>>> +We have a number of peripherals that after probing (using some undefined method)
>>>> +should result in different device tree configuration.
>>>> +
>>>> +We cannot boot with this static tree because due to the configuration of the
>>>> +foo platform there exist multiple conficting peripherals DT fragments.
>>> 
>>>                                    ^^^^^^^^^^  conflicting
>>> 
>>> I assume conflicting because, for instance, the different peripherals might
>>> occupy the same address space, use the same interrupt, or use the same gpio.
>>> Mentioning that would provide a fuller picture for the neophyte.
>>> 
>> 
>> Yes, thanks for bringing this to my attention. This document is heavy on the neophyte for sure.
>> 
>>>> +
>>>> +So for the bar peripheral we would have this:
>>>> +
>>>> +---- foo+bar.dts -------------------------------------------------------------
>>>> +	/* FOO platform + bar peripheral */
>>>> +	/ {
>>>> +		compatible = "corp,foo";
>>>> +
>>>> +		/* shared resources */
>>>> +		res: res {
>>>> +		};
>>>> +
>>>> +		/* On chip peripherals */
>>>> +		ocp: ocp {
>>>> +			/* peripherals that are always instantiated */
>>>> +			peripheral1 { ... };
>>>> +
>>>> +			/* bar peripheral */
>>>> +			bar {
>>>> +				compatible = "corp,bar";
>>>> +				... /* various properties and child nodes */
>>>> +			};
>>>> +		};
>>>> +	};
>>>> +---- foo+bar.dts -------------------------------------------------------------
>>>> +
>>>> +While for the baz peripheral we would have this:
>>>> +
>>>> +---- foo+baz.dts -------------------------------------------------------------
>>>> +	/* FOO platform + baz peripheral */
>>>> +	/ {
>>>> +		compatible = "corp,foo";
>>>> +
>>>> +		/* shared resources */
>>>> +		res: res {
>>>> +			/* baz resources */
>>>> +			baz_res: res_baz { ... };
>>>> +		};
>>>> +
>>>> +		/* On chip peripherals */
>>>> +		ocp: ocp {
>>>> +			/* peripherals that are always instantiated */
>>>> +			peripheral1 { ... };
>>>> +
>>>> +			/* baz peripheral */
>>>> +			baz {
>>>> +				compatible = "corp,baz";
>>>> +				/* reference to another point in the tree */
>>>> +				ref-to-res = <&baz_res>;
>>>> +				... /* various properties and child nodes */
>>>> +			};
>>>> +		};
>>>> +	};
>>>> +---- foo+baz.dts -------------------------------------------------------------
>>>> +
>>>> +We note that the baz case is more complicated, since the baz peripheral needs to
>>>> +reference another node in the DT tree.
>>> 
>>> I know that there are other situations that can justify overlays, so not
>>> contesting the basic need with this comment.  But the above situation could
>>> be handled in a much simpler fashion by setting the status property of each
>>> of the conflicting devices to disabled, then after probing setting the status
>>> to ok.  That method removes a lot of complexity.
>>> 
>>> A big driver for the concept of overlays was being able to describe different
>>> add on boards at run time, instead of when the base dtb was created.  I think
>>> we have agreed that moving to a connector model instead of a raw overlay is
>>> the proper way to address add on boards.
>>> 
>>> Can you address how an overlay can be created that will work for a board
>>> plugged into any of the identical sockets that is compatible with the
>>> board?
>>> 
>>> 
>> 
>> Yes, I will try to do so.
>> 
>>>> +
>>>> +2. Device Tree Object Format Requirements
>>>> +-----------------------------------------
>>>> +
>>>> +Since the device tree is used for booting a number of very different hardware
>>>> +platforms it is imperative that we tread very carefully.
>>>> +
>>>> +2.a) No changes to the Device Tree binary format for the base tree. We cannot
>>>> +modify the tree format at all and all the information we require should be
>>>> +encoded using device tree itself. We can add nodes that can be safely ignored
>>>> +by both bootloaders and the kernel. The plugin dtb's are optionally tagged
>>>> +with a different magic number in the header but otherwise they too are simple
>>>> +blobs.
>>>> +
>>>> +2.b) Changes to the DTS source format should be absolutely minimal, and should
>>>> +only be needed for the DT fragment definitions, and not the base boot DT.
>>>> +
>>>> +2.c) An explicit option should be used to instruct DTC to generate the required
>>>> +information needed for object resolution. Platforms that don't use the
>>>> +dynamic object format can safely ignore it.
>>>> +
>>>> +2.d) Finally, DT syntax changes should be kept to a minimum. It should be
>>>> +possible to express everything using the existing DT syntax.
>>>> +
>>>> +3. Implementation
>>>> +-----------------
>>>> +
>>>> +The basic unit of addressing in Device Tree is the phandle. Turns out it's
>>>> +relatively simple to extend the way phandles are generated and referenced
>>>> +so that it's possible to dynamically convert symbolic references (labels)
>>>> +to phandle values. This is a valid assumption as long as the author uses
>>>> +reference syntax and does not assign phandle values manually (which might
>>>> +be a problem with decompiled source files).
>>>> +
>>>> +We can roughly divide the operation into two steps.
>>>> +
>>>> +3.a) Compilation of the base board DTS file using the '-@' option
>>>> +generates a valid DT blob with an added __symbols__ node at the root node,
>>>> +containing a list of all nodes that are marked with a label.
>>>> +
>>>> +Using the foo.dts file above the following node will be generated;
>>>> +
>>>> +$ dtc -@ -O dtb -o foo.dtb -b 0 foo.dts
>>>> +$ fdtdump foo.dtb
>>>> +...
>>>> +/ {
>>>> +	...
>>>> +	res {
>>>> +		...
>>>> +		phandle = <0x00000001>;
>>>> +		...
>>>> +	};
>>>> +	ocp {
>>>> +		...
>>>> +		phandle = <0x00000002>;
>>>> +		...
>>>> +	};
>>>> +	__symbols__ {
>>>> +		res="/res";
>>>> +		ocp="/ocp";
>>>> +	};
>>>> +};
>>>> +
>>>> +Notice that all the nodes that had a label have been recorded, and that
>>>> +phandles have been generated for them.
>>>> +
>>>> +This blob can be used to boot the board normally, the __symbols__ node will
>>>> +be safely ignored both by the bootloader and the kernel (the only loss will
>>>> +be a few bytes of memory and disk space).
>>>> +
>>>> +3.b) The Device Tree fragments must be compiled with the same option but they
>>>> +must also have a tag (/plugin/) that allows undefined references to nodes
>>>> +that are not present at compilation time to be recorded so that the runtime
>>>> +loader can fix them.
>>>> +
>>>> +So the bar peripheral's DTS format would be of the form:
>>>> +
>>>> +/dts-v1/ /plugin/;	/* allow undefined references and record them */
>>>> +/ {
>>>> +	....	/* various properties for loader use; i.e. part id etc. */
>>>> +	fragment@0 {
>>>> +		target = <&ocp>;
>>>> +		__overlay__ {
>>>> +			/* bar peripheral */
>>>> +			bar {
>>>> +				compatible = "corp,bar";
>>>> +				... /* various properties and child nodes */
>>>> +			}
>>>> +		};
>>>> +	};
>>>> +};
>>> 
>>> The last version of your patches that I tested did not require specifying
>>> the target property, the fragment node, and the __overlay__ node.  dtc
>>> properly created all of those items automatically.  For example, I could
>>> go to all of the trouble of creating those items in a dts like:
>>> 
>>> $ cat example_1_hand_coded.dts
>>> /dts-v1/;
>>> /plugin/;
>>> 
>>> / {
>>> 
>>> 	fragment@0 {
>>> 		target = <&am3353x_pinmux>;
>>> 
>>> 		__overlay__ {
>>> 
>>> 			i2c1_pins: pinmux_i2c1_pins {
>>> 				pinctrl-single,pins = <
>>> 					0x158 0x72
>>> 					0x15c 0x72
>>> 				>;
>>> 			};
>>> 		};
>>> 	};
>>> 
>>> 	fragment@1 {
>>> 		target = <&i2c1>;
>>> 
>>> 		__overlay__ {
>>> 			pinctrl-names = "default";
>>> 			pinctrl-0 = <&i2c1_pins>;
>>> 			clock-frequency = <400000>;
>>> 			status = "okay";
>>> 
>>> 			at24@50 {
>>> 				compatible = "at,24c256";
>>> 				pagesize = <64>;
>>> 				reg = <0x50>;
>>> 			};
>>> 		};
>>> 	};
>>> };
>>> 
>>> 
>>> Or I could let dtc automagically create all the special features
>>> (target, fragment, __overlay__) from an equivalent dts:
>>> 
>>> $ cat example_1.dts
>>> /dts-v1/;
>>> /plugin/;
>>> 
>>> 
>>> 		&am3353x_pinmux {
>>> 			i2c1_pins: pinmux_i2c1_pins {
>>> 				pinctrl-single,pins = <
>>> 					0x158 0x72
>>> 					0x15c 0x72
>>> 				>;
>>> 			};
>>> 		};
>>> 
>>> 		&i2c1 {
>>> 			#address-cells = <1>;
>>> 			#size-cells = <0>;
>>> 			pinctrl-names = "default";
>>> 			pinctrl-0 = <&i2c1_pins>;
>>> 			clock-frequency = <400000>;
>>> 			status = "okay";
>>> 
>>> 			at24@50 {
>>> 				compatible = "at,24c256";
>>> 				pagesize = <64>;
>>> 				reg = <0x50>;
>>> 			};
>>> 		};
>>> 
>>> 
>>> I would much prefer that people never hand code the target, fragment, and
>>> __overlay__ in a dts source file.  Exposing them at the source level adds
>>> complexity, confusion, and an increased chance of creating an invalid
>>> overlay dtb.
>>> 
>>> If possible, I would prefer target, fragment, and __overlay__ not be valid
>>> input to dtc.  It would probably be difficult to prohibit target and fragment,
>>> because however unlikely they are as property and node names, they are valid
>>> dts syntax before adding the overlay enhancements to dtc.  However __overlay__
>>> is not a valid node name without the overlay enhancements and could remain
>>> invalid dts input.
>>> 
>>> I prefer that target, fragment, and __overlay__ be documented as a dtb to
>>> target system API.  In this case, for the normal developer, they are
>>> hidden in the binary dtb format and in the kernel (or boot loader)
>>> overlay framework code.
>>> 
>>> I do recognize that if __overlay__ is not valid dtc input then it is not
>>> possible to decompile an overlay into a dts containing __overlay__ and
>>> then recompile that dts.  This could be resolved by a more complex
>>> decompile that turned the overlay dtb back into the form of example_1.dts
>>> above.
>>> 
>>> After reading to the end of this patch, I see that the simpler form of
>>> .dts (like example_1.dts) is also noted as "an alternative syntax to
>>> the expanded form for overlays".
>>> 
>>> 
>> 
>> Phew.
>> 
>> Let me address all that.
>> 
>> When I started on this the main problem was that there was no support for applying
>> overlays in the kernel. The original patch series for dtc is meant to support the
>> encoding of the required information into device tree format.
>> 
>> The syntax of overlays like this '&foo { };’ is a new thing that can be subject to
>> change.
> 
> Well.. yes and no.  What I'm going to call "compile time overlays"
> using that syntax have been around for ages (rather longer than
> dynamic overlays).  The semantics you hve for runtime overlay
> application are pretty much identical to those for compile time
> overlays, except (duh) applied later.
> 
> That's why I want to unify the syntax between the two.  And, up to a
> point, to unify the concepts as well.  This is why I want to treat
> this as having dtc parse the source into a bundle of overlays which it
> then decides whether it needs to apply immediately (compile time
> overlay) or encode them into the dtbo format for the bootloader or
> kernel to apply later (dynamic overlay).
> 

It is a worthy goal, but it will require quite a lot of work (and time).

One thing that comes to mind is mapping the semantics of the compile time
‘overlays’ to run time will require changes in the blob format.

For instance the syntax for deleting nodes/properties has no mapping to
runtime. We will need to figure out how to encode them, etc.

It is something that can wait while we get things right, we don’t have a
pressing need right now.

>> On the last patchset I’ve split it out so that it is clear.
> 
> Yeah, but you're splitting it based on the history, rather than what I
> think is the conceptually clearer approach:  first, allow the overlay
> (&ref { ... }) syntax to be either compile-time or dynamic.  second,
> add in backwards compatiblity hacks for manually encoded dts files.
> 

I’m not sure what your point here is. First things first; encoding of runtime
overlays without changes in syntax. Next comes the new syntax for defining them.

There is no backward compatibility hack. The hack _is_ the &ref { } syntax since
nothing uses it now.

We intent to use from now on, true, but it’s been holding up the rest of the
patchset for years now.

>> Now, since we’ve settled on the internal encoding format (__overlays__, target, etc)
>> we can tackle the syntax cases and alternative target options.
> 
> But that's not an internal encoding format, it's an _external_
> encoding format.

I concede that for the definition of internal/external from the viewpoint
of the dtc compiler.

> 
>> So, yes we should forbid __overlay__ to be a valid node name eventually along with
>> a bunch of other syntax stuff.
>> 
>> Having come to mind, we should see what we need for the connector
>> format to work.
> 
> No argument there.
> 
>> 
>>>> +
>>>> +Note that there's a target property that specifies the location where the
>>>> +contents of the overlay node will be placed, and it references the node
>>>> +in the foo.dts file.
>>>> +
>>>> +$ dtc -@ -O dtb -o bar.dtbo -b 0 bar.dts
>>>> +$ fdtdump bar.dtbo
>>>> +...
>>>> +/ {
>>>> +	... /* properties */
>>>> +	fragment@0 {
>>>> +		target = <0xffffffff>;
>>>> +		__overlay__ {
>>>> +			bar {
>>>> +				compatible = "corp,bar";
>>>> +				... /* various properties and child nodes */
>>>> +			}
>>>> +		};
>>>> +	};
>>>> +	__fixups__ {
>>>> +	    ocp = "/fragment@0:target:0";
>>>> +	};
>>>> +};
>>>> +
>>>> +No __symbols__ has been generated (no label in bar.dts).
>>>> +Note that the target's ocp label is undefined, so the phandle handle
>>>> +value is filled with the illegal value '0xffffffff', while a __fixups__
>>>> +node has been generated, which marks the location in the tree where
>>>> +the label lookup should store the runtime phandle value of the ocp node.
>>>> +
>>>> +The format of the __fixups__ node entry is
>>>> +
>>>> +	<label> = "<local-full-path>:<property-name>:<offset>";
>>>> +
>>>> +<label> 		Is the label we're referring
>>>> +<local-full-path>	Is the full path of the node the reference is
>>>> +<property-name>		Is the name of the property containing the
>>>> +			reference
>>>> +<offset>		The offset (in bytes) of where the property's
>>>> +			phandle value is located.
>>>> +
>>>> +Doing the same with the baz peripheral's DTS format is a little bit more
>>>> +involved, since baz contains references to local labels which require
>>>> +local fixups.
>>>> +
>>>> +/dts-v1/ /plugin/;	/* allow undefined label references and record them */
>>>> +/ {
>>>> +	....	/* various properties for loader use; i.e. part id etc. */
>>>> +	fragment@0 {
>>>> +		target = <&res>;
>>>> +		__overlay__ {
>>>> +			/* baz resources */
>>>> +			baz_res: res_baz { ... };
>>>> +		};
>>>> +	};
>>>> +	fragment@1 {
>>>> +		target = <&ocp>;
>>>> +		__overlay__ {
>>>> +			/* baz peripheral */
>>>> +			baz {
>>>> +				compatible = "corp,baz";
>>>> +				/* reference to another point in the tree */
>>>> +				ref-to-res = <&baz_res>;
>>>> +				... /* various properties and child nodes */
>>>> +			}
>>>> +		};
>>>> +	};
>>>> +};
>>>> +
>>>> +Note that &bar_res reference.
>>>> +
>>>> +$ dtc -@ -O dtb -o baz.dtbo -b 0 baz.dts
>>>> +$ fdtdump baz.dtbo
>>>> +...
>>>> +/ {
>>>> +	... /* properties */
>>>> +	fragment@0 {
>>>> +		target = <0xffffffff>;
>>>> +		__overlay__ {
>>>> +			res_baz {
>>>> +				....
>>>> +				phandle = <0x00000001>;
>>>> +			};
>>>> +		};
>>>> +	};
>>>> +	fragment@1 {
>>>> +		target = <0xffffffff>;
>>>> +		__overlay__ {
>>>> +			baz {
>>>> +				compatible = "corp,baz";
>>>> +				... /* various properties and child nodes */
>>>> +				ref-to-res = <0x00000001>;
>>>> +			}
>>>> +		};
>>>> +	};
>>>> +	__fixups__ {
>>>> +		res = "/fragment@0:target:0";
>>>> +		ocp = "/fragment@1:target:0";
>>>> +	};
>>>> +	__local_fixups__ {
>>>> +		fragment@1 {
>>>> +			__overlay__ {
>>>> +				baz {
>>>> +					ref-to-res = <0>;
>>>> +				};
>>>> +			};
>>>> +		};
>>>> +	};
>>>> +};
>>>> +
>>>> +This is similar to the bar case, but the reference of a local label by the
>>>> +baz node generates a __local_fixups__ entry that records the place that the
>>>> +local reference is being made. No matter how phandles are allocated from dtc
>>>> +the run time loader must apply an offset to each phandle in every dynamic
>>>> +DT object loaded. The __local_fixups__ node records the place of every
>>>> +local reference so that the loader can apply the offset.
>>>> +
>>>> +There is an alternative syntax to the expanded form for overlays with phandle
>>>> +targets which makes the format similar to the one using in .dtsi include files.
>>>> +
>>>> +So for the &ocp target example above one can simply write:
>>>> +
>>>> +/dts-v1/ /plugin/;
>>>> +&ocp {
>>>> +	/* bar peripheral */
>>>> +	bar {
>>>> +		compatible = "corp,bar";
>>>> +		... /* various properties and child nodes */
>>>> +	}
>>>> +};
>>>> +
>>>> +The resulting dtb object is identical.
>>>> 
>>> 
>> 
>> Regards
>> 
>> — Pantelis
>> 
> 
> -- 
> David Gibson			| I'll have my music baroque, and my code
> david AT gibson.dropbear.id.au	| minimalist, thank you.  NOT _the_ _other_
> 				| _way_ _around_!
> http://www.ozlabs.org/~dgibson

Regards

— Pantelis

--
To unsubscribe from this list: send the line "unsubscribe devicetree" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v10 2/4] dtc: Document the dynamic plugin internals

[David Gibson]

[-- Attachment #1: Type: text/plain, Size: 21684 bytes --]

On Fri, Dec 02, 2016 at 11:09:49AM +0200, Pantelis Antoniou wrote:
> Hi David,
> 
> > On Dec 2, 2016, at 05:25 , David Gibson <david-xT8FGy+AXnRB3Ne2BGzF6laj5H9X9Tb+@public.gmane.org> wrote:
> > 
> > On Tue, Nov 29, 2016 at 01:21:40PM +0200, Pantelis Antoniou wrote:
> >> Hi Frank,
> >> 
> >>> On Nov 29, 2016, at 03:36 , Frank Rowand <frowand.list-Re5JQEeQqe8AvxtiuMwx3w@public.gmane.org> wrote:
> >>> 
> >>> On 11/25/16 04:32, Pantelis Antoniou wrote:
> >>>> Provides the document explaining the internal mechanics of
> >>>> plugins and options.
> >>>> 
> >>>> Signed-off-by: Pantelis Antoniou <pantelis.antoniou-OWPKS81ov/FWk0Htik3J/w@public.gmane.org>
> >>>> ---
> >>>> Documentation/dt-object-internal.txt | 318 +++++++++++++++++++++++++++++++++++
> >>>> 1 file changed, 318 insertions(+)
> >>>> create mode 100644 Documentation/dt-object-internal.txt
> >>>> 
> >>>> diff --git a/Documentation/dt-object-internal.txt b/Documentation/dt-object-internal.txt
> >>>> new file mode 100644
> >>>> index 0000000..d5b841e
> >>>> --- /dev/null
> >>>> +++ b/Documentation/dt-object-internal.txt
> >>>> @@ -0,0 +1,318 @@
> >>>> +Device Tree Dynamic Object format internals
> >>>> +-------------------------------------------
> >>>> +
> >>>> +The Device Tree for most platforms is a static representation of
> >>>> +the hardware capabilities. This is insufficient for many platforms
> >>>> +that need to dynamically insert device tree fragments to the
> >>>> +running kernel's live tree.
> >>>> +
> >>>> +This document explains the the device tree object format and the
> >>>> +modifications made to the device tree compiler, which make it possible.
> >>>> +
> >>>> +1. Simplified Problem Definition
> >>>> +--------------------------------
> >>>> +
> >>>> +Assume we have a platform which boots using following simplified device tree.
> >>>> +
> >>>> +---- foo.dts -----------------------------------------------------------------
> >>>> +	/* FOO platform */
> >>>> +	/ {
> >>>> +		compatible = "corp,foo";
> >>>> +
> >>>> +		/* shared resources */
> >>>> +		res: res {
> >>>> +		};
> >>>> +
> >>>> +		/* On chip peripherals */
> >>>> +		ocp: ocp {
> >>>> +			/* peripherals that are always instantiated */
> >>>> +			peripheral1 { ... };
> >>>> +		};
> >>>> +	};
> >>>> +---- foo.dts -----------------------------------------------------------------
> >>>> +
> >>>> +We have a number of peripherals that after probing (using some undefined method)
> >>>> +should result in different device tree configuration.
> >>>> +
> >>>> +We cannot boot with this static tree because due to the configuration of the
> >>>> +foo platform there exist multiple conficting peripherals DT fragments.
> >>> 
> >>>                                    ^^^^^^^^^^  conflicting
> >>> 
> >>> I assume conflicting because, for instance, the different peripherals might
> >>> occupy the same address space, use the same interrupt, or use the same gpio.
> >>> Mentioning that would provide a fuller picture for the neophyte.
> >>> 
> >> 
> >> Yes, thanks for bringing this to my attention. This document is heavy on the neophyte for sure.
> >> 
> >>>> +
> >>>> +So for the bar peripheral we would have this:
> >>>> +
> >>>> +---- foo+bar.dts -------------------------------------------------------------
> >>>> +	/* FOO platform + bar peripheral */
> >>>> +	/ {
> >>>> +		compatible = "corp,foo";
> >>>> +
> >>>> +		/* shared resources */
> >>>> +		res: res {
> >>>> +		};
> >>>> +
> >>>> +		/* On chip peripherals */
> >>>> +		ocp: ocp {
> >>>> +			/* peripherals that are always instantiated */
> >>>> +			peripheral1 { ... };
> >>>> +
> >>>> +			/* bar peripheral */
> >>>> +			bar {
> >>>> +				compatible = "corp,bar";
> >>>> +				... /* various properties and child nodes */
> >>>> +			};
> >>>> +		};
> >>>> +	};
> >>>> +---- foo+bar.dts -------------------------------------------------------------
> >>>> +
> >>>> +While for the baz peripheral we would have this:
> >>>> +
> >>>> +---- foo+baz.dts -------------------------------------------------------------
> >>>> +	/* FOO platform + baz peripheral */
> >>>> +	/ {
> >>>> +		compatible = "corp,foo";
> >>>> +
> >>>> +		/* shared resources */
> >>>> +		res: res {
> >>>> +			/* baz resources */
> >>>> +			baz_res: res_baz { ... };
> >>>> +		};
> >>>> +
> >>>> +		/* On chip peripherals */
> >>>> +		ocp: ocp {
> >>>> +			/* peripherals that are always instantiated */
> >>>> +			peripheral1 { ... };
> >>>> +
> >>>> +			/* baz peripheral */
> >>>> +			baz {
> >>>> +				compatible = "corp,baz";
> >>>> +				/* reference to another point in the tree */
> >>>> +				ref-to-res = <&baz_res>;
> >>>> +				... /* various properties and child nodes */
> >>>> +			};
> >>>> +		};
> >>>> +	};
> >>>> +---- foo+baz.dts -------------------------------------------------------------
> >>>> +
> >>>> +We note that the baz case is more complicated, since the baz peripheral needs to
> >>>> +reference another node in the DT tree.
> >>> 
> >>> I know that there are other situations that can justify overlays, so not
> >>> contesting the basic need with this comment.  But the above situation could
> >>> be handled in a much simpler fashion by setting the status property of each
> >>> of the conflicting devices to disabled, then after probing setting the status
> >>> to ok.  That method removes a lot of complexity.
> >>> 
> >>> A big driver for the concept of overlays was being able to describe different
> >>> add on boards at run time, instead of when the base dtb was created.  I think
> >>> we have agreed that moving to a connector model instead of a raw overlay is
> >>> the proper way to address add on boards.
> >>> 
> >>> Can you address how an overlay can be created that will work for a board
> >>> plugged into any of the identical sockets that is compatible with the
> >>> board?
> >>> 
> >>> 
> >> 
> >> Yes, I will try to do so.
> >> 
> >>>> +
> >>>> +2. Device Tree Object Format Requirements
> >>>> +-----------------------------------------
> >>>> +
> >>>> +Since the device tree is used for booting a number of very different hardware
> >>>> +platforms it is imperative that we tread very carefully.
> >>>> +
> >>>> +2.a) No changes to the Device Tree binary format for the base tree. We cannot
> >>>> +modify the tree format at all and all the information we require should be
> >>>> +encoded using device tree itself. We can add nodes that can be safely ignored
> >>>> +by both bootloaders and the kernel. The plugin dtb's are optionally tagged
> >>>> +with a different magic number in the header but otherwise they too are simple
> >>>> +blobs.
> >>>> +
> >>>> +2.b) Changes to the DTS source format should be absolutely minimal, and should
> >>>> +only be needed for the DT fragment definitions, and not the base boot DT.
> >>>> +
> >>>> +2.c) An explicit option should be used to instruct DTC to generate the required
> >>>> +information needed for object resolution. Platforms that don't use the
> >>>> +dynamic object format can safely ignore it.
> >>>> +
> >>>> +2.d) Finally, DT syntax changes should be kept to a minimum. It should be
> >>>> +possible to express everything using the existing DT syntax.
> >>>> +
> >>>> +3. Implementation
> >>>> +-----------------
> >>>> +
> >>>> +The basic unit of addressing in Device Tree is the phandle. Turns out it's
> >>>> +relatively simple to extend the way phandles are generated and referenced
> >>>> +so that it's possible to dynamically convert symbolic references (labels)
> >>>> +to phandle values. This is a valid assumption as long as the author uses
> >>>> +reference syntax and does not assign phandle values manually (which might
> >>>> +be a problem with decompiled source files).
> >>>> +
> >>>> +We can roughly divide the operation into two steps.
> >>>> +
> >>>> +3.a) Compilation of the base board DTS file using the '-@' option
> >>>> +generates a valid DT blob with an added __symbols__ node at the root node,
> >>>> +containing a list of all nodes that are marked with a label.
> >>>> +
> >>>> +Using the foo.dts file above the following node will be generated;
> >>>> +
> >>>> +$ dtc -@ -O dtb -o foo.dtb -b 0 foo.dts
> >>>> +$ fdtdump foo.dtb
> >>>> +...
> >>>> +/ {
> >>>> +	...
> >>>> +	res {
> >>>> +		...
> >>>> +		phandle = <0x00000001>;
> >>>> +		...
> >>>> +	};
> >>>> +	ocp {
> >>>> +		...
> >>>> +		phandle = <0x00000002>;
> >>>> +		...
> >>>> +	};
> >>>> +	__symbols__ {
> >>>> +		res="/res";
> >>>> +		ocp="/ocp";
> >>>> +	};
> >>>> +};
> >>>> +
> >>>> +Notice that all the nodes that had a label have been recorded, and that
> >>>> +phandles have been generated for them.
> >>>> +
> >>>> +This blob can be used to boot the board normally, the __symbols__ node will
> >>>> +be safely ignored both by the bootloader and the kernel (the only loss will
> >>>> +be a few bytes of memory and disk space).
> >>>> +
> >>>> +3.b) The Device Tree fragments must be compiled with the same option but they
> >>>> +must also have a tag (/plugin/) that allows undefined references to nodes
> >>>> +that are not present at compilation time to be recorded so that the runtime
> >>>> +loader can fix them.
> >>>> +
> >>>> +So the bar peripheral's DTS format would be of the form:
> >>>> +
> >>>> +/dts-v1/ /plugin/;	/* allow undefined references and record them */
> >>>> +/ {
> >>>> +	....	/* various properties for loader use; i.e. part id etc. */
> >>>> +	fragment@0 {
> >>>> +		target = <&ocp>;
> >>>> +		__overlay__ {
> >>>> +			/* bar peripheral */
> >>>> +			bar {
> >>>> +				compatible = "corp,bar";
> >>>> +				... /* various properties and child nodes */
> >>>> +			}
> >>>> +		};
> >>>> +	};
> >>>> +};
> >>> 
> >>> The last version of your patches that I tested did not require specifying
> >>> the target property, the fragment node, and the __overlay__ node.  dtc
> >>> properly created all of those items automatically.  For example, I could
> >>> go to all of the trouble of creating those items in a dts like:
> >>> 
> >>> $ cat example_1_hand_coded.dts
> >>> /dts-v1/;
> >>> /plugin/;
> >>> 
> >>> / {
> >>> 
> >>> 	fragment@0 {
> >>> 		target = <&am3353x_pinmux>;
> >>> 
> >>> 		__overlay__ {
> >>> 
> >>> 			i2c1_pins: pinmux_i2c1_pins {
> >>> 				pinctrl-single,pins = <
> >>> 					0x158 0x72
> >>> 					0x15c 0x72
> >>> 				>;
> >>> 			};
> >>> 		};
> >>> 	};
> >>> 
> >>> 	fragment@1 {
> >>> 		target = <&i2c1>;
> >>> 
> >>> 		__overlay__ {
> >>> 			pinctrl-names = "default";
> >>> 			pinctrl-0 = <&i2c1_pins>;
> >>> 			clock-frequency = <400000>;
> >>> 			status = "okay";
> >>> 
> >>> 			at24@50 {
> >>> 				compatible = "at,24c256";
> >>> 				pagesize = <64>;
> >>> 				reg = <0x50>;
> >>> 			};
> >>> 		};
> >>> 	};
> >>> };
> >>> 
> >>> 
> >>> Or I could let dtc automagically create all the special features
> >>> (target, fragment, __overlay__) from an equivalent dts:
> >>> 
> >>> $ cat example_1.dts
> >>> /dts-v1/;
> >>> /plugin/;
> >>> 
> >>> 
> >>> 		&am3353x_pinmux {
> >>> 			i2c1_pins: pinmux_i2c1_pins {
> >>> 				pinctrl-single,pins = <
> >>> 					0x158 0x72
> >>> 					0x15c 0x72
> >>> 				>;
> >>> 			};
> >>> 		};
> >>> 
> >>> 		&i2c1 {
> >>> 			#address-cells = <1>;
> >>> 			#size-cells = <0>;
> >>> 			pinctrl-names = "default";
> >>> 			pinctrl-0 = <&i2c1_pins>;
> >>> 			clock-frequency = <400000>;
> >>> 			status = "okay";
> >>> 
> >>> 			at24@50 {
> >>> 				compatible = "at,24c256";
> >>> 				pagesize = <64>;
> >>> 				reg = <0x50>;
> >>> 			};
> >>> 		};
> >>> 
> >>> 
> >>> I would much prefer that people never hand code the target, fragment, and
> >>> __overlay__ in a dts source file.  Exposing them at the source level adds
> >>> complexity, confusion, and an increased chance of creating an invalid
> >>> overlay dtb.
> >>> 
> >>> If possible, I would prefer target, fragment, and __overlay__ not be valid
> >>> input to dtc.  It would probably be difficult to prohibit target and fragment,
> >>> because however unlikely they are as property and node names, they are valid
> >>> dts syntax before adding the overlay enhancements to dtc.  However __overlay__
> >>> is not a valid node name without the overlay enhancements and could remain
> >>> invalid dts input.
> >>> 
> >>> I prefer that target, fragment, and __overlay__ be documented as a dtb to
> >>> target system API.  In this case, for the normal developer, they are
> >>> hidden in the binary dtb format and in the kernel (or boot loader)
> >>> overlay framework code.
> >>> 
> >>> I do recognize that if __overlay__ is not valid dtc input then it is not
> >>> possible to decompile an overlay into a dts containing __overlay__ and
> >>> then recompile that dts.  This could be resolved by a more complex
> >>> decompile that turned the overlay dtb back into the form of example_1.dts
> >>> above.
> >>> 
> >>> After reading to the end of this patch, I see that the simpler form of
> >>> .dts (like example_1.dts) is also noted as "an alternative syntax to
> >>> the expanded form for overlays".
> >>> 
> >>> 
> >> 
> >> Phew.
> >> 
> >> Let me address all that.
> >> 
> >> When I started on this the main problem was that there was no support for applying
> >> overlays in the kernel. The original patch series for dtc is meant to support the
> >> encoding of the required information into device tree format.
> >> 
> >> The syntax of overlays like this '&foo { };’ is a new thing that can be subject to
> >> change.
> > 
> > Well.. yes and no.  What I'm going to call "compile time overlays"
> > using that syntax have been around for ages (rather longer than
> > dynamic overlays).  The semantics you hve for runtime overlay
> > application are pretty much identical to those for compile time
> > overlays, except (duh) applied later.
> > 
> > That's why I want to unify the syntax between the two.  And, up to a
> > point, to unify the concepts as well.  This is why I want to treat
> > this as having dtc parse the source into a bundle of overlays which it
> > then decides whether it needs to apply immediately (compile time
> > overlay) or encode them into the dtbo format for the bootloader or
> > kernel to apply later (dynamic overlay).
> > 
> 
> It is a worthy goal, but it will require quite a lot of work (and time).

Hm.. I think you're overestimating the complexity of it.  A reasonable
first chunk I've already done in that 'overlay' branch.

> One thing that comes to mind is mapping the semantics of the compile time
> ‘overlays’ to run time will require changes in the blob format.
> 
> For instance the syntax for deleting nodes/properties has no mapping to
> runtime. We will need to figure out how to encode them, etc.

No support for deletions is a difference yes, but we could just give
an error when trying to encode a compile time overlay into a runtime
fragment if there are any deletions within.  At least until we come up
with a runtime overlay encoding for deletions.

.. and, I'm pretty sure that's the *only* semantic difference between
compile time overlays and dtbo fragments.

> It is something that can wait while we get things right, we don’t have a
> pressing need right now.
> 
> >> On the last patchset I’ve split it out so that it is clear.
> > 
> > Yeah, but you're splitting it based on the history, rather than what I
> > think is the conceptually clearer approach:  first, allow the overlay
> > (&ref { ... }) syntax to be either compile-time or dynamic.  second,
> > add in backwards compatiblity hacks for manually encoded dts files.
> > 
> 
> I’m not sure what your point here is. First things first; encoding of runtime
> overlays without changes in syntax. Next comes the new syntax for defining them.
> 
> There is no backward compatibility hack. The hack _is_ the &ref { } syntax since
> nothing uses it now.
> 
> We intent to use from now on, true, but it’s been holding up the rest of the
> patchset for years now.

Hm, yeah, I guess.

> >> Now, since we’ve settled on the internal encoding format (__overlays__, target, etc)
> >> we can tackle the syntax cases and alternative target options.
> > 
> > But that's not an internal encoding format, it's an _external_
> > encoding format.
> 
> I concede that for the definition of internal/external from the viewpoint
> of the dtc compiler.
> 
> > 
> >> So, yes we should forbid __overlay__ to be a valid node name eventually along with
> >> a bunch of other syntax stuff.
> >> 
> >> Having come to mind, we should see what we need for the connector
> >> format to work.
> > 
> > No argument there.
> > 
> >> 
> >>>> +
> >>>> +Note that there's a target property that specifies the location where the
> >>>> +contents of the overlay node will be placed, and it references the node
> >>>> +in the foo.dts file.
> >>>> +
> >>>> +$ dtc -@ -O dtb -o bar.dtbo -b 0 bar.dts
> >>>> +$ fdtdump bar.dtbo
> >>>> +...
> >>>> +/ {
> >>>> +	... /* properties */
> >>>> +	fragment@0 {
> >>>> +		target = <0xffffffff>;
> >>>> +		__overlay__ {
> >>>> +			bar {
> >>>> +				compatible = "corp,bar";
> >>>> +				... /* various properties and child nodes */
> >>>> +			}
> >>>> +		};
> >>>> +	};
> >>>> +	__fixups__ {
> >>>> +	    ocp = "/fragment@0:target:0";
> >>>> +	};
> >>>> +};
> >>>> +
> >>>> +No __symbols__ has been generated (no label in bar.dts).
> >>>> +Note that the target's ocp label is undefined, so the phandle handle
> >>>> +value is filled with the illegal value '0xffffffff', while a __fixups__
> >>>> +node has been generated, which marks the location in the tree where
> >>>> +the label lookup should store the runtime phandle value of the ocp node.
> >>>> +
> >>>> +The format of the __fixups__ node entry is
> >>>> +
> >>>> +	<label> = "<local-full-path>:<property-name>:<offset>";
> >>>> +
> >>>> +<label> 		Is the label we're referring
> >>>> +<local-full-path>	Is the full path of the node the reference is
> >>>> +<property-name>		Is the name of the property containing the
> >>>> +			reference
> >>>> +<offset>		The offset (in bytes) of where the property's
> >>>> +			phandle value is located.
> >>>> +
> >>>> +Doing the same with the baz peripheral's DTS format is a little bit more
> >>>> +involved, since baz contains references to local labels which require
> >>>> +local fixups.
> >>>> +
> >>>> +/dts-v1/ /plugin/;	/* allow undefined label references and record them */
> >>>> +/ {
> >>>> +	....	/* various properties for loader use; i.e. part id etc. */
> >>>> +	fragment@0 {
> >>>> +		target = <&res>;
> >>>> +		__overlay__ {
> >>>> +			/* baz resources */
> >>>> +			baz_res: res_baz { ... };
> >>>> +		};
> >>>> +	};
> >>>> +	fragment@1 {
> >>>> +		target = <&ocp>;
> >>>> +		__overlay__ {
> >>>> +			/* baz peripheral */
> >>>> +			baz {
> >>>> +				compatible = "corp,baz";
> >>>> +				/* reference to another point in the tree */
> >>>> +				ref-to-res = <&baz_res>;
> >>>> +				... /* various properties and child nodes */
> >>>> +			}
> >>>> +		};
> >>>> +	};
> >>>> +};
> >>>> +
> >>>> +Note that &bar_res reference.
> >>>> +
> >>>> +$ dtc -@ -O dtb -o baz.dtbo -b 0 baz.dts
> >>>> +$ fdtdump baz.dtbo
> >>>> +...
> >>>> +/ {
> >>>> +	... /* properties */
> >>>> +	fragment@0 {
> >>>> +		target = <0xffffffff>;
> >>>> +		__overlay__ {
> >>>> +			res_baz {
> >>>> +				....
> >>>> +				phandle = <0x00000001>;
> >>>> +			};
> >>>> +		};
> >>>> +	};
> >>>> +	fragment@1 {
> >>>> +		target = <0xffffffff>;
> >>>> +		__overlay__ {
> >>>> +			baz {
> >>>> +				compatible = "corp,baz";
> >>>> +				... /* various properties and child nodes */
> >>>> +				ref-to-res = <0x00000001>;
> >>>> +			}
> >>>> +		};
> >>>> +	};
> >>>> +	__fixups__ {
> >>>> +		res = "/fragment@0:target:0";
> >>>> +		ocp = "/fragment@1:target:0";
> >>>> +	};
> >>>> +	__local_fixups__ {
> >>>> +		fragment@1 {
> >>>> +			__overlay__ {
> >>>> +				baz {
> >>>> +					ref-to-res = <0>;
> >>>> +				};
> >>>> +			};
> >>>> +		};
> >>>> +	};
> >>>> +};
> >>>> +
> >>>> +This is similar to the bar case, but the reference of a local label by the
> >>>> +baz node generates a __local_fixups__ entry that records the place that the
> >>>> +local reference is being made. No matter how phandles are allocated from dtc
> >>>> +the run time loader must apply an offset to each phandle in every dynamic
> >>>> +DT object loaded. The __local_fixups__ node records the place of every
> >>>> +local reference so that the loader can apply the offset.
> >>>> +
> >>>> +There is an alternative syntax to the expanded form for overlays with phandle
> >>>> +targets which makes the format similar to the one using in .dtsi include files.
> >>>> +
> >>>> +So for the &ocp target example above one can simply write:
> >>>> +
> >>>> +/dts-v1/ /plugin/;
> >>>> +&ocp {
> >>>> +	/* bar peripheral */
> >>>> +	bar {
> >>>> +		compatible = "corp,bar";
> >>>> +		... /* various properties and child nodes */
> >>>> +	}
> >>>> +};
> >>>> +
> >>>> +The resulting dtb object is identical.
> >>>> 
> >>> 
> >> 
> >> Regards
> >> 
> >> — Pantelis
> >> 
> > 
> 
> Regards
> 
> — Pantelis
> 

-- 
David Gibson			| I'll have my music baroque, and my code
david AT gibson.dropbear.id.au	| minimalist, thank you.  NOT _the_ _other_
				| _way_ _around_!
http://www.ozlabs.org/~dgibson

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 819 bytes --]

[PATCH v10 3/4] dtc: Plugin and fixup support

[Pantelis Antoniou]

This patch enable the generation of symbols & local fixup information
for trees compiled with the -@ (--symbols) option.

Using this patch labels in the tree and their users emit information
in __symbols__ and __local_fixups__ nodes.

The __fixups__ node make possible the dynamic resolution of phandle
references which are present in the plugin tree but lie in the
tree that are applying the overlay against.

While there is a new magic number for dynamic device tree/overlays blobs
it is by default enabled. Remember to use -M to generate compatible
blobs.

Signed-off-by: Pantelis Antoniou <pantelis.antoniou-OWPKS81ov/FWk0Htik3J/w@public.gmane.org>
Signed-off-by: Sascha Hauer <s.hauer-bIcnvbaLZ9MEGnE8C9+IrQ@public.gmane.org>
Signed-off-by: Jan Luebbe <jlu-bIcnvbaLZ9MEGnE8C9+IrQ@public.gmane.org>
---
 Documentation/manual.txt |  25 +++++-
 checks.c                 |   8 +-
 dtc-lexer.l              |   5 ++
 dtc-parser.y             |  50 +++++++++--
 dtc.c                    |  39 +++++++-
 dtc.h                    |  20 ++++-
 fdtdump.c                |   2 +-
 flattree.c               |  17 ++--
 fstree.c                 |   2 +-
 libfdt/fdt.c             |   2 +-
 libfdt/fdt.h             |   3 +-
 livetree.c               | 225 ++++++++++++++++++++++++++++++++++++++++++++++-
 tests/mangle-layout.c    |   7 +-
 13 files changed, 375 insertions(+), 30 deletions(-)

diff --git a/Documentation/manual.txt b/Documentation/manual.txt
index 398de32..094893b 100644
--- a/Documentation/manual.txt
+++ b/Documentation/manual.txt
@@ -119,6 +119,24 @@ Options:
 	Make space for <number> reserve map entries
 	Relevant for dtb and asm output only.
 
+    -@
+	Generates a __symbols__ node at the root node of the resulting blob
+	for any node labels used, and for any local references using phandles
+	it also generates a __local_fixups__ node that tracks them.
+
+	When using the /plugin/ tag all unresolved label references to
+	be tracked in the __fixups__ node, making dynamic resolution possible.
+
+    -A
+	Generate automatically aliases for all node labels. This is similar to
+	the -@ option (the __symbols__ node contain identical information) but
+	the semantics are slightly different since no phandles are automatically
+	generated for labeled nodes.
+
+    -M
+	Generate blobs with the old FDT magic number for device tree objects.
+	By default blobs use the DTBO FDT magic number instead.
+
     -S <bytes>
 	Ensure the blob at least <bytes> long, adding additional
 	space if needed.
@@ -146,13 +164,18 @@ Additionally, dtc performs various sanity checks on the tree.
 Here is a very rough overview of the layout of a DTS source file:
 
 
-    sourcefile:   list_of_memreserve devicetree
+    sourcefile:   versioninfo plugindecl list_of_memreserve devicetree
 
     memreserve:   label 'memreserve' ADDR ADDR ';'
 		| label 'memreserve' ADDR '-' ADDR ';'
 
     devicetree:   '/' nodedef
 
+    versioninfo:  '/' 'dts-v1' '/' ';'
+
+    plugindecl:   '/' 'plugin' '/' ';'
+                | /* empty */
+
     nodedef:      '{' list_of_property list_of_subnode '}' ';'
 
     property:     label PROPNAME '=' propdata ';'
diff --git a/checks.c b/checks.c
index 2bd27a4..4292f4b 100644
--- a/checks.c
+++ b/checks.c
@@ -487,8 +487,12 @@ static void fixup_phandle_references(struct check *c, struct boot_info *bi,
 
 			refnode = get_node_by_ref(dt, m->ref);
 			if (! refnode) {
-				FAIL(c, "Reference to non-existent node or label \"%s\"\n",
-				     m->ref);
+				if (!(bi->versionflags & VF_PLUGIN))
+					FAIL(c, "Reference to non-existent node or "
+							"label \"%s\"\n", m->ref);
+				else /* mark the entry as unresolved */
+					*((cell_t *)(prop->val.val + m->offset)) =
+						cpu_to_fdt32(0xffffffff);
 				continue;
 			}
 
diff --git a/dtc-lexer.l b/dtc-lexer.l
index 790fbf6..40bbc87 100644
--- a/dtc-lexer.l
+++ b/dtc-lexer.l
@@ -121,6 +121,11 @@ static void lexical_error(const char *fmt, ...);
 			return DT_V1;
 		}
 
+<*>"/plugin/"	{
+			DPRINT("Keyword: /plugin/\n");
+			return DT_PLUGIN;
+		}
+
 <*>"/memreserve/"	{
 			DPRINT("Keyword: /memreserve/\n");
 			BEGIN_DEFAULT();
diff --git a/dtc-parser.y b/dtc-parser.y
index 14aaf2e..1a1f660 100644
--- a/dtc-parser.y
+++ b/dtc-parser.y
@@ -19,6 +19,7 @@
  */
 %{
 #include <stdio.h>
+#include <inttypes.h>
 
 #include "dtc.h"
 #include "srcpos.h"
@@ -33,6 +34,7 @@ extern void yyerror(char const *s);
 
 extern struct boot_info *the_boot_info;
 extern bool treesource_error;
+
 %}
 
 %union {
@@ -52,9 +54,11 @@ extern bool treesource_error;
 	struct node *nodelist;
 	struct reserve_info *re;
 	uint64_t integer;
+	unsigned int flags;
 }
 
 %token DT_V1
+%token DT_PLUGIN
 %token DT_MEMRESERVE
 %token DT_LSHIFT DT_RSHIFT DT_LE DT_GE DT_EQ DT_NE DT_AND DT_OR
 %token DT_BITS
@@ -71,6 +75,8 @@ extern bool treesource_error;
 
 %type <data> propdata
 %type <data> propdataprefix
+%type <flags> versioninfo
+%type <flags> plugindecl
 %type <re> memreserve
 %type <re> memreserves
 %type <array> arrayprefix
@@ -101,16 +107,34 @@ extern bool treesource_error;
 %%
 
 sourcefile:
-	  v1tag memreserves devicetree
+	  versioninfo plugindecl memreserves devicetree
+		{
+			the_boot_info = build_boot_info($1 | $2, $3, $4,
+							guess_boot_cpuid($4));
+		}
+	;
+
+versioninfo:
+	v1tag
 		{
-			the_boot_info = build_boot_info($2, $3,
-							guess_boot_cpuid($3));
+			$$ = VF_DT_V1;
 		}
 	;
 
 v1tag:
 	  DT_V1 ';'
+	| DT_V1
 	| DT_V1 ';' v1tag
+
+plugindecl:
+	DT_PLUGIN ';'
+		{
+			$$ = VF_PLUGIN;
+		}
+	| /* empty */
+		{
+			$$ = 0;
+		}
 	;
 
 memreserves:
@@ -161,10 +185,19 @@ devicetree:
 		{
 			struct node *target = get_node_by_ref($1, $2);
 
-			if (target)
+			if (target) {
 				merge_nodes(target, $3);
-			else
-				ERROR(&@2, "Label or path %s not found", $2);
+			} else {
+				/*
+				 * We rely on the rule being always:
+				 *   versioninfo plugindecl memreserves devicetree
+				 * so $-1 is what we want (plugindecl)
+				 */
+				if ($<flags>-1 & VF_PLUGIN)
+					add_orphan_node($1, $3, $2);
+				else
+					ERROR(&@2, "Label or path %s not found", $2);
+			}
 			$$ = $1;
 		}
 	| devicetree DT_DEL_NODE DT_REF ';'
@@ -179,6 +212,11 @@ devicetree:
 
 			$$ = $1;
 		}
+	| /* empty */
+		{
+			/* build empty node */
+			$$ = name_node(build_node(NULL, NULL), "");
+		}
 	;
 
 nodedef:
diff --git a/dtc.c b/dtc.c
index 9dcf640..06e91bc 100644
--- a/dtc.c
+++ b/dtc.c
@@ -32,6 +32,9 @@ int minsize;		/* Minimum blob size */
 int padsize;		/* Additional padding to blob */
 int alignsize;		/* Additional padding to blob accroding to the alignsize */
 int phandle_format = PHANDLE_BOTH;	/* Use linux,phandle or phandle properties */
+int symbol_fixup_support;	/* enable symbols & fixup support */
+int auto_label_aliases;		/* auto generate labels -> aliases */
+int no_dtbo_magic;		/* use old FDT magic values for objects */
 
 static int is_power_of_2(int x)
 {
@@ -59,7 +62,7 @@ static void fill_fullpaths(struct node *tree, const char *prefix)
 #define FDT_VERSION(version)	_FDT_VERSION(version)
 #define _FDT_VERSION(version)	#version
 static const char usage_synopsis[] = "dtc [options] <input file>";
-static const char usage_short_opts[] = "qI:O:o:V:d:R:S:p:a:fb:i:H:sW:E:hv";
+static const char usage_short_opts[] = "qI:O:o:V:d:R:S:p:a:fb:i:H:sW:E:@AMhv";
 static struct option const usage_long_opts[] = {
 	{"quiet",            no_argument, NULL, 'q'},
 	{"in-format",         a_argument, NULL, 'I'},
@@ -78,6 +81,9 @@ static struct option const usage_long_opts[] = {
 	{"phandle",           a_argument, NULL, 'H'},
 	{"warning",           a_argument, NULL, 'W'},
 	{"error",             a_argument, NULL, 'E'},
+	{"symbols",	     no_argument, NULL, '@'},
+	{"auto-alias",       no_argument, NULL, 'A'},
+	{"no-dtbo-magic",    no_argument, NULL, 'M'},
 	{"help",             no_argument, NULL, 'h'},
 	{"version",          no_argument, NULL, 'v'},
 	{NULL,               no_argument, NULL, 0x0},
@@ -109,6 +115,9 @@ static const char * const usage_opts_help[] = {
 	 "\t\tboth   - Both \"linux,phandle\" and \"phandle\" properties",
 	"\n\tEnable/disable warnings (prefix with \"no-\")",
 	"\n\tEnable/disable errors (prefix with \"no-\")",
+	"\n\tEnable symbols/fixup support",
+	"\n\tEnable auto-alias of labels",
+	"\n\tDo not use DTBO magic value for plugin objects",
 	"\n\tPrint this help and exit",
 	"\n\tPrint version and exit",
 	NULL,
@@ -153,7 +162,7 @@ static const char *guess_input_format(const char *fname, const char *fallback)
 	fclose(f);
 
 	magic = fdt32_to_cpu(magic);
-	if (magic == FDT_MAGIC)
+	if (magic == FDT_MAGIC || magic == FDT_MAGIC_DTBO)
 		return "dtb";
 
 	return guess_type_by_name(fname, fallback);
@@ -172,6 +181,7 @@ int main(int argc, char *argv[])
 	FILE *outf = NULL;
 	int outversion = DEFAULT_FDT_VERSION;
 	long long cmdline_boot_cpuid = -1;
+	fdt32_t out_magic = FDT_MAGIC;
 
 	quiet      = 0;
 	reservenum = 0;
@@ -249,6 +259,16 @@ int main(int argc, char *argv[])
 			parse_checks_option(false, true, optarg);
 			break;
 
+		case '@':
+			symbol_fixup_support = 1;
+			break;
+		case 'A':
+			auto_label_aliases = 1;
+			break;
+		case 'M':
+			no_dtbo_magic = 1;
+			break;
+
 		case 'h':
 			usage(NULL);
 		default:
@@ -306,6 +326,14 @@ int main(int argc, char *argv[])
 	fill_fullpaths(bi->dt, "");
 	process_checks(force, bi);
 
+	if (auto_label_aliases)
+		generate_label_tree(bi->dt, "aliases", false);
+
+	if (symbol_fixup_support) {
+		generate_label_tree(bi->dt, "__symbols__", true);
+		generate_fixups_tree(bi->dt);
+	}
+
 	if (sort)
 		sort_tree(bi);
 
@@ -318,12 +346,15 @@ int main(int argc, char *argv[])
 			    outname, strerror(errno));
 	}
 
+	if (!no_dtbo_magic && (bi->versionflags & VF_PLUGIN))
+		out_magic = FDT_MAGIC_DTBO;
+
 	if (streq(outform, "dts")) {
 		dt_to_source(outf, bi);
 	} else if (streq(outform, "dtb")) {
-		dt_to_blob(outf, bi, outversion);
+		dt_to_blob(outf, bi, out_magic, outversion);
 	} else if (streq(outform, "asm")) {
-		dt_to_asm(outf, bi, outversion);
+		dt_to_asm(outf, bi, out_magic, outversion);
 	} else if (streq(outform, "null")) {
 		/* do nothing */
 	} else {
diff --git a/dtc.h b/dtc.h
index 32009bc..581b3bf 100644
--- a/dtc.h
+++ b/dtc.h
@@ -55,6 +55,9 @@ extern int minsize;		/* Minimum blob size */
 extern int padsize;		/* Additional padding to blob */
 extern int alignsize;		/* Additional padding to blob accroding to the alignsize */
 extern int phandle_format;	/* Use linux,phandle or phandle properties */
+extern int symbol_fixup_support;/* enable symbols & fixup support */
+extern int auto_label_aliases;	/* auto generate labels -> aliases */
+extern int no_dtbo_magic;	/* use old FDT magic values for objects */
 
 #define PHANDLE_LEGACY	0x1
 #define PHANDLE_EPAPR	0x2
@@ -195,6 +198,7 @@ struct node *build_node_delete(void);
 struct node *name_node(struct node *node, char *name);
 struct node *chain_node(struct node *first, struct node *list);
 struct node *merge_nodes(struct node *old_node, struct node *new_node);
+void add_orphan_node(struct node *old_node, struct node *new_node, char *ref);
 
 void add_property(struct node *node, struct property *prop);
 void delete_property_by_name(struct node *node, char *name);
@@ -202,6 +206,8 @@ void delete_property(struct property *prop);
 void add_child(struct node *parent, struct node *child);
 void delete_node_by_name(struct node *parent, char *name);
 void delete_node(struct node *node);
+void append_to_property(struct node *node,
+			char *name, const void *data, int len);
 
 const char *get_unitname(struct node *node);
 struct property *get_property(struct node *node, const char *propname);
@@ -237,14 +243,22 @@ struct reserve_info *add_reserve_entry(struct reserve_info *list,
 
 
 struct boot_info {
+	unsigned int versionflags;
 	struct reserve_info *reservelist;
 	struct node *dt;		/* the device tree */
 	uint32_t boot_cpuid_phys;
 };
 
-struct boot_info *build_boot_info(struct reserve_info *reservelist,
+/* version flags definitions */
+#define VF_DT_V1	0x0001	/* /dts-v1/ */
+#define VF_PLUGIN	0x0002	/* /plugin/ */
+
+struct boot_info *build_boot_info(unsigned int versionflags,
+				  struct reserve_info *reservelist,
 				  struct node *tree, uint32_t boot_cpuid_phys);
 void sort_tree(struct boot_info *bi);
+void generate_label_tree(struct node *dt, char *gen_node_name, bool allocph);
+void generate_fixups_tree(struct node *dt);
 
 /* Checks */
 
@@ -253,8 +267,8 @@ void process_checks(bool force, struct boot_info *bi);
 
 /* Flattened trees */
 
-void dt_to_blob(FILE *f, struct boot_info *bi, int version);
-void dt_to_asm(FILE *f, struct boot_info *bi, int version);
+void dt_to_blob(FILE *f, struct boot_info *bi, fdt32_t magic, int version);
+void dt_to_asm(FILE *f, struct boot_info *bi, fdt32_t magic, int version);
 
 struct boot_info *dt_from_blob(const char *fname);
 
diff --git a/fdtdump.c b/fdtdump.c
index a9a2484..dd63ac2 100644
--- a/fdtdump.c
+++ b/fdtdump.c
@@ -201,7 +201,7 @@ int main(int argc, char *argv[])
 			p = memchr(p, smagic[0], endp - p - FDT_MAGIC_SIZE);
 			if (!p)
 				break;
-			if (fdt_magic(p) == FDT_MAGIC) {
+			if (fdt_magic(p) == FDT_MAGIC || fdt_magic(p) == FDT_MAGIC_DTBO) {
 				/* try and validate the main struct */
 				off_t this_len = endp - p;
 				fdt32_t max_version = 17;
diff --git a/flattree.c b/flattree.c
index a9d9520..57d76cf 100644
--- a/flattree.c
+++ b/flattree.c
@@ -335,6 +335,7 @@ static struct data flatten_reserve_list(struct reserve_info *reservelist,
 }
 
 static void make_fdt_header(struct fdt_header *fdt,
+			    fdt32_t magic,
 			    struct version_info *vi,
 			    int reservesize, int dtsize, int strsize,
 			    int boot_cpuid_phys)
@@ -345,7 +346,7 @@ static void make_fdt_header(struct fdt_header *fdt,
 
 	memset(fdt, 0xff, sizeof(*fdt));
 
-	fdt->magic = cpu_to_fdt32(FDT_MAGIC);
+	fdt->magic = cpu_to_fdt32(magic);
 	fdt->version = cpu_to_fdt32(vi->version);
 	fdt->last_comp_version = cpu_to_fdt32(vi->last_comp_version);
 
@@ -366,7 +367,7 @@ static void make_fdt_header(struct fdt_header *fdt,
 		fdt->size_dt_struct = cpu_to_fdt32(dtsize);
 }
 
-void dt_to_blob(FILE *f, struct boot_info *bi, int version)
+void dt_to_blob(FILE *f, struct boot_info *bi, fdt32_t magic, int version)
 {
 	struct version_info *vi = NULL;
 	int i;
@@ -390,7 +391,7 @@ void dt_to_blob(FILE *f, struct boot_info *bi, int version)
 	reservebuf = flatten_reserve_list(bi->reservelist, vi);
 
 	/* Make header */
-	make_fdt_header(&fdt, vi, reservebuf.len, dtbuf.len, strbuf.len,
+	make_fdt_header(&fdt, magic, vi, reservebuf.len, dtbuf.len, strbuf.len,
 			bi->boot_cpuid_phys);
 
 	/*
@@ -467,7 +468,7 @@ static void dump_stringtable_asm(FILE *f, struct data strbuf)
 	}
 }
 
-void dt_to_asm(FILE *f, struct boot_info *bi, int version)
+void dt_to_asm(FILE *f, struct boot_info *bi, fdt32_t magic, int version)
 {
 	struct version_info *vi = NULL;
 	int i;
@@ -830,6 +831,7 @@ struct boot_info *dt_from_blob(const char *fname)
 	struct node *tree;
 	uint32_t val;
 	int flags = 0;
+	unsigned int versionflags = VF_DT_V1;
 
 	f = srcfile_relative_open(fname, NULL);
 
@@ -845,9 +847,12 @@ struct boot_info *dt_from_blob(const char *fname)
 	}
 
 	magic = fdt32_to_cpu(magic);
-	if (magic != FDT_MAGIC)
+	if (magic != FDT_MAGIC && magic != FDT_MAGIC_DTBO)
 		die("Blob has incorrect magic number\n");
 
+	if (magic == FDT_MAGIC_DTBO)
+		versionflags |= VF_PLUGIN;
+
 	rc = fread(&totalsize, sizeof(totalsize), 1, f);
 	if (ferror(f))
 		die("Error reading DT blob size: %s\n", strerror(errno));
@@ -942,5 +947,5 @@ struct boot_info *dt_from_blob(const char *fname)
 
 	fclose(f);
 
-	return build_boot_info(reservelist, tree, boot_cpuid_phys);
+	return build_boot_info(versionflags, reservelist, tree, boot_cpuid_phys);
 }
diff --git a/fstree.c b/fstree.c
index 6d1beec..54f520b 100644
--- a/fstree.c
+++ b/fstree.c
@@ -86,6 +86,6 @@ struct boot_info *dt_from_fs(const char *dirname)
 	tree = read_fstree(dirname);
 	tree = name_node(tree, "");
 
-	return build_boot_info(NULL, tree, guess_boot_cpuid(tree));
+	return build_boot_info(VF_DT_V1, NULL, tree, guess_boot_cpuid(tree));
 }
 
diff --git a/libfdt/fdt.c b/libfdt/fdt.c
index 22286a1..28d422c 100644
--- a/libfdt/fdt.c
+++ b/libfdt/fdt.c
@@ -57,7 +57,7 @@
 
 int fdt_check_header(const void *fdt)
 {
-	if (fdt_magic(fdt) == FDT_MAGIC) {
+	if (fdt_magic(fdt) == FDT_MAGIC || fdt_magic(fdt) == FDT_MAGIC_DTBO) {
 		/* Complete tree */
 		if (fdt_version(fdt) < FDT_FIRST_SUPPORTED_VERSION)
 			return -FDT_ERR_BADVERSION;
diff --git a/libfdt/fdt.h b/libfdt/fdt.h
index 526aedb..493cd55 100644
--- a/libfdt/fdt.h
+++ b/libfdt/fdt.h
@@ -55,7 +55,7 @@
 #ifndef __ASSEMBLY__
 
 struct fdt_header {
-	fdt32_t magic;			 /* magic word FDT_MAGIC */
+	fdt32_t magic;			 /* magic word FDT_MAGIC[|_DTBO] */
 	fdt32_t totalsize;		 /* total size of DT block */
 	fdt32_t off_dt_struct;		 /* offset to structure */
 	fdt32_t off_dt_strings;		 /* offset to strings */
@@ -93,6 +93,7 @@ struct fdt_property {
 #endif /* !__ASSEMBLY */
 
 #define FDT_MAGIC	0xd00dfeed	/* 4: version, 4: total size */
+#define FDT_MAGIC_DTBO	0xd00dfdb0	/* DTBO magic */
 #define FDT_TAGSIZE	sizeof(fdt32_t)
 
 #define FDT_BEGIN_NODE	0x1		/* Start node: full name */
diff --git a/livetree.c b/livetree.c
index 3dc7559..f2c86bd 100644
--- a/livetree.c
+++ b/livetree.c
@@ -216,6 +216,31 @@ struct node *merge_nodes(struct node *old_node, struct node *new_node)
 	return old_node;
 }
 
+void add_orphan_node(struct node *dt, struct node *new_node, char *ref)
+{
+	static unsigned int next_orphan_fragment = 0;
+	struct node *node = build_node(NULL, NULL);
+	struct property *p;
+	struct data d = empty_data;
+	char *name;
+
+	memset(node, 0, sizeof(*node));
+
+	d = data_add_marker(d, REF_PHANDLE, ref);
+	d = data_append_integer(d, 0xffffffff, 32);
+
+	p = build_property("target", d);
+	add_property(node, p);
+
+	xasprintf(&name, "fragment@%u",
+			next_orphan_fragment++);
+	name_node(node, name);
+	name_node(new_node, "__overlay__");
+
+	add_child(dt, node);
+	add_child(node, new_node);
+}
+
 struct node *chain_node(struct node *first, struct node *list)
 {
 	assert(first->next_sibling == NULL);
@@ -296,6 +321,23 @@ void delete_node(struct node *node)
 	delete_labels(&node->labels);
 }
 
+void append_to_property(struct node *node,
+				    char *name, const void *data, int len)
+{
+	struct data d;
+	struct property *p;
+
+	p = get_property(node, name);
+	if (p) {
+		d = data_append_data(p->val, data, len);
+		p->val = d;
+	} else {
+		d = data_append_data(empty_data, data, len);
+		p = build_property(name, d);
+		add_property(node, p);
+	}
+}
+
 struct reserve_info *build_reserve_entry(uint64_t address, uint64_t size)
 {
 	struct reserve_info *new = xmalloc(sizeof(*new));
@@ -335,12 +377,14 @@ struct reserve_info *add_reserve_entry(struct reserve_info *list,
 	return list;
 }
 
-struct boot_info *build_boot_info(struct reserve_info *reservelist,
+struct boot_info *build_boot_info(unsigned int versionflags,
+				  struct reserve_info *reservelist,
 				  struct node *tree, uint32_t boot_cpuid_phys)
 {
 	struct boot_info *bi;
 
 	bi = xmalloc(sizeof(*bi));
+	bi->versionflags = versionflags;
 	bi->reservelist = reservelist;
 	bi->dt = tree;
 	bi->boot_cpuid_phys = boot_cpuid_phys;
@@ -709,3 +753,182 @@ void sort_tree(struct boot_info *bi)
 	sort_reserve_entries(bi);
 	sort_node(bi->dt);
 }
+
+/* utility helper to avoid code duplication */
+static struct node *build_and_name_child_node(struct node *parent, char *name)
+{
+	struct node *node;
+
+	node = build_node(NULL, NULL);
+	name_node(node, xstrdup(name));
+	add_child(parent, node);
+
+	return node;
+}
+
+static void generate_label_tree_internal(struct node *dt, struct node *node,
+					 struct node *an, bool allocph)
+{
+	struct node *c;
+	struct property *p;
+	struct label *l;
+
+	/* if if there are labels */
+	if (node->labels) {
+		/* now add the label in the node */
+		for_each_label(node->labels, l) {
+			/* check whether the label already exists */
+			p = get_property(an, l->label);
+			if (p) {
+				fprintf(stderr, "WARNING: label %s already"
+					" exists in /%s", l->label,
+					an->name);
+				continue;
+			}
+
+			/* insert it */
+			p = build_property(l->label,
+				data_copy_mem(node->fullpath,
+						strlen(node->fullpath) + 1));
+			add_property(an, p);
+		}
+
+		/* force allocation of a phandle for this node */
+		if (allocph)
+			(void)get_node_phandle(dt, node);
+	}
+
+	for_each_child(node, c)
+		generate_label_tree_internal(dt, c, an, allocph);
+}
+
+void generate_label_tree(struct node *dt, char *gen_node_name, bool allocph)
+{
+	struct node *an;
+
+	for_each_child(dt, an)
+		if (streq(gen_node_name, an->name))
+			break;
+
+	if (!an)
+		an = build_and_name_child_node(dt, gen_node_name);
+	if (!an)
+		die("Could not build label node /%s\n", gen_node_name);
+
+	generate_label_tree_internal(dt, dt, an, allocph);
+}
+
+#define FIXUPS	"__fixups__"
+#define LOCAL_FIXUPS "__local_fixups__"
+
+static void add_fixup_entry(struct node *dt, struct node *node,
+		struct property *prop, struct marker *m)
+{
+	struct node *fn;	/* fixup node */
+	char *entry;
+
+	/* m->ref can only be a REF_PHANDLE, but check anyway */
+	assert(m->type == REF_PHANDLE);
+
+	/* fn is the node we're putting entries in */
+	fn = get_subnode(dt, FIXUPS);
+	assert(fn != NULL);
+
+	/* there shouldn't be any ':' in the arguments */
+	if (strchr(node->fullpath, ':') || strchr(prop->name, ':'))
+		die("arguments should not contain ':'\n");
+
+	xasprintf(&entry, "%s:%s:%u",
+			node->fullpath, prop->name, m->offset);
+	append_to_property(fn, m->ref, entry, strlen(entry) + 1);
+}
+
+static void add_local_fixup_entry(struct node *dt, struct node *node,
+		struct property *prop, struct marker *m,
+		struct node *refnode)
+{
+	struct node *lfn, *wn, *nwn;	/* local fixup node, walk node, new */
+	uint32_t value_32;
+	char *s, *e, *comp;
+	int len;
+
+	/* fn is the node we're putting entries in */
+	lfn = get_subnode(dt, LOCAL_FIXUPS);
+	assert(lfn != NULL);
+
+	/* walk the path components creating nodes if they don't exist */
+	comp = xmalloc(strlen(node->fullpath) + 1);
+	/* start skipping the first / */
+	s = node->fullpath + 1;
+	wn = lfn;
+	while (*s) {
+		/* retrieve path component */
+		e = strchr(s, '/');
+		if (e == NULL)
+			e = s + strlen(s);
+		len = e - s;
+		memcpy(comp, s, len);
+		comp[len] = '\0';
+
+		/* if no node exists, create it */
+		nwn = get_subnode(wn, comp);
+		if (!nwn)
+			nwn = build_and_name_child_node(wn, comp);
+		wn = nwn;
+
+		/* last path component */
+		if (!*e)
+			break;
+
+		/* next path component */
+		s = e + 1;
+	}
+	free(comp);
+
+	value_32 = cpu_to_fdt32(m->offset);
+	append_to_property(wn, prop->name, &value_32, sizeof(value_32));
+}
+
+static void generate_fixups_tree_internal(struct node *dt, struct node *node)
+{
+	struct node *c;
+	struct property *prop;
+	struct marker *m;
+	struct node *refnode;
+
+	for_each_property(node, prop) {
+		m = prop->val.markers;
+		for_each_marker_of_type(m, REF_PHANDLE) {
+			refnode = get_node_by_ref(dt, m->ref);
+			if (!refnode)
+				add_fixup_entry(dt, node, prop, m);
+			else
+				add_local_fixup_entry(dt, node, prop, m,
+						refnode);
+		}
+	}
+
+	for_each_child(node, c)
+		generate_fixups_tree_internal(dt, c);
+}
+
+void generate_fixups_tree(struct node *dt)
+{
+	struct node *an;
+
+	for_each_child(dt, an)
+		if (streq(FIXUPS, an->name))
+			break;
+
+	if (!an)
+		build_and_name_child_node(dt, FIXUPS);
+
+	for_each_child(dt, an)
+		if (streq(LOCAL_FIXUPS, an->name))
+			break;
+
+	if (!an)
+		build_and_name_child_node(dt, LOCAL_FIXUPS);
+
+	generate_fixups_tree_internal(dt, dt);
+}
diff --git a/tests/mangle-layout.c b/tests/mangle-layout.c
index a76e51e..d29ebc6 100644
--- a/tests/mangle-layout.c
+++ b/tests/mangle-layout.c
@@ -42,7 +42,8 @@ static void expand_buf(struct bufstate *buf, int newsize)
 	buf->size = newsize;
 }
 
-static void new_header(struct bufstate *buf, int version, const void *fdt)
+static void new_header(struct bufstate *buf, fdt32_t magic, int version,
+		       const void *fdt)
 {
 	int hdrsize;
 
@@ -56,7 +57,7 @@ static void new_header(struct bufstate *buf, int version, const void *fdt)
 	expand_buf(buf, hdrsize);
 	memset(buf->buf, 0, hdrsize);
 
-	fdt_set_magic(buf->buf, FDT_MAGIC);
+	fdt_set_magic(buf->buf, magic);
 	fdt_set_version(buf->buf, version);
 	fdt_set_last_comp_version(buf->buf, 16);
 	fdt_set_boot_cpuid_phys(buf->buf, fdt_boot_cpuid_phys(fdt));
@@ -145,7 +146,7 @@ int main(int argc, char *argv[])
 	if (fdt_version(fdt) < 17)
 		CONFIG("Input tree must be v17");
 
-	new_header(&buf, version, fdt);
+	new_header(&buf, FDT_MAGIC, version, fdt);
 
 	while (*blockorder) {
 		add_block(&buf, version, *blockorder, fdt);
-- 
2.1.4

Re: [PATCH v10 3/4] dtc: Plugin and fixup support

[David Gibson]

[-- Attachment #1: Type: text/plain, Size: 28762 bytes --]

On Fri, Nov 25, 2016 at 02:32:10PM +0200, Pantelis Antoniou wrote:
> This patch enable the generation of symbols & local fixup information
> for trees compiled with the -@ (--symbols) option.
> 
> Using this patch labels in the tree and their users emit information
> in __symbols__ and __local_fixups__ nodes.
> 
> The __fixups__ node make possible the dynamic resolution of phandle
> references which are present in the plugin tree but lie in the
> tree that are applying the overlay against.
> 
> While there is a new magic number for dynamic device tree/overlays blobs
> it is by default enabled. Remember to use -M to generate compatible
> blobs.
> 
> Signed-off-by: Pantelis Antoniou <pantelis.antoniou-OWPKS81ov/FWk0Htik3J/w@public.gmane.org>
> Signed-off-by: Sascha Hauer <s.hauer-bIcnvbaLZ9MEGnE8C9+IrQ@public.gmane.org>
> Signed-off-by: Jan Luebbe <jlu-bIcnvbaLZ9MEGnE8C9+IrQ@public.gmane.org>
> ---
>  Documentation/manual.txt |  25 +++++-
>  checks.c                 |   8 +-
>  dtc-lexer.l              |   5 ++
>  dtc-parser.y             |  50 +++++++++--
>  dtc.c                    |  39 +++++++-
>  dtc.h                    |  20 ++++-
>  fdtdump.c                |   2 +-
>  flattree.c               |  17 ++--
>  fstree.c                 |   2 +-
>  libfdt/fdt.c             |   2 +-
>  libfdt/fdt.h             |   3 +-
>  livetree.c               | 225 ++++++++++++++++++++++++++++++++++++++++++++++-
>  tests/mangle-layout.c    |   7 +-
>  13 files changed, 375 insertions(+), 30 deletions(-)
> 
> diff --git a/Documentation/manual.txt b/Documentation/manual.txt
> index 398de32..094893b 100644
> --- a/Documentation/manual.txt
> +++ b/Documentation/manual.txt
> @@ -119,6 +119,24 @@ Options:
>  	Make space for <number> reserve map entries
>  	Relevant for dtb and asm output only.
>  
> +    -@
> +	Generates a __symbols__ node at the root node of the resulting blob
> +	for any node labels used, and for any local references using phandles
> +	it also generates a __local_fixups__ node that tracks them.
> +
> +	When using the /plugin/ tag all unresolved label references to
> +	be tracked in the __fixups__ node, making dynamic resolution possible.
> +
> +    -A
> +	Generate automatically aliases for all node labels. This is similar to
> +	the -@ option (the __symbols__ node contain identical information) but
> +	the semantics are slightly different since no phandles are automatically
> +	generated for labeled nodes.
> +
> +    -M
> +	Generate blobs with the old FDT magic number for device tree objects.
> +	By default blobs use the DTBO FDT magic number instead.
> +
>      -S <bytes>
>  	Ensure the blob at least <bytes> long, adding additional
>  	space if needed.
> @@ -146,13 +164,18 @@ Additionally, dtc performs various sanity checks on the tree.
>  Here is a very rough overview of the layout of a DTS source file:
>  
>  
> -    sourcefile:   list_of_memreserve devicetree
> +    sourcefile:   versioninfo plugindecl list_of_memreserve devicetree
>  
>      memreserve:   label 'memreserve' ADDR ADDR ';'
>  		| label 'memreserve' ADDR '-' ADDR ';'
>  
>      devicetree:   '/' nodedef
>  
> +    versioninfo:  '/' 'dts-v1' '/' ';'
> +
> +    plugindecl:   '/' 'plugin' '/' ';'
> +                | /* empty */
> +
>      nodedef:      '{' list_of_property list_of_subnode '}' ';'
>  
>      property:     label PROPNAME '=' propdata ';'
> diff --git a/checks.c b/checks.c
> index 2bd27a4..4292f4b 100644
> --- a/checks.c
> +++ b/checks.c
> @@ -487,8 +487,12 @@ static void fixup_phandle_references(struct check *c, struct boot_info *bi,
>  
>  			refnode = get_node_by_ref(dt, m->ref);
>  			if (! refnode) {
> -				FAIL(c, "Reference to non-existent node or label \"%s\"\n",
> -				     m->ref);
> +				if (!(bi->versionflags & VF_PLUGIN))
> +					FAIL(c, "Reference to non-existent node or "
> +							"label \"%s\"\n", m->ref);
> +				else /* mark the entry as unresolved */
> +					*((cell_t *)(prop->val.val + m->offset)) =
> +						cpu_to_fdt32(0xffffffff);
>  				continue;
>  			}
>  
> diff --git a/dtc-lexer.l b/dtc-lexer.l
> index 790fbf6..40bbc87 100644
> --- a/dtc-lexer.l
> +++ b/dtc-lexer.l
> @@ -121,6 +121,11 @@ static void lexical_error(const char *fmt, ...);
>  			return DT_V1;
>  		}
>  
> +<*>"/plugin/"	{
> +			DPRINT("Keyword: /plugin/\n");
> +			return DT_PLUGIN;
> +		}
> +
>  <*>"/memreserve/"	{
>  			DPRINT("Keyword: /memreserve/\n");
>  			BEGIN_DEFAULT();
> diff --git a/dtc-parser.y b/dtc-parser.y
> index 14aaf2e..1a1f660 100644
> --- a/dtc-parser.y
> +++ b/dtc-parser.y
> @@ -19,6 +19,7 @@
>   */
>  %{
>  #include <stdio.h>
> +#include <inttypes.h>
>  
>  #include "dtc.h"
>  #include "srcpos.h"
> @@ -33,6 +34,7 @@ extern void yyerror(char const *s);
>  
>  extern struct boot_info *the_boot_info;
>  extern bool treesource_error;
> +

Extraneous whitespace change here

>  %}
>  
>  %union {
> @@ -52,9 +54,11 @@ extern bool treesource_error;
>  	struct node *nodelist;
>  	struct reserve_info *re;
>  	uint64_t integer;
> +	unsigned int flags;
>  }
>  
>  %token DT_V1
> +%token DT_PLUGIN
>  %token DT_MEMRESERVE
>  %token DT_LSHIFT DT_RSHIFT DT_LE DT_GE DT_EQ DT_NE DT_AND DT_OR
>  %token DT_BITS
> @@ -71,6 +75,8 @@ extern bool treesource_error;
>  
>  %type <data> propdata
>  %type <data> propdataprefix
> +%type <flags> versioninfo
> +%type <flags> plugindecl
>  %type <re> memreserve
>  %type <re> memreserves
>  %type <array> arrayprefix
> @@ -101,16 +107,34 @@ extern bool treesource_error;
>  %%
>  
>  sourcefile:
> -	  v1tag memreserves devicetree
> +	  versioninfo plugindecl memreserves devicetree
> +		{
> +			the_boot_info = build_boot_info($1 | $2, $3, $4,
> +							guess_boot_cpuid($4));
> +		}
> +	;
> +
> +versioninfo:
> +	v1tag
>  		{
> -			the_boot_info = build_boot_info($2, $3,
> -							guess_boot_cpuid($3));
> +			$$ = VF_DT_V1;
>  		}
>  	;
>  
>  v1tag:
>  	  DT_V1 ';'
> +	| DT_V1
>  	| DT_V1 ';' v1tag
> +
> +plugindecl:
> +	DT_PLUGIN ';'
> +		{
> +			$$ = VF_PLUGIN;
> +		}
> +	| /* empty */
> +		{
> +			$$ = 0;
> +		}
>  	;
>  
>  memreserves:
> @@ -161,10 +185,19 @@ devicetree:
>  		{
>  			struct node *target = get_node_by_ref($1, $2);
>  
> -			if (target)
> +			if (target) {
>  				merge_nodes(target, $3);
> -			else
> -				ERROR(&@2, "Label or path %s not found", $2);
> +			} else {
> +				/*
> +				 * We rely on the rule being always:
> +				 *   versioninfo plugindecl memreserves devicetree
> +				 * so $-1 is what we want (plugindecl)
> +				 */
> +				if ($<flags>-1 & VF_PLUGIN)

o_O... ok.  I've never seen negative value references before.  Can you
provide a link to some documentation saying this is actually supported
usage in bison?  I wasn't able to find it when I looked.

> +					add_orphan_node($1, $3, $2);
> +				else
> +					ERROR(&@2, "Label or path %s not found", $2);
> +			}
>  			$$ = $1;
>  		}
>  	| devicetree DT_DEL_NODE DT_REF ';'
> @@ -179,6 +212,11 @@ devicetree:
>  
>  			$$ = $1;
>  		}
> +	| /* empty */
> +		{
> +			/* build empty node */
> +			$$ = name_node(build_node(NULL, NULL), "");
> +		}
>  	;
>  
>  nodedef:
> diff --git a/dtc.c b/dtc.c
> index 9dcf640..06e91bc 100644
> --- a/dtc.c
> +++ b/dtc.c
> @@ -32,6 +32,9 @@ int minsize;		/* Minimum blob size */
>  int padsize;		/* Additional padding to blob */
>  int alignsize;		/* Additional padding to blob accroding to the alignsize */
>  int phandle_format = PHANDLE_BOTH;	/* Use linux,phandle or phandle properties */
> +int symbol_fixup_support;	/* enable symbols & fixup support */
> +int auto_label_aliases;		/* auto generate labels -> aliases */
> +int no_dtbo_magic;		/* use old FDT magic values for objects */
>  
>  static int is_power_of_2(int x)
>  {
> @@ -59,7 +62,7 @@ static void fill_fullpaths(struct node *tree, const char *prefix)
>  #define FDT_VERSION(version)	_FDT_VERSION(version)
>  #define _FDT_VERSION(version)	#version
>  static const char usage_synopsis[] = "dtc [options] <input file>";
> -static const char usage_short_opts[] = "qI:O:o:V:d:R:S:p:a:fb:i:H:sW:E:hv";
> +static const char usage_short_opts[] = "qI:O:o:V:d:R:S:p:a:fb:i:H:sW:E:@AMhv";
>  static struct option const usage_long_opts[] = {
>  	{"quiet",            no_argument, NULL, 'q'},
>  	{"in-format",         a_argument, NULL, 'I'},
> @@ -78,6 +81,9 @@ static struct option const usage_long_opts[] = {
>  	{"phandle",           a_argument, NULL, 'H'},
>  	{"warning",           a_argument, NULL, 'W'},
>  	{"error",             a_argument, NULL, 'E'},
> +	{"symbols",	     no_argument, NULL, '@'},
> +	{"auto-alias",       no_argument, NULL, 'A'},
> +	{"no-dtbo-magic",    no_argument, NULL, 'M'},
>  	{"help",             no_argument, NULL, 'h'},
>  	{"version",          no_argument, NULL, 'v'},
>  	{NULL,               no_argument, NULL, 0x0},
> @@ -109,6 +115,9 @@ static const char * const usage_opts_help[] = {
>  	 "\t\tboth   - Both \"linux,phandle\" and \"phandle\" properties",
>  	"\n\tEnable/disable warnings (prefix with \"no-\")",
>  	"\n\tEnable/disable errors (prefix with \"no-\")",
> +	"\n\tEnable symbols/fixup support",
> +	"\n\tEnable auto-alias of labels",
> +	"\n\tDo not use DTBO magic value for plugin objects",
>  	"\n\tPrint this help and exit",
>  	"\n\tPrint version and exit",
>  	NULL,
> @@ -153,7 +162,7 @@ static const char *guess_input_format(const char *fname, const char *fallback)
>  	fclose(f);
>  
>  	magic = fdt32_to_cpu(magic);
> -	if (magic == FDT_MAGIC)
> +	if (magic == FDT_MAGIC || magic == FDT_MAGIC_DTBO)
>  		return "dtb";
>  
>  	return guess_type_by_name(fname, fallback);
> @@ -172,6 +181,7 @@ int main(int argc, char *argv[])
>  	FILE *outf = NULL;
>  	int outversion = DEFAULT_FDT_VERSION;
>  	long long cmdline_boot_cpuid = -1;
> +	fdt32_t out_magic = FDT_MAGIC;
>  
>  	quiet      = 0;
>  	reservenum = 0;
> @@ -249,6 +259,16 @@ int main(int argc, char *argv[])
>  			parse_checks_option(false, true, optarg);
>  			break;
>  
> +		case '@':
> +			symbol_fixup_support = 1;
> +			break;
> +		case 'A':
> +			auto_label_aliases = 1;
> +			break;
> +		case 'M':
> +			no_dtbo_magic = 1;
> +			break;
> +
>  		case 'h':
>  			usage(NULL);
>  		default:
> @@ -306,6 +326,14 @@ int main(int argc, char *argv[])
>  	fill_fullpaths(bi->dt, "");
>  	process_checks(force, bi);
>  
> +	if (auto_label_aliases)
> +		generate_label_tree(bi->dt, "aliases", false);
> +
> +	if (symbol_fixup_support) {
> +		generate_label_tree(bi->dt, "__symbols__", true);
> +		generate_fixups_tree(bi->dt);

Hang on.. this doesn't seem right.  I thought -@ controlled the
__symbols__ side (i.e. the part upon which we overlay) rather than the
fixups side (the part which overlays).  A dtbo could certainly have
both, of course, but for base trees, wouldn't you have symbols without
fixups?  And should it be illegal to try to build a /plugin/ without
-@?

> +	}
> +
>  	if (sort)
>  		sort_tree(bi);
>  
> @@ -318,12 +346,15 @@ int main(int argc, char *argv[])
>  			    outname, strerror(errno));
>  	}
>  
> +	if (!no_dtbo_magic && (bi->versionflags & VF_PLUGIN))
> +		out_magic = FDT_MAGIC_DTBO;
> +
>  	if (streq(outform, "dts")) {
>  		dt_to_source(outf, bi);
>  	} else if (streq(outform, "dtb")) {
> -		dt_to_blob(outf, bi, outversion);
> +		dt_to_blob(outf, bi, out_magic, outversion);
>  	} else if (streq(outform, "asm")) {
> -		dt_to_asm(outf, bi, outversion);
> +		dt_to_asm(outf, bi, out_magic, outversion);
>  	} else if (streq(outform, "null")) {
>  		/* do nothing */
>  	} else {
> diff --git a/dtc.h b/dtc.h
> index 32009bc..581b3bf 100644
> --- a/dtc.h
> +++ b/dtc.h
> @@ -55,6 +55,9 @@ extern int minsize;		/* Minimum blob size */
>  extern int padsize;		/* Additional padding to blob */
>  extern int alignsize;		/* Additional padding to blob accroding to the alignsize */
>  extern int phandle_format;	/* Use linux,phandle or phandle properties */
> +extern int symbol_fixup_support;/* enable symbols & fixup support */
> +extern int auto_label_aliases;	/* auto generate labels -> aliases */
> +extern int no_dtbo_magic;	/* use old FDT magic values for objects */
>  
>  #define PHANDLE_LEGACY	0x1
>  #define PHANDLE_EPAPR	0x2
> @@ -195,6 +198,7 @@ struct node *build_node_delete(void);
>  struct node *name_node(struct node *node, char *name);
>  struct node *chain_node(struct node *first, struct node *list);
>  struct node *merge_nodes(struct node *old_node, struct node *new_node);
> +void add_orphan_node(struct node *old_node, struct node *new_node, char *ref);
>  
>  void add_property(struct node *node, struct property *prop);
>  void delete_property_by_name(struct node *node, char *name);
> @@ -202,6 +206,8 @@ void delete_property(struct property *prop);
>  void add_child(struct node *parent, struct node *child);
>  void delete_node_by_name(struct node *parent, char *name);
>  void delete_node(struct node *node);
> +void append_to_property(struct node *node,
> +			char *name, const void *data, int len);
>  
>  const char *get_unitname(struct node *node);
>  struct property *get_property(struct node *node, const char *propname);
> @@ -237,14 +243,22 @@ struct reserve_info *add_reserve_entry(struct reserve_info *list,
>  
>  
>  struct boot_info {
> +	unsigned int versionflags;
>  	struct reserve_info *reservelist;
>  	struct node *dt;		/* the device tree */
>  	uint32_t boot_cpuid_phys;
>  };
>  
> -struct boot_info *build_boot_info(struct reserve_info *reservelist,
> +/* version flags definitions */
> +#define VF_DT_V1	0x0001	/* /dts-v1/ */
> +#define VF_PLUGIN	0x0002	/* /plugin/ */
> +
> +struct boot_info *build_boot_info(unsigned int versionflags,
> +				  struct reserve_info *reservelist,
>  				  struct node *tree, uint32_t boot_cpuid_phys);
>  void sort_tree(struct boot_info *bi);
> +void generate_label_tree(struct node *dt, char *gen_node_name, bool allocph);
> +void generate_fixups_tree(struct node *dt);
>  
>  /* Checks */
>  
> @@ -253,8 +267,8 @@ void process_checks(bool force, struct boot_info *bi);
>  
>  /* Flattened trees */
>  
> -void dt_to_blob(FILE *f, struct boot_info *bi, int version);
> -void dt_to_asm(FILE *f, struct boot_info *bi, int version);
> +void dt_to_blob(FILE *f, struct boot_info *bi, fdt32_t magic, int version);
> +void dt_to_asm(FILE *f, struct boot_info *bi, fdt32_t magic, int version);
>  
>  struct boot_info *dt_from_blob(const char *fname);
>  
> diff --git a/fdtdump.c b/fdtdump.c
> index a9a2484..dd63ac2 100644
> --- a/fdtdump.c
> +++ b/fdtdump.c
> @@ -201,7 +201,7 @@ int main(int argc, char *argv[])
>  			p = memchr(p, smagic[0], endp - p - FDT_MAGIC_SIZE);
>  			if (!p)
>  				break;
> -			if (fdt_magic(p) == FDT_MAGIC) {
> +			if (fdt_magic(p) == FDT_MAGIC || fdt_magic(p) == FDT_MAGIC_DTBO) {
>  				/* try and validate the main struct */
>  				off_t this_len = endp - p;
>  				fdt32_t max_version = 17;
> diff --git a/flattree.c b/flattree.c
> index a9d9520..57d76cf 100644
> --- a/flattree.c
> +++ b/flattree.c
> @@ -335,6 +335,7 @@ static struct data flatten_reserve_list(struct reserve_info *reservelist,
>  }
>  
>  static void make_fdt_header(struct fdt_header *fdt,
> +			    fdt32_t magic,
>  			    struct version_info *vi,
>  			    int reservesize, int dtsize, int strsize,
>  			    int boot_cpuid_phys)
> @@ -345,7 +346,7 @@ static void make_fdt_header(struct fdt_header *fdt,
>  
>  	memset(fdt, 0xff, sizeof(*fdt));
>  
> -	fdt->magic = cpu_to_fdt32(FDT_MAGIC);
> +	fdt->magic = cpu_to_fdt32(magic);
>  	fdt->version = cpu_to_fdt32(vi->version);
>  	fdt->last_comp_version = cpu_to_fdt32(vi->last_comp_version);
>  
> @@ -366,7 +367,7 @@ static void make_fdt_header(struct fdt_header *fdt,
>  		fdt->size_dt_struct = cpu_to_fdt32(dtsize);
>  }
>  
> -void dt_to_blob(FILE *f, struct boot_info *bi, int version)
> +void dt_to_blob(FILE *f, struct boot_info *bi, fdt32_t magic, int version)
>  {
>  	struct version_info *vi = NULL;
>  	int i;
> @@ -390,7 +391,7 @@ void dt_to_blob(FILE *f, struct boot_info *bi, int version)
>  	reservebuf = flatten_reserve_list(bi->reservelist, vi);
>  
>  	/* Make header */
> -	make_fdt_header(&fdt, vi, reservebuf.len, dtbuf.len, strbuf.len,
> +	make_fdt_header(&fdt, magic, vi, reservebuf.len, dtbuf.len, strbuf.len,
>  			bi->boot_cpuid_phys);
>  
>  	/*
> @@ -467,7 +468,7 @@ static void dump_stringtable_asm(FILE *f, struct data strbuf)
>  	}
>  }
>  
> -void dt_to_asm(FILE *f, struct boot_info *bi, int version)
> +void dt_to_asm(FILE *f, struct boot_info *bi, fdt32_t magic, int version)
>  {
>  	struct version_info *vi = NULL;
>  	int i;
> @@ -830,6 +831,7 @@ struct boot_info *dt_from_blob(const char *fname)
>  	struct node *tree;
>  	uint32_t val;
>  	int flags = 0;
> +	unsigned int versionflags = VF_DT_V1;
>  
>  	f = srcfile_relative_open(fname, NULL);
>  
> @@ -845,9 +847,12 @@ struct boot_info *dt_from_blob(const char *fname)
>  	}
>  
>  	magic = fdt32_to_cpu(magic);
> -	if (magic != FDT_MAGIC)
> +	if (magic != FDT_MAGIC && magic != FDT_MAGIC_DTBO)
>  		die("Blob has incorrect magic number\n");
>  
> +	if (magic == FDT_MAGIC_DTBO)
> +		versionflags |= VF_PLUGIN;

Not particularly useful yet, but I wonder if we'll want some option to
force treating dtb input as a plugin, for the case of old plugins
which don't have the new magic number.

> +
>  	rc = fread(&totalsize, sizeof(totalsize), 1, f);
>  	if (ferror(f))
>  		die("Error reading DT blob size: %s\n", strerror(errno));
> @@ -942,5 +947,5 @@ struct boot_info *dt_from_blob(const char *fname)
>  
>  	fclose(f);
>  
> -	return build_boot_info(reservelist, tree, boot_cpuid_phys);
> +	return build_boot_info(versionflags, reservelist, tree, boot_cpuid_phys);
>  }
> diff --git a/fstree.c b/fstree.c
> index 6d1beec..54f520b 100644
> --- a/fstree.c
> +++ b/fstree.c
> @@ -86,6 +86,6 @@ struct boot_info *dt_from_fs(const char *dirname)
>  	tree = read_fstree(dirname);
>  	tree = name_node(tree, "");
>  
> -	return build_boot_info(NULL, tree, guess_boot_cpuid(tree));
> +	return build_boot_info(VF_DT_V1, NULL, tree, guess_boot_cpuid(tree));
>  }
>  
> diff --git a/libfdt/fdt.c b/libfdt/fdt.c
> index 22286a1..28d422c 100644
> --- a/libfdt/fdt.c
> +++ b/libfdt/fdt.c
> @@ -57,7 +57,7 @@
>  
>  int fdt_check_header(const void *fdt)
>  {
> -	if (fdt_magic(fdt) == FDT_MAGIC) {
> +	if (fdt_magic(fdt) == FDT_MAGIC || fdt_magic(fdt) == FDT_MAGIC_DTBO) {
>  		/* Complete tree */
>  		if (fdt_version(fdt) < FDT_FIRST_SUPPORTED_VERSION)
>  			return -FDT_ERR_BADVERSION;
> diff --git a/libfdt/fdt.h b/libfdt/fdt.h
> index 526aedb..493cd55 100644
> --- a/libfdt/fdt.h
> +++ b/libfdt/fdt.h
> @@ -55,7 +55,7 @@
>  #ifndef __ASSEMBLY__
>  
>  struct fdt_header {
> -	fdt32_t magic;			 /* magic word FDT_MAGIC */
> +	fdt32_t magic;			 /* magic word FDT_MAGIC[|_DTBO] */
>  	fdt32_t totalsize;		 /* total size of DT block */
>  	fdt32_t off_dt_struct;		 /* offset to structure */
>  	fdt32_t off_dt_strings;		 /* offset to strings */
> @@ -93,6 +93,7 @@ struct fdt_property {
>  #endif /* !__ASSEMBLY */
>  
>  #define FDT_MAGIC	0xd00dfeed	/* 4: version, 4: total size */
> +#define FDT_MAGIC_DTBO	0xd00dfdb0	/* DTBO magic */
>  #define FDT_TAGSIZE	sizeof(fdt32_t)
>  
>  #define FDT_BEGIN_NODE	0x1		/* Start node: full name */
> diff --git a/livetree.c b/livetree.c
> index 3dc7559..f2c86bd 100644
> --- a/livetree.c
> +++ b/livetree.c
> @@ -216,6 +216,31 @@ struct node *merge_nodes(struct node *old_node, struct node *new_node)
>  	return old_node;
>  }
>  
> +void add_orphan_node(struct node *dt, struct node *new_node, char *ref)
> +{
> +	static unsigned int next_orphan_fragment = 0;
> +	struct node *node = build_node(NULL, NULL);
> +	struct property *p;
> +	struct data d = empty_data;
> +	char *name;
> +
> +	memset(node, 0, sizeof(*node));

You don't need the memset() now that you're using build_node() above.

> +
> +	d = data_add_marker(d, REF_PHANDLE, ref);
> +	d = data_append_integer(d, 0xffffffff, 32);
> +
> +	p = build_property("target", d);
> +	add_property(node, p);
> +
> +	xasprintf(&name, "fragment@%u",
> +			next_orphan_fragment++);
> +	name_node(node, name);
> +	name_node(new_node, "__overlay__");

You can do this more naturally if you do the name_node() here, then
you can just pass the __overlay__ node into build_node() for the
fragment@ node instead of having to explicitly add_child.

> +
> +	add_child(dt, node);
> +	add_child(node, new_node);
> +}
> +
>  struct node *chain_node(struct node *first, struct node *list)
>  {
>  	assert(first->next_sibling == NULL);
> @@ -296,6 +321,23 @@ void delete_node(struct node *node)
>  	delete_labels(&node->labels);
>  }
>  
> +void append_to_property(struct node *node,
> +				    char *name, const void *data, int len)
> +{
> +	struct data d;
> +	struct property *p;
> +
> +	p = get_property(node, name);
> +	if (p) {
> +		d = data_append_data(p->val, data, len);
> +		p->val = d;
> +	} else {
> +		d = data_append_data(empty_data, data, len);
> +		p = build_property(name, d);
> +		add_property(node, p);
> +	}
> +}
> +
>  struct reserve_info *build_reserve_entry(uint64_t address, uint64_t size)
>  {
>  	struct reserve_info *new = xmalloc(sizeof(*new));
> @@ -335,12 +377,14 @@ struct reserve_info *add_reserve_entry(struct reserve_info *list,
>  	return list;
>  }
>  
> -struct boot_info *build_boot_info(struct reserve_info *reservelist,
> +struct boot_info *build_boot_info(unsigned int versionflags,
> +				  struct reserve_info *reservelist,
>  				  struct node *tree, uint32_t boot_cpuid_phys)
>  {
>  	struct boot_info *bi;
>  
>  	bi = xmalloc(sizeof(*bi));
> +	bi->versionflags = versionflags;
>  	bi->reservelist = reservelist;
>  	bi->dt = tree;
>  	bi->boot_cpuid_phys = boot_cpuid_phys;
> @@ -709,3 +753,182 @@ void sort_tree(struct boot_info *bi)
>  	sort_reserve_entries(bi);
>  	sort_node(bi->dt);
>  }
> +
> +/* utility helper to avoid code duplication */
> +static struct node *build_and_name_child_node(struct node *parent, char *name)
> +{
> +	struct node *node;
> +
> +	node = build_node(NULL, NULL);
> +	name_node(node, xstrdup(name));
> +	add_child(parent, node);
> +
> +	return node;
> +}
> +
> +static void generate_label_tree_internal(struct node *dt, struct node *node,
> +					 struct node *an, bool allocph)
> +{
> +	struct node *c;
> +	struct property *p;
> +	struct label *l;
> +
> +	/* if if there are labels */
> +	if (node->labels) {
> +		/* now add the label in the node */
> +		for_each_label(node->labels, l) {
> +			/* check whether the label already exists */
> +			p = get_property(an, l->label);
> +			if (p) {
> +				fprintf(stderr, "WARNING: label %s already"
> +					" exists in /%s", l->label,
> +					an->name);
> +				continue;
> +			}
> +
> +			/* insert it */
> +			p = build_property(l->label,
> +				data_copy_mem(node->fullpath,
> +						strlen(node->fullpath) + 1));
> +			add_property(an, p);
> +		}
> +
> +		/* force allocation of a phandle for this node */
> +		if (allocph)
> +			(void)get_node_phandle(dt, node);
> +	}
> +
> +	for_each_child(node, c)
> +		generate_label_tree_internal(dt, c, an, allocph);
> +}
> +
> +void generate_label_tree(struct node *dt, char *gen_node_name, bool allocph)
> +{
> +	struct node *an;
> +
> +	for_each_child(dt, an)
> +		if (streq(gen_node_name, an->name))
> +			break;
> +
> +	if (!an)
> +		an = build_and_name_child_node(dt, gen_node_name);
> +	if (!an)
> +		die("Could not build label node /%s\n", gen_node_name);
> +
> +	generate_label_tree_internal(dt, dt, an, allocph);
> +}
> +
> +#define FIXUPS	"__fixups__"
> +#define LOCAL_FIXUPS "__local_fixups__"
> +
> +static void add_fixup_entry(struct node *dt, struct node *node,
> +		struct property *prop, struct marker *m)
> +{
> +	struct node *fn;	/* fixup node */
> +	char *entry;
> +
> +	/* m->ref can only be a REF_PHANDLE, but check anyway */
> +	assert(m->type == REF_PHANDLE);
> +
> +	/* fn is the node we're putting entries in */
> +	fn = get_subnode(dt, FIXUPS);
> +	assert(fn != NULL);
> +
> +	/* there shouldn't be any ':' in the arguments */
> +	if (strchr(node->fullpath, ':') || strchr(prop->name, ':'))
> +		die("arguments should not contain ':'\n");
> +
> +	xasprintf(&entry, "%s:%s:%u",
> +			node->fullpath, prop->name, m->offset);
> +	append_to_property(fn, m->ref, entry, strlen(entry) + 1);
> +}
> +
> +static void add_local_fixup_entry(struct node *dt, struct node *node,
> +		struct property *prop, struct marker *m,
> +		struct node *refnode)
> +{
> +	struct node *lfn, *wn, *nwn;	/* local fixup node, walk node, new */
> +	uint32_t value_32;
> +	char *s, *e, *comp;
> +	int len;
> +
> +	/* fn is the node we're putting entries in */
> +	lfn = get_subnode(dt, LOCAL_FIXUPS);
> +	assert(lfn != NULL);
> +
> +	/* walk the path components creating nodes if they don't exist */
> +	comp = xmalloc(strlen(node->fullpath) + 1);
> +	/* start skipping the first / */
> +	s = node->fullpath + 1;
> +	wn = lfn;
> +	while (*s) {
> +		/* retrieve path component */
> +		e = strchr(s, '/');
> +		if (e == NULL)
> +			e = s + strlen(s);
> +		len = e - s;
> +		memcpy(comp, s, len);
> +		comp[len] = '\0';

Parsing the fullpath into components seems an odd way of doing this.
We have an actual handle on the node, and therefore all it's parents,
which already have the individual path components split out.

> +
> +		/* if no node exists, create it */
> +		nwn = get_subnode(wn, comp);
> +		if (!nwn)
> +			nwn = build_and_name_child_node(wn, comp);
> +		wn = nwn;
> +
> +		/* last path component */
> +		if (!*e)
> +			break;
> +
> +		/* next path component */
> +		s = e + 1;
> +	}
> +	free(comp);
> +
> +	value_32 = cpu_to_fdt32(m->offset);
> +	append_to_property(wn, prop->name, &value_32, sizeof(value_32));
> +}
> +
> +static void generate_fixups_tree_internal(struct node *dt, struct node *node)
> +{
> +	struct node *c;
> +	struct property *prop;
> +	struct marker *m;
> +	struct node *refnode;
> +
> +	for_each_property(node, prop) {
> +		m = prop->val.markers;
> +		for_each_marker_of_type(m, REF_PHANDLE) {
> +			refnode = get_node_by_ref(dt, m->ref);
> +			if (!refnode)
> +				add_fixup_entry(dt, node, prop, m);
> +			else
> +				add_local_fixup_entry(dt, node, prop, m,
> +						refnode);
> +		}
> +	}
> +
> +	for_each_child(node, c)
> +		generate_fixups_tree_internal(dt, c);
> +}
> +
> +void generate_fixups_tree(struct node *dt)
> +{
> +	struct node *an;
> +
> +	for_each_child(dt, an)
> +		if (streq(FIXUPS, an->name))
> +			break;
> +
> +	if (!an)
> +		build_and_name_child_node(dt, FIXUPS);
> +
> +	for_each_child(dt, an)
> +		if (streq(LOCAL_FIXUPS, an->name))
> +			break;
> +
> +	if (!an)
> +		build_and_name_child_node(dt, LOCAL_FIXUPS);
> +
> +	generate_fixups_tree_internal(dt, dt);
> +}
> diff --git a/tests/mangle-layout.c b/tests/mangle-layout.c
> index a76e51e..d29ebc6 100644
> --- a/tests/mangle-layout.c
> +++ b/tests/mangle-layout.c
> @@ -42,7 +42,8 @@ static void expand_buf(struct bufstate *buf, int newsize)
>  	buf->size = newsize;
>  }
>  
> -static void new_header(struct bufstate *buf, int version, const void *fdt)
> +static void new_header(struct bufstate *buf, fdt32_t magic, int version,
> +		       const void *fdt)
>  {
>  	int hdrsize;
>  
> @@ -56,7 +57,7 @@ static void new_header(struct bufstate *buf, int version, const void *fdt)
>  	expand_buf(buf, hdrsize);
>  	memset(buf->buf, 0, hdrsize);
>  
> -	fdt_set_magic(buf->buf, FDT_MAGIC);
> +	fdt_set_magic(buf->buf, magic);
>  	fdt_set_version(buf->buf, version);
>  	fdt_set_last_comp_version(buf->buf, 16);
>  	fdt_set_boot_cpuid_phys(buf->buf, fdt_boot_cpuid_phys(fdt));
> @@ -145,7 +146,7 @@ int main(int argc, char *argv[])
>  	if (fdt_version(fdt) < 17)
>  		CONFIG("Input tree must be v17");
>  
> -	new_header(&buf, version, fdt);
> +	new_header(&buf, FDT_MAGIC, version, fdt);
>  
>  	while (*blockorder) {
>  		add_block(&buf, version, *blockorder, fdt);

-- 
David Gibson			| I'll have my music baroque, and my code
david AT gibson.dropbear.id.au	| minimalist, thank you.  NOT _the_ _other_
				| _way_ _around_!
http://www.ozlabs.org/~dgibson

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 819 bytes --]

Re: [PATCH v10 3/4] dtc: Plugin and fixup support

[Pantelis Antoniou]

> On Nov 28, 2016, at 06:12 , David Gibson <david-xT8FGy+AXnRB3Ne2BGzF6laj5H9X9Tb+@public.gmane.org> wrote:
> 
> On Fri, Nov 25, 2016 at 02:32:10PM +0200, Pantelis Antoniou wrote:
>> This patch enable the generation of symbols & local fixup information
>> for trees compiled with the -@ (--symbols) option.
>> 
>> Using this patch labels in the tree and their users emit information
>> in __symbols__ and __local_fixups__ nodes.
>> 
>> The __fixups__ node make possible the dynamic resolution of phandle
>> references which are present in the plugin tree but lie in the
>> tree that are applying the overlay against.
>> 
>> While there is a new magic number for dynamic device tree/overlays blobs
>> it is by default enabled. Remember to use -M to generate compatible
>> blobs.
>> 
>> Signed-off-by: Pantelis Antoniou <pantelis.antoniou-OWPKS81ov/FWk0Htik3J/w@public.gmane.org>
>> Signed-off-by: Sascha Hauer <s.hauer-bIcnvbaLZ9MEGnE8C9+IrQ@public.gmane.org>
>> Signed-off-by: Jan Luebbe <jlu-bIcnvbaLZ9MEGnE8C9+IrQ@public.gmane.org>
>> ---
>> Documentation/manual.txt |  25 +++++-
>> checks.c                 |   8 +-
>> dtc-lexer.l              |   5 ++
>> dtc-parser.y             |  50 +++++++++--
>> dtc.c                    |  39 +++++++-
>> dtc.h                    |  20 ++++-
>> fdtdump.c                |   2 +-
>> flattree.c               |  17 ++--
>> fstree.c                 |   2 +-
>> libfdt/fdt.c             |   2 +-
>> libfdt/fdt.h             |   3 +-
>> livetree.c               | 225 ++++++++++++++++++++++++++++++++++++++++++++++-
>> tests/mangle-layout.c    |   7 +-
>> 13 files changed, 375 insertions(+), 30 deletions(-)
>> 
>> diff --git a/Documentation/manual.txt b/Documentation/manual.txt
>> index 398de32..094893b 100644
>> --- a/Documentation/manual.txt
>> +++ b/Documentation/manual.txt
>> @@ -119,6 +119,24 @@ Options:
>> 	Make space for <number> reserve map entries
>> 	Relevant for dtb and asm output only.
>> 
>> +    -@
>> +	Generates a __symbols__ node at the root node of the resulting blob
>> +	for any node labels used, and for any local references using phandles
>> +	it also generates a __local_fixups__ node that tracks them.
>> +
>> +	When using the /plugin/ tag all unresolved label references to
>> +	be tracked in the __fixups__ node, making dynamic resolution possible.
>> +
>> +    -A
>> +	Generate automatically aliases for all node labels. This is similar to
>> +	the -@ option (the __symbols__ node contain identical information) but
>> +	the semantics are slightly different since no phandles are automatically
>> +	generated for labeled nodes.
>> +
>> +    -M
>> +	Generate blobs with the old FDT magic number for device tree objects.
>> +	By default blobs use the DTBO FDT magic number instead.
>> +
>>     -S <bytes>
>> 	Ensure the blob at least <bytes> long, adding additional
>> 	space if needed.
>> @@ -146,13 +164,18 @@ Additionally, dtc performs various sanity checks on the tree.
>> Here is a very rough overview of the layout of a DTS source file:
>> 
>> 
>> -    sourcefile:   list_of_memreserve devicetree
>> +    sourcefile:   versioninfo plugindecl list_of_memreserve devicetree
>> 
>>     memreserve:   label 'memreserve' ADDR ADDR ';'
>> 		| label 'memreserve' ADDR '-' ADDR ';'
>> 
>>     devicetree:   '/' nodedef
>> 
>> +    versioninfo:  '/' 'dts-v1' '/' ';'
>> +
>> +    plugindecl:   '/' 'plugin' '/' ';'
>> +                | /* empty */
>> +
>>     nodedef:      '{' list_of_property list_of_subnode '}' ';'
>> 
>>     property:     label PROPNAME '=' propdata ';'
>> diff --git a/checks.c b/checks.c
>> index 2bd27a4..4292f4b 100644
>> --- a/checks.c
>> +++ b/checks.c
>> @@ -487,8 +487,12 @@ static void fixup_phandle_references(struct check *c, struct boot_info *bi,
>> 
>> 			refnode = get_node_by_ref(dt, m->ref);
>> 			if (! refnode) {
>> -				FAIL(c, "Reference to non-existent node or label \"%s\"\n",
>> -				     m->ref);
>> +				if (!(bi->versionflags & VF_PLUGIN))
>> +					FAIL(c, "Reference to non-existent node or "
>> +							"label \"%s\"\n", m->ref);
>> +				else /* mark the entry as unresolved */
>> +					*((cell_t *)(prop->val.val + m->offset)) =
>> +						cpu_to_fdt32(0xffffffff);
>> 				continue;
>> 			}
>> 
>> diff --git a/dtc-lexer.l b/dtc-lexer.l
>> index 790fbf6..40bbc87 100644
>> --- a/dtc-lexer.l
>> +++ b/dtc-lexer.l
>> @@ -121,6 +121,11 @@ static void lexical_error(const char *fmt, ...);
>> 			return DT_V1;
>> 		}
>> 
>> +<*>"/plugin/"	{
>> +			DPRINT("Keyword: /plugin/\n");
>> +			return DT_PLUGIN;
>> +		}
>> +
>> <*>"/memreserve/"	{
>> 			DPRINT("Keyword: /memreserve/\n");
>> 			BEGIN_DEFAULT();
>> diff --git a/dtc-parser.y b/dtc-parser.y
>> index 14aaf2e..1a1f660 100644
>> --- a/dtc-parser.y
>> +++ b/dtc-parser.y
>> @@ -19,6 +19,7 @@
>>  */
>> %{
>> #include <stdio.h>
>> +#include <inttypes.h>
>> 
>> #include "dtc.h"
>> #include "srcpos.h"
>> @@ -33,6 +34,7 @@ extern void yyerror(char const *s);
>> 
>> extern struct boot_info *the_boot_info;
>> extern bool treesource_error;
>> +
> 
> Extraneous whitespace change here
> 

OK.

>> %}
>> 
>> %union {
>> @@ -52,9 +54,11 @@ extern bool treesource_error;
>> 	struct node *nodelist;
>> 	struct reserve_info *re;
>> 	uint64_t integer;
>> +	unsigned int flags;
>> }
>> 
>> %token DT_V1
>> +%token DT_PLUGIN
>> %token DT_MEMRESERVE
>> %token DT_LSHIFT DT_RSHIFT DT_LE DT_GE DT_EQ DT_NE DT_AND DT_OR
>> %token DT_BITS
>> @@ -71,6 +75,8 @@ extern bool treesource_error;
>> 
>> %type <data> propdata
>> %type <data> propdataprefix
>> +%type <flags> versioninfo
>> +%type <flags> plugindecl
>> %type <re> memreserve
>> %type <re> memreserves
>> %type <array> arrayprefix
>> @@ -101,16 +107,34 @@ extern bool treesource_error;
>> %%
>> 
>> sourcefile:
>> -	  v1tag memreserves devicetree
>> +	  versioninfo plugindecl memreserves devicetree
>> +		{
>> +			the_boot_info = build_boot_info($1 | $2, $3, $4,
>> +							guess_boot_cpuid($4));
>> +		}
>> +	;
>> +
>> +versioninfo:
>> +	v1tag
>> 		{
>> -			the_boot_info = build_boot_info($2, $3,
>> -							guess_boot_cpuid($3));
>> +			$$ = VF_DT_V1;
>> 		}
>> 	;
>> 
>> v1tag:
>> 	  DT_V1 ';'
>> +	| DT_V1
>> 	| DT_V1 ';' v1tag
>> +
>> +plugindecl:
>> +	DT_PLUGIN ';'
>> +		{
>> +			$$ = VF_PLUGIN;
>> +		}
>> +	| /* empty */
>> +		{
>> +			$$ = 0;
>> +		}
>> 	;
>> 
>> memreserves:
>> @@ -161,10 +185,19 @@ devicetree:
>> 		{
>> 			struct node *target = get_node_by_ref($1, $2);
>> 
>> -			if (target)
>> +			if (target) {
>> 				merge_nodes(target, $3);
>> -			else
>> -				ERROR(&@2, "Label or path %s not found", $2);
>> +			} else {
>> +				/*
>> +				 * We rely on the rule being always:
>> +				 *   versioninfo plugindecl memreserves devicetree
>> +				 * so $-1 is what we want (plugindecl)
>> +				 */
>> +				if ($<flags>-1 & VF_PLUGIN)
> 
> o_O... ok.  I've never seen negative value references before.  Can you
> provide a link to some documentation saying this is actually supported
> usage in bison?  I wasn't able to find it when I looked.
> 

There is a section about inherited attributes in the flex & bison book by O’Reily.

https://books.google.gr/books?id=3Sr1V5J9_qMC&lpg=PP1&dq=flex%20bison&hl=el&pg=PP1#v=onepage&q=flex%20bison&f=false

There’s a direct link to the 2nd Edition of lex & yacc:

https://books.google.gr/books?id=fMPxfWfe67EC&lpg=PA183&ots=RcRSji2NAT&dq=yacc%20inherited%20attributes&hl=el&pg=PA183#v=onepage&q=yacc%20inherited%20attributes&f=false

>> +					add_orphan_node($1, $3, $2);
>> +				else
>> +					ERROR(&@2, "Label or path %s not found", $2);
>> +			}
>> 			$$ = $1;
>> 		}
>> 	| devicetree DT_DEL_NODE DT_REF ';'
>> @@ -179,6 +212,11 @@ devicetree:
>> 
>> 			$$ = $1;
>> 		}
>> +	| /* empty */
>> +		{
>> +			/* build empty node */
>> +			$$ = name_node(build_node(NULL, NULL), "");
>> +		}
>> 	;
>> 
>> nodedef:
>> diff --git a/dtc.c b/dtc.c
>> index 9dcf640..06e91bc 100644
>> --- a/dtc.c
>> +++ b/dtc.c
>> @@ -32,6 +32,9 @@ int minsize;		/* Minimum blob size */
>> int padsize;		/* Additional padding to blob */
>> int alignsize;		/* Additional padding to blob accroding to the alignsize */
>> int phandle_format = PHANDLE_BOTH;	/* Use linux,phandle or phandle properties */
>> +int symbol_fixup_support;	/* enable symbols & fixup support */
>> +int auto_label_aliases;		/* auto generate labels -> aliases */
>> +int no_dtbo_magic;		/* use old FDT magic values for objects */
>> 
>> static int is_power_of_2(int x)
>> {
>> @@ -59,7 +62,7 @@ static void fill_fullpaths(struct node *tree, const char *prefix)
>> #define FDT_VERSION(version)	_FDT_VERSION(version)
>> #define _FDT_VERSION(version)	#version
>> static const char usage_synopsis[] = "dtc [options] <input file>";
>> -static const char usage_short_opts[] = "qI:O:o:V:d:R:S:p:a:fb:i:H:sW:E:hv";
>> +static const char usage_short_opts[] = "qI:O:o:V:d:R:S:p:a:fb:i:H:sW:E:@AMhv";
>> static struct option const usage_long_opts[] = {
>> 	{"quiet",            no_argument, NULL, 'q'},
>> 	{"in-format",         a_argument, NULL, 'I'},
>> @@ -78,6 +81,9 @@ static struct option const usage_long_opts[] = {
>> 	{"phandle",           a_argument, NULL, 'H'},
>> 	{"warning",           a_argument, NULL, 'W'},
>> 	{"error",             a_argument, NULL, 'E'},
>> +	{"symbols",	     no_argument, NULL, '@'},
>> +	{"auto-alias",       no_argument, NULL, 'A'},
>> +	{"no-dtbo-magic",    no_argument, NULL, 'M'},
>> 	{"help",             no_argument, NULL, 'h'},
>> 	{"version",          no_argument, NULL, 'v'},
>> 	{NULL,               no_argument, NULL, 0x0},
>> @@ -109,6 +115,9 @@ static const char * const usage_opts_help[] = {
>> 	 "\t\tboth   - Both \"linux,phandle\" and \"phandle\" properties",
>> 	"\n\tEnable/disable warnings (prefix with \"no-\")",
>> 	"\n\tEnable/disable errors (prefix with \"no-\")",
>> +	"\n\tEnable symbols/fixup support",
>> +	"\n\tEnable auto-alias of labels",
>> +	"\n\tDo not use DTBO magic value for plugin objects",
>> 	"\n\tPrint this help and exit",
>> 	"\n\tPrint version and exit",
>> 	NULL,
>> @@ -153,7 +162,7 @@ static const char *guess_input_format(const char *fname, const char *fallback)
>> 	fclose(f);
>> 
>> 	magic = fdt32_to_cpu(magic);
>> -	if (magic == FDT_MAGIC)
>> +	if (magic == FDT_MAGIC || magic == FDT_MAGIC_DTBO)
>> 		return "dtb";
>> 
>> 	return guess_type_by_name(fname, fallback);
>> @@ -172,6 +181,7 @@ int main(int argc, char *argv[])
>> 	FILE *outf = NULL;
>> 	int outversion = DEFAULT_FDT_VERSION;
>> 	long long cmdline_boot_cpuid = -1;
>> +	fdt32_t out_magic = FDT_MAGIC;
>> 
>> 	quiet      = 0;
>> 	reservenum = 0;
>> @@ -249,6 +259,16 @@ int main(int argc, char *argv[])
>> 			parse_checks_option(false, true, optarg);
>> 			break;
>> 
>> +		case '@':
>> +			symbol_fixup_support = 1;
>> +			break;
>> +		case 'A':
>> +			auto_label_aliases = 1;
>> +			break;
>> +		case 'M':
>> +			no_dtbo_magic = 1;
>> +			break;
>> +
>> 		case 'h':
>> 			usage(NULL);
>> 		default:
>> @@ -306,6 +326,14 @@ int main(int argc, char *argv[])
>> 	fill_fullpaths(bi->dt, "");
>> 	process_checks(force, bi);
>> 
>> +	if (auto_label_aliases)
>> +		generate_label_tree(bi->dt, "aliases", false);
>> +
>> +	if (symbol_fixup_support) {
>> +		generate_label_tree(bi->dt, "__symbols__", true);
>> +		generate_fixups_tree(bi->dt);
> 
> Hang on.. this doesn't seem right.  I thought -@ controlled the
> __symbols__ side (i.e. the part upon which we overlay) rather than the
> fixups side (the part which overlays).  A dtbo could certainly have
> both, of course, but for base trees, wouldn't you have symbols without
> fixups?  And should it be illegal to try to build a /plugin/ without
> -@?

It does control both for now. For base trees having the fixup nodes
will allow us to do probe order dependency tracking in the future.
For plugins we need the __symbols__ node to support stacked overlays, i.e.
overlays referring label that were introduced by a previous overlay.  

For plugins there is no requirement for now to actually contain references to
be resolved. It can easily be enforced though. 

> 
>> +	}
>> +
>> 	if (sort)
>> 		sort_tree(bi);
>> 
>> @@ -318,12 +346,15 @@ int main(int argc, char *argv[])
>> 			    outname, strerror(errno));
>> 	}
>> 
>> +	if (!no_dtbo_magic && (bi->versionflags & VF_PLUGIN))
>> +		out_magic = FDT_MAGIC_DTBO;
>> +
>> 	if (streq(outform, "dts")) {
>> 		dt_to_source(outf, bi);
>> 	} else if (streq(outform, "dtb")) {
>> -		dt_to_blob(outf, bi, outversion);
>> +		dt_to_blob(outf, bi, out_magic, outversion);
>> 	} else if (streq(outform, "asm")) {
>> -		dt_to_asm(outf, bi, outversion);
>> +		dt_to_asm(outf, bi, out_magic, outversion);
>> 	} else if (streq(outform, "null")) {
>> 		/* do nothing */
>> 	} else {
>> diff --git a/dtc.h b/dtc.h
>> index 32009bc..581b3bf 100644
>> --- a/dtc.h
>> +++ b/dtc.h
>> @@ -55,6 +55,9 @@ extern int minsize;		/* Minimum blob size */
>> extern int padsize;		/* Additional padding to blob */
>> extern int alignsize;		/* Additional padding to blob accroding to the alignsize */
>> extern int phandle_format;	/* Use linux,phandle or phandle properties */
>> +extern int symbol_fixup_support;/* enable symbols & fixup support */
>> +extern int auto_label_aliases;	/* auto generate labels -> aliases */
>> +extern int no_dtbo_magic;	/* use old FDT magic values for objects */
>> 
>> #define PHANDLE_LEGACY	0x1
>> #define PHANDLE_EPAPR	0x2
>> @@ -195,6 +198,7 @@ struct node *build_node_delete(void);
>> struct node *name_node(struct node *node, char *name);
>> struct node *chain_node(struct node *first, struct node *list);
>> struct node *merge_nodes(struct node *old_node, struct node *new_node);
>> +void add_orphan_node(struct node *old_node, struct node *new_node, char *ref);
>> 
>> void add_property(struct node *node, struct property *prop);
>> void delete_property_by_name(struct node *node, char *name);
>> @@ -202,6 +206,8 @@ void delete_property(struct property *prop);
>> void add_child(struct node *parent, struct node *child);
>> void delete_node_by_name(struct node *parent, char *name);
>> void delete_node(struct node *node);
>> +void append_to_property(struct node *node,
>> +			char *name, const void *data, int len);
>> 
>> const char *get_unitname(struct node *node);
>> struct property *get_property(struct node *node, const char *propname);
>> @@ -237,14 +243,22 @@ struct reserve_info *add_reserve_entry(struct reserve_info *list,
>> 
>> 
>> struct boot_info {
>> +	unsigned int versionflags;
>> 	struct reserve_info *reservelist;
>> 	struct node *dt;		/* the device tree */
>> 	uint32_t boot_cpuid_phys;
>> };
>> 
>> -struct boot_info *build_boot_info(struct reserve_info *reservelist,
>> +/* version flags definitions */
>> +#define VF_DT_V1	0x0001	/* /dts-v1/ */
>> +#define VF_PLUGIN	0x0002	/* /plugin/ */
>> +
>> +struct boot_info *build_boot_info(unsigned int versionflags,
>> +				  struct reserve_info *reservelist,
>> 				  struct node *tree, uint32_t boot_cpuid_phys);
>> void sort_tree(struct boot_info *bi);
>> +void generate_label_tree(struct node *dt, char *gen_node_name, bool allocph);
>> +void generate_fixups_tree(struct node *dt);
>> 
>> /* Checks */
>> 
>> @@ -253,8 +267,8 @@ void process_checks(bool force, struct boot_info *bi);
>> 
>> /* Flattened trees */
>> 
>> -void dt_to_blob(FILE *f, struct boot_info *bi, int version);
>> -void dt_to_asm(FILE *f, struct boot_info *bi, int version);
>> +void dt_to_blob(FILE *f, struct boot_info *bi, fdt32_t magic, int version);
>> +void dt_to_asm(FILE *f, struct boot_info *bi, fdt32_t magic, int version);
>> 
>> struct boot_info *dt_from_blob(const char *fname);
>> 
>> diff --git a/fdtdump.c b/fdtdump.c
>> index a9a2484..dd63ac2 100644
>> --- a/fdtdump.c
>> +++ b/fdtdump.c
>> @@ -201,7 +201,7 @@ int main(int argc, char *argv[])
>> 			p = memchr(p, smagic[0], endp - p - FDT_MAGIC_SIZE);
>> 			if (!p)
>> 				break;
>> -			if (fdt_magic(p) == FDT_MAGIC) {
>> +			if (fdt_magic(p) == FDT_MAGIC || fdt_magic(p) == FDT_MAGIC_DTBO) {
>> 				/* try and validate the main struct */
>> 				off_t this_len = endp - p;
>> 				fdt32_t max_version = 17;
>> diff --git a/flattree.c b/flattree.c
>> index a9d9520..57d76cf 100644
>> --- a/flattree.c
>> +++ b/flattree.c
>> @@ -335,6 +335,7 @@ static struct data flatten_reserve_list(struct reserve_info *reservelist,
>> }
>> 
>> static void make_fdt_header(struct fdt_header *fdt,
>> +			    fdt32_t magic,
>> 			    struct version_info *vi,
>> 			    int reservesize, int dtsize, int strsize,
>> 			    int boot_cpuid_phys)
>> @@ -345,7 +346,7 @@ static void make_fdt_header(struct fdt_header *fdt,
>> 
>> 	memset(fdt, 0xff, sizeof(*fdt));
>> 
>> -	fdt->magic = cpu_to_fdt32(FDT_MAGIC);
>> +	fdt->magic = cpu_to_fdt32(magic);
>> 	fdt->version = cpu_to_fdt32(vi->version);
>> 	fdt->last_comp_version = cpu_to_fdt32(vi->last_comp_version);
>> 
>> @@ -366,7 +367,7 @@ static void make_fdt_header(struct fdt_header *fdt,
>> 		fdt->size_dt_struct = cpu_to_fdt32(dtsize);
>> }
>> 
>> -void dt_to_blob(FILE *f, struct boot_info *bi, int version)
>> +void dt_to_blob(FILE *f, struct boot_info *bi, fdt32_t magic, int version)
>> {
>> 	struct version_info *vi = NULL;
>> 	int i;
>> @@ -390,7 +391,7 @@ void dt_to_blob(FILE *f, struct boot_info *bi, int version)
>> 	reservebuf = flatten_reserve_list(bi->reservelist, vi);
>> 
>> 	/* Make header */
>> -	make_fdt_header(&fdt, vi, reservebuf.len, dtbuf.len, strbuf.len,
>> +	make_fdt_header(&fdt, magic, vi, reservebuf.len, dtbuf.len, strbuf.len,
>> 			bi->boot_cpuid_phys);
>> 
>> 	/*
>> @@ -467,7 +468,7 @@ static void dump_stringtable_asm(FILE *f, struct data strbuf)
>> 	}
>> }
>> 
>> -void dt_to_asm(FILE *f, struct boot_info *bi, int version)
>> +void dt_to_asm(FILE *f, struct boot_info *bi, fdt32_t magic, int version)
>> {
>> 	struct version_info *vi = NULL;
>> 	int i;
>> @@ -830,6 +831,7 @@ struct boot_info *dt_from_blob(const char *fname)
>> 	struct node *tree;
>> 	uint32_t val;
>> 	int flags = 0;
>> +	unsigned int versionflags = VF_DT_V1;
>> 
>> 	f = srcfile_relative_open(fname, NULL);
>> 
>> @@ -845,9 +847,12 @@ struct boot_info *dt_from_blob(const char *fname)
>> 	}
>> 
>> 	magic = fdt32_to_cpu(magic);
>> -	if (magic != FDT_MAGIC)
>> +	if (magic != FDT_MAGIC && magic != FDT_MAGIC_DTBO)
>> 		die("Blob has incorrect magic number\n");
>> 
>> +	if (magic == FDT_MAGIC_DTBO)
>> +		versionflags |= VF_PLUGIN;
> 
> Not particularly useful yet, but I wonder if we'll want some option to
> force treating dtb input as a plugin, for the case of old plugins
> which don't have the new magic number.
> 

It can easily be added.

>> +
>> 	rc = fread(&totalsize, sizeof(totalsize), 1, f);
>> 	if (ferror(f))
>> 		die("Error reading DT blob size: %s\n", strerror(errno));
>> @@ -942,5 +947,5 @@ struct boot_info *dt_from_blob(const char *fname)
>> 
>> 	fclose(f);
>> 
>> -	return build_boot_info(reservelist, tree, boot_cpuid_phys);
>> +	return build_boot_info(versionflags, reservelist, tree, boot_cpuid_phys);
>> }
>> diff --git a/fstree.c b/fstree.c
>> index 6d1beec..54f520b 100644
>> --- a/fstree.c
>> +++ b/fstree.c
>> @@ -86,6 +86,6 @@ struct boot_info *dt_from_fs(const char *dirname)
>> 	tree = read_fstree(dirname);
>> 	tree = name_node(tree, "");
>> 
>> -	return build_boot_info(NULL, tree, guess_boot_cpuid(tree));
>> +	return build_boot_info(VF_DT_V1, NULL, tree, guess_boot_cpuid(tree));
>> }
>> 
>> diff --git a/libfdt/fdt.c b/libfdt/fdt.c
>> index 22286a1..28d422c 100644
>> --- a/libfdt/fdt.c
>> +++ b/libfdt/fdt.c
>> @@ -57,7 +57,7 @@
>> 
>> int fdt_check_header(const void *fdt)
>> {
>> -	if (fdt_magic(fdt) == FDT_MAGIC) {
>> +	if (fdt_magic(fdt) == FDT_MAGIC || fdt_magic(fdt) == FDT_MAGIC_DTBO) {
>> 		/* Complete tree */
>> 		if (fdt_version(fdt) < FDT_FIRST_SUPPORTED_VERSION)
>> 			return -FDT_ERR_BADVERSION;
>> diff --git a/libfdt/fdt.h b/libfdt/fdt.h
>> index 526aedb..493cd55 100644
>> --- a/libfdt/fdt.h
>> +++ b/libfdt/fdt.h
>> @@ -55,7 +55,7 @@
>> #ifndef __ASSEMBLY__
>> 
>> struct fdt_header {
>> -	fdt32_t magic;			 /* magic word FDT_MAGIC */
>> +	fdt32_t magic;			 /* magic word FDT_MAGIC[|_DTBO] */
>> 	fdt32_t totalsize;		 /* total size of DT block */
>> 	fdt32_t off_dt_struct;		 /* offset to structure */
>> 	fdt32_t off_dt_strings;		 /* offset to strings */
>> @@ -93,6 +93,7 @@ struct fdt_property {
>> #endif /* !__ASSEMBLY */
>> 
>> #define FDT_MAGIC	0xd00dfeed	/* 4: version, 4: total size */
>> +#define FDT_MAGIC_DTBO	0xd00dfdb0	/* DTBO magic */
>> #define FDT_TAGSIZE	sizeof(fdt32_t)
>> 
>> #define FDT_BEGIN_NODE	0x1		/* Start node: full name */
>> diff --git a/livetree.c b/livetree.c
>> index 3dc7559..f2c86bd 100644
>> --- a/livetree.c
>> +++ b/livetree.c
>> @@ -216,6 +216,31 @@ struct node *merge_nodes(struct node *old_node, struct node *new_node)
>> 	return old_node;
>> }
>> 
>> +void add_orphan_node(struct node *dt, struct node *new_node, char *ref)
>> +{
>> +	static unsigned int next_orphan_fragment = 0;
>> +	struct node *node = build_node(NULL, NULL);
>> +	struct property *p;
>> +	struct data d = empty_data;
>> +	char *name;
>> +
>> +	memset(node, 0, sizeof(*node));
> 
> You don't need the memset() now that you're using build_node() above.
> 

OK

>> +
>> +	d = data_add_marker(d, REF_PHANDLE, ref);
>> +	d = data_append_integer(d, 0xffffffff, 32);
>> +
>> +	p = build_property("target", d);
>> +	add_property(node, p);
>> +
>> +	xasprintf(&name, "fragment@%u",
>> +			next_orphan_fragment++);
>> +	name_node(node, name);
>> +	name_node(new_node, "__overlay__");
> 
> You can do this more naturally if you do the name_node() here, then
> you can just pass the __overlay__ node into build_node() for the
> fragment@ node instead of having to explicitly add_child.
> 

Hmm, I’ll see if I can rework it like this.

>> +
>> +	add_child(dt, node);
>> +	add_child(node, new_node);
>> +}
>> +
>> struct node *chain_node(struct node *first, struct node *list)
>> {
>> 	assert(first->next_sibling == NULL);
>> @@ -296,6 +321,23 @@ void delete_node(struct node *node)
>> 	delete_labels(&node->labels);
>> }
>> 
>> +void append_to_property(struct node *node,
>> +				    char *name, const void *data, int len)
>> +{
>> +	struct data d;
>> +	struct property *p;
>> +
>> +	p = get_property(node, name);
>> +	if (p) {
>> +		d = data_append_data(p->val, data, len);
>> +		p->val = d;
>> +	} else {
>> +		d = data_append_data(empty_data, data, len);
>> +		p = build_property(name, d);
>> +		add_property(node, p);
>> +	}
>> +}
>> +
>> struct reserve_info *build_reserve_entry(uint64_t address, uint64_t size)
>> {
>> 	struct reserve_info *new = xmalloc(sizeof(*new));
>> @@ -335,12 +377,14 @@ struct reserve_info *add_reserve_entry(struct reserve_info *list,
>> 	return list;
>> }
>> 
>> -struct boot_info *build_boot_info(struct reserve_info *reservelist,
>> +struct boot_info *build_boot_info(unsigned int versionflags,
>> +				  struct reserve_info *reservelist,
>> 				  struct node *tree, uint32_t boot_cpuid_phys)
>> {
>> 	struct boot_info *bi;
>> 
>> 	bi = xmalloc(sizeof(*bi));
>> +	bi->versionflags = versionflags;
>> 	bi->reservelist = reservelist;
>> 	bi->dt = tree;
>> 	bi->boot_cpuid_phys = boot_cpuid_phys;
>> @@ -709,3 +753,182 @@ void sort_tree(struct boot_info *bi)
>> 	sort_reserve_entries(bi);
>> 	sort_node(bi->dt);
>> }
>> +
>> +/* utility helper to avoid code duplication */
>> +static struct node *build_and_name_child_node(struct node *parent, char *name)
>> +{
>> +	struct node *node;
>> +
>> +	node = build_node(NULL, NULL);
>> +	name_node(node, xstrdup(name));
>> +	add_child(parent, node);
>> +
>> +	return node;
>> +}
>> +
>> +static void generate_label_tree_internal(struct node *dt, struct node *node,
>> +					 struct node *an, bool allocph)
>> +{
>> +	struct node *c;
>> +	struct property *p;
>> +	struct label *l;
>> +
>> +	/* if if there are labels */
>> +	if (node->labels) {
>> +		/* now add the label in the node */
>> +		for_each_label(node->labels, l) {
>> +			/* check whether the label already exists */
>> +			p = get_property(an, l->label);
>> +			if (p) {
>> +				fprintf(stderr, "WARNING: label %s already"
>> +					" exists in /%s", l->label,
>> +					an->name);
>> +				continue;
>> +			}
>> +
>> +			/* insert it */
>> +			p = build_property(l->label,
>> +				data_copy_mem(node->fullpath,
>> +						strlen(node->fullpath) + 1));
>> +			add_property(an, p);
>> +		}
>> +
>> +		/* force allocation of a phandle for this node */
>> +		if (allocph)
>> +			(void)get_node_phandle(dt, node);
>> +	}
>> +
>> +	for_each_child(node, c)
>> +		generate_label_tree_internal(dt, c, an, allocph);
>> +}
>> +
>> +void generate_label_tree(struct node *dt, char *gen_node_name, bool allocph)
>> +{
>> +	struct node *an;
>> +
>> +	for_each_child(dt, an)
>> +		if (streq(gen_node_name, an->name))
>> +			break;
>> +
>> +	if (!an)
>> +		an = build_and_name_child_node(dt, gen_node_name);
>> +	if (!an)
>> +		die("Could not build label node /%s\n", gen_node_name);
>> +
>> +	generate_label_tree_internal(dt, dt, an, allocph);
>> +}
>> +
>> +#define FIXUPS	"__fixups__"
>> +#define LOCAL_FIXUPS "__local_fixups__"
>> +
>> +static void add_fixup_entry(struct node *dt, struct node *node,
>> +		struct property *prop, struct marker *m)
>> +{
>> +	struct node *fn;	/* fixup node */
>> +	char *entry;
>> +
>> +	/* m->ref can only be a REF_PHANDLE, but check anyway */
>> +	assert(m->type == REF_PHANDLE);
>> +
>> +	/* fn is the node we're putting entries in */
>> +	fn = get_subnode(dt, FIXUPS);
>> +	assert(fn != NULL);
>> +
>> +	/* there shouldn't be any ':' in the arguments */
>> +	if (strchr(node->fullpath, ':') || strchr(prop->name, ':'))
>> +		die("arguments should not contain ':'\n");
>> +
>> +	xasprintf(&entry, "%s:%s:%u",
>> +			node->fullpath, prop->name, m->offset);
>> +	append_to_property(fn, m->ref, entry, strlen(entry) + 1);
>> +}
>> +
>> +static void add_local_fixup_entry(struct node *dt, struct node *node,
>> +		struct property *prop, struct marker *m,
>> +		struct node *refnode)
>> +{
>> +	struct node *lfn, *wn, *nwn;	/* local fixup node, walk node, new */
>> +	uint32_t value_32;
>> +	char *s, *e, *comp;
>> +	int len;
>> +
>> +	/* fn is the node we're putting entries in */
>> +	lfn = get_subnode(dt, LOCAL_FIXUPS);
>> +	assert(lfn != NULL);
>> +
>> +	/* walk the path components creating nodes if they don't exist */
>> +	comp = xmalloc(strlen(node->fullpath) + 1);
>> +	/* start skipping the first / */
>> +	s = node->fullpath + 1;
>> +	wn = lfn;
>> +	while (*s) {
>> +		/* retrieve path component */
>> +		e = strchr(s, '/');
>> +		if (e == NULL)
>> +			e = s + strlen(s);
>> +		len = e - s;
>> +		memcpy(comp, s, len);
>> +		comp[len] = '\0';
> 
> Parsing the fullpath into components seems an odd way of doing this.
> We have an actual handle on the node, and therefore all it's parents,
> which already have the individual path components split out.
> 

Hmm, I’ll see if it can be done. I don’t remember what was the original
cause for using this form.

>> +
>> +		/* if no node exists, create it */
>> +		nwn = get_subnode(wn, comp);
>> +		if (!nwn)
>> +			nwn = build_and_name_child_node(wn, comp);
>> +		wn = nwn;
>> +
>> +		/* last path component */
>> +		if (!*e)
>> +			break;
>> +
>> +		/* next path component */
>> +		s = e + 1;
>> +	}
>> +	free(comp);
>> +
>> +	value_32 = cpu_to_fdt32(m->offset);
>> +	append_to_property(wn, prop->name, &value_32, sizeof(value_32));
>> +}
>> +
>> +static void generate_fixups_tree_internal(struct node *dt, struct node *node)
>> +{
>> +	struct node *c;
>> +	struct property *prop;
>> +	struct marker *m;
>> +	struct node *refnode;
>> +
>> +	for_each_property(node, prop) {
>> +		m = prop->val.markers;
>> +		for_each_marker_of_type(m, REF_PHANDLE) {
>> +			refnode = get_node_by_ref(dt, m->ref);
>> +			if (!refnode)
>> +				add_fixup_entry(dt, node, prop, m);
>> +			else
>> +				add_local_fixup_entry(dt, node, prop, m,
>> +						refnode);
>> +		}
>> +	}
>> +
>> +	for_each_child(node, c)
>> +		generate_fixups_tree_internal(dt, c);
>> +}
>> +
>> +void generate_fixups_tree(struct node *dt)
>> +{
>> +	struct node *an;
>> +
>> +	for_each_child(dt, an)
>> +		if (streq(FIXUPS, an->name))
>> +			break;
>> +
>> +	if (!an)
>> +		build_and_name_child_node(dt, FIXUPS);
>> +
>> +	for_each_child(dt, an)
>> +		if (streq(LOCAL_FIXUPS, an->name))
>> +			break;
>> +
>> +	if (!an)
>> +		build_and_name_child_node(dt, LOCAL_FIXUPS);
>> +
>> +	generate_fixups_tree_internal(dt, dt);
>> +}
>> diff --git a/tests/mangle-layout.c b/tests/mangle-layout.c
>> index a76e51e..d29ebc6 100644
>> --- a/tests/mangle-layout.c
>> +++ b/tests/mangle-layout.c
>> @@ -42,7 +42,8 @@ static void expand_buf(struct bufstate *buf, int newsize)
>> 	buf->size = newsize;
>> }
>> 
>> -static void new_header(struct bufstate *buf, int version, const void *fdt)
>> +static void new_header(struct bufstate *buf, fdt32_t magic, int version,
>> +		       const void *fdt)
>> {
>> 	int hdrsize;
>> 
>> @@ -56,7 +57,7 @@ static void new_header(struct bufstate *buf, int version, const void *fdt)
>> 	expand_buf(buf, hdrsize);
>> 	memset(buf->buf, 0, hdrsize);
>> 
>> -	fdt_set_magic(buf->buf, FDT_MAGIC);
>> +	fdt_set_magic(buf->buf, magic);
>> 	fdt_set_version(buf->buf, version);
>> 	fdt_set_last_comp_version(buf->buf, 16);
>> 	fdt_set_boot_cpuid_phys(buf->buf, fdt_boot_cpuid_phys(fdt));
>> @@ -145,7 +146,7 @@ int main(int argc, char *argv[])
>> 	if (fdt_version(fdt) < 17)
>> 		CONFIG("Input tree must be v17");
>> 
>> -	new_header(&buf, version, fdt);
>> +	new_header(&buf, FDT_MAGIC, version, fdt);
>> 
>> 	while (*blockorder) {
>> 		add_block(&buf, version, *blockorder, fdt);
> 
> -- 
> David Gibson			| I'll have my music baroque, and my code
> david AT gibson.dropbear.id.au	| minimalist, thank you.  NOT _the_ _other_
> 				| _way_ _around_!
> http://www.ozlabs.org/~dgibson

Regards

— Pantelis

Re: [PATCH v10 3/4] dtc: Plugin and fixup support

[Phil Elwell]

On 28/11/2016 12:10, Pantelis Antoniou wrote:
> For plugins we need the __symbols__ node to support stacked overlays, i.e.
> overlays referring label that were introduced by a previous overlay.
Although it is arguably useful to be able to refer to symbols created by
one overlay from within another, do we really want all symbols to be
global? Isn't there a call for a new syntax or usage pattern to indicate
either that a symbol should be local to the overlay or, my preferred
option, global?

Phil

Re: [PATCH v10 3/4] dtc: Plugin and fixup support

[David Gibson]

[-- Attachment #1: Type: text/plain, Size: 1065 bytes --]

On Mon, Nov 28, 2016 at 12:24:20PM +0000, Phil Elwell wrote:
> On 28/11/2016 12:10, Pantelis Antoniou wrote:
> > For plugins we need the __symbols__ node to support stacked overlays, i.e.
> > overlays referring label that were introduced by a previous overlay.
> Although it is arguably useful to be able to refer to symbols created by
> one overlay from within another, do we really want all symbols to be
> global? Isn't there a call for a new syntax or usage pattern to indicate
> either that a symbol should be local to the overlay or, my preferred
> option, global?

So, this is back to a design question about the overlay format.  As
noted in the initial discussions about possible "connector" formats, I
think we will want some sort of local symbols.  But the current
overlay format with all global symbols is out there and we need to
support it.

-- 
David Gibson			| I'll have my music baroque, and my code
david AT gibson.dropbear.id.au	| minimalist, thank you.  NOT _the_ _other_
				| _way_ _around_!
http://www.ozlabs.org/~dgibson

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 819 bytes --]

Re: [PATCH v10 3/4] dtc: Plugin and fixup support

[Phil Elwell]

On 29/11/2016 02:11, David Gibson wrote:
> On Mon, Nov 28, 2016 at 12:24:20PM +0000, Phil Elwell wrote:
>> On 28/11/2016 12:10, Pantelis Antoniou wrote:
>>> For plugins we need the __symbols__ node to support stacked overlays, i.e.
>>> overlays referring label that were introduced by a previous overlay.
>> Although it is arguably useful to be able to refer to symbols created by
>> one overlay from within another, do we really want all symbols to be
>> global? Isn't there a call for a new syntax or usage pattern to indicate
>> either that a symbol should be local to the overlay or, my preferred
>> option, global?
> So, this is back to a design question about the overlay format.  As
> noted in the initial discussions about possible "connector" formats, I
> think we will want some sort of local symbols.  But the current
> overlay format with all global symbols is out there and we need to
> support it.
The overlay format we have does not dictate the scope of the symbols.

In all implementations I know of - the Raspberry Pi loader, the current
Linux kernel, the latest dtc patch set - there is a completely
asymmetric relationship between the base DTB and an overlay:
* the base DTB exports __symbols__ to resolve the overlays unresolved
label references, as recorded by the __fixups__ node
* the overlay's phandles are renumbered so as not to clash with the base
tree using the __local_fixups__
* the contents of the __overlay__ nodes are applied to the base tree, as
directed by the "target" or "target-path" properties

The __symbols__ node of the overlay is ignored and discarded. The
__fixups__ and __local_fixups__ in the base DTB (if present - the RPi
dtc only generates them for /plugins/) are ignored.

In the set of RPi overlays only one exports a global symbol, which it
achieves with an overlay aimed at target-path = "/__symbols__" that adds
a new symbol (in this case "i2c_gpio").

If the __symbols__ in an overlay are automatically merged with the base
symbols, that is a significant change in semantics which needs to be
discussed.

Phil

Re: [PATCH v10 3/4] dtc: Plugin and fixup support

[Pantelis Antoniou]

Hi Phil,

> On Nov 29, 2016, at 12:32 , Phil Elwell <phil-FnsA7b+Nu9XbIbC87yuRow@public.gmane.org> wrote:
> 
> On 29/11/2016 02:11, David Gibson wrote:
>> On Mon, Nov 28, 2016 at 12:24:20PM +0000, Phil Elwell wrote:
>>> On 28/11/2016 12:10, Pantelis Antoniou wrote:
>>>> For plugins we need the __symbols__ node to support stacked overlays, i.e.
>>>> overlays referring label that were introduced by a previous overlay.
>>> Although it is arguably useful to be able to refer to symbols created by
>>> one overlay from within another, do we really want all symbols to be
>>> global? Isn't there a call for a new syntax or usage pattern to indicate
>>> either that a symbol should be local to the overlay or, my preferred
>>> option, global?
>> So, this is back to a design question about the overlay format.  As
>> noted in the initial discussions about possible "connector" formats, I
>> think we will want some sort of local symbols.  But the current
>> overlay format with all global symbols is out there and we need to
>> support it.
> The overlay format we have does not dictate the scope of the symbols.
> 
> In all implementations I know of - the Raspberry Pi loader, the current
> Linux kernel, the latest dtc patch set - there is a completely
> asymmetric relationship between the base DTB and an overlay:
> * the base DTB exports __symbols__ to resolve the overlays unresolved
> label references, as recorded by the __fixups__ node
> * the overlay's phandles are renumbered so as not to clash with the base
> tree using the __local_fixups__
> * the contents of the __overlay__ nodes are applied to the base tree, as
> directed by the "target" or "target-path” properties
> 

Yes

> The __symbols__ node of the overlay is ignored and discarded. The
> __fixups__ and __local_fixups__ in the base DTB (if present - the RPi
> dtc only generates them for /plugins/) are ignored.
> 

That was a limitation that no-longer applies. Overlay symbols can be added
to the base tree symbol list with a small patch I have already posted.

> In the set of RPi overlays only one exports a global symbol, which it
> achieves with an overlay aimed at target-path = "/__symbols__" that adds
> a new symbol (in this case "i2c_gpio").
> 
> If the __symbols__ in an overlay are automatically merged with the base
> symbols, that is a significant change in semantics which needs to be
> discussed.
> 

You no longer need to do this anymore. Hope this helps :)

> Phil

Regards

— Pantelis

--
To unsubscribe from this list: send the line "unsubscribe devicetree" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v10 3/4] dtc: Plugin and fixup support

[Phil Elwell]

On 29/11/2016 10:39, Pantelis Antoniou wrote:
> Hi Phil,
>
>> On Nov 29, 2016, at 12:32 , Phil Elwell <phil-FnsA7b+Nu9XbIbC87yuRow@public.gmane.org> wrote:
>>
>> On 29/11/2016 02:11, David Gibson wrote:
>>> On Mon, Nov 28, 2016 at 12:24:20PM +0000, Phil Elwell wrote:
>>>> On 28/11/2016 12:10, Pantelis Antoniou wrote:
>>>>> For plugins we need the __symbols__ node to support stacked overlays, i.e.
>>>>> overlays referring label that were introduced by a previous overlay.
>>>> Although it is arguably useful to be able to refer to symbols created by
>>>> one overlay from within another, do we really want all symbols to be
>>>> global? Isn't there a call for a new syntax or usage pattern to indicate
>>>> either that a symbol should be local to the overlay or, my preferred
>>>> option, global?
>>> So, this is back to a design question about the overlay format.  As
>>> noted in the initial discussions about possible "connector" formats, I
>>> think we will want some sort of local symbols.  But the current
>>> overlay format with all global symbols is out there and we need to
>>> support it.
>> The overlay format we have does not dictate the scope of the symbols.
>>
>> In all implementations I know of - the Raspberry Pi loader, the current
>> Linux kernel, the latest dtc patch set - there is a completely
>> asymmetric relationship between the base DTB and an overlay:
>> * the base DTB exports __symbols__ to resolve the overlays unresolved
>> label references, as recorded by the __fixups__ node
>> * the overlay's phandles are renumbered so as not to clash with the base
>> tree using the __local_fixups__
>> * the contents of the __overlay__ nodes are applied to the base tree, as
>> directed by the "target" or "target-path” properties
>>
> Yes
>
>> The __symbols__ node of the overlay is ignored and discarded. The
>> __fixups__ and __local_fixups__ in the base DTB (if present - the RPi
>> dtc only generates them for /plugins/) are ignored.
>>
> That was a limitation that no-longer applies. Overlay symbols can be added
> to the base tree symbol list with a small patch I have already posted.
The fact that they can doesn't mean they necessarily should.
>
>> In the set of RPi overlays only one exports a global symbol, which it
>> achieves with an overlay aimed at target-path = "/__symbols__" that adds
>> a new symbol (in this case "i2c_gpio").
>>
>> If the __symbols__ in an overlay are automatically merged with the base
>> symbols, that is a significant change in semantics which needs to be
>> discussed.
>>
> You no longer need to do this anymore. Hope this helps :)

Not really, no. With the current scheme we have control over the scope,
but now there is none.

How does your patch handle duplicate symbols?

Phil

Re: [PATCH v10 3/4] dtc: Plugin and fixup support

[Pantelis Antoniou]

Hi Phil,

> On Nov 29, 2016, at 12:50 , Phil Elwell <phil-FnsA7b+Nu9XbIbC87yuRow@public.gmane.org> wrote:
> 
> On 29/11/2016 10:39, Pantelis Antoniou wrote:
>> Hi Phil,
>> 
>>> On Nov 29, 2016, at 12:32 , Phil Elwell <phil-FnsA7b+Nu9XbIbC87yuRow@public.gmane.org> wrote:
>>> 
>>> On 29/11/2016 02:11, David Gibson wrote:
>>>> On Mon, Nov 28, 2016 at 12:24:20PM +0000, Phil Elwell wrote:
>>>>> On 28/11/2016 12:10, Pantelis Antoniou wrote:
>>>>>> For plugins we need the __symbols__ node to support stacked overlays, i.e.
>>>>>> overlays referring label that were introduced by a previous overlay.
>>>>> Although it is arguably useful to be able to refer to symbols created by
>>>>> one overlay from within another, do we really want all symbols to be
>>>>> global? Isn't there a call for a new syntax or usage pattern to indicate
>>>>> either that a symbol should be local to the overlay or, my preferred
>>>>> option, global?
>>>> So, this is back to a design question about the overlay format.  As
>>>> noted in the initial discussions about possible "connector" formats, I
>>>> think we will want some sort of local symbols.  But the current
>>>> overlay format with all global symbols is out there and we need to
>>>> support it.
>>> The overlay format we have does not dictate the scope of the symbols.
>>> 
>>> In all implementations I know of - the Raspberry Pi loader, the current
>>> Linux kernel, the latest dtc patch set - there is a completely
>>> asymmetric relationship between the base DTB and an overlay:
>>> * the base DTB exports __symbols__ to resolve the overlays unresolved
>>> label references, as recorded by the __fixups__ node
>>> * the overlay's phandles are renumbered so as not to clash with the base
>>> tree using the __local_fixups__
>>> * the contents of the __overlay__ nodes are applied to the base tree, as
>>> directed by the "target" or "target-path” properties
>>> 
>> Yes
>> 
>>> The __symbols__ node of the overlay is ignored and discarded. The
>>> __fixups__ and __local_fixups__ in the base DTB (if present - the RPi
>>> dtc only generates them for /plugins/) are ignored.
>>> 
>> That was a limitation that no-longer applies. Overlay symbols can be added
>> to the base tree symbol list with a small patch I have already posted.
> The fact that they can doesn't mean they necessarily should.
>> 
>>> In the set of RPi overlays only one exports a global symbol, which it
>>> achieves with an overlay aimed at target-path = "/__symbols__" that adds
>>> a new symbol (in this case "i2c_gpio").
>>> 
>>> If the __symbols__ in an overlay are automatically merged with the base
>>> symbols, that is a significant change in semantics which needs to be
>>> discussed.
>>> 
>> You no longer need to do this anymore. Hope this helps :)
> 
> Not really, no. With the current scheme we have control over the scope,
> but now there is none.
> 

Manually adding symbols by targeting __symbols__ is just bad. There is absolutely
no guarantee that the symbol/fixup node(s) will still be there in following iterations
of the patches.

I am thinking of parsing them, recording the information in kernel structures and then
deleting them altogether.

> How does your patch handle duplicate symbols?
> 

It doesn’t. Having duplicate global symbols is bad. 

It appears you want scoping rules instead. Care to paste a concrete example?

> Phil

Regards

— Pantelis

Re: [PATCH v10 3/4] dtc: Plugin and fixup support

[Phil Elwell]

Pantelis,

On 29/11/2016 10:55, Pantelis Antoniou wrote:
> Manually adding symbols by targeting __symbols__ is just bad. There is absolutely
> no guarantee that the symbol/fixup node(s) will still be there in following iterations
> of the patches.
Remember that this is now part of the Linux kernel - it isn't something
you can just change at will.
> I am thinking of parsing them, recording the information in kernel structures and then
> deleting them altogether.
>
>> How does your patch handle duplicate symbols?
>>
> It doesn’t. Having duplicate global symbols is bad. 
>
> It appears you want scoping rules instead. Care to paste a concrete example?

Concrete non-trivial examples are hard to come by. There are some simple
cases where we've attached labels to __overlay__ nodes so that the
contents can be patched by our overlay parameter mechanism - they could
just be given unique names instead of just "frag0", "frag1" etc. I'm
more concerned about parameterised macro-expanded overlays.

Consider an overlay that defines a CAN controller on an SPI bus. We
currently have two such overlays in the RPi tree, one for SPI 0.0 and
one for SPI 0.1. Here's an extract from one of them:

    /* the interrupt pin of the can-controller */
    fragment@2 {
        target = <&gpio>;
        __overlay__ {
            can0_pins: can0_pins {
                brcm,pins = <25>;
                brcm,function = <0>; /* input */
            };
        };
    };
...
    fragment@4 {
        target = <&spi0>;
        __overlay__ {
            /* needed to avoid dtc warning */
            #address-cells = <1>;
            #size-cells = <0>;
            can0: mcp2515@0 {
                reg = <0>;
                compatible = "microchip,mcp2515";
                pinctrl-names = "default";
                pinctrl-0 = <&can0_pins>;
                spi-max-frequency = <10000000>;
                interrupt-parent = <&gpio>;
                interrupts = <25 0x2>;
                clocks = <&can0_osc>;
            };
        };
    };

One day I'd like to merge these into a single parameterised version that
could target any CS line on any SPI controller. This requires that any
created node names are unique with the scope of the parent ("mcp2515@0",
"can0_pins"), and that the name of the target label (spi0) is patched to
select the correct SPI bus. Our existing, limited overlay parameter
mechanism uses labels to identify properties to patch:

        spimaxfrequency = <&can0>,"spi-max-frequency:0";

(I would have attached labels to the properties themselves, but that
doesn't seem to work, contrary to the ePAPR spec.)

If the labels that locate the node, property and label names to change
also themselves have to be made unique then that adds an extra level of
complexity.

The parameter application is a pre-processing step before the overlay is
merged, so there is nothing preventing me from filtering the symbols
node before passing it on based on rules of my own choosing, but I
wanted to make more people aware of this change.

Phil

Re: [PATCH v10 3/4] dtc: Plugin and fixup support

[Pantelis Antoniou]

Hi Phil,

> On Nov 29, 2016, at 14:11 , Phil Elwell <phil-FnsA7b+Nu9XbIbC87yuRow@public.gmane.org> wrote:
> 
> Pantelis,
> 
> On 29/11/2016 10:55, Pantelis Antoniou wrote:
>> Manually adding symbols by targeting __symbols__ is just bad. There is absolutely
>> no guarantee that the symbol/fixup node(s) will still be there in following iterations
>> of the patches.
> Remember that this is now part of the Linux kernel - it isn't something
> you can just change at will.

Since I’m the one that put it there, I’m sure I have a little bit of leverage.

>> I am thinking of parsing them, recording the information in kernel structures and then
>> deleting them altogether.
>> 
>>> How does your patch handle duplicate symbols?
>>> 
>> It doesn’t. Having duplicate global symbols is bad. 
>> 
>> It appears you want scoping rules instead. Care to paste a concrete example?
> 
> Concrete non-trivial examples are hard to come by. There are some simple
> cases where we've attached labels to __overlay__ nodes so that the
> contents can be patched by our overlay parameter mechanism - they could
> just be given unique names instead of just "frag0", "frag1" etc. I'm
> more concerned about parameterised macro-expanded overlays.
> 
> Consider an overlay that defines a CAN controller on an SPI bus. We
> currently have two such overlays in the RPi tree, one for SPI 0.0 and
> one for SPI 0.1. Here's an extract from one of them:
> 
>    /* the interrupt pin of the can-controller */
>    fragment@2 {
>        target = <&gpio>;
>        __overlay__ {
>            can0_pins: can0_pins {
>                brcm,pins = <25>;
>                brcm,function = <0>; /* input */
>            };
>        };
>    };
> ...
>    fragment@4 {
>        target = <&spi0>;
>        __overlay__ {
>            /* needed to avoid dtc warning */
>            #address-cells = <1>;
>            #size-cells = <0>;
>            can0: mcp2515@0 {
>                reg = <0>;
>                compatible = "microchip,mcp2515";
>                pinctrl-names = "default";
>                pinctrl-0 = <&can0_pins>;
>                spi-max-frequency = <10000000>;
>                interrupt-parent = <&gpio>;
>                interrupts = <25 0x2>;
>                clocks = <&can0_osc>;
>            };
>        };
>    };
> 
> One day I'd like to merge these into a single parameterised version that
> could target any CS line on any SPI controller. This requires that any
> created node names are unique with the scope of the parent ("mcp2515@0",
> "can0_pins"), and that the name of the target label (spi0) is patched to
> select the correct SPI bus. Our existing, limited overlay parameter
> mechanism uses labels to identify properties to patch:
> 
>        spimaxfrequency = <&can0>,"spi-max-frequency:0";
> 
> (I would have attached labels to the properties themselves, but that
> doesn't seem to work, contrary to the ePAPR spec.)
> 
> If the labels that locate the node, property and label names to change
> also themselves have to be made unique then that adds an extra level of
> complexity.
> 
> The parameter application is a pre-processing step before the overlay is
> merged, so there is nothing preventing me from filtering the symbols
> node before passing it on based on rules of my own choosing, but I
> wanted to make more people aware of this change.
> 

First note is that this is exactly what the portable connector is supposed to
do; abstract away the SoC differences.

Second note is that it’s not the overlay application that’s having problems, it’s
your parameter patching method.

The 'spimaxfrequency = <&can0>,"spi-max-frequency:0”’ form could more easily be
done by targeting aliases instead of node labels.

I.e. You can apply the overlay, set an alias to the node and instead of referencing
the label, reference the alias.

Again, this is a stop-gap until the portable connector is done, but what I take out
of this is the need for a parameterization step in which an overlay is modified before
it is applied according to an external parameter.

> Phil
> 

Regards

— Pantelis

Re: [PATCH v10 3/4] dtc: Plugin and fixup support

[Phil Elwell]

On 29/11/2016 12:24, Pantelis Antoniou wrote:
> First note is that this is exactly what the portable connector is supposed to
> do; abstract away the SoC differences.
>
> Second note is that it’s not the overlay application that’s having problems, it’s
> your parameter patching method.
>
> The 'spimaxfrequency = <&can0>,"spi-max-frequency:0”’ form could more easily be
> done by targeting aliases instead of node labels.
>
> I.e. You can apply the overlay, set an alias to the node and instead of referencing
> the label, reference the alias.
But the patching is done before the overlays are applied, in isolation
from the base tree, so that they can still be used with the kernel
configfs overlay mechanism. How do aliases (which associated symbols
with absolute paths) help?

> Again, this is a stop-gap until the portable connector is done, but what I take out
> of this is the need for a parameterization step in which an overlay is modified before
> it is applied according to an external parameter.
Yes, absolutely.

Phil

Re: [PATCH v10 3/4] dtc: Plugin and fixup support

[Pantelis Antoniou]

Hi Phil,

> On Nov 29, 2016, at 14:57 , Phil Elwell <phil-FnsA7b+Nu9XbIbC87yuRow@public.gmane.org> wrote:
> 
> On 29/11/2016 12:24, Pantelis Antoniou wrote:
>> First note is that this is exactly what the portable connector is supposed to
>> do; abstract away the SoC differences.
>> 
>> Second note is that it’s not the overlay application that’s having problems, it’s
>> your parameter patching method.
>> 
>> The 'spimaxfrequency = <&can0>,"spi-max-frequency:0”’ form could more easily be
>> done by targeting aliases instead of node labels.
>> 
>> I.e. You can apply the overlay, set an alias to the node and instead of referencing
>> the label, reference the alias.
> But the patching is done before the overlays are applied, in isolation
> from the base tree, so that they can still be used with the kernel
> configfs overlay mechanism. How do aliases (which associated symbols
> with absolute paths) help?
> 

An alias is the standard way to refer to nodes symbolically. They can be overwritten at
runtime without triggering an error.

>> Again, this is a stop-gap until the portable connector is done, but what I take out
>> of this is the need for a parameterization step in which an overlay is modified before
>> it is applied according to an external parameter.
> Yes, absolutely.
> 

Speaking of which, since these overlays are applied at runtime, why not build them with a script
and have a #define passed to the c preprocessor before compiling them?

It doesn’t appear to be a problem doing it this way. 

> Phil
> —

Regards

— Pantelis

Re: [PATCH v10 3/4] dtc: Plugin and fixup support

[Phil Elwell]

On 29/11/2016 13:00, Pantelis Antoniou wrote:
> An alias is the standard way to refer to nodes symbolically. They can be overwritten at
> runtime without triggering an error.
Can you give me a concrete example of how this would look?
> Speaking of which, since these overlays are applied at runtime, why not build them with a script
> and have a #define passed to the c preprocessor before compiling them?
Because the parameters are applied at run time, not compile time.

Phil

--
To unsubscribe from this list: send the line "unsubscribe devicetree" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v10 3/4] dtc: Plugin and fixup support

[Pantelis Antoniou]

> On Nov 29, 2016, at 15:05 , Phil Elwell <phil-FnsA7b+Nu9XbIbC87yuRow@public.gmane.org> wrote:
> 
> On 29/11/2016 13:00, Pantelis Antoniou wrote:
>> An alias is the standard way to refer to nodes symbolically. They can be overwritten at
>> runtime without triggering an error.
> Can you give me a concrete example of how this would look?

Maybe later in the day, kinda busy right now.

>> Speaking of which, since these overlays are applied at runtime, why not build them with a script
>> and have a #define passed to the c preprocessor before compiling them?
> Because the parameters are applied at run time, not compile time.
> 

There’s no difference; you can cpp -IFOO_VALUE=foo in.dts | dtc | cat >/config/foo/dtb in a nutshell.

And have foo-property = FOO_VALUE; in in.dts.

> Phil
> 

Regards

— Pantelis

Re: [PATCH v10 3/4] dtc: Plugin and fixup support

[Phil Elwell]

On 29/11/2016 13:08, Pantelis Antoniou wrote:
> There’s no difference; you can cpp -IFOO_VALUE=foo in.dts | dtc | cat >/config/foo/dtb in a nutshell.
>
> And have foo-property = FOO_VALUE; in in.dts.
In a boot loader?

--
To unsubscribe from this list: send the line "unsubscribe devicetree" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v10 3/4] dtc: Plugin and fixup support

[Pantelis Antoniou]

Hi Phil,

> On Nov 29, 2016, at 15:11 , Phil Elwell <phil-FnsA7b+Nu9XbIbC87yuRow@public.gmane.org> wrote:
> 
> On 29/11/2016 13:08, Pantelis Antoniou wrote:
>> There’s no difference; you can cpp -IFOO_VALUE=foo in.dts | dtc | cat >/config/foo/dtb in a nutshell.
>> 
>> And have foo-property = FOO_VALUE; in in.dts.
> In a boot loader?
> 

Ah, that why you don’t do that in the bootloader :)

Regards

— Pantelis

Re: [PATCH v10 3/4] dtc: Plugin and fixup support

[David Gibson]

[-- Attachment #1: Type: text/plain, Size: 3560 bytes --]

On Tue, Nov 29, 2016 at 12:11:15PM +0000, Phil Elwell wrote:
> Pantelis,
> 
> On 29/11/2016 10:55, Pantelis Antoniou wrote:
> > Manually adding symbols by targeting __symbols__ is just bad. There is absolutely
> > no guarantee that the symbol/fixup node(s) will still be there in following iterations
> > of the patches.
> Remember that this is now part of the Linux kernel - it isn't something
> you can just change at will.
> > I am thinking of parsing them, recording the information in kernel structures and then
> > deleting them altogether.
> >
> >> How does your patch handle duplicate symbols?
> >>
> > It doesn’t. Having duplicate global symbols is bad. 
> >
> > It appears you want scoping rules instead. Care to paste a concrete example?
> 
> Concrete non-trivial examples are hard to come by. There are some simple
> cases where we've attached labels to __overlay__ nodes so that the
> contents can be patched by our overlay parameter mechanism - they could
> just be given unique names instead of just "frag0", "frag1" etc. I'm
> more concerned about parameterised macro-expanded overlays.
> 
> Consider an overlay that defines a CAN controller on an SPI bus. We
> currently have two such overlays in the RPi tree, one for SPI 0.0 and
> one for SPI 0.1. Here's an extract from one of them:
> 
>     /* the interrupt pin of the can-controller */
>     fragment@2 {
>         target = <&gpio>;
>         __overlay__ {
>             can0_pins: can0_pins {
>                 brcm,pins = <25>;
>                 brcm,function = <0>; /* input */
>             };
>         };
>     };
> ...
>     fragment@4 {
>         target = <&spi0>;
>         __overlay__ {
>             /* needed to avoid dtc warning */
>             #address-cells = <1>;
>             #size-cells = <0>;
>             can0: mcp2515@0 {
>                 reg = <0>;
>                 compatible = "microchip,mcp2515";
>                 pinctrl-names = "default";
>                 pinctrl-0 = <&can0_pins>;
>                 spi-max-frequency = <10000000>;
>                 interrupt-parent = <&gpio>;
>                 interrupts = <25 0x2>;
>                 clocks = <&can0_osc>;
>             };
>         };
>     };
> 
> One day I'd like to merge these into a single parameterised version that
> could target any CS line on any SPI controller. This requires that any
> created node names are unique with the scope of the parent ("mcp2515@0",
> "can0_pins"), and that the name of the target label (spi0) is patched to
> select the correct SPI bus. Our existing, limited overlay parameter
> mechanism uses labels to identify properties to patch:

Yeah, I think this is basically out of scope for the simple overlay
format.  It's really not designed to do this - it works directly with
the global namespace everywhere.

The connector proposals are designed to address this sort of case.

This sort of limitation is, incidentally, why I objected to the dtb
overlay stuff when it was originally proposed (I would have preferred
something closer to the connector proposals which are now floating
again).  That in turn is why it's taken so long to get on its way into
mainline dtc.  But despite the limitations, people wanted it badly
enough that it's become widely used, so now we have to cope with it.

-- 
David Gibson			| I'll have my music baroque, and my code
david AT gibson.dropbear.id.au	| minimalist, thank you.  NOT _the_ _other_
				| _way_ _around_!
http://www.ozlabs.org/~dgibson

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 819 bytes --]

Re: [PATCH v10 3/4] dtc: Plugin and fixup support

[David Gibson]

[-- Attachment #1: Type: text/plain, Size: 3404 bytes --]

On Tue, Nov 29, 2016 at 10:50:53AM +0000, Phil Elwell wrote:
> On 29/11/2016 10:39, Pantelis Antoniou wrote:
> > Hi Phil,
> >
> >> On Nov 29, 2016, at 12:32 , Phil Elwell <phil-FnsA7b+Nu9XbIbC87yuRow@public.gmane.org> wrote:
> >>
> >> On 29/11/2016 02:11, David Gibson wrote:
> >>> On Mon, Nov 28, 2016 at 12:24:20PM +0000, Phil Elwell wrote:
> >>>> On 28/11/2016 12:10, Pantelis Antoniou wrote:
> >>>>> For plugins we need the __symbols__ node to support stacked overlays, i.e.
> >>>>> overlays referring label that were introduced by a previous overlay.
> >>>> Although it is arguably useful to be able to refer to symbols created by
> >>>> one overlay from within another, do we really want all symbols to be
> >>>> global? Isn't there a call for a new syntax or usage pattern to indicate
> >>>> either that a symbol should be local to the overlay or, my preferred
> >>>> option, global?
> >>> So, this is back to a design question about the overlay format.  As
> >>> noted in the initial discussions about possible "connector" formats, I
> >>> think we will want some sort of local symbols.  But the current
> >>> overlay format with all global symbols is out there and we need to
> >>> support it.
> >> The overlay format we have does not dictate the scope of the symbols.
> >>
> >> In all implementations I know of - the Raspberry Pi loader, the current
> >> Linux kernel, the latest dtc patch set - there is a completely
> >> asymmetric relationship between the base DTB and an overlay:
> >> * the base DTB exports __symbols__ to resolve the overlays unresolved
> >> label references, as recorded by the __fixups__ node
> >> * the overlay's phandles are renumbered so as not to clash with the base
> >> tree using the __local_fixups__
> >> * the contents of the __overlay__ nodes are applied to the base tree, as
> >> directed by the "target" or "target-path” properties
> >>
> > Yes
> >
> >> The __symbols__ node of the overlay is ignored and discarded. The
> >> __fixups__ and __local_fixups__ in the base DTB (if present - the RPi
> >> dtc only generates them for /plugins/) are ignored.
> >>
> > That was a limitation that no-longer applies. Overlay symbols can be added
> > to the base tree symbol list with a small patch I have already posted.
> The fact that they can doesn't mean they necessarily should.

They should - at least as long as we're using this simple overlay
format.  What else would you do with them?  If they're not merged,
they're useless in the overlay.

> >
> >> In the set of RPi overlays only one exports a global symbol, which it
> >> achieves with an overlay aimed at target-path = "/__symbols__" that adds
> >> a new symbol (in this case "i2c_gpio").
> >>
> >> If the __symbols__ in an overlay are automatically merged with the base
> >> symbols, that is a significant change in semantics which needs to be
> >> discussed.
> >>
> > You no longer need to do this anymore. Hope this helps :)
> 
> Not really, no. With the current scheme we have control over the scope,
> but now there is none.

You really don't, symbols were always global.

> 
> How does your patch handle duplicate symbols?
> 
> Phil

-- 
David Gibson			| I'll have my music baroque, and my code
david AT gibson.dropbear.id.au	| minimalist, thank you.  NOT _the_ _other_
				| _way_ _around_!
http://www.ozlabs.org/~dgibson

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 819 bytes --]

Re: [PATCH v10 3/4] dtc: Plugin and fixup support

[David Gibson]

[-- Attachment #1: Type: text/plain, Size: 2625 bytes --]

On Tue, Nov 29, 2016 at 10:32:28AM +0000, Phil Elwell wrote:
> On 29/11/2016 02:11, David Gibson wrote:
> > On Mon, Nov 28, 2016 at 12:24:20PM +0000, Phil Elwell wrote:
> >> On 28/11/2016 12:10, Pantelis Antoniou wrote:
> >>> For plugins we need the __symbols__ node to support stacked overlays, i.e.
> >>> overlays referring label that were introduced by a previous overlay.
> >> Although it is arguably useful to be able to refer to symbols created by
> >> one overlay from within another, do we really want all symbols to be
> >> global? Isn't there a call for a new syntax or usage pattern to indicate
> >> either that a symbol should be local to the overlay or, my preferred
> >> option, global?
> > So, this is back to a design question about the overlay format.  As
> > noted in the initial discussions about possible "connector" formats, I
> > think we will want some sort of local symbols.  But the current
> > overlay format with all global symbols is out there and we need to
> > support it.
> The overlay format we have does not dictate the scope of the symbols.
> 
> In all implementations I know of - the Raspberry Pi loader, the current
> Linux kernel, the latest dtc patch set - there is a completely
> asymmetric relationship between the base DTB and an overlay:
> * the base DTB exports __symbols__ to resolve the overlays unresolved
> label references, as recorded by the __fixups__ node
> * the overlay's phandles are renumbered so as not to clash with the base
> tree using the __local_fixups__
> * the contents of the __overlay__ nodes are applied to the base tree, as
> directed by the "target" or "target-path" properties
> 
> The __symbols__ node of the overlay is ignored and discarded. The
> __fixups__ and __local_fixups__ in the base DTB (if present - the RPi
> dtc only generates them for /plugins/) are ignored.
> 
> In the set of RPi overlays only one exports a global symbol, which it
> achieves with an overlay aimed at target-path = "/__symbols__" that adds
> a new symbol (in this case "i2c_gpio").
> 
> If the __symbols__ in an overlay are automatically merged with the base
> symbols, that is a significant change in semantics which needs to be
> discussed.

Symbols are global, they really can't be anything else in this format.

This is why I think the connector approach - though substantially more
complex - is a better idea in the wrong run.

-- 
David Gibson			| I'll have my music baroque, and my code
david AT gibson.dropbear.id.au	| minimalist, thank you.  NOT _the_ _other_
				| _way_ _around_!
http://www.ozlabs.org/~dgibson

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 819 bytes --]

Re: [PATCH v10 3/4] dtc: Plugin and fixup support

[David Gibson]

[-- Attachment #1: Type: text/plain, Size: 13831 bytes --]

On Mon, Nov 28, 2016 at 02:10:35PM +0200, Pantelis Antoniou wrote:
> 
> > On Nov 28, 2016, at 06:12 , David Gibson <david-xT8FGy+AXnRB3Ne2BGzF6laj5H9X9Tb+@public.gmane.org> wrote:
> > 
> > On Fri, Nov 25, 2016 at 02:32:10PM +0200, Pantelis Antoniou wrote:
> >> This patch enable the generation of symbols & local fixup information
> >> for trees compiled with the -@ (--symbols) option.
> >> 
> >> Using this patch labels in the tree and their users emit information
> >> in __symbols__ and __local_fixups__ nodes.
> >> 
> >> The __fixups__ node make possible the dynamic resolution of phandle
> >> references which are present in the plugin tree but lie in the
> >> tree that are applying the overlay against.
> >> 
> >> While there is a new magic number for dynamic device tree/overlays blobs
> >> it is by default enabled. Remember to use -M to generate compatible
> >> blobs.
> >> 
> >> Signed-off-by: Pantelis Antoniou <pantelis.antoniou-OWPKS81ov/FWk0Htik3J/w@public.gmane.org>
> >> Signed-off-by: Sascha Hauer <s.hauer-bIcnvbaLZ9MEGnE8C9+IrQ@public.gmane.org>
> >> Signed-off-by: Jan Luebbe <jlu-bIcnvbaLZ9MEGnE8C9+IrQ@public.gmane.org>
> >> ---
> >> Documentation/manual.txt |  25 +++++-
> >> checks.c                 |   8 +-
> >> dtc-lexer.l              |   5 ++
> >> dtc-parser.y             |  50 +++++++++--
> >> dtc.c                    |  39 +++++++-
> >> dtc.h                    |  20 ++++-
> >> fdtdump.c                |   2 +-
> >> flattree.c               |  17 ++--
> >> fstree.c                 |   2 +-
> >> libfdt/fdt.c             |   2 +-
> >> libfdt/fdt.h             |   3 +-
> >> livetree.c               | 225 ++++++++++++++++++++++++++++++++++++++++++++++-
> >> tests/mangle-layout.c    |   7 +-
> >> 13 files changed, 375 insertions(+), 30 deletions(-)
> >> 
> >> diff --git a/Documentation/manual.txt b/Documentation/manual.txt
> >> index 398de32..094893b 100644
> >> --- a/Documentation/manual.txt
> >> +++ b/Documentation/manual.txt
> >> @@ -119,6 +119,24 @@ Options:
> >> 	Make space for <number> reserve map entries
> >> 	Relevant for dtb and asm output only.
> >> 
> >> +    -@
> >> +	Generates a __symbols__ node at the root node of the resulting blob
> >> +	for any node labels used, and for any local references using phandles
> >> +	it also generates a __local_fixups__ node that tracks them.
> >> +
> >> +	When using the /plugin/ tag all unresolved label references to
> >> +	be tracked in the __fixups__ node, making dynamic resolution possible.
> >> +
> >> +    -A
> >> +	Generate automatically aliases for all node labels. This is similar to
> >> +	the -@ option (the __symbols__ node contain identical information) but
> >> +	the semantics are slightly different since no phandles are automatically
> >> +	generated for labeled nodes.
> >> +
> >> +    -M
> >> +	Generate blobs with the old FDT magic number for device tree objects.
> >> +	By default blobs use the DTBO FDT magic number instead.
> >> +
> >>     -S <bytes>
> >> 	Ensure the blob at least <bytes> long, adding additional
> >> 	space if needed.
> >> @@ -146,13 +164,18 @@ Additionally, dtc performs various sanity checks on the tree.
> >> Here is a very rough overview of the layout of a DTS source file:
> >> 
> >> 
> >> -    sourcefile:   list_of_memreserve devicetree
> >> +    sourcefile:   versioninfo plugindecl list_of_memreserve devicetree
> >> 
> >>     memreserve:   label 'memreserve' ADDR ADDR ';'
> >> 		| label 'memreserve' ADDR '-' ADDR ';'
> >> 
> >>     devicetree:   '/' nodedef
> >> 
> >> +    versioninfo:  '/' 'dts-v1' '/' ';'
> >> +
> >> +    plugindecl:   '/' 'plugin' '/' ';'
> >> +                | /* empty */
> >> +
> >>     nodedef:      '{' list_of_property list_of_subnode '}' ';'
> >> 
> >>     property:     label PROPNAME '=' propdata ';'
> >> diff --git a/checks.c b/checks.c
> >> index 2bd27a4..4292f4b 100644
> >> --- a/checks.c
> >> +++ b/checks.c
> >> @@ -487,8 +487,12 @@ static void fixup_phandle_references(struct check *c, struct boot_info *bi,
> >> 
> >> 			refnode = get_node_by_ref(dt, m->ref);
> >> 			if (! refnode) {
> >> -				FAIL(c, "Reference to non-existent node or label \"%s\"\n",
> >> -				     m->ref);
> >> +				if (!(bi->versionflags & VF_PLUGIN))
> >> +					FAIL(c, "Reference to non-existent node or "
> >> +							"label \"%s\"\n", m->ref);
> >> +				else /* mark the entry as unresolved */
> >> +					*((cell_t *)(prop->val.val + m->offset)) =
> >> +						cpu_to_fdt32(0xffffffff);
> >> 				continue;
> >> 			}
> >> 
> >> diff --git a/dtc-lexer.l b/dtc-lexer.l
> >> index 790fbf6..40bbc87 100644
> >> --- a/dtc-lexer.l
> >> +++ b/dtc-lexer.l
> >> @@ -121,6 +121,11 @@ static void lexical_error(const char *fmt, ...);
> >> 			return DT_V1;
> >> 		}
> >> 
> >> +<*>"/plugin/"	{
> >> +			DPRINT("Keyword: /plugin/\n");
> >> +			return DT_PLUGIN;
> >> +		}
> >> +
> >> <*>"/memreserve/"	{
> >> 			DPRINT("Keyword: /memreserve/\n");
> >> 			BEGIN_DEFAULT();
> >> diff --git a/dtc-parser.y b/dtc-parser.y
> >> index 14aaf2e..1a1f660 100644
> >> --- a/dtc-parser.y
> >> +++ b/dtc-parser.y
> >> @@ -19,6 +19,7 @@
> >>  */
> >> %{
> >> #include <stdio.h>
> >> +#include <inttypes.h>
> >> 
> >> #include "dtc.h"
> >> #include "srcpos.h"
> >> @@ -33,6 +34,7 @@ extern void yyerror(char const *s);
> >> 
> >> extern struct boot_info *the_boot_info;
> >> extern bool treesource_error;
> >> +
> > 
> > Extraneous whitespace change here
> > 
> 
> OK.
> 
> >> %}
> >> 
> >> %union {
> >> @@ -52,9 +54,11 @@ extern bool treesource_error;
> >> 	struct node *nodelist;
> >> 	struct reserve_info *re;
> >> 	uint64_t integer;
> >> +	unsigned int flags;
> >> }
> >> 
> >> %token DT_V1
> >> +%token DT_PLUGIN
> >> %token DT_MEMRESERVE
> >> %token DT_LSHIFT DT_RSHIFT DT_LE DT_GE DT_EQ DT_NE DT_AND DT_OR
> >> %token DT_BITS
> >> @@ -71,6 +75,8 @@ extern bool treesource_error;
> >> 
> >> %type <data> propdata
> >> %type <data> propdataprefix
> >> +%type <flags> versioninfo
> >> +%type <flags> plugindecl
> >> %type <re> memreserve
> >> %type <re> memreserves
> >> %type <array> arrayprefix
> >> @@ -101,16 +107,34 @@ extern bool treesource_error;
> >> %%
> >> 
> >> sourcefile:
> >> -	  v1tag memreserves devicetree
> >> +	  versioninfo plugindecl memreserves devicetree
> >> +		{
> >> +			the_boot_info = build_boot_info($1 | $2, $3, $4,
> >> +							guess_boot_cpuid($4));
> >> +		}
> >> +	;
> >> +
> >> +versioninfo:
> >> +	v1tag
> >> 		{
> >> -			the_boot_info = build_boot_info($2, $3,
> >> -							guess_boot_cpuid($3));
> >> +			$$ = VF_DT_V1;
> >> 		}
> >> 	;
> >> 
> >> v1tag:
> >> 	  DT_V1 ';'
> >> +	| DT_V1
> >> 	| DT_V1 ';' v1tag
> >> +
> >> +plugindecl:
> >> +	DT_PLUGIN ';'
> >> +		{
> >> +			$$ = VF_PLUGIN;
> >> +		}
> >> +	| /* empty */
> >> +		{
> >> +			$$ = 0;
> >> +		}
> >> 	;
> >> 
> >> memreserves:
> >> @@ -161,10 +185,19 @@ devicetree:
> >> 		{
> >> 			struct node *target = get_node_by_ref($1, $2);
> >> 
> >> -			if (target)
> >> +			if (target) {
> >> 				merge_nodes(target, $3);
> >> -			else
> >> -				ERROR(&@2, "Label or path %s not found", $2);
> >> +			} else {
> >> +				/*
> >> +				 * We rely on the rule being always:
> >> +				 *   versioninfo plugindecl memreserves devicetree
> >> +				 * so $-1 is what we want (plugindecl)
> >> +				 */
> >> +				if ($<flags>-1 & VF_PLUGIN)
> > 
> > o_O... ok.  I've never seen negative value references before.  Can you
> > provide a link to some documentation saying this is actually supported
> > usage in bison?  I wasn't able to find it when I looked.
> > 
> 
> There is a section about inherited attributes in the flex & bison book by O’Reily.
> 
> https://books.google.gr/books?id=3Sr1V5J9_qMC&lpg=PP1&dq=flex%20bison&hl=el&pg=PP1#v=onepage&q=flex%20bison&f=false
> 
> There’s a direct link to the 2nd Edition of lex & yacc:
> 
> https://books.google.gr/books?id=fMPxfWfe67EC&lpg=PA183&ots=RcRSji2NAT&dq=yacc%20inherited%20attributes&hl=el&pg=PA183#v=onepage&q=yacc%20inherited%20attributes&f=false

Thanks for the link.  I still think moving the fragment assembly out
of the parser will be a better idea long term, but this does address
the main concern I had, so it will do for now.

> >> +					add_orphan_node($1, $3, $2);
> >> +				else
> >> +					ERROR(&@2, "Label or path %s not found", $2);
> >> +			}
> >> 			$$ = $1;
> >> 		}
> >> 	| devicetree DT_DEL_NODE DT_REF ';'
> >> @@ -179,6 +212,11 @@ devicetree:
> >> 
> >> 			$$ = $1;
> >> 		}
> >> +	| /* empty */
> >> +		{
> >> +			/* build empty node */
> >> +			$$ = name_node(build_node(NULL, NULL), "");
> >> +		}
> >> 	;
> >> 
> >> nodedef:
> >> diff --git a/dtc.c b/dtc.c
> >> index 9dcf640..06e91bc 100644
> >> --- a/dtc.c
> >> +++ b/dtc.c
> >> @@ -32,6 +32,9 @@ int minsize;		/* Minimum blob size */
> >> int padsize;		/* Additional padding to blob */
> >> int alignsize;		/* Additional padding to blob accroding to the alignsize */
> >> int phandle_format = PHANDLE_BOTH;	/* Use linux,phandle or phandle properties */
> >> +int symbol_fixup_support;	/* enable symbols & fixup support */
> >> +int auto_label_aliases;		/* auto generate labels -> aliases */
> >> +int no_dtbo_magic;		/* use old FDT magic values for objects */
> >> 
> >> static int is_power_of_2(int x)
> >> {
> >> @@ -59,7 +62,7 @@ static void fill_fullpaths(struct node *tree, const char *prefix)
> >> #define FDT_VERSION(version)	_FDT_VERSION(version)
> >> #define _FDT_VERSION(version)	#version
> >> static const char usage_synopsis[] = "dtc [options] <input file>";
> >> -static const char usage_short_opts[] = "qI:O:o:V:d:R:S:p:a:fb:i:H:sW:E:hv";
> >> +static const char usage_short_opts[] = "qI:O:o:V:d:R:S:p:a:fb:i:H:sW:E:@AMhv";
> >> static struct option const usage_long_opts[] = {
> >> 	{"quiet",            no_argument, NULL, 'q'},
> >> 	{"in-format",         a_argument, NULL, 'I'},
> >> @@ -78,6 +81,9 @@ static struct option const usage_long_opts[] = {
> >> 	{"phandle",           a_argument, NULL, 'H'},
> >> 	{"warning",           a_argument, NULL, 'W'},
> >> 	{"error",             a_argument, NULL, 'E'},
> >> +	{"symbols",	     no_argument, NULL, '@'},
> >> +	{"auto-alias",       no_argument, NULL, 'A'},
> >> +	{"no-dtbo-magic",    no_argument, NULL, 'M'},
> >> 	{"help",             no_argument, NULL, 'h'},
> >> 	{"version",          no_argument, NULL, 'v'},
> >> 	{NULL,               no_argument, NULL, 0x0},
> >> @@ -109,6 +115,9 @@ static const char * const usage_opts_help[] = {
> >> 	 "\t\tboth   - Both \"linux,phandle\" and \"phandle\" properties",
> >> 	"\n\tEnable/disable warnings (prefix with \"no-\")",
> >> 	"\n\tEnable/disable errors (prefix with \"no-\")",
> >> +	"\n\tEnable symbols/fixup support",
> >> +	"\n\tEnable auto-alias of labels",
> >> +	"\n\tDo not use DTBO magic value for plugin objects",
> >> 	"\n\tPrint this help and exit",
> >> 	"\n\tPrint version and exit",
> >> 	NULL,
> >> @@ -153,7 +162,7 @@ static const char *guess_input_format(const char *fname, const char *fallback)
> >> 	fclose(f);
> >> 
> >> 	magic = fdt32_to_cpu(magic);
> >> -	if (magic == FDT_MAGIC)
> >> +	if (magic == FDT_MAGIC || magic == FDT_MAGIC_DTBO)
> >> 		return "dtb";
> >> 
> >> 	return guess_type_by_name(fname, fallback);
> >> @@ -172,6 +181,7 @@ int main(int argc, char *argv[])
> >> 	FILE *outf = NULL;
> >> 	int outversion = DEFAULT_FDT_VERSION;
> >> 	long long cmdline_boot_cpuid = -1;
> >> +	fdt32_t out_magic = FDT_MAGIC;
> >> 
> >> 	quiet      = 0;
> >> 	reservenum = 0;
> >> @@ -249,6 +259,16 @@ int main(int argc, char *argv[])
> >> 			parse_checks_option(false, true, optarg);
> >> 			break;
> >> 
> >> +		case '@':
> >> +			symbol_fixup_support = 1;
> >> +			break;
> >> +		case 'A':
> >> +			auto_label_aliases = 1;
> >> +			break;
> >> +		case 'M':
> >> +			no_dtbo_magic = 1;
> >> +			break;
> >> +
> >> 		case 'h':
> >> 			usage(NULL);
> >> 		default:
> >> @@ -306,6 +326,14 @@ int main(int argc, char *argv[])
> >> 	fill_fullpaths(bi->dt, "");
> >> 	process_checks(force, bi);
> >> 
> >> +	if (auto_label_aliases)
> >> +		generate_label_tree(bi->dt, "aliases", false);
> >> +
> >> +	if (symbol_fixup_support) {
> >> +		generate_label_tree(bi->dt, "__symbols__", true);
> >> +		generate_fixups_tree(bi->dt);
> > 
> > Hang on.. this doesn't seem right.  I thought -@ controlled the
> > __symbols__ side (i.e. the part upon which we overlay) rather than the
> > fixups side (the part which overlays).  A dtbo could certainly have
> > both, of course, but for base trees, wouldn't you have symbols without
> > fixups?  And should it be illegal to try to build a /plugin/ without
> > -@?
> 
> It does control both for now. For base trees having the fixup nodes
> will allow us to do probe order dependency tracking in the future.

Erm.. how?

> For plugins we need the __symbols__ node to support stacked overlays, i.e.
> overlays referring label that were introduced by a previous overlay.

Yes, I realise that an overlay may well want __symbols__ as well.  But
they still seem conceptually different.  I think -@ should control
__symbols__ whereas /plugin/ should control __fixups__.

> For plugins there is no requirement for now to actually contain references to
> be resolved. It can easily be enforced though.

Sure, but I don't see the relevance of that here.  You could just omit
the __fixups__ node if there's nothing to go into them.

-- 
David Gibson			| I'll have my music baroque, and my code
david AT gibson.dropbear.id.au	| minimalist, thank you.  NOT _the_ _other_
				| _way_ _around_!
http://www.ozlabs.org/~dgibson

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 819 bytes --]

Re: [PATCH v10 3/4] dtc: Plugin and fixup support

[Pantelis Antoniou]

Hi David,

> On Nov 29, 2016, at 04:10 , David Gibson <david-xT8FGy+AXnRB3Ne2BGzF6laj5H9X9Tb+@public.gmane.org> wrote:
> 
> On Mon, Nov 28, 2016 at 02:10:35PM +0200, Pantelis Antoniou wrote:
>> 
>>> On Nov 28, 2016, at 06:12 , David Gibson <david-xT8FGy+AXnRB3Ne2BGzF6laj5H9X9Tb+@public.gmane.org> wrote:
>>> 
>>> On Fri, Nov 25, 2016 at 02:32:10PM +0200, Pantelis Antoniou wrote:
>>>> This patch enable the generation of symbols & local fixup information
>>>> for trees compiled with the -@ (--symbols) option.
>>>> 
>>>> Using this patch labels in the tree and their users emit information
>>>> in __symbols__ and __local_fixups__ nodes.
>>>> 
>>>> The __fixups__ node make possible the dynamic resolution of phandle
>>>> references which are present in the plugin tree but lie in the
>>>> tree that are applying the overlay against.
>>>> 
>>>> While there is a new magic number for dynamic device tree/overlays blobs
>>>> it is by default enabled. Remember to use -M to generate compatible
>>>> blobs.
>>>> 
>>>> Signed-off-by: Pantelis Antoniou <pantelis.antoniou-OWPKS81ov/FWk0Htik3J/w@public.gmane.org>
>>>> Signed-off-by: Sascha Hauer <s.hauer-bIcnvbaLZ9MEGnE8C9+IrQ@public.gmane.org>
>>>> Signed-off-by: Jan Luebbe <jlu-bIcnvbaLZ9MEGnE8C9+IrQ@public.gmane.org>
>>>> ---
>>>> Documentation/manual.txt |  25 +++++-
>>>> checks.c                 |   8 +-
>>>> dtc-lexer.l              |   5 ++
>>>> dtc-parser.y             |  50 +++++++++--
>>>> dtc.c                    |  39 +++++++-
>>>> dtc.h                    |  20 ++++-
>>>> fdtdump.c                |   2 +-
>>>> flattree.c               |  17 ++--
>>>> fstree.c                 |   2 +-
>>>> libfdt/fdt.c             |   2 +-
>>>> libfdt/fdt.h             |   3 +-
>>>> livetree.c               | 225 ++++++++++++++++++++++++++++++++++++++++++++++-
>>>> tests/mangle-layout.c    |   7 +-
>>>> 13 files changed, 375 insertions(+), 30 deletions(-)
>>>> 
>>>> diff --git a/Documentation/manual.txt b/Documentation/manual.txt
>>>> index 398de32..094893b 100644
>>>> --- a/Documentation/manual.txt
>>>> +++ b/Documentation/manual.txt
>>>> @@ -119,6 +119,24 @@ Options:
>>>> 	Make space for <number> reserve map entries
>>>> 	Relevant for dtb and asm output only.
>>>> 
>>>> +    -@
>>>> +	Generates a __symbols__ node at the root node of the resulting blob
>>>> +	for any node labels used, and for any local references using phandles
>>>> +	it also generates a __local_fixups__ node that tracks them.
>>>> +
>>>> +	When using the /plugin/ tag all unresolved label references to
>>>> +	be tracked in the __fixups__ node, making dynamic resolution possible.
>>>> +
>>>> +    -A
>>>> +	Generate automatically aliases for all node labels. This is similar to
>>>> +	the -@ option (the __symbols__ node contain identical information) but
>>>> +	the semantics are slightly different since no phandles are automatically
>>>> +	generated for labeled nodes.
>>>> +
>>>> +    -M
>>>> +	Generate blobs with the old FDT magic number for device tree objects.
>>>> +	By default blobs use the DTBO FDT magic number instead.
>>>> +
>>>>    -S <bytes>
>>>> 	Ensure the blob at least <bytes> long, adding additional
>>>> 	space if needed.
>>>> @@ -146,13 +164,18 @@ Additionally, dtc performs various sanity checks on the tree.
>>>> Here is a very rough overview of the layout of a DTS source file:
>>>> 
>>>> 
>>>> -    sourcefile:   list_of_memreserve devicetree
>>>> +    sourcefile:   versioninfo plugindecl list_of_memreserve devicetree
>>>> 
>>>>    memreserve:   label 'memreserve' ADDR ADDR ';'
>>>> 		| label 'memreserve' ADDR '-' ADDR ';'
>>>> 
>>>>    devicetree:   '/' nodedef
>>>> 
>>>> +    versioninfo:  '/' 'dts-v1' '/' ';'
>>>> +
>>>> +    plugindecl:   '/' 'plugin' '/' ';'
>>>> +                | /* empty */
>>>> +
>>>>    nodedef:      '{' list_of_property list_of_subnode '}' ';'
>>>> 
>>>>    property:     label PROPNAME '=' propdata ';'
>>>> diff --git a/checks.c b/checks.c
>>>> index 2bd27a4..4292f4b 100644
>>>> --- a/checks.c
>>>> +++ b/checks.c
>>>> @@ -487,8 +487,12 @@ static void fixup_phandle_references(struct check *c, struct boot_info *bi,
>>>> 
>>>> 			refnode = get_node_by_ref(dt, m->ref);
>>>> 			if (! refnode) {
>>>> -				FAIL(c, "Reference to non-existent node or label \"%s\"\n",
>>>> -				     m->ref);
>>>> +				if (!(bi->versionflags & VF_PLUGIN))
>>>> +					FAIL(c, "Reference to non-existent node or "
>>>> +							"label \"%s\"\n", m->ref);
>>>> +				else /* mark the entry as unresolved */
>>>> +					*((cell_t *)(prop->val.val + m->offset)) =
>>>> +						cpu_to_fdt32(0xffffffff);
>>>> 				continue;
>>>> 			}
>>>> 
>>>> diff --git a/dtc-lexer.l b/dtc-lexer.l
>>>> index 790fbf6..40bbc87 100644
>>>> --- a/dtc-lexer.l
>>>> +++ b/dtc-lexer.l
>>>> @@ -121,6 +121,11 @@ static void lexical_error(const char *fmt, ...);
>>>> 			return DT_V1;
>>>> 		}
>>>> 
>>>> +<*>"/plugin/"	{
>>>> +			DPRINT("Keyword: /plugin/\n");
>>>> +			return DT_PLUGIN;
>>>> +		}
>>>> +
>>>> <*>"/memreserve/"	{
>>>> 			DPRINT("Keyword: /memreserve/\n");
>>>> 			BEGIN_DEFAULT();
>>>> diff --git a/dtc-parser.y b/dtc-parser.y
>>>> index 14aaf2e..1a1f660 100644
>>>> --- a/dtc-parser.y
>>>> +++ b/dtc-parser.y
>>>> @@ -19,6 +19,7 @@
>>>> */
>>>> %{
>>>> #include <stdio.h>
>>>> +#include <inttypes.h>
>>>> 
>>>> #include "dtc.h"
>>>> #include "srcpos.h"
>>>> @@ -33,6 +34,7 @@ extern void yyerror(char const *s);
>>>> 
>>>> extern struct boot_info *the_boot_info;
>>>> extern bool treesource_error;
>>>> +
>>> 
>>> Extraneous whitespace change here
>>> 
>> 
>> OK.
>> 
>>>> %}
>>>> 
>>>> %union {
>>>> @@ -52,9 +54,11 @@ extern bool treesource_error;
>>>> 	struct node *nodelist;
>>>> 	struct reserve_info *re;
>>>> 	uint64_t integer;
>>>> +	unsigned int flags;
>>>> }
>>>> 
>>>> %token DT_V1
>>>> +%token DT_PLUGIN
>>>> %token DT_MEMRESERVE
>>>> %token DT_LSHIFT DT_RSHIFT DT_LE DT_GE DT_EQ DT_NE DT_AND DT_OR
>>>> %token DT_BITS
>>>> @@ -71,6 +75,8 @@ extern bool treesource_error;
>>>> 
>>>> %type <data> propdata
>>>> %type <data> propdataprefix
>>>> +%type <flags> versioninfo
>>>> +%type <flags> plugindecl
>>>> %type <re> memreserve
>>>> %type <re> memreserves
>>>> %type <array> arrayprefix
>>>> @@ -101,16 +107,34 @@ extern bool treesource_error;
>>>> %%
>>>> 
>>>> sourcefile:
>>>> -	  v1tag memreserves devicetree
>>>> +	  versioninfo plugindecl memreserves devicetree
>>>> +		{
>>>> +			the_boot_info = build_boot_info($1 | $2, $3, $4,
>>>> +							guess_boot_cpuid($4));
>>>> +		}
>>>> +	;
>>>> +
>>>> +versioninfo:
>>>> +	v1tag
>>>> 		{
>>>> -			the_boot_info = build_boot_info($2, $3,
>>>> -							guess_boot_cpuid($3));
>>>> +			$$ = VF_DT_V1;
>>>> 		}
>>>> 	;
>>>> 
>>>> v1tag:
>>>> 	  DT_V1 ';'
>>>> +	| DT_V1
>>>> 	| DT_V1 ';' v1tag
>>>> +
>>>> +plugindecl:
>>>> +	DT_PLUGIN ';'
>>>> +		{
>>>> +			$$ = VF_PLUGIN;
>>>> +		}
>>>> +	| /* empty */
>>>> +		{
>>>> +			$$ = 0;
>>>> +		}
>>>> 	;
>>>> 
>>>> memreserves:
>>>> @@ -161,10 +185,19 @@ devicetree:
>>>> 		{
>>>> 			struct node *target = get_node_by_ref($1, $2);
>>>> 
>>>> -			if (target)
>>>> +			if (target) {
>>>> 				merge_nodes(target, $3);
>>>> -			else
>>>> -				ERROR(&@2, "Label or path %s not found", $2);
>>>> +			} else {
>>>> +				/*
>>>> +				 * We rely on the rule being always:
>>>> +				 *   versioninfo plugindecl memreserves devicetree
>>>> +				 * so $-1 is what we want (plugindecl)
>>>> +				 */
>>>> +				if ($<flags>-1 & VF_PLUGIN)
>>> 
>>> o_O... ok.  I've never seen negative value references before.  Can you
>>> provide a link to some documentation saying this is actually supported
>>> usage in bison?  I wasn't able to find it when I looked.
>>> 
>> 
>> There is a section about inherited attributes in the flex & bison book by O’Reily.
>> 
>> https://books.google.gr/books?id=3Sr1V5J9_qMC&lpg=PP1&dq=flex%20bison&hl=el&pg=PP1#v=onepage&q=flex%20bison&f=false
>> 
>> There’s a direct link to the 2nd Edition of lex & yacc:
>> 
>> https://books.google.gr/books?id=fMPxfWfe67EC&lpg=PA183&ots=RcRSji2NAT&dq=yacc%20inherited%20attributes&hl=el&pg=PA183#v=onepage&q=yacc%20inherited%20attributes&f=false
> 
> Thanks for the link.  I still think moving the fragment assembly out
> of the parser will be a better idea long term, but this does address
> the main concern I had, so it will do for now.
> 
>>>> +					add_orphan_node($1, $3, $2);
>>>> +				else
>>>> +					ERROR(&@2, "Label or path %s not found", $2);
>>>> +			}
>>>> 			$$ = $1;
>>>> 		}
>>>> 	| devicetree DT_DEL_NODE DT_REF ';'
>>>> @@ -179,6 +212,11 @@ devicetree:
>>>> 
>>>> 			$$ = $1;
>>>> 		}
>>>> +	| /* empty */
>>>> +		{
>>>> +			/* build empty node */
>>>> +			$$ = name_node(build_node(NULL, NULL), "");
>>>> +		}
>>>> 	;
>>>> 
>>>> nodedef:
>>>> diff --git a/dtc.c b/dtc.c
>>>> index 9dcf640..06e91bc 100644
>>>> --- a/dtc.c
>>>> +++ b/dtc.c
>>>> @@ -32,6 +32,9 @@ int minsize;		/* Minimum blob size */
>>>> int padsize;		/* Additional padding to blob */
>>>> int alignsize;		/* Additional padding to blob accroding to the alignsize */
>>>> int phandle_format = PHANDLE_BOTH;	/* Use linux,phandle or phandle properties */
>>>> +int symbol_fixup_support;	/* enable symbols & fixup support */
>>>> +int auto_label_aliases;		/* auto generate labels -> aliases */
>>>> +int no_dtbo_magic;		/* use old FDT magic values for objects */
>>>> 
>>>> static int is_power_of_2(int x)
>>>> {
>>>> @@ -59,7 +62,7 @@ static void fill_fullpaths(struct node *tree, const char *prefix)
>>>> #define FDT_VERSION(version)	_FDT_VERSION(version)
>>>> #define _FDT_VERSION(version)	#version
>>>> static const char usage_synopsis[] = "dtc [options] <input file>";
>>>> -static const char usage_short_opts[] = "qI:O:o:V:d:R:S:p:a:fb:i:H:sW:E:hv";
>>>> +static const char usage_short_opts[] = "qI:O:o:V:d:R:S:p:a:fb:i:H:sW:E:@AMhv";
>>>> static struct option const usage_long_opts[] = {
>>>> 	{"quiet",            no_argument, NULL, 'q'},
>>>> 	{"in-format",         a_argument, NULL, 'I'},
>>>> @@ -78,6 +81,9 @@ static struct option const usage_long_opts[] = {
>>>> 	{"phandle",           a_argument, NULL, 'H'},
>>>> 	{"warning",           a_argument, NULL, 'W'},
>>>> 	{"error",             a_argument, NULL, 'E'},
>>>> +	{"symbols",	     no_argument, NULL, '@'},
>>>> +	{"auto-alias",       no_argument, NULL, 'A'},
>>>> +	{"no-dtbo-magic",    no_argument, NULL, 'M'},
>>>> 	{"help",             no_argument, NULL, 'h'},
>>>> 	{"version",          no_argument, NULL, 'v'},
>>>> 	{NULL,               no_argument, NULL, 0x0},
>>>> @@ -109,6 +115,9 @@ static const char * const usage_opts_help[] = {
>>>> 	 "\t\tboth   - Both \"linux,phandle\" and \"phandle\" properties",
>>>> 	"\n\tEnable/disable warnings (prefix with \"no-\")",
>>>> 	"\n\tEnable/disable errors (prefix with \"no-\")",
>>>> +	"\n\tEnable symbols/fixup support",
>>>> +	"\n\tEnable auto-alias of labels",
>>>> +	"\n\tDo not use DTBO magic value for plugin objects",
>>>> 	"\n\tPrint this help and exit",
>>>> 	"\n\tPrint version and exit",
>>>> 	NULL,
>>>> @@ -153,7 +162,7 @@ static const char *guess_input_format(const char *fname, const char *fallback)
>>>> 	fclose(f);
>>>> 
>>>> 	magic = fdt32_to_cpu(magic);
>>>> -	if (magic == FDT_MAGIC)
>>>> +	if (magic == FDT_MAGIC || magic == FDT_MAGIC_DTBO)
>>>> 		return "dtb";
>>>> 
>>>> 	return guess_type_by_name(fname, fallback);
>>>> @@ -172,6 +181,7 @@ int main(int argc, char *argv[])
>>>> 	FILE *outf = NULL;
>>>> 	int outversion = DEFAULT_FDT_VERSION;
>>>> 	long long cmdline_boot_cpuid = -1;
>>>> +	fdt32_t out_magic = FDT_MAGIC;
>>>> 
>>>> 	quiet      = 0;
>>>> 	reservenum = 0;
>>>> @@ -249,6 +259,16 @@ int main(int argc, char *argv[])
>>>> 			parse_checks_option(false, true, optarg);
>>>> 			break;
>>>> 
>>>> +		case '@':
>>>> +			symbol_fixup_support = 1;
>>>> +			break;
>>>> +		case 'A':
>>>> +			auto_label_aliases = 1;
>>>> +			break;
>>>> +		case 'M':
>>>> +			no_dtbo_magic = 1;
>>>> +			break;
>>>> +
>>>> 		case 'h':
>>>> 			usage(NULL);
>>>> 		default:
>>>> @@ -306,6 +326,14 @@ int main(int argc, char *argv[])
>>>> 	fill_fullpaths(bi->dt, "");
>>>> 	process_checks(force, bi);
>>>> 
>>>> +	if (auto_label_aliases)
>>>> +		generate_label_tree(bi->dt, "aliases", false);
>>>> +
>>>> +	if (symbol_fixup_support) {
>>>> +		generate_label_tree(bi->dt, "__symbols__", true);
>>>> +		generate_fixups_tree(bi->dt);
>>> 
>>> Hang on.. this doesn't seem right.  I thought -@ controlled the
>>> __symbols__ side (i.e. the part upon which we overlay) rather than the
>>> fixups side (the part which overlays).  A dtbo could certainly have
>>> both, of course, but for base trees, wouldn't you have symbols without
>>> fixups?  And should it be illegal to try to build a /plugin/ without
>>> -@?
>> 
>> It does control both for now. For base trees having the fixup nodes
>> will allow us to do probe order dependency tracking in the future.
> 
> Erm.. how?
> 
>> For plugins we need the __symbols__ node to support stacked overlays, i.e.
>> overlays referring label that were introduced by a previous overlay.
> 
> Yes, I realise that an overlay may well want __symbols__ as well.  But
> they still seem conceptually different.  I think -@ should control
> __symbols__ whereas /plugin/ should control __fixups__.
> 

It is easily done. Although using /plugin/ as an auto-magic option does both
just fine.

>> For plugins there is no requirement for now to actually contain references to
>> be resolved. It can easily be enforced though.
> 
> Sure, but I don't see the relevance of that here.  You could just omit
> the __fixups__ node if there's nothing to go into them.
> 

Hmm, yeah.

> -- 
> David Gibson			| I'll have my music baroque, and my code
> david AT gibson.dropbear.id.au	| minimalist, thank you.  NOT _the_ _other_
> 				| _way_ _around_!
> http://www.ozlabs.org/~dgibson

Regards

— Pantelis

--
To unsubscribe from this list: send the line "unsubscribe devicetree" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v10 3/4] dtc: Plugin and fixup support

[David Gibson]

[-- Attachment #1: Type: text/plain, Size: 15036 bytes --]

On Tue, Nov 29, 2016 at 01:09:11PM +0200, Pantelis Antoniou wrote:
> Hi David,
> 
> > On Nov 29, 2016, at 04:10 , David Gibson <david-xT8FGy+AXnRB3Ne2BGzF6laj5H9X9Tb+@public.gmane.org> wrote:
> > 
> > On Mon, Nov 28, 2016 at 02:10:35PM +0200, Pantelis Antoniou wrote:
> >> 
> >>> On Nov 28, 2016, at 06:12 , David Gibson <david-xT8FGy+AXnRB3Ne2BGzF6laj5H9X9Tb+@public.gmane.org> wrote:
> >>> 
> >>> On Fri, Nov 25, 2016 at 02:32:10PM +0200, Pantelis Antoniou wrote:
> >>>> This patch enable the generation of symbols & local fixup information
> >>>> for trees compiled with the -@ (--symbols) option.
> >>>> 
> >>>> Using this patch labels in the tree and their users emit information
> >>>> in __symbols__ and __local_fixups__ nodes.
> >>>> 
> >>>> The __fixups__ node make possible the dynamic resolution of phandle
> >>>> references which are present in the plugin tree but lie in the
> >>>> tree that are applying the overlay against.
> >>>> 
> >>>> While there is a new magic number for dynamic device tree/overlays blobs
> >>>> it is by default enabled. Remember to use -M to generate compatible
> >>>> blobs.
> >>>> 
> >>>> Signed-off-by: Pantelis Antoniou <pantelis.antoniou-OWPKS81ov/FWk0Htik3J/w@public.gmane.org>
> >>>> Signed-off-by: Sascha Hauer <s.hauer-bIcnvbaLZ9MEGnE8C9+IrQ@public.gmane.org>
> >>>> Signed-off-by: Jan Luebbe <jlu-bIcnvbaLZ9MEGnE8C9+IrQ@public.gmane.org>
> >>>> ---
> >>>> Documentation/manual.txt |  25 +++++-
> >>>> checks.c                 |   8 +-
> >>>> dtc-lexer.l              |   5 ++
> >>>> dtc-parser.y             |  50 +++++++++--
> >>>> dtc.c                    |  39 +++++++-
> >>>> dtc.h                    |  20 ++++-
> >>>> fdtdump.c                |   2 +-
> >>>> flattree.c               |  17 ++--
> >>>> fstree.c                 |   2 +-
> >>>> libfdt/fdt.c             |   2 +-
> >>>> libfdt/fdt.h             |   3 +-
> >>>> livetree.c               | 225 ++++++++++++++++++++++++++++++++++++++++++++++-
> >>>> tests/mangle-layout.c    |   7 +-
> >>>> 13 files changed, 375 insertions(+), 30 deletions(-)
> >>>> 
> >>>> diff --git a/Documentation/manual.txt b/Documentation/manual.txt
> >>>> index 398de32..094893b 100644
> >>>> --- a/Documentation/manual.txt
> >>>> +++ b/Documentation/manual.txt
> >>>> @@ -119,6 +119,24 @@ Options:
> >>>> 	Make space for <number> reserve map entries
> >>>> 	Relevant for dtb and asm output only.
> >>>> 
> >>>> +    -@
> >>>> +	Generates a __symbols__ node at the root node of the resulting blob
> >>>> +	for any node labels used, and for any local references using phandles
> >>>> +	it also generates a __local_fixups__ node that tracks them.
> >>>> +
> >>>> +	When using the /plugin/ tag all unresolved label references to
> >>>> +	be tracked in the __fixups__ node, making dynamic resolution possible.
> >>>> +
> >>>> +    -A
> >>>> +	Generate automatically aliases for all node labels. This is similar to
> >>>> +	the -@ option (the __symbols__ node contain identical information) but
> >>>> +	the semantics are slightly different since no phandles are automatically
> >>>> +	generated for labeled nodes.
> >>>> +
> >>>> +    -M
> >>>> +	Generate blobs with the old FDT magic number for device tree objects.
> >>>> +	By default blobs use the DTBO FDT magic number instead.
> >>>> +
> >>>>    -S <bytes>
> >>>> 	Ensure the blob at least <bytes> long, adding additional
> >>>> 	space if needed.
> >>>> @@ -146,13 +164,18 @@ Additionally, dtc performs various sanity checks on the tree.
> >>>> Here is a very rough overview of the layout of a DTS source file:
> >>>> 
> >>>> 
> >>>> -    sourcefile:   list_of_memreserve devicetree
> >>>> +    sourcefile:   versioninfo plugindecl list_of_memreserve devicetree
> >>>> 
> >>>>    memreserve:   label 'memreserve' ADDR ADDR ';'
> >>>> 		| label 'memreserve' ADDR '-' ADDR ';'
> >>>> 
> >>>>    devicetree:   '/' nodedef
> >>>> 
> >>>> +    versioninfo:  '/' 'dts-v1' '/' ';'
> >>>> +
> >>>> +    plugindecl:   '/' 'plugin' '/' ';'
> >>>> +                | /* empty */
> >>>> +
> >>>>    nodedef:      '{' list_of_property list_of_subnode '}' ';'
> >>>> 
> >>>>    property:     label PROPNAME '=' propdata ';'
> >>>> diff --git a/checks.c b/checks.c
> >>>> index 2bd27a4..4292f4b 100644
> >>>> --- a/checks.c
> >>>> +++ b/checks.c
> >>>> @@ -487,8 +487,12 @@ static void fixup_phandle_references(struct check *c, struct boot_info *bi,
> >>>> 
> >>>> 			refnode = get_node_by_ref(dt, m->ref);
> >>>> 			if (! refnode) {
> >>>> -				FAIL(c, "Reference to non-existent node or label \"%s\"\n",
> >>>> -				     m->ref);
> >>>> +				if (!(bi->versionflags & VF_PLUGIN))
> >>>> +					FAIL(c, "Reference to non-existent node or "
> >>>> +							"label \"%s\"\n", m->ref);
> >>>> +				else /* mark the entry as unresolved */
> >>>> +					*((cell_t *)(prop->val.val + m->offset)) =
> >>>> +						cpu_to_fdt32(0xffffffff);
> >>>> 				continue;
> >>>> 			}
> >>>> 
> >>>> diff --git a/dtc-lexer.l b/dtc-lexer.l
> >>>> index 790fbf6..40bbc87 100644
> >>>> --- a/dtc-lexer.l
> >>>> +++ b/dtc-lexer.l
> >>>> @@ -121,6 +121,11 @@ static void lexical_error(const char *fmt, ...);
> >>>> 			return DT_V1;
> >>>> 		}
> >>>> 
> >>>> +<*>"/plugin/"	{
> >>>> +			DPRINT("Keyword: /plugin/\n");
> >>>> +			return DT_PLUGIN;
> >>>> +		}
> >>>> +
> >>>> <*>"/memreserve/"	{
> >>>> 			DPRINT("Keyword: /memreserve/\n");
> >>>> 			BEGIN_DEFAULT();
> >>>> diff --git a/dtc-parser.y b/dtc-parser.y
> >>>> index 14aaf2e..1a1f660 100644
> >>>> --- a/dtc-parser.y
> >>>> +++ b/dtc-parser.y
> >>>> @@ -19,6 +19,7 @@
> >>>> */
> >>>> %{
> >>>> #include <stdio.h>
> >>>> +#include <inttypes.h>
> >>>> 
> >>>> #include "dtc.h"
> >>>> #include "srcpos.h"
> >>>> @@ -33,6 +34,7 @@ extern void yyerror(char const *s);
> >>>> 
> >>>> extern struct boot_info *the_boot_info;
> >>>> extern bool treesource_error;
> >>>> +
> >>> 
> >>> Extraneous whitespace change here
> >>> 
> >> 
> >> OK.
> >> 
> >>>> %}
> >>>> 
> >>>> %union {
> >>>> @@ -52,9 +54,11 @@ extern bool treesource_error;
> >>>> 	struct node *nodelist;
> >>>> 	struct reserve_info *re;
> >>>> 	uint64_t integer;
> >>>> +	unsigned int flags;
> >>>> }
> >>>> 
> >>>> %token DT_V1
> >>>> +%token DT_PLUGIN
> >>>> %token DT_MEMRESERVE
> >>>> %token DT_LSHIFT DT_RSHIFT DT_LE DT_GE DT_EQ DT_NE DT_AND DT_OR
> >>>> %token DT_BITS
> >>>> @@ -71,6 +75,8 @@ extern bool treesource_error;
> >>>> 
> >>>> %type <data> propdata
> >>>> %type <data> propdataprefix
> >>>> +%type <flags> versioninfo
> >>>> +%type <flags> plugindecl
> >>>> %type <re> memreserve
> >>>> %type <re> memreserves
> >>>> %type <array> arrayprefix
> >>>> @@ -101,16 +107,34 @@ extern bool treesource_error;
> >>>> %%
> >>>> 
> >>>> sourcefile:
> >>>> -	  v1tag memreserves devicetree
> >>>> +	  versioninfo plugindecl memreserves devicetree
> >>>> +		{
> >>>> +			the_boot_info = build_boot_info($1 | $2, $3, $4,
> >>>> +							guess_boot_cpuid($4));
> >>>> +		}
> >>>> +	;
> >>>> +
> >>>> +versioninfo:
> >>>> +	v1tag
> >>>> 		{
> >>>> -			the_boot_info = build_boot_info($2, $3,
> >>>> -							guess_boot_cpuid($3));
> >>>> +			$$ = VF_DT_V1;
> >>>> 		}
> >>>> 	;
> >>>> 
> >>>> v1tag:
> >>>> 	  DT_V1 ';'
> >>>> +	| DT_V1
> >>>> 	| DT_V1 ';' v1tag
> >>>> +
> >>>> +plugindecl:
> >>>> +	DT_PLUGIN ';'
> >>>> +		{
> >>>> +			$$ = VF_PLUGIN;
> >>>> +		}
> >>>> +	| /* empty */
> >>>> +		{
> >>>> +			$$ = 0;
> >>>> +		}
> >>>> 	;
> >>>> 
> >>>> memreserves:
> >>>> @@ -161,10 +185,19 @@ devicetree:
> >>>> 		{
> >>>> 			struct node *target = get_node_by_ref($1, $2);
> >>>> 
> >>>> -			if (target)
> >>>> +			if (target) {
> >>>> 				merge_nodes(target, $3);
> >>>> -			else
> >>>> -				ERROR(&@2, "Label or path %s not found", $2);
> >>>> +			} else {
> >>>> +				/*
> >>>> +				 * We rely on the rule being always:
> >>>> +				 *   versioninfo plugindecl memreserves devicetree
> >>>> +				 * so $-1 is what we want (plugindecl)
> >>>> +				 */
> >>>> +				if ($<flags>-1 & VF_PLUGIN)
> >>> 
> >>> o_O... ok.  I've never seen negative value references before.  Can you
> >>> provide a link to some documentation saying this is actually supported
> >>> usage in bison?  I wasn't able to find it when I looked.
> >>> 
> >> 
> >> There is a section about inherited attributes in the flex & bison book by O’Reily.
> >> 
> >> https://books.google.gr/books?id=3Sr1V5J9_qMC&lpg=PP1&dq=flex%20bison&hl=el&pg=PP1#v=onepage&q=flex%20bison&f=false
> >> 
> >> There’s a direct link to the 2nd Edition of lex & yacc:
> >> 
> >> https://books.google.gr/books?id=fMPxfWfe67EC&lpg=PA183&ots=RcRSji2NAT&dq=yacc%20inherited%20attributes&hl=el&pg=PA183#v=onepage&q=yacc%20inherited%20attributes&f=false
> > 
> > Thanks for the link.  I still think moving the fragment assembly out
> > of the parser will be a better idea long term, but this does address
> > the main concern I had, so it will do for now.
> > 
> >>>> +					add_orphan_node($1, $3, $2);
> >>>> +				else
> >>>> +					ERROR(&@2, "Label or path %s not found", $2);
> >>>> +			}
> >>>> 			$$ = $1;
> >>>> 		}
> >>>> 	| devicetree DT_DEL_NODE DT_REF ';'
> >>>> @@ -179,6 +212,11 @@ devicetree:
> >>>> 
> >>>> 			$$ = $1;
> >>>> 		}
> >>>> +	| /* empty */
> >>>> +		{
> >>>> +			/* build empty node */
> >>>> +			$$ = name_node(build_node(NULL, NULL), "");
> >>>> +		}
> >>>> 	;
> >>>> 
> >>>> nodedef:
> >>>> diff --git a/dtc.c b/dtc.c
> >>>> index 9dcf640..06e91bc 100644
> >>>> --- a/dtc.c
> >>>> +++ b/dtc.c
> >>>> @@ -32,6 +32,9 @@ int minsize;		/* Minimum blob size */
> >>>> int padsize;		/* Additional padding to blob */
> >>>> int alignsize;		/* Additional padding to blob accroding to the alignsize */
> >>>> int phandle_format = PHANDLE_BOTH;	/* Use linux,phandle or phandle properties */
> >>>> +int symbol_fixup_support;	/* enable symbols & fixup support */
> >>>> +int auto_label_aliases;		/* auto generate labels -> aliases */
> >>>> +int no_dtbo_magic;		/* use old FDT magic values for objects */
> >>>> 
> >>>> static int is_power_of_2(int x)
> >>>> {
> >>>> @@ -59,7 +62,7 @@ static void fill_fullpaths(struct node *tree, const char *prefix)
> >>>> #define FDT_VERSION(version)	_FDT_VERSION(version)
> >>>> #define _FDT_VERSION(version)	#version
> >>>> static const char usage_synopsis[] = "dtc [options] <input file>";
> >>>> -static const char usage_short_opts[] = "qI:O:o:V:d:R:S:p:a:fb:i:H:sW:E:hv";
> >>>> +static const char usage_short_opts[] = "qI:O:o:V:d:R:S:p:a:fb:i:H:sW:E:@AMhv";
> >>>> static struct option const usage_long_opts[] = {
> >>>> 	{"quiet",            no_argument, NULL, 'q'},
> >>>> 	{"in-format",         a_argument, NULL, 'I'},
> >>>> @@ -78,6 +81,9 @@ static struct option const usage_long_opts[] = {
> >>>> 	{"phandle",           a_argument, NULL, 'H'},
> >>>> 	{"warning",           a_argument, NULL, 'W'},
> >>>> 	{"error",             a_argument, NULL, 'E'},
> >>>> +	{"symbols",	     no_argument, NULL, '@'},
> >>>> +	{"auto-alias",       no_argument, NULL, 'A'},
> >>>> +	{"no-dtbo-magic",    no_argument, NULL, 'M'},
> >>>> 	{"help",             no_argument, NULL, 'h'},
> >>>> 	{"version",          no_argument, NULL, 'v'},
> >>>> 	{NULL,               no_argument, NULL, 0x0},
> >>>> @@ -109,6 +115,9 @@ static const char * const usage_opts_help[] = {
> >>>> 	 "\t\tboth   - Both \"linux,phandle\" and \"phandle\" properties",
> >>>> 	"\n\tEnable/disable warnings (prefix with \"no-\")",
> >>>> 	"\n\tEnable/disable errors (prefix with \"no-\")",
> >>>> +	"\n\tEnable symbols/fixup support",
> >>>> +	"\n\tEnable auto-alias of labels",
> >>>> +	"\n\tDo not use DTBO magic value for plugin objects",
> >>>> 	"\n\tPrint this help and exit",
> >>>> 	"\n\tPrint version and exit",
> >>>> 	NULL,
> >>>> @@ -153,7 +162,7 @@ static const char *guess_input_format(const char *fname, const char *fallback)
> >>>> 	fclose(f);
> >>>> 
> >>>> 	magic = fdt32_to_cpu(magic);
> >>>> -	if (magic == FDT_MAGIC)
> >>>> +	if (magic == FDT_MAGIC || magic == FDT_MAGIC_DTBO)
> >>>> 		return "dtb";
> >>>> 
> >>>> 	return guess_type_by_name(fname, fallback);
> >>>> @@ -172,6 +181,7 @@ int main(int argc, char *argv[])
> >>>> 	FILE *outf = NULL;
> >>>> 	int outversion = DEFAULT_FDT_VERSION;
> >>>> 	long long cmdline_boot_cpuid = -1;
> >>>> +	fdt32_t out_magic = FDT_MAGIC;
> >>>> 
> >>>> 	quiet      = 0;
> >>>> 	reservenum = 0;
> >>>> @@ -249,6 +259,16 @@ int main(int argc, char *argv[])
> >>>> 			parse_checks_option(false, true, optarg);
> >>>> 			break;
> >>>> 
> >>>> +		case '@':
> >>>> +			symbol_fixup_support = 1;
> >>>> +			break;
> >>>> +		case 'A':
> >>>> +			auto_label_aliases = 1;
> >>>> +			break;
> >>>> +		case 'M':
> >>>> +			no_dtbo_magic = 1;
> >>>> +			break;
> >>>> +
> >>>> 		case 'h':
> >>>> 			usage(NULL);
> >>>> 		default:
> >>>> @@ -306,6 +326,14 @@ int main(int argc, char *argv[])
> >>>> 	fill_fullpaths(bi->dt, "");
> >>>> 	process_checks(force, bi);
> >>>> 
> >>>> +	if (auto_label_aliases)
> >>>> +		generate_label_tree(bi->dt, "aliases", false);
> >>>> +
> >>>> +	if (symbol_fixup_support) {
> >>>> +		generate_label_tree(bi->dt, "__symbols__", true);
> >>>> +		generate_fixups_tree(bi->dt);
> >>> 
> >>> Hang on.. this doesn't seem right.  I thought -@ controlled the
> >>> __symbols__ side (i.e. the part upon which we overlay) rather than the
> >>> fixups side (the part which overlays).  A dtbo could certainly have
> >>> both, of course, but for base trees, wouldn't you have symbols without
> >>> fixups?  And should it be illegal to try to build a /plugin/ without
> >>> -@?
> >> 
> >> It does control both for now. For base trees having the fixup nodes
> >> will allow us to do probe order dependency tracking in the future.
> > 
> > Erm.. how?
> > 
> >> For plugins we need the __symbols__ node to support stacked overlays, i.e.
> >> overlays referring label that were introduced by a previous overlay.
> > 
> > Yes, I realise that an overlay may well want __symbols__ as well.  But
> > they still seem conceptually different.  I think -@ should control
> > __symbols__ whereas /plugin/ should control __fixups__.
> > 
> 
> It is easily done. Although using /plugin/ as an auto-magic option does both
> just fine.

Sorry, I don't follow what you're saying.

> >> For plugins there is no requirement for now to actually contain references to
> >> be resolved. It can easily be enforced though.
> > 
> > Sure, but I don't see the relevance of that here.  You could just omit
> > the __fixups__ node if there's nothing to go into them.
> > 
> 
> Hmm, yeah.
> 
> 
> Regards
> 
> — Pantelis
> 

-- 
David Gibson			| I'll have my music baroque, and my code
david AT gibson.dropbear.id.au	| minimalist, thank you.  NOT _the_ _other_
				| _way_ _around_!
http://www.ozlabs.org/~dgibson

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 819 bytes --]

Re: [PATCH v10 3/4] dtc: Plugin and fixup support

[Pantelis Antoniou]

Hi David,

> On Nov 30, 2016, at 03:50 , David Gibson <david-xT8FGy+AXnRB3Ne2BGzF6laj5H9X9Tb+@public.gmane.org> wrote:
> 
> On Tue, Nov 29, 2016 at 01:09:11PM +0200, Pantelis Antoniou wrote:
>> Hi David,
>> 
>>> On Nov 29, 2016, at 04:10 , David Gibson <david-xT8FGy+AXnRB3Ne2BGzF6laj5H9X9Tb+@public.gmane.org> wrote:
>>> 
>>> On Mon, Nov 28, 2016 at 02:10:35PM +0200, Pantelis Antoniou wrote:
>>>> 
>>>>> On Nov 28, 2016, at 06:12 , David Gibson <david-xT8FGy+AXnRB3Ne2BGzF6laj5H9X9Tb+@public.gmane.org> wrote:
>>>>> 
>>>>> On Fri, Nov 25, 2016 at 02:32:10PM +0200, Pantelis Antoniou wrote:
>>>>>> This patch enable the generation of symbols & local fixup information
>>>>>> for trees compiled with the -@ (--symbols) option.
>>>>>> 
>>>>>> Using this patch labels in the tree and their users emit information
>>>>>> in __symbols__ and __local_fixups__ nodes.
>>>>>> 
>>>>>> The __fixups__ node make possible the dynamic resolution of phandle
>>>>>> references which are present in the plugin tree but lie in the
>>>>>> tree that are applying the overlay against.
>>>>>> 
>>>>>> While there is a new magic number for dynamic device tree/overlays blobs
>>>>>> it is by default enabled. Remember to use -M to generate compatible
>>>>>> blobs.
>>>>>> 
>>>>>> Signed-off-by: Pantelis Antoniou <pantelis.antoniou-OWPKS81ov/FWk0Htik3J/w@public.gmane.org>
>>>>>> Signed-off-by: Sascha Hauer <s.hauer-bIcnvbaLZ9MEGnE8C9+IrQ@public.gmane.org>
>>>>>> Signed-off-by: Jan Luebbe <jlu-bIcnvbaLZ9MEGnE8C9+IrQ@public.gmane.org>
>>>>>> ---
>>>>>> Documentation/manual.txt |  25 +++++-
>>>>>> checks.c                 |   8 +-
>>>>>> dtc-lexer.l              |   5 ++
>>>>>> dtc-parser.y             |  50 +++++++++--
>>>>>> dtc.c                    |  39 +++++++-
>>>>>> dtc.h                    |  20 ++++-
>>>>>> fdtdump.c                |   2 +-
>>>>>> flattree.c               |  17 ++--
>>>>>> fstree.c                 |   2 +-
>>>>>> libfdt/fdt.c             |   2 +-
>>>>>> libfdt/fdt.h             |   3 +-
>>>>>> livetree.c               | 225 ++++++++++++++++++++++++++++++++++++++++++++++-
>>>>>> tests/mangle-layout.c    |   7 +-
>>>>>> 13 files changed, 375 insertions(+), 30 deletions(-)
>>>>>> 
>>>>>> diff --git a/Documentation/manual.txt b/Documentation/manual.txt
>>>>>> index 398de32..094893b 100644
>>>>>> --- a/Documentation/manual.txt
>>>>>> +++ b/Documentation/manual.txt
>>>>>> @@ -119,6 +119,24 @@ Options:
>>>>>> 	Make space for <number> reserve map entries
>>>>>> 	Relevant for dtb and asm output only.
>>>>>> 
>>>>>> +    -@
>>>>>> +	Generates a __symbols__ node at the root node of the resulting blob
>>>>>> +	for any node labels used, and for any local references using phandles
>>>>>> +	it also generates a __local_fixups__ node that tracks them.
>>>>>> +
>>>>>> +	When using the /plugin/ tag all unresolved label references to
>>>>>> +	be tracked in the __fixups__ node, making dynamic resolution possible.
>>>>>> +
>>>>>> +    -A
>>>>>> +	Generate automatically aliases for all node labels. This is similar to
>>>>>> +	the -@ option (the __symbols__ node contain identical information) but
>>>>>> +	the semantics are slightly different since no phandles are automatically
>>>>>> +	generated for labeled nodes.
>>>>>> +
>>>>>> +    -M
>>>>>> +	Generate blobs with the old FDT magic number for device tree objects.
>>>>>> +	By default blobs use the DTBO FDT magic number instead.
>>>>>> +
>>>>>>   -S <bytes>
>>>>>> 	Ensure the blob at least <bytes> long, adding additional
>>>>>> 	space if needed.
>>>>>> @@ -146,13 +164,18 @@ Additionally, dtc performs various sanity checks on the tree.
>>>>>> Here is a very rough overview of the layout of a DTS source file:
>>>>>> 
>>>>>> 
>>>>>> -    sourcefile:   list_of_memreserve devicetree
>>>>>> +    sourcefile:   versioninfo plugindecl list_of_memreserve devicetree
>>>>>> 
>>>>>>   memreserve:   label 'memreserve' ADDR ADDR ';'
>>>>>> 		| label 'memreserve' ADDR '-' ADDR ';'
>>>>>> 
>>>>>>   devicetree:   '/' nodedef
>>>>>> 
>>>>>> +    versioninfo:  '/' 'dts-v1' '/' ';'
>>>>>> +
>>>>>> +    plugindecl:   '/' 'plugin' '/' ';'
>>>>>> +                | /* empty */
>>>>>> +
>>>>>>   nodedef:      '{' list_of_property list_of_subnode '}' ';'
>>>>>> 
>>>>>>   property:     label PROPNAME '=' propdata ';'
>>>>>> diff --git a/checks.c b/checks.c
>>>>>> index 2bd27a4..4292f4b 100644
>>>>>> --- a/checks.c
>>>>>> +++ b/checks.c
>>>>>> @@ -487,8 +487,12 @@ static void fixup_phandle_references(struct check *c, struct boot_info *bi,
>>>>>> 
>>>>>> 			refnode = get_node_by_ref(dt, m->ref);
>>>>>> 			if (! refnode) {
>>>>>> -				FAIL(c, "Reference to non-existent node or label \"%s\"\n",
>>>>>> -				     m->ref);
>>>>>> +				if (!(bi->versionflags & VF_PLUGIN))
>>>>>> +					FAIL(c, "Reference to non-existent node or "
>>>>>> +							"label \"%s\"\n", m->ref);
>>>>>> +				else /* mark the entry as unresolved */
>>>>>> +					*((cell_t *)(prop->val.val + m->offset)) =
>>>>>> +						cpu_to_fdt32(0xffffffff);
>>>>>> 				continue;
>>>>>> 			}
>>>>>> 
>>>>>> diff --git a/dtc-lexer.l b/dtc-lexer.l
>>>>>> index 790fbf6..40bbc87 100644
>>>>>> --- a/dtc-lexer.l
>>>>>> +++ b/dtc-lexer.l
>>>>>> @@ -121,6 +121,11 @@ static void lexical_error(const char *fmt, ...);
>>>>>> 			return DT_V1;
>>>>>> 		}
>>>>>> 
>>>>>> +<*>"/plugin/"	{
>>>>>> +			DPRINT("Keyword: /plugin/\n");
>>>>>> +			return DT_PLUGIN;
>>>>>> +		}
>>>>>> +
>>>>>> <*>"/memreserve/"	{
>>>>>> 			DPRINT("Keyword: /memreserve/\n");
>>>>>> 			BEGIN_DEFAULT();
>>>>>> diff --git a/dtc-parser.y b/dtc-parser.y
>>>>>> index 14aaf2e..1a1f660 100644
>>>>>> --- a/dtc-parser.y
>>>>>> +++ b/dtc-parser.y
>>>>>> @@ -19,6 +19,7 @@
>>>>>> */
>>>>>> %{
>>>>>> #include <stdio.h>
>>>>>> +#include <inttypes.h>
>>>>>> 
>>>>>> #include "dtc.h"
>>>>>> #include "srcpos.h"
>>>>>> @@ -33,6 +34,7 @@ extern void yyerror(char const *s);
>>>>>> 
>>>>>> extern struct boot_info *the_boot_info;
>>>>>> extern bool treesource_error;
>>>>>> +
>>>>> 
>>>>> Extraneous whitespace change here
>>>>> 
>>>> 
>>>> OK.
>>>> 
>>>>>> %}
>>>>>> 
>>>>>> %union {
>>>>>> @@ -52,9 +54,11 @@ extern bool treesource_error;
>>>>>> 	struct node *nodelist;
>>>>>> 	struct reserve_info *re;
>>>>>> 	uint64_t integer;
>>>>>> +	unsigned int flags;
>>>>>> }
>>>>>> 
>>>>>> %token DT_V1
>>>>>> +%token DT_PLUGIN
>>>>>> %token DT_MEMRESERVE
>>>>>> %token DT_LSHIFT DT_RSHIFT DT_LE DT_GE DT_EQ DT_NE DT_AND DT_OR
>>>>>> %token DT_BITS
>>>>>> @@ -71,6 +75,8 @@ extern bool treesource_error;
>>>>>> 
>>>>>> %type <data> propdata
>>>>>> %type <data> propdataprefix
>>>>>> +%type <flags> versioninfo
>>>>>> +%type <flags> plugindecl
>>>>>> %type <re> memreserve
>>>>>> %type <re> memreserves
>>>>>> %type <array> arrayprefix
>>>>>> @@ -101,16 +107,34 @@ extern bool treesource_error;
>>>>>> %%
>>>>>> 
>>>>>> sourcefile:
>>>>>> -	  v1tag memreserves devicetree
>>>>>> +	  versioninfo plugindecl memreserves devicetree
>>>>>> +		{
>>>>>> +			the_boot_info = build_boot_info($1 | $2, $3, $4,
>>>>>> +							guess_boot_cpuid($4));
>>>>>> +		}
>>>>>> +	;
>>>>>> +
>>>>>> +versioninfo:
>>>>>> +	v1tag
>>>>>> 		{
>>>>>> -			the_boot_info = build_boot_info($2, $3,
>>>>>> -							guess_boot_cpuid($3));
>>>>>> +			$$ = VF_DT_V1;
>>>>>> 		}
>>>>>> 	;
>>>>>> 
>>>>>> v1tag:
>>>>>> 	  DT_V1 ';'
>>>>>> +	| DT_V1
>>>>>> 	| DT_V1 ';' v1tag
>>>>>> +
>>>>>> +plugindecl:
>>>>>> +	DT_PLUGIN ';'
>>>>>> +		{
>>>>>> +			$$ = VF_PLUGIN;
>>>>>> +		}
>>>>>> +	| /* empty */
>>>>>> +		{
>>>>>> +			$$ = 0;
>>>>>> +		}
>>>>>> 	;
>>>>>> 
>>>>>> memreserves:
>>>>>> @@ -161,10 +185,19 @@ devicetree:
>>>>>> 		{
>>>>>> 			struct node *target = get_node_by_ref($1, $2);
>>>>>> 
>>>>>> -			if (target)
>>>>>> +			if (target) {
>>>>>> 				merge_nodes(target, $3);
>>>>>> -			else
>>>>>> -				ERROR(&@2, "Label or path %s not found", $2);
>>>>>> +			} else {
>>>>>> +				/*
>>>>>> +				 * We rely on the rule being always:
>>>>>> +				 *   versioninfo plugindecl memreserves devicetree
>>>>>> +				 * so $-1 is what we want (plugindecl)
>>>>>> +				 */
>>>>>> +				if ($<flags>-1 & VF_PLUGIN)
>>>>> 
>>>>> o_O... ok.  I've never seen negative value references before.  Can you
>>>>> provide a link to some documentation saying this is actually supported
>>>>> usage in bison?  I wasn't able to find it when I looked.
>>>>> 
>>>> 
>>>> There is a section about inherited attributes in the flex & bison book by O’Reily.
>>>> 
>>>> https://books.google.gr/books?id=3Sr1V5J9_qMC&lpg=PP1&dq=flex%20bison&hl=el&pg=PP1#v=onepage&q=flex%20bison&f=false
>>>> 
>>>> There’s a direct link to the 2nd Edition of lex & yacc:
>>>> 
>>>> https://books.google.gr/books?id=fMPxfWfe67EC&lpg=PA183&ots=RcRSji2NAT&dq=yacc%20inherited%20attributes&hl=el&pg=PA183#v=onepage&q=yacc%20inherited%20attributes&f=false
>>> 
>>> Thanks for the link.  I still think moving the fragment assembly out
>>> of the parser will be a better idea long term, but this does address
>>> the main concern I had, so it will do for now.
>>> 
>>>>>> +					add_orphan_node($1, $3, $2);
>>>>>> +				else
>>>>>> +					ERROR(&@2, "Label or path %s not found", $2);
>>>>>> +			}
>>>>>> 			$$ = $1;
>>>>>> 		}
>>>>>> 	| devicetree DT_DEL_NODE DT_REF ';'
>>>>>> @@ -179,6 +212,11 @@ devicetree:
>>>>>> 
>>>>>> 			$$ = $1;
>>>>>> 		}
>>>>>> +	| /* empty */
>>>>>> +		{
>>>>>> +			/* build empty node */
>>>>>> +			$$ = name_node(build_node(NULL, NULL), "");
>>>>>> +		}
>>>>>> 	;
>>>>>> 
>>>>>> nodedef:
>>>>>> diff --git a/dtc.c b/dtc.c
>>>>>> index 9dcf640..06e91bc 100644
>>>>>> --- a/dtc.c
>>>>>> +++ b/dtc.c
>>>>>> @@ -32,6 +32,9 @@ int minsize;		/* Minimum blob size */
>>>>>> int padsize;		/* Additional padding to blob */
>>>>>> int alignsize;		/* Additional padding to blob accroding to the alignsize */
>>>>>> int phandle_format = PHANDLE_BOTH;	/* Use linux,phandle or phandle properties */
>>>>>> +int symbol_fixup_support;	/* enable symbols & fixup support */
>>>>>> +int auto_label_aliases;		/* auto generate labels -> aliases */
>>>>>> +int no_dtbo_magic;		/* use old FDT magic values for objects */
>>>>>> 
>>>>>> static int is_power_of_2(int x)
>>>>>> {
>>>>>> @@ -59,7 +62,7 @@ static void fill_fullpaths(struct node *tree, const char *prefix)
>>>>>> #define FDT_VERSION(version)	_FDT_VERSION(version)
>>>>>> #define _FDT_VERSION(version)	#version
>>>>>> static const char usage_synopsis[] = "dtc [options] <input file>";
>>>>>> -static const char usage_short_opts[] = "qI:O:o:V:d:R:S:p:a:fb:i:H:sW:E:hv";
>>>>>> +static const char usage_short_opts[] = "qI:O:o:V:d:R:S:p:a:fb:i:H:sW:E:@AMhv";
>>>>>> static struct option const usage_long_opts[] = {
>>>>>> 	{"quiet",            no_argument, NULL, 'q'},
>>>>>> 	{"in-format",         a_argument, NULL, 'I'},
>>>>>> @@ -78,6 +81,9 @@ static struct option const usage_long_opts[] = {
>>>>>> 	{"phandle",           a_argument, NULL, 'H'},
>>>>>> 	{"warning",           a_argument, NULL, 'W'},
>>>>>> 	{"error",             a_argument, NULL, 'E'},
>>>>>> +	{"symbols",	     no_argument, NULL, '@'},
>>>>>> +	{"auto-alias",       no_argument, NULL, 'A'},
>>>>>> +	{"no-dtbo-magic",    no_argument, NULL, 'M'},
>>>>>> 	{"help",             no_argument, NULL, 'h'},
>>>>>> 	{"version",          no_argument, NULL, 'v'},
>>>>>> 	{NULL,               no_argument, NULL, 0x0},
>>>>>> @@ -109,6 +115,9 @@ static const char * const usage_opts_help[] = {
>>>>>> 	 "\t\tboth   - Both \"linux,phandle\" and \"phandle\" properties",
>>>>>> 	"\n\tEnable/disable warnings (prefix with \"no-\")",
>>>>>> 	"\n\tEnable/disable errors (prefix with \"no-\")",
>>>>>> +	"\n\tEnable symbols/fixup support",
>>>>>> +	"\n\tEnable auto-alias of labels",
>>>>>> +	"\n\tDo not use DTBO magic value for plugin objects",
>>>>>> 	"\n\tPrint this help and exit",
>>>>>> 	"\n\tPrint version and exit",
>>>>>> 	NULL,
>>>>>> @@ -153,7 +162,7 @@ static const char *guess_input_format(const char *fname, const char *fallback)
>>>>>> 	fclose(f);
>>>>>> 
>>>>>> 	magic = fdt32_to_cpu(magic);
>>>>>> -	if (magic == FDT_MAGIC)
>>>>>> +	if (magic == FDT_MAGIC || magic == FDT_MAGIC_DTBO)
>>>>>> 		return "dtb";
>>>>>> 
>>>>>> 	return guess_type_by_name(fname, fallback);
>>>>>> @@ -172,6 +181,7 @@ int main(int argc, char *argv[])
>>>>>> 	FILE *outf = NULL;
>>>>>> 	int outversion = DEFAULT_FDT_VERSION;
>>>>>> 	long long cmdline_boot_cpuid = -1;
>>>>>> +	fdt32_t out_magic = FDT_MAGIC;
>>>>>> 
>>>>>> 	quiet      = 0;
>>>>>> 	reservenum = 0;
>>>>>> @@ -249,6 +259,16 @@ int main(int argc, char *argv[])
>>>>>> 			parse_checks_option(false, true, optarg);
>>>>>> 			break;
>>>>>> 
>>>>>> +		case '@':
>>>>>> +			symbol_fixup_support = 1;
>>>>>> +			break;
>>>>>> +		case 'A':
>>>>>> +			auto_label_aliases = 1;
>>>>>> +			break;
>>>>>> +		case 'M':
>>>>>> +			no_dtbo_magic = 1;
>>>>>> +			break;
>>>>>> +
>>>>>> 		case 'h':
>>>>>> 			usage(NULL);
>>>>>> 		default:
>>>>>> @@ -306,6 +326,14 @@ int main(int argc, char *argv[])
>>>>>> 	fill_fullpaths(bi->dt, "");
>>>>>> 	process_checks(force, bi);
>>>>>> 
>>>>>> +	if (auto_label_aliases)
>>>>>> +		generate_label_tree(bi->dt, "aliases", false);
>>>>>> +
>>>>>> +	if (symbol_fixup_support) {
>>>>>> +		generate_label_tree(bi->dt, "__symbols__", true);
>>>>>> +		generate_fixups_tree(bi->dt);
>>>>> 
>>>>> Hang on.. this doesn't seem right.  I thought -@ controlled the
>>>>> __symbols__ side (i.e. the part upon which we overlay) rather than the
>>>>> fixups side (the part which overlays).  A dtbo could certainly have
>>>>> both, of course, but for base trees, wouldn't you have symbols without
>>>>> fixups?  And should it be illegal to try to build a /plugin/ without
>>>>> -@?
>>>> 
>>>> It does control both for now. For base trees having the fixup nodes
>>>> will allow us to do probe order dependency tracking in the future.
>>> 
>>> Erm.. how?
>>> 
>>>> For plugins we need the __symbols__ node to support stacked overlays, i.e.
>>>> overlays referring label that were introduced by a previous overlay.
>>> 
>>> Yes, I realise that an overlay may well want __symbols__ as well.  But
>>> they still seem conceptually different.  I think -@ should control
>>> __symbols__ whereas /plugin/ should control __fixups__.
>>> 
>> 
>> It is easily done. Although using /plugin/ as an auto-magic option does both
>> just fine.
> 
> Sorry, I don't follow what you're saying.
> 

The last patch uses the /plugin/ tag to turn on the options that are required
(both symbols & fixups)

>>>> For plugins there is no requirement for now to actually contain references to
>>>> be resolved. It can easily be enforced though.
>>> 
>>> Sure, but I don't see the relevance of that here.  You could just omit
>>> the __fixups__ node if there's nothing to go into them.
>>> 
>> 
>> Hmm, yeah.
>> 
>> 
>> Regards
>> 
>> — Pantelis
>> 
> 
> -- 
> David Gibson			| I'll have my music baroque, and my code
> david AT gibson.dropbear.id.au	| minimalist, thank you.  NOT _the_ _other_
> 				| _way_ _around_!
> http://www.ozlabs.org/~dgibson

Regards

— Pantelis

--
To unsubscribe from this list: send the line "unsubscribe devicetree" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH v10 4/4] tests: Add overlay tests

[Pantelis Antoniou]

Add a number of tests for dynamic objects/overlays.

Re-use the original test by moving the contents to a .dtsi include

Signed-off-by: Pantelis Antoniou <pantelis.antoniou-OWPKS81ov/FWk0Htik3J/w@public.gmane.org>
---
 tests/overlay_overlay_dtc.dts     | 76 +----------------------------------
 tests/overlay_overlay_dtc.dtsi    | 83 +++++++++++++++++++++++++++++++++++++++
 tests/overlay_overlay_new_dtc.dts | 11 ++++++
 tests/overlay_overlay_simple.dts  | 12 ++++++
 tests/run_tests.sh                | 20 ++++++++++
 5 files changed, 127 insertions(+), 75 deletions(-)
 create mode 100644 tests/overlay_overlay_dtc.dtsi
 create mode 100644 tests/overlay_overlay_new_dtc.dts
 create mode 100644 tests/overlay_overlay_simple.dts

diff --git a/tests/overlay_overlay_dtc.dts b/tests/overlay_overlay_dtc.dts
index 30d2362..ca943ea 100644
--- a/tests/overlay_overlay_dtc.dts
+++ b/tests/overlay_overlay_dtc.dts
@@ -8,78 +8,4 @@
 /dts-v1/;
 /plugin/;
 
-/ {
-	/* Test that we can change an int by another */
-	fragment@0 {
-		target = <&test>;
-
-		__overlay__ {
-			test-int-property = <43>;
-		};
-	};
-
-	/* Test that we can replace a string by a longer one */
-	fragment@1 {
-		target = <&test>;
-
-		__overlay__ {
-			test-str-property = "foobar";
-		};
-	};
-
-	/* Test that we add a new property */
-	fragment@2 {
-		target = <&test>;
-
-		__overlay__ {
-			test-str-property-2 = "foobar2";
-		};
-	};
-
-	/* Test that we add a new node (by phandle) */
-	fragment@3 {
-		target = <&test>;
-
-		__overlay__ {
-			new-node {
-				new-property;
-			};
-		};
-	};
-
-	fragment@5 {
-		target = <&test>;
-
-		__overlay__ {
-			local: new-local-node {
-				new-property;
-			};
-		};
-	};
-
-	fragment@6 {
-		target = <&test>;
-
-		__overlay__ {
-			test-phandle = <&test>, <&local>;
-		};
-	};
-
-	fragment@7 {
-		target = <&test>;
-
-		__overlay__ {
-			test-several-phandle = <&local>, <&local>;
-		};
-	};
-
-	fragment@8 {
-		target = <&test>;
-
-		__overlay__ {
-			sub-test-node {
-				new-sub-test-property;
-			};
-		};
-	};
-};
+/include/ "overlay_overlay_dtc.dtsi"
diff --git a/tests/overlay_overlay_dtc.dtsi b/tests/overlay_overlay_dtc.dtsi
new file mode 100644
index 0000000..8ea8d5d
--- /dev/null
+++ b/tests/overlay_overlay_dtc.dtsi
@@ -0,0 +1,83 @@
+/*
+ * Copyright (c) 2016 NextThing Co
+ * Copyright (c) 2016 Free Electrons
+ * Copyright (c) 2016 Konsulko Inc.
+ *
+ * SPDX-License-Identifier:	GPL-2.0+
+ */
+
+/ {
+	/* Test that we can change an int by another */
+	fragment@0 {
+		target = <&test>;
+
+		__overlay__ {
+			test-int-property = <43>;
+		};
+	};
+
+	/* Test that we can replace a string by a longer one */
+	fragment@1 {
+		target = <&test>;
+
+		__overlay__ {
+			test-str-property = "foobar";
+		};
+	};
+
+	/* Test that we add a new property */
+	fragment@2 {
+		target = <&test>;
+
+		__overlay__ {
+			test-str-property-2 = "foobar2";
+		};
+	};
+
+	/* Test that we add a new node (by phandle) */
+	fragment@3 {
+		target = <&test>;
+
+		__overlay__ {
+			new-node {
+				new-property;
+			};
+		};
+	};
+
+	fragment@5 {
+		target = <&test>;
+
+		__overlay__ {
+			local: new-local-node {
+				new-property;
+			};
+		};
+	};
+
+	fragment@6 {
+		target = <&test>;
+
+		__overlay__ {
+			test-phandle = <&test>, <&local>;
+		};
+	};
+
+	fragment@7 {
+		target = <&test>;
+
+		__overlay__ {
+			test-several-phandle = <&local>, <&local>;
+		};
+	};
+
+	fragment@8 {
+		target = <&test>;
+
+		__overlay__ {
+			sub-test-node {
+				new-sub-test-property;
+			};
+		};
+	};
+};
diff --git a/tests/overlay_overlay_new_dtc.dts b/tests/overlay_overlay_new_dtc.dts
new file mode 100644
index 0000000..14d3f54
--- /dev/null
+++ b/tests/overlay_overlay_new_dtc.dts
@@ -0,0 +1,11 @@
+/*
+ * Copyright (c) 2016 NextThing Co
+ * Copyright (c) 2016 Free Electrons
+ * Copyright (c) 2016 Konsulko Inc.
+ *
+ * SPDX-License-Identifier:	GPL-2.0+
+ */
+
+/dts-v1/ /plugin/;
+
+/include/ "overlay_overlay_dtc.dtsi"
diff --git a/tests/overlay_overlay_simple.dts b/tests/overlay_overlay_simple.dts
new file mode 100644
index 0000000..8657e1e
--- /dev/null
+++ b/tests/overlay_overlay_simple.dts
@@ -0,0 +1,12 @@
+/*
+ * Copyright (c) 2016 Konsulko Inc.
+ *
+ * SPDX-License-Identifier:	GPL-2.0+
+ */
+
+/dts-v1/;
+/plugin/;
+
+&test {
+	test-int-property = <43>;
+};
diff --git a/tests/run_tests.sh b/tests/run_tests.sh
index e4139dd..99c09bc 100755
--- a/tests/run_tests.sh
+++ b/tests/run_tests.sh
@@ -181,6 +181,26 @@ overlay_tests () {
         run_dtc_test -@ -I dts -O dtb -o overlay_base_with_symbols.test.dtb overlay_base.dts
         run_dtc_test -@ -I dts -O dtb -o overlay_overlay_with_symbols.test.dtb overlay_overlay_dtc.dts
         run_test overlay overlay_base_with_symbols.test.dtb overlay_overlay_with_symbols.test.dtb
+
+        # new /plugin/ format
+        run_dtc_test -@ -I dts -O dtb -o overlay_overlay_new_with_symbols.test.dtb overlay_overlay_new_dtc.dts
+
+        # test new magic option
+        run_dtc_test -M@ -I dts -O dtb -o overlay_overlay_with_symbols_new_magic.test.dtb overlay_overlay_dtc.dts
+
+        # test plugin source to dtb and back
+        run_dtc_test -@ -I dtb -O dts -o overlay_overlay_dtc.test.dts overlay_overlay_with_symbols.test.dtb
+        run_dtc_test -@ -I dts -O dtb -o overlay_overlay_with_symbols.test.test.dtb overlay_overlay_dtc.test.dts
+        run_test dtbs_equal_ordered overlay_overlay_with_symbols.test.dtb overlay_overlay_with_symbols.test.test.dtb
+
+	# test simplified plugin syntax
+        run_dtc_test -@ -I dts -O dtb -o overlay_overlay_simple.dtb overlay_overlay_simple.dts
+
+	# test plugin source to dtb and back (with new magic)
+        run_dtc_test -@ -I dtb -O dts -o overlay_overlay_dtc_new_magic.test.dts overlay_overlay_with_symbols_new_magic.test.dtb
+        run_dtc_test -@ -I dts -O dtb -o overlay_overlay_with_symbols_new_magic.test.test.dtb overlay_overlay_dtc_new_magic.test.dts
+        run_test dtbs_equal_ordered overlay_overlay_with_symbols_new_magic.test.dtb overlay_overlay_with_symbols_new_magic.test.test.dtb
+
     fi
 
     # Bad fixup tests
-- 
2.1.4

--
To unsubscribe from this list: send the line "unsubscribe devicetree" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html