[PATCH 0/4] Add support for Hyperlinks and Markup on kernel-doc

[PATCH 0/4] Add support for Hyperlinks and Markup on kernel-doc

[Danilo Cesar Lemes de Paula]

This series add supports for hyperlink cross-references on Docbooks and
an optional markup syntax for in-source Documentation.

Danilo Cesar Lemes de Paula (4):
  scripts/kernel-doc: Adding cross-reference links to html
    documentation.
  scripts/kernel-doc: Replacing highlights hash by an array
  scripts/kernel-doc: Adding infrastructure for markdown support
  drm/doc: Convert to markdown

 Documentation/DocBook/Makefile     |  68 +++++++++----
 Documentation/DocBook/drm.tmpl     |  86 ----------------
 drivers/gpu/drm/drm_modeset_lock.c |  14 ++-
 drivers/gpu/drm/drm_prime.c        |  16 ++-
 drivers/gpu/drm/i915/i915_reg.h    |  48 ++++-----
 include/drm/drm_vma_manager.h      |  10 +-
 scripts/docproc.c                  |  49 ++++++---
 scripts/kernel-doc                 | 170 ++++++++++++++++++++++---------
 scripts/kernel-doc-xml-ref         | 198 +++++++++++++++++++++++++++++++++++++
 9 files changed, 444 insertions(+), 215 deletions(-)
 create mode 100755 scripts/kernel-doc-xml-ref

-- 
2.1.4

[PATCH 1/4] scripts/kernel-doc: Adding cross-reference links to html documentation.

[Danilo Cesar Lemes de Paula]

Functions, Structs and Parameters definitions on kernel documentation
are pure cosmetic, it only highlights the element.

To ease the navigation in the documentation we should use <links> inside
those tags so readers can easily jump between methods directly.

This was discussed in 2014[1] and is implemented by getting a list
of <refentries> from the DocBook XML to generate a database. Then it looks
for <function>,<structnames> and <paramdef> tags that matches the ones in
the database. As it only links existent references, no broken links are
added.

[1] - lists.freedesktop.org/archives/dri-devel/2014-August/065404.html

Signed-off-by: Danilo Cesar Lemes de Paula <danilo.cesar@collabora.co.uk>
Cc: Randy Dunlap <rdunlap@infradead.org>
Cc: Daniel Vetter <daniel.vetter@ffwll.ch>
Cc: Laurent Pinchart <laurent.pinchart@ideasonboard.com>
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: Herbert Xu <herbert@gondor.apana.org.au>
Cc: Stephan Mueller <smueller@chronox.de>
Cc: Michal Marek <mmarek@suse.cz>
Cc: linux-kernel@vger.kernel.org
Cc: linux-doc@vger.kernel.org
Cc: intel-gfx <intel-gfx@lists.freedesktop.org>
Cc: dri-devel <dri-devel@lists.freedesktop.org>
---
 Documentation/DocBook/Makefile |  43 ++++++---
 scripts/kernel-doc-xml-ref     | 198 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 228 insertions(+), 13 deletions(-)
 create mode 100755 scripts/kernel-doc-xml-ref

diff --git a/Documentation/DocBook/Makefile b/Documentation/DocBook/Makefile
index b6a6a2e..322255b 100644
--- a/Documentation/DocBook/Makefile
+++ b/Documentation/DocBook/Makefile
@@ -64,8 +64,9 @@ installmandocs: mandocs
 
 ###
 #External programs used
-KERNELDOC = $(srctree)/scripts/kernel-doc
-DOCPROC   = $(objtree)/scripts/docproc
+KERNELDOCXMLREF = $(srctree)/scripts/kernel-doc-xml-ref
+KERNELDOC       = $(srctree)/scripts/kernel-doc
+DOCPROC         = $(objtree)/scripts/docproc
 
 XMLTOFLAGS = -m $(srctree)/$(src)/stylesheet.xsl
 XMLTOFLAGS += --skip-validation
@@ -89,7 +90,7 @@ define rule_docproc
         ) > $(dir $@).$(notdir $@).cmd
 endef
 
-%.xml: %.tmpl $(KERNELDOC) $(DOCPROC) FORCE
+%.xml: %.tmpl $(KERNELDOC) $(DOCPROC) $(KERNELDOCXMLREF) FORCE
 	$(call if_changed_rule,docproc)
 
 # Tell kbuild to always build the programs
@@ -140,7 +141,20 @@ quiet_cmd_db2html = HTML    $@
 		echo '<a HREF="$(patsubst %.html,%,$(notdir $@))/index.html"> \
 		$(patsubst %.html,%,$(notdir $@))</a><p>' > $@
 
-%.html:	%.xml
+###
+# Rules to create an aux XML and .db, and use them to re-process the DocBook XML
+# to fill internal hyperlinks
+       gen_aux_xml = :
+ quiet_gen_aux_xml = echo '  XMLREF  $@'
+silent_gen_aux_xml = :
+%.aux.xml: %.xml
+	@$($(quiet)gen_aux_xml)
+	@rm -rf $@
+	@(cat $< | egrep "^<refentry id" | egrep -o "\".*\"" | cut -f 2 -d \" > $<.db)
+	@$(KERNELDOCXMLREF) -db $<.db $< > $@
+.PRECIOUS: %.aux.xml
+
+%.html:	%.aux.xml
 	@(which xmlto > /dev/null 2>&1) || \
 	 (echo "*** You need to install xmlto ***"; \
 	  exit 1)
@@ -209,15 +223,18 @@ dochelp:
 ###
 # Temporary files left by various tools
 clean-files := $(DOCBOOKS) \
-	$(patsubst %.xml, %.dvi,  $(DOCBOOKS)) \
-	$(patsubst %.xml, %.aux,  $(DOCBOOKS)) \
-	$(patsubst %.xml, %.tex,  $(DOCBOOKS)) \
-	$(patsubst %.xml, %.log,  $(DOCBOOKS)) \
-	$(patsubst %.xml, %.out,  $(DOCBOOKS)) \
-	$(patsubst %.xml, %.ps,   $(DOCBOOKS)) \
-	$(patsubst %.xml, %.pdf,  $(DOCBOOKS)) \
-	$(patsubst %.xml, %.html, $(DOCBOOKS)) \
-	$(patsubst %.xml, %.9,    $(DOCBOOKS)) \
+	$(patsubst %.xml, %.dvi,     $(DOCBOOKS)) \
+	$(patsubst %.xml, %.aux,     $(DOCBOOKS)) \
+	$(patsubst %.xml, %.tex,     $(DOCBOOKS)) \
+	$(patsubst %.xml, %.log,     $(DOCBOOKS)) \
+	$(patsubst %.xml, %.out,     $(DOCBOOKS)) \
+	$(patsubst %.xml, %.ps,      $(DOCBOOKS)) \
+	$(patsubst %.xml, %.pdf,     $(DOCBOOKS)) \
+	$(patsubst %.xml, %.html,    $(DOCBOOKS)) \
+	$(patsubst %.xml, %.9,       $(DOCBOOKS)) \
+	$(patsubst %.xml, %.aux.xml, $(DOCBOOKS)) \
+	$(patsubst %.xml, %.xml.db,  $(DOCBOOKS)) \
+	$(patsubst %.xml, %.xml,     $(DOCBOOKS)) \
 	$(index)
 
 clean-dirs := $(patsubst %.xml,%,$(DOCBOOKS)) man
diff --git a/scripts/kernel-doc-xml-ref b/scripts/kernel-doc-xml-ref
new file mode 100755
index 0000000..104a5a5
--- /dev/null
+++ b/scripts/kernel-doc-xml-ref
@@ -0,0 +1,198 @@
+#!/usr/bin/perl -w
+
+use strict;
+
+## Copyright (C) 2015  Intel Corporation                         ##
+#                                                                ##
+## This software falls under the GNU General Public License.     ##
+## Please read the COPYING file for more information             ##
+#
+#
+# This software reads a XML file and a list of valid interal
+# references to replace Docbook tags with links.
+#
+# The list of "valid internal references" must be one-per-line in the following format:
+#      API-struct-foo
+#      API-enum-bar
+#      API-my-function
+#
+# The software walks over the XML file looking for xml tags representing possible references
+# to the Document. Each reference will be cross checked against the "Valid Internal Reference" list. If
+# the referece is found it replaces its content by a <link> tag.
+#
+# usage:
+# kernel-doc-xml-ref -db filename
+#		     xml filename > outputfile
+
+# read arguments
+if ($#ARGV != 2) {
+	usage();
+}
+
+#Holds the database filename
+my $databasefile;
+my @database;
+
+#holds the inputfile
+my $inputfile;
+my $errors = 0;
+
+my %highlights = (
+	"<function>(.*?)</function>",
+	    "\"<function>\" . convert_function(\$1, \$line) . \"</function>\"",
+	"<structname>(.*?)</structname>",
+	    "\"<structname>\" . convert_struct(\$1) . \"</structname>\"",
+	"<funcdef>(.*?)<function>(.*?)</function></funcdef>",
+	    "\"<funcdef>\" . convert_param(\$1) . \"<function>\$2</function></funcdef>\"",
+	"<paramdef>(.*?)<parameter>(.*?)</parameter></paramdef>",
+	    "\"<paramdef>\" . convert_param(\$1) . \"<parameter>\$2</parameter></paramdef>\"");
+
+while($ARGV[0] =~ m/^-(.*)/) {
+	my $cmd = shift @ARGV;
+	if ($cmd eq "-db") {
+		$databasefile = shift @ARGV
+	} else {
+		usage();
+	}
+}
+$inputfile = shift @ARGV;
+
+sub open_database {
+	open (my $handle, '<', $databasefile) or die "Cannot open $databasefile";
+	chomp(my @lines = <$handle>);
+	close $handle;
+
+	@database = @lines;
+}
+
+sub process_file {
+	open_database();
+
+	my $dohighlight;
+	foreach my $pattern (keys %highlights) {
+		$dohighlight .=  "\$line =~ s:$pattern:$highlights{$pattern}:eg;\n";
+	}
+
+	open(FILE, $inputfile) or die("Could not open $inputfile") or die ("Cannot open $inputfile");
+	foreach my $line (<FILE>)  {
+		eval $dohighlight;
+		print $line;
+	}
+}
+
+sub trim($_)
+{
+	my $str = $_[0];
+	$str =~ s/^\s+|\s+$//g;
+	return $str
+}
+
+sub has_key_defined($_)
+{
+	if ( grep( /^$_[0]$/, @database)) {
+		return 1;
+	}
+	return 0;
+}
+
+# Gets a <function> content and add it a hyperlink if possible.
+sub convert_function($_)
+{
+	my $arg = $_[0];
+	my $key = $_[0];
+
+	my $line = $_[1];
+
+	$key = trim($key);
+
+	$key =~ s/[^A-Za-z0-9]/-/g;
+	$key = "API-" . $key;
+
+	# We shouldn't add links to <funcdef> prototype
+	if (!has_key_defined($key) || $line =~ m/\s+<funcdef/i) {
+		return $arg;
+	}
+
+	my $head = $arg;
+	my $tail = "";
+	if ($arg =~ /(.*?)( ?)$/) {
+		$head = $1;
+		$tail = $2;
+	}
+	return "<link linkend=\"$key\">$head</link>$tail";
+}
+
+# Converting a struct text to link
+sub convert_struct($_)
+{
+	my $arg = $_[0];
+	my $key = $_[0];
+	$key =~ s/(struct )?(\w)/$2/g;
+	$key =~ s/[^A-Za-z0-9]/-/g;
+	$key = "API-struct-" . $key;
+
+	if (!has_key_defined($key)) {
+		return $arg;
+	}
+
+	my ($head, $tail) = split_pointer($arg);
+	return "<link linkend=\"$key\">$head</link>$tail";
+}
+
+# Identify "object *" elements
+sub split_pointer($_)
+{
+	my $arg = $_[0];
+	if ($arg =~ /(.*?)( ?\* ?)/) {
+		return ($1, $2);
+	}
+	return ($arg, "");
+}
+
+sub convert_param($_)
+{
+	my $type = $_[0];
+	my $keyname = convert_key_name($type);
+
+	if (!has_key_defined($keyname)) {
+		return $type;
+	}
+
+	my ($head, $tail) = split_pointer($type);
+	return "<link linkend=\"$keyname\">$head</link>$tail";
+
+}
+
+# DocBook links are in the API-<TYPE>-<STRUCT-NAME> format
+# This method gets an element and returns a valid DocBook reference for it.
+sub convert_key_name($_)
+{
+	#Pattern $2 is optional and might be uninitialized
+	no warnings 'uninitialized';
+
+	my $str = $_[0];
+	$str =~ s/(const|static)? ?(struct)? ?([a-zA-Z0-9_]+) ?(\*|&)?/$2 $3/g ;
+
+	# trim
+	$str =~ s/^\s+|\s+$//g;
+
+	# spaces and _ to -
+	$str =~ s/[^A-Za-z0-9]/-/g;
+
+	return "API-" . $str;
+}
+
+sub usage {
+	print "Usage: $0 -db database filename\n";
+	print "         xml source file(s) > outputfile\n";
+	exit 1;
+}
+
+# starting point
+process_file();
+
+if ($errors) {
+	print STDERR "$errors errors\n";
+}
+
+exit($errors);
-- 
2.1.4

[PATCH 2/4] scripts/kernel-doc: Replacing highlights hash by an array

[Danilo Cesar Lemes de Paula]

The "highlight" code is very sensible to the order of the hash keys,
but the order of the keys cannot be predicted on Perl. It generates
faulty DocBook entries like:
	- @<function>device_for_each_child</function>

We should use an array for that job, so we can guarantee that the order
of the regex execution on dohighlight won't change.

Signed-off-by: Danilo Cesar Lemes de Paula <danilo.cesar@collabora.co.uk>
Cc: Randy Dunlap <rdunlap@infradead.org>
Cc: Daniel Vetter <daniel.vetter@ffwll.ch>
Cc: Laurent Pinchart <laurent.pinchart@ideasonboard.com>
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: Herbert Xu <herbert@gondor.apana.org.au>
Cc: Stephan Mueller <smueller@chronox.de>
Cc: Michal Marek <mmarek@suse.cz>
Cc: linux-kernel@vger.kernel.org
Cc: linux-doc@vger.kernel.org
Cc: intel-gfx <intel-gfx@lists.freedesktop.org>
Cc: dri-devel <dri-devel@lists.freedesktop.org>
---
 scripts/kernel-doc | 104 ++++++++++++++++++++++++++++++-----------------------
 1 file changed, 60 insertions(+), 44 deletions(-)

diff --git a/scripts/kernel-doc b/scripts/kernel-doc
index 9922e66..a38a69a 100755
--- a/scripts/kernel-doc
+++ b/scripts/kernel-doc
@@ -182,59 +182,73 @@ my $type_env = '(\$\w+)';
 #  One for each output format
 
 # these work fairly well
-my %highlights_html = ( $type_constant, "<i>\$1</i>",
-			$type_func, "<b>\$1</b>",
-			$type_struct_xml, "<i>\$1</i>",
-			$type_env, "<b><i>\$1</i></b>",
-			$type_param, "<tt><b>\$1</b></tt>" );
+my @highlights_html = (
+                       [$type_constant, "<i>\$1</i>"],
+                       [$type_func, "<b>\$1</b>"],
+                       [$type_struct_xml, "<i>\$1</i>"],
+                       [$type_env, "<b><i>\$1</i></b>"],
+                       [$type_param, "<tt><b>\$1</b></tt>"]
+                      );
 my $local_lt = "\\\\\\\\lt:";
 my $local_gt = "\\\\\\\\gt:";
 my $blankline_html = $local_lt . "p" . $local_gt;	# was "<p>"
 
 # html version 5
-my %highlights_html5 = ( $type_constant, "<span class=\"const\">\$1</span>",
-			$type_func, "<span class=\"func\">\$1</span>",
-			$type_struct_xml, "<span class=\"struct\">\$1</span>",
-			$type_env, "<span class=\"env\">\$1</span>",
-			$type_param, "<span class=\"param\">\$1</span>" );
+my @highlights_html5 = (
+                        [$type_constant, "<span class=\"const\">\$1</span>"],
+                        [$type_func, "<span class=\"func\">\$1</span>"],
+                        [$type_struct_xml, "<span class=\"struct\">\$1</span>"],
+                        [$type_env, "<span class=\"env\">\$1</span>"],
+                        [$type_param, "<span class=\"param\">\$1</span>]"]
+		       );
 my $blankline_html5 = $local_lt . "br /" . $local_gt;
 
 # XML, docbook format
-my %highlights_xml = ( "([^=])\\\"([^\\\"<]+)\\\"", "\$1<quote>\$2</quote>",
-			$type_constant, "<constant>\$1</constant>",
-			$type_func, "<function>\$1</function>",
-			$type_struct_xml, "<structname>\$1</structname>",
-			$type_env, "<envar>\$1</envar>",
-			$type_param, "<parameter>\$1</parameter>" );
+my @highlights_xml = (
+                      ["([^=])\\\"([^\\\"<]+)\\\"", "\$1<quote>\$2</quote>"],
+                      [$type_constant, "<constant>\$1</constant>"],
+                      [$type_struct_xml, "<structname>\$1</structname>"],
+                      [$type_param, "<parameter>\$1</parameter>"],
+                      [$type_func, "<function>\$1</function>"],
+                      [$type_env, "<envar>\$1</envar>"]
+		     );
 my $blankline_xml = $local_lt . "/para" . $local_gt . $local_lt . "para" . $local_gt . "\n";
 
 # gnome, docbook format
-my %highlights_gnome = ( $type_constant, "<replaceable class=\"option\">\$1</replaceable>",
-			 $type_func, "<function>\$1</function>",
-			 $type_struct, "<structname>\$1</structname>",
-			 $type_env, "<envar>\$1</envar>",
-			 $type_param, "<parameter>\$1</parameter>" );
+my @highlights_gnome = (
+                        [$type_constant, "<replaceable class=\"option\">\$1</replaceable>"],
+                        [$type_func, "<function>\$1</function>"],
+                        [$type_struct, "<structname>\$1</structname>"],
+                        [$type_env, "<envar>\$1</envar>"],
+                        [$type_param, "<parameter>\$1</parameter>" ]
+		       );
 my $blankline_gnome = "</para><para>\n";
 
 # these are pretty rough
-my %highlights_man = ( $type_constant, "\$1",
-		       $type_func, "\\\\fB\$1\\\\fP",
-		       $type_struct, "\\\\fI\$1\\\\fP",
-		       $type_param, "\\\\fI\$1\\\\fP" );
+my @highlights_man = (
+                      [$type_constant, "\$1"],
+                      [$type_func, "\\\\fB\$1\\\\fP"],
+                      [$type_struct, "\\\\fI\$1\\\\fP"],
+                      [$type_param, "\\\\fI\$1\\\\fP"]
+		     );
 my $blankline_man = "";
 
 # text-mode
-my %highlights_text = ( $type_constant, "\$1",
-			$type_func, "\$1",
-			$type_struct, "\$1",
-			$type_param, "\$1" );
+my @highlights_text = (
+                       [$type_constant, "\$1"],
+                       [$type_func, "\$1"],
+                       [$type_struct, "\$1"],
+                       [$type_param, "\$1"]
+		      );
 my $blankline_text = "";
 
 # list mode
-my %highlights_list = ( $type_constant, "\$1",
-			$type_func, "\$1",
-			$type_struct, "\$1",
-			$type_param, "\$1" );
+my @highlights_list = (
+                       [$type_constant, "\$1"],
+                       [$type_func, "\$1"],
+                       [$type_struct, "\$1"],
+                       [$type_param, "\$1"]
+		      );
 my $blankline_list = "";
 
 # read arguments
@@ -249,7 +263,7 @@ my $verbose = 0;
 my $output_mode = "man";
 my $output_preformatted = 0;
 my $no_doc_sections = 0;
-my %highlights = %highlights_man;
+my @highlights = @highlights_man;
 my $blankline = $blankline_man;
 my $modulename = "Kernel API";
 my $function_only = 0;
@@ -328,31 +342,31 @@ while ($ARGV[0] =~ m/^-(.*)/) {
     my $cmd = shift @ARGV;
     if ($cmd eq "-html") {
 	$output_mode = "html";
-	%highlights = %highlights_html;
+	@highlights = @highlights_html;
 	$blankline = $blankline_html;
     } elsif ($cmd eq "-html5") {
 	$output_mode = "html5";
-	%highlights = %highlights_html5;
+	@highlights = @highlights_html5;
 	$blankline = $blankline_html5;
     } elsif ($cmd eq "-man") {
 	$output_mode = "man";
-	%highlights = %highlights_man;
+	@highlights = @highlights_man;
 	$blankline = $blankline_man;
     } elsif ($cmd eq "-text") {
 	$output_mode = "text";
-	%highlights = %highlights_text;
+	@highlights = @highlights_text;
 	$blankline = $blankline_text;
     } elsif ($cmd eq "-docbook") {
 	$output_mode = "xml";
-	%highlights = %highlights_xml;
+	@highlights = @highlights_xml;
 	$blankline = $blankline_xml;
     } elsif ($cmd eq "-list") {
 	$output_mode = "list";
-	%highlights = %highlights_list;
+	@highlights = @highlights_list;
 	$blankline = $blankline_list;
     } elsif ($cmd eq "-gnome") {
 	$output_mode = "gnome";
-	%highlights = %highlights_gnome;
+	@highlights = @highlights_gnome;
 	$blankline = $blankline_gnome;
     } elsif ($cmd eq "-module") { # not needed for XML, inherits from calling document
 	$modulename = shift @ARGV;
@@ -2587,9 +2601,11 @@ $kernelversion = get_kernel_version();
 
 # generate a sequence of code that will splice in highlighting information
 # using the s// operator.
-foreach my $pattern (keys %highlights) {
-#   print STDERR "scanning pattern:$pattern, highlight:($highlights{$pattern})\n";
-    $dohighlight .=  "\$contents =~ s:$pattern:$highlights{$pattern}:gs;\n";
+foreach my $k (keys @highlights) {
+    my $pattern = $highlights[$k][0];
+    my $result = $highlights[$k][1];
+#   print STDERR "scanning pattern:$pattern, highlight:($result)\n";
+    $dohighlight .=  "\$contents =~ s:$pattern:$result:gs;\n";
 }
 
 # Read the file that maps relative names to absolute names for
-- 
2.1.4

[PATCH 3/4] scripts/kernel-doc: Adding infrastructure for markdown support

[Danilo Cesar Lemes de Paula]

Markdown support is given by calling an external tool, pandoc, for all
highlighted text on kernel-doc.

Pandoc converts Markdown text to proper Docbook tags, which will be
later translated to pdf, html or other targets.

This adds the capability of adding human-readle text highlight (bold,
underline, etc), bullet and numbered lists, simple tables, fixed-width
text (including asciiart), requiring minimal changes to current documentation.

At this moment, pandoc is totally optional. Docbooks ready for markdown
should be added to the MARKDOWNREADY variable inside the Makefile. In
case the developer doesn't have pandoc installed, Make will throw a
warning and the documentation build will continue, generating
simple Documentation without the features brought by pandoc.

Signed-off-by: Danilo Cesar Lemes de Paula <danilo.cesar@collabora.co.uk>
Cc: Randy Dunlap <rdunlap@infradead.org>
Cc: Daniel Vetter <daniel.vetter@ffwll.ch>
Cc: Laurent Pinchart <laurent.pinchart@ideasonboard.com>
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: Herbert Xu <herbert@gondor.apana.org.au>
Cc: Stephan Mueller <smueller@chronox.de>
Cc: Michal Marek <mmarek@suse.cz>
Cc: linux-kernel@vger.kernel.org
Cc: linux-doc@vger.kernel.org
Cc: intel-gfx <intel-gfx@lists.freedesktop.org>
Cc: dri-devel <dri-devel@lists.freedesktop.org>
---
 Documentation/DocBook/Makefile | 25 +++++++++++-----
 scripts/docproc.c              | 49 +++++++++++++++++++++----------
 scripts/kernel-doc             | 66 ++++++++++++++++++++++++++++++++++++++++--
 3 files changed, 115 insertions(+), 25 deletions(-)

diff --git a/Documentation/DocBook/Makefile b/Documentation/DocBook/Makefile
index 322255b..7c404b3 100644
--- a/Documentation/DocBook/Makefile
+++ b/Documentation/DocBook/Makefile
@@ -17,6 +17,8 @@ DOCBOOKS := z8530book.xml device-drivers.xml \
 	    tracepoint.xml drm.xml media_api.xml w1.xml \
 	    writing_musb_glue_layer.xml crypto-API.xml
 
+MARKDOWNREADY := 
+
 include Documentation/DocBook/media/Makefile
 
 ###
@@ -79,18 +81,23 @@ XMLTOFLAGS += --skip-validation
 # The following rules are used to generate the .xml documentation
 # required to generate the final targets. (ps, pdf, html).
 quiet_cmd_docproc = DOCPROC $@
-      cmd_docproc = SRCTREE=$(srctree)/ $(DOCPROC) doc $< >$@
+      cmd_docproc = SRCTREE=$(srctree)/ $(DOCPROC) doc $< $$USEMARKDOWN >$@
 define rule_docproc
-	set -e;								\
-        $(if $($(quiet)cmd_$(1)),echo '  $($(quiet)cmd_$(1))';) 	\
-        $(cmd_$(1)); 							\
-        ( 								\
-          echo 'cmd_$@ := $(cmd_$(1))'; 				\
-          echo $@: `SRCTREE=$(srctree) $(DOCPROC) depend $<`; 		\
+	set -e;									\
+	USEMARKDOWN="";								\
+	FILE=`basename $@`;							\
+	[[ "$(MARKDOWNREADY)" =~ "$${FILE}" ]] && USEMARKDOWN="-use-markdown";	\
+        $(if $($(quiet)cmd_$(1)),echo '  $($(quiet)cmd_$(1))';) 		\
+        $(cmd_$(1)); 								\
+        ( 									\
+          echo 'cmd_$@ := $(cmd_$(1))'; 					\
+          echo $@: `SRCTREE=$(srctree) $(DOCPROC) depend $<`; 			\
         ) > $(dir $@).$(notdir $@).cmd
 endef
 
 %.xml: %.tmpl $(KERNELDOC) $(DOCPROC) $(KERNELDOCXMLREF) FORCE
+	@(which pandoc > /dev/null 2>&1) || \
+	(echo "*** To get propper documentation you need to install pandoc ***";)
 	$(call if_changed_rule,docproc)
 
 # Tell kbuild to always build the programs
@@ -101,6 +108,10 @@ notfoundtemplate = echo "*** You have to install docbook-utils or xmlto ***"; \
 db2xtemplate = db2TYPE -o $(dir $@) $<
 xmltotemplate = xmlto TYPE $(XMLTOFLAGS) -o $(dir $@) $<
 
+ifneq ($(shell which pandoc >/dev/null 2>&1 && echo found),found)
+	MARKDOWNREADY := "";
+endif
+
 # determine which methods are available
 ifeq ($(shell which db2ps >/dev/null 2>&1 && echo found),found)
 	use-db2x = db2x
diff --git a/scripts/docproc.c b/scripts/docproc.c
index e267e621..45140b2 100644
--- a/scripts/docproc.c
+++ b/scripts/docproc.c
@@ -73,12 +73,15 @@ FILELINE * docsection;
 #define NOFUNCTION    "-nofunction"
 #define NODOCSECTIONS "-no-doc-sections"
 #define SHOWNOTFOUND  "-show-not-found"
+#define USEMARKDOWN   "-use-markdown"
 
 static char *srctree, *kernsrctree;
 
 static char **all_list = NULL;
 static int all_list_len = 0;
 
+static int use_markdown = 0;
+
 static void consume_symbol(const char *sym)
 {
 	int i;
@@ -95,10 +98,11 @@ static void consume_symbol(const char *sym)
 
 static void usage (void)
 {
-	fprintf(stderr, "Usage: docproc {doc|depend} file\n");
+	fprintf(stderr, "Usage: docproc {doc|depend} [--use-markdown] file\n");
 	fprintf(stderr, "Input is read from file.tmpl. Output is sent to stdout\n");
 	fprintf(stderr, "doc: frontend when generating kernel documentation\n");
 	fprintf(stderr, "depend: generate list of files referenced within file\n");
+	fprintf(stderr, "--use-markdown: pass --use-markdown to kernel-doc call\n");
 	fprintf(stderr, "Environment variable SRCTREE: absolute path to sources.\n");
 	fprintf(stderr, "                     KBUILD_SRC: absolute path to kernel source tree.\n");
 }
@@ -294,6 +298,9 @@ static void singfunc(char * filename, char * line)
 	int i, idx = 0;
 	int startofsym = 1;
 	vec[idx++] = KERNELDOC;
+	if (use_markdown) {
+		vec[idx++] = USEMARKDOWN;
+	}
 	vec[idx++] = DOCBOOK;
 	vec[idx++] = SHOWNOTFOUND;
 
@@ -328,8 +335,9 @@ static void singfunc(char * filename, char * line)
 static void docsect(char *filename, char *line)
 {
 	/* kerneldoc -docbook -show-not-found -function "section" file NULL */
-	char *vec[7];
+	char *vec[8];
 	char *s;
+	int idx = 0;
 
 	for (s = line; *s; s++)
 		if (*s == '\n')
@@ -342,30 +350,37 @@ static void docsect(char *filename, char *line)
 	consume_symbol(s);
 	free(s);
 
-	vec[0] = KERNELDOC;
-	vec[1] = DOCBOOK;
-	vec[2] = SHOWNOTFOUND;
-	vec[3] = FUNCTION;
-	vec[4] = line;
-	vec[5] = filename;
-	vec[6] = NULL;
+	vec[idx++] = KERNELDOC;
+	if (use_markdown) {
+		vec[idx++] = USEMARKDOWN;
+	}
+	vec[idx++] = DOCBOOK;
+	vec[idx++] = SHOWNOTFOUND;
+	vec[idx++] = FUNCTION;
+	vec[idx++] = line;
+	vec[idx++] = filename;
+	vec[idx] = NULL;
 	exec_kernel_doc(vec);
 }
 
 static void find_all_symbols(char *filename)
 {
-	char *vec[4]; /* kerneldoc -list file NULL */
+	char *vec[5]; /* kerneldoc -list file NULL */
 	pid_t pid;
 	int ret, i, count, start;
 	char real_filename[PATH_MAX + 1];
 	int pipefd[2];
 	char *data, *str;
 	size_t data_len = 0;
+	int idx = 0;
 
-	vec[0] = KERNELDOC;
-	vec[1] = LIST;
-	vec[2] = filename;
-	vec[3] = NULL;
+	vec[idx++] = KERNELDOC;
+	if (use_markdown) {
+		vec[idx++] = USEMARKDOWN;
+	}
+	vec[idx++] = LIST;
+	vec[idx++] = filename;
+	vec[idx] = NULL;
 
 	if (pipe(pipefd)) {
 		perror("pipe");
@@ -509,7 +524,7 @@ int main(int argc, char *argv[])
 	kernsrctree = getenv("KBUILD_SRC");
 	if (!kernsrctree || !*kernsrctree)
 		kernsrctree = srctree;
-	if (argc != 3) {
+	if (argc < 3 || argc > 4) {
 		usage();
 		exit(1);
 	}
@@ -521,6 +536,10 @@ int main(int argc, char *argv[])
 		exit(2);
 	}
 
+	if (argc == 4 && strcmp("-use-markdown", argv[3]) == 0) {
+		use_markdown = 1;
+	}
+
 	if (strcmp("doc", argv[1]) == 0) {
 		/* Need to do this in two passes.
 		 * First pass is used to collect all symbols exported
diff --git a/scripts/kernel-doc b/scripts/kernel-doc
index a38a69a..ab2e875 100755
--- a/scripts/kernel-doc
+++ b/scripts/kernel-doc
@@ -1,6 +1,7 @@
 #!/usr/bin/perl -w
 
 use strict;
+use IPC::Open3;
 
 ## Copyright (c) 1998 Michael Zucchi, All Rights Reserved        ##
 ## Copyright (C) 2000, 1  Tim Waugh <twaugh@redhat.com>          ##
@@ -258,6 +259,7 @@ if ($#ARGV == -1) {
 
 my $kernelversion;
 my $dohighlight = "";
+my $use_markdown = 0;
 
 my $verbose = 0;
 my $output_mode = "man";
@@ -378,6 +380,8 @@ while ($ARGV[0] =~ m/^-(.*)/) {
 	$function_only = 2;
 	$function = shift @ARGV;
 	$function_table{$function} = 1;
+    } elsif ($cmd eq "-use-markdown") {
+	$use_markdown = 1;
     } elsif ($cmd eq "-v") {
 	$verbose = 1;
     } elsif (($cmd eq "-h") || ($cmd eq "--help")) {
@@ -396,6 +400,7 @@ sub usage {
     print "         [ -no-doc-sections ]\n";
     print "         [ -function funcname [ -function funcname ...] ]\n";
     print "         [ -nofunction funcname [ -nofunction funcname ...] ]\n";
+    print "         [ -use-markdown ]\n";
     print "         [ -v ]\n";
     print "         c source file(s) > outputfile\n";
     print "         -v : verbose output, more warnings & other info listed\n";
@@ -469,6 +474,49 @@ sub dump_doc_section {
     }
 }
 
+sub markdown_to_docbook {
+	my $orig_content = $_[0];
+
+	my $pid = open3( \*CHLD_IN, \*CHLD_OUT, \*CHLD_ERR, "pandoc  --columns=80 -f markdown -t docbook" );
+
+	print CHLD_IN "$orig_content";
+	close(CHLD_IN);
+
+	waitpid($pid, 0);
+
+	my $content = "";
+	chomp(my @lines = <CHLD_OUT>);
+	foreach my $line (@lines) {
+		$content .= $line . "\n";
+	}
+	close(CHLD_OUT);
+	close(CHLD_ERR);
+
+	# pandoc insists in adding Main <para></para>, we should remove them.
+	$content =~ s:\A\s*<para>\s*\n(.*)\n</para>\Z$:$1:egsm;
+
+	return $content;
+}
+
+# Markdown->Docbook conversion by pandoc requires unescaped text
+# Kernel-doc converts every & to "&amp;", we need to convert it back.
+sub markdown_unescape
+{
+	my $text = shift;
+	my @lines = split /\n/, $text;
+
+	my @result;
+	foreach my $line (@lines) {
+		if ( $line =~ m /^    /s ) {
+			$line =~ s:\&amp;:\&:gs
+		}
+		push @result, $line;
+	}
+
+	return join "\n",@result;
+
+}
+
 ##
 # output function
 #
@@ -495,11 +543,19 @@ sub output_highlight {
 	$contents = local_unescape($contents);
 	# convert data read & converted thru xml_escape() into &xyz; format:
 	$contents =~ s/\\\\\\/\&/g;
+
+	if ($use_markdown) {
+		$contents = markdown_unescape($contents);
+	}
     }
+
 #   print STDERR "contents b4:$contents\n";
     eval $dohighlight;
     die $@ if $@;
 #   print STDERR "contents af:$contents\n";
+    if ($use_markdown) {
+        $contents = markdown_to_docbook($contents);
+    }
 
 #   strip whitespaces when generating html5
     if ($output_mode eq "html5") {
@@ -507,7 +563,8 @@ sub output_highlight {
 	$contents =~ s/\s+$//;
     }
     foreach $line (split "\n", $contents) {
-	if (! $output_preformatted) {
+	if (! $output_preformatted &&
+	    !($use_markdown && $line =~ m /^    /s)) {
 	    $line =~ s/^\s*//;
 	}
 	if ($line eq ""){
@@ -928,7 +985,9 @@ sub output_section_xml(%) {
 	print "<refsect1>\n";
 	print "<title>$section</title>\n";
 	if ($section =~ m/EXAMPLE/i) {
-	    print "<informalexample><programlisting>\n";
+	    print "<informalexample>\n";
+	    # programlisting is already included by pandoc
+	    print "<programlisting>\n" unless $use_markdown;
 	    $output_preformatted = 1;
 	} else {
 	    print "<para>\n";
@@ -936,7 +995,8 @@ sub output_section_xml(%) {
 	output_highlight($args{'sections'}{$section});
 	$output_preformatted = 0;
 	if ($section =~ m/EXAMPLE/i) {
-	    print "</programlisting></informalexample>\n";
+	    print "</programlisting>\n" unless $use_markdown;
+	    print "</informalexample>\n";
 	} else {
 	    print "</para>\n";
 	}
-- 
2.1.4

[PATCH 4/4] drm/doc: Convert to markdown

[Danilo Cesar Lemes de Paula]

DRM Docbook is now Markdown ready. This means its doc is able to
use markdown text on it.

* Documentation/DocBook/drm.tmpl: Contains a table duplicated from
  drivers/gpu/drm/i915/i915_reg.h. This is not needed anymore

* drivers/gpu/drm/drm_modeset_lock.c: had a code example that used
  to look pretty bad on html. Fixed by using proper code markup.

* drivers/gpu/drm/drm_prime.c: Remove spaces between lines to make
  a proper markup list.

* drivers/gpu/drm/i915/i915_reg.h: Altought pandoc supports tables,
  it doesn't support table cell spanning. But we can use fixed-width
  for those special cases.

* include/drm/drm_vma_manager.h: Another code example that should be
  proper indented with four spaces.

Signed-off-by: Danilo Cesar Lemes de Paula <danilo.cesar@collabora.co.uk>
Cc: Randy Dunlap <rdunlap@infradead.org>
Cc: Daniel Vetter <daniel.vetter@ffwll.ch>
Cc: Laurent Pinchart <laurent.pinchart@ideasonboard.com>
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: Herbert Xu <herbert@gondor.apana.org.au>
Cc: Stephan Mueller <smueller@chronox.de>
Cc: Michal Marek <mmarek@suse.cz>
Cc: linux-kernel@vger.kernel.org
Cc: linux-doc@vger.kernel.org
Cc: intel-gfx <intel-gfx@lists.freedesktop.org>
Cc: dri-devel <dri-devel@lists.freedesktop.org>
---
 Documentation/DocBook/Makefile     |  2 +-
 Documentation/DocBook/drm.tmpl     | 86 --------------------------------------
 drivers/gpu/drm/drm_modeset_lock.c | 14 +++----
 drivers/gpu/drm/drm_prime.c        | 16 +++----
 drivers/gpu/drm/i915/i915_reg.h    | 48 ++++++++++-----------
 include/drm/drm_vma_manager.h      | 10 ++---
 6 files changed, 42 insertions(+), 134 deletions(-)

diff --git a/Documentation/DocBook/Makefile b/Documentation/DocBook/Makefile
index 7c404b3..702e812 100644
--- a/Documentation/DocBook/Makefile
+++ b/Documentation/DocBook/Makefile
@@ -17,7 +17,7 @@ DOCBOOKS := z8530book.xml device-drivers.xml \
 	    tracepoint.xml drm.xml media_api.xml w1.xml \
 	    writing_musb_glue_layer.xml crypto-API.xml
 
-MARKDOWNREADY := 
+MARKDOWNREADY := drm.xml
 
 include Documentation/DocBook/media/Makefile
 
diff --git a/Documentation/DocBook/drm.tmpl b/Documentation/DocBook/drm.tmpl
index 2fb9a54..4fb4636 100644
--- a/Documentation/DocBook/drm.tmpl
+++ b/Documentation/DocBook/drm.tmpl
@@ -4073,92 +4073,6 @@ int num_ioctls;</synopsis>
       <sect2>
         <title>DPIO</title>
 !Pdrivers/gpu/drm/i915/i915_reg.h DPIO
-	<table id="dpiox2">
-	  <title>Dual channel PHY (VLV/CHV/BXT)</title>
-	  <tgroup cols="8">
-	    <colspec colname="c0" />
-	    <colspec colname="c1" />
-	    <colspec colname="c2" />
-	    <colspec colname="c3" />
-	    <colspec colname="c4" />
-	    <colspec colname="c5" />
-	    <colspec colname="c6" />
-	    <colspec colname="c7" />
-	    <spanspec spanname="ch0" namest="c0" nameend="c3" />
-	    <spanspec spanname="ch1" namest="c4" nameend="c7" />
-	    <spanspec spanname="ch0pcs01" namest="c0" nameend="c1" />
-	    <spanspec spanname="ch0pcs23" namest="c2" nameend="c3" />
-	    <spanspec spanname="ch1pcs01" namest="c4" nameend="c5" />
-	    <spanspec spanname="ch1pcs23" namest="c6" nameend="c7" />
-	    <thead>
-	      <row>
-		<entry spanname="ch0">CH0</entry>
-		<entry spanname="ch1">CH1</entry>
-	      </row>
-	    </thead>
-	    <tbody valign="top" align="center">
-	      <row>
-		<entry spanname="ch0">CMN/PLL/REF</entry>
-		<entry spanname="ch1">CMN/PLL/REF</entry>
-	      </row>
-	      <row>
-		<entry spanname="ch0pcs01">PCS01</entry>
-		<entry spanname="ch0pcs23">PCS23</entry>
-		<entry spanname="ch1pcs01">PCS01</entry>
-		<entry spanname="ch1pcs23">PCS23</entry>
-	      </row>
-	      <row>
-		<entry>TX0</entry>
-		<entry>TX1</entry>
-		<entry>TX2</entry>
-		<entry>TX3</entry>
-		<entry>TX0</entry>
-		<entry>TX1</entry>
-		<entry>TX2</entry>
-		<entry>TX3</entry>
-	      </row>
-	      <row>
-		<entry spanname="ch0">DDI0</entry>
-		<entry spanname="ch1">DDI1</entry>
-	      </row>
-	    </tbody>
-	  </tgroup>
-	</table>
-	<table id="dpiox1">
-	  <title>Single channel PHY (CHV/BXT)</title>
-	  <tgroup cols="4">
-	    <colspec colname="c0" />
-	    <colspec colname="c1" />
-	    <colspec colname="c2" />
-	    <colspec colname="c3" />
-	    <spanspec spanname="ch0" namest="c0" nameend="c3" />
-	    <spanspec spanname="ch0pcs01" namest="c0" nameend="c1" />
-	    <spanspec spanname="ch0pcs23" namest="c2" nameend="c3" />
-	    <thead>
-	      <row>
-		<entry spanname="ch0">CH0</entry>
-	      </row>
-	    </thead>
-	    <tbody valign="top" align="center">
-	      <row>
-		<entry spanname="ch0">CMN/PLL/REF</entry>
-	      </row>
-	      <row>
-		<entry spanname="ch0pcs01">PCS01</entry>
-		<entry spanname="ch0pcs23">PCS23</entry>
-	      </row>
-	      <row>
-		<entry>TX0</entry>
-		<entry>TX1</entry>
-		<entry>TX2</entry>
-		<entry>TX3</entry>
-	      </row>
-	      <row>
-		<entry spanname="ch0">DDI2</entry>
-	      </row>
-	    </tbody>
-	  </tgroup>
-	</table>
       </sect2>
 
       <sect2>
diff --git a/drivers/gpu/drm/drm_modeset_lock.c b/drivers/gpu/drm/drm_modeset_lock.c
index c0a5cd8..de59b0c 100644
--- a/drivers/gpu/drm/drm_modeset_lock.c
+++ b/drivers/gpu/drm/drm_modeset_lock.c
@@ -40,17 +40,15 @@
  * The basic usage pattern is to:
  *
  *     drm_modeset_acquire_init(&ctx)
- *   retry:
+ *     retry:
  *     foreach (lock in random_ordered_set_of_locks) {
- *       ret = drm_modeset_lock(lock, &ctx)
- *       if (ret == -EDEADLK) {
- *          drm_modeset_backoff(&ctx);
- *          goto retry;
- *       }
+ *         ret = drm_modeset_lock(lock, &ctx)
+ *         if (ret == -EDEADLK) {
+ *             drm_modeset_backoff(&ctx);
+ *             goto retry;
+ *         }
  *     }
- *
  *     ... do stuff ...
- *
  *     drm_modeset_drop_locks(&ctx);
  *     drm_modeset_acquire_fini(&ctx);
  */
diff --git a/drivers/gpu/drm/drm_prime.c b/drivers/gpu/drm/drm_prime.c
index 9f935f5..27aa718 100644
--- a/drivers/gpu/drm/drm_prime.c
+++ b/drivers/gpu/drm/drm_prime.c
@@ -313,19 +313,15 @@ static const struct dma_buf_ops drm_gem_prime_dmabuf_ops =  {
  *
  * Export callbacks:
  *
- *  - @gem_prime_pin (optional): prepare a GEM object for exporting
- *
- *  - @gem_prime_get_sg_table: provide a scatter/gather table of pinned pages
- *
- *  - @gem_prime_vmap: vmap a buffer exported by your driver
- *
- *  - @gem_prime_vunmap: vunmap a buffer exported by your driver
- *
- *  - @gem_prime_mmap (optional): mmap a buffer exported by your driver
+ *  * @gem_prime_pin (optional): prepare a GEM object for exporting
+ *  * @gem_prime_get_sg_table: provide a scatter/gather table of pinned pages
+ *  * @gem_prime_vmap: vmap a buffer exported by your driver
+ *  * @gem_prime_vunmap: vunmap a buffer exported by your driver
+ *  * @gem_prime_mmap (optional): mmap a buffer exported by your driver
  *
  * Import callback:
  *
- *  - @gem_prime_import_sg_table (import): produce a GEM object from another
+ *  * @gem_prime_import_sg_table (import): produce a GEM object from another
  *    driver's scatter/gather table
  */
 
diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h
index 2030f60..07c757c 100644
--- a/drivers/gpu/drm/i915/i915_reg.h
+++ b/drivers/gpu/drm/i915/i915_reg.h
@@ -778,31 +778,31 @@ enum skl_disp_power_wells {
  *
  * Note: DDI0 is digital port B, DD1 is digital port C, and DDI2 is
  * digital port D (CHV) or port A (BXT).
- */
-/*
- * Dual channel PHY (VLV/CHV/BXT)
- * ---------------------------------
- * |      CH0      |      CH1      |
- * |  CMN/PLL/REF  |  CMN/PLL/REF  |
- * |---------------|---------------| Display PHY
- * | PCS01 | PCS23 | PCS01 | PCS23 |
- * |-------|-------|-------|-------|
- * |TX0|TX1|TX2|TX3|TX0|TX1|TX2|TX3|
- * ---------------------------------
- * |     DDI0      |     DDI1      | DP/HDMI ports
- * ---------------------------------
  *
- * Single channel PHY (CHV/BXT)
- * -----------------
- * |      CH0      |
- * |  CMN/PLL/REF  |
- * |---------------| Display PHY
- * | PCS01 | PCS23 |
- * |-------|-------|
- * |TX0|TX1|TX2|TX3|
- * -----------------
- * |     DDI2      | DP/HDMI port
- * -----------------
+ *
+ *     Dual channel PHY (VLV/CHV/BXT)
+ *     ---------------------------------
+ *     |      CH0      |      CH1      |
+ *     |  CMN/PLL/REF  |  CMN/PLL/REF  |
+ *     |---------------|---------------| Display PHY
+ *     | PCS01 | PCS23 | PCS01 | PCS23 |
+ *     |-------|-------|-------|-------|
+ *     |TX0|TX1|TX2|TX3|TX0|TX1|TX2|TX3|
+ *     ---------------------------------
+ *     |     DDI0      |     DDI1      | DP/HDMI ports
+ *     ---------------------------------
+ *
+ *     Single channel PHY (CHV/BXT)
+ *     -----------------
+ *     |      CH0      |
+ *     |  CMN/PLL/REF  |
+ *     |---------------| Display PHY
+ *     | PCS01 | PCS23 |
+ *     |-------|-------|
+ *     |TX0|TX1|TX2|TX3|
+ *     -----------------
+ *     |     DDI2      | DP/HDMI port
+ *     -----------------
  */
 #define DPIO_DEVFN			0
 
diff --git a/include/drm/drm_vma_manager.h b/include/drm/drm_vma_manager.h
index 8cd402c..2ca44db 100644
--- a/include/drm/drm_vma_manager.h
+++ b/include/drm/drm_vma_manager.h
@@ -110,11 +110,11 @@ drm_vma_offset_exact_lookup(struct drm_vma_offset_manager *mgr,
  * Note: You're in atomic-context while holding this lock!
  *
  * Example:
- *   drm_vma_offset_lock_lookup(mgr);
- *   node = drm_vma_offset_lookup_locked(mgr);
- *   if (node)
- *       kref_get_unless_zero(container_of(node, sth, entr));
- *   drm_vma_offset_unlock_lookup(mgr);
+ *     drm_vma_offset_lock_lookup(mgr);
+ *     node = drm_vma_offset_lookup_locked(mgr);
+ *     if (node)
+ *         kref_get_unless_zero(container_of(node, sth, entr));
+ *     drm_vma_offset_unlock_lookup(mgr);
  */
 static inline void drm_vma_offset_lock_lookup(struct drm_vma_offset_manager *mgr)
 {
-- 
2.1.4

Re: [PATCH 0/4] Add support for Hyperlinks and Markup on kernel-doc

[Jonathan Corbet]

On Thu, 23 Jul 2015 15:16:23 -0300
Danilo Cesar Lemes de Paula <danilo.cesar@collabora.co.uk> wrote:

> This series add supports for hyperlink cross-references on Docbooks and
> an optional markup syntax for in-source Documentation.

I like the idea; just be warned that it's likely to be a week or two and
one more ocean crossing before I can take a serious look at this...

Thanks,

jon

Re: [PATCH 0/4] Add support for Hyperlinks and Markup on kernel-doc

[Danilo Cesar Lemes de Paula]

On 07/23/2015 05:29 PM, Jonathan Corbet wrote:
> On Thu, 23 Jul 2015 15:16:23 -0300
> Danilo Cesar Lemes de Paula <danilo.cesar@collabora.co.uk> wrote:
> 
>> This series add supports for hyperlink cross-references on Docbooks and
>> an optional markup syntax for in-source Documentation.
> 
> I like the idea; just be warned that it's likely to be a week or two and
> one more ocean crossing before I can take a serious look at this...
> 
> Thanks,

Hey,

Did you find time to take a look on this?

I understand that there's some discussion behind the curtains regarding
the markdown support, but the cross-reference-hyperlink patch is also in
the same patch series. It doesn't change any text in the docbook unless
there's really a cross-reference link to it. Different from the markdown
support (when people start to use markdown to write docs it will be hard
to go back), the cross-link stuff doesn't require/create any change to
current documentation, it is pretty safe to use.

Would you mind to share your plans about this?

Thanks,

Danilo Cesar

Re: [PATCH 0/4] Add support for Hyperlinks and Markup on kernel-doc

[Jonathan Corbet]

On Thu, 13 Aug 2015 20:09:35 -0300
Danilo Cesar Lemes de Paula <danilo.cesar@collabora.co.uk> wrote:

> Did you find time to take a look on this?

No.  Just when I thought things couldn't get crazier, my laptop died.  

	https://plus.google.com/+JonathanCorbet/posts/FBHp48dPb95

What spare time I had has been dedicated to recovering from that in time
to give my talk next week.
 
> I understand that there's some discussion behind the curtains regarding
> the markdown support, but the cross-reference-hyperlink patch is also in
> the same patch series. It doesn't change any text in the docbook unless
> there's really a cross-reference link to it. Different from the markdown
> support (when people start to use markdown to write docs it will be hard
> to go back), the cross-link stuff doesn't require/create any change to
> current documentation, it is pretty safe to use.
> 
> Would you mind to share your plans about this?

No behind-the-curtains discussions happening, or, let's say, if there
are, I've not been invited either.

I'd like to get back to the cross-reference stuff.  Last I tried, it
failed while building the media docs; have you been able to look at
that?  

Longer term, as you may know from the kernel summit discussions, I'd
really like to get rid of a lot of the XML gunk and put in something more
straightforward, be it based on Markdown or something else.  Doing that,
however, requires that I find the time to implement something that's
convincingly better.  It may happen soon, but I sure can't guarantee it.

Meanwhile, I think it would be a horrible mistake to delay useful work
because I have a gleam in my eye to do something different one of these
years, so I'll not do that.  I fully expect to merge all of the stuff
you've done, I just need to have a good look at it and test it out a
bit.  As I said before, I can't promise that for the 4.3 merge window,
but I'll try.

Apologies,

jon

Re: [PATCH 0/4] Add support for Hyperlinks and Markup on kernel-doc

[Danilo Cesar Lemes de Paula]

On 08/13/2015 08:20 PM, Jonathan Corbet wrote:
> On Thu, 13 Aug 2015 20:09:35 -0300
> Danilo Cesar Lemes de Paula <danilo.cesar@collabora.co.uk> wrote:
> 
>> Did you find time to take a look on this?
> 
> No.  Just when I thought things couldn't get crazier, my laptop died.  
> 
> 	https://plus.google.com/+JonathanCorbet/posts/FBHp48dPb95
> 
> What spare time I had has been dedicated to recovering from that in time
> to give my talk next week.

Those evil machines... =)

>  
>> I understand that there's some discussion behind the curtains regarding
>> the markdown support, but the cross-reference-hyperlink patch is also in
>> the same patch series. It doesn't change any text in the docbook unless
>> there's really a cross-reference link to it. Different from the markdown
>> support (when people start to use markdown to write docs it will be hard
>> to go back), the cross-link stuff doesn't require/create any change to
>> current documentation, it is pretty safe to use.
>>
>> Would you mind to share your plans about this?
> 
> No behind-the-curtains discussions happening, or, let's say, if there
> are, I've not been invited either.

I meant the Kernel summit mailing list, but looks like that thread died
even before my patches.

> 
> I'd like to get back to the cross-reference stuff.  Last I tried, it
> failed while building the media docs; have you been able to look at
> that?  

I did, and I didn't find anything.
Media API already spits lots of warnings without my patch. I did send
those warnings to a file and count them. My patch produces the same
amount of warnings as the original branch.
I did, however, use a clean build to test that. Daniel Vetter complained
that the Documentation was being rebuilt all the time, something wrong
with the dependencies. I did fix that in v2. Maybe the errors you got
were related to it?


> 
> Longer term, as you may know from the kernel summit discussions, I'd
> really like to get rid of a lot of the XML gunk and put in something more
> straightforward, be it based on Markdown or something else.  Doing that,
> however, requires that I find the time to implement something that's
> convincingly better.  It may happen soon, but I sure can't guarantee it.
> 
> Meanwhile, I think it would be a horrible mistake to delay useful work
> because I have a gleam in my eye to do something different one of these
> years, so I'll not do that.  I fully expect to merge all of the stuff
> you've done, I just need to have a good look at it and test it out a
> bit.  As I said before, I can't promise that for the 4.3 merge window,
> but I'll try.
> 
> Apologies,

No need to apologize, just wanted to know what was going on.

Danilo Cesar

Re: [PATCH 0/4] Add support for Hyperlinks and Markup on kernel-doc

[Stephan Mueller]

Am Donnerstag, 23. Juli 2015, 15:16:23 schrieb Danilo Cesar Lemes de Paula:

Hi Danilo,

>This series add supports for hyperlink cross-references on Docbooks and
>an optional markup syntax for in-source Documentation.

Can you please give an example what you mean with the latter?

Thanks for updating the documentation generation.


Ciao
Stephan

Re: [PATCH 0/4] Add support for Hyperlinks and Markup on kernel-doc

[Danilo Cesar Lemes de Paula]

On 07/25/2015 07:20 AM, Stephan Mueller wrote:
> Am Donnerstag, 23. Juli 2015, 15:16:23 schrieb Danilo Cesar Lemes de Paula:
> 
> Hi Danilo,
> 
>> This series add supports for hyperlink cross-references on Docbooks and
>> an optional markup syntax for in-source Documentation.
> 
> Can you please give an example what you mean with the latter?

Sure,

take a look on
https://people.collabora.com/~danilo/intel/Documentation.MarkDown/DocBook/drm/API-drm-dev-ref.html
The words "must" and "any" are emphasised, it's different from the original
https://people.collabora.com/~danilo/intel/Documentation.old/DocBook/drm/API-drm-dev-ref.html

That page is created from drivers/gpu/drm/drm_drv.c, part of drm_dev_ref Documentation, where the text is "You *must* already own".

This is a ridiculous simple example but there are some bullet lists, numbered lists and code example in my patch 4/4.
It can also do asciiart and tables.

I'm sending a v2 of this patch series later as I have some fixes for it.

Danilo Cesar

[PATCH v2 0/4] Add support for Hyperlinks and Markup on kernel-doc

[Danilo Cesar Lemes de Paula]

This series add supports for hyperlink cross-references on
Docbooks and an optional markup syntax for in-source
Documentation.

eg:
  https://people.collabora.com/~danilo/intel/Documentation.MarkDown/DocBook/drm/API-drm-dev-ref.html
old:
  https://people.collabora.com/~danilo/intel/Documentation.old/DocBook/drm/API-drm-dev-ref.html

Danilo Cesar Lemes de Paula (4):
  scripts/kernel-doc: Adding cross-reference links to html
    documentation.
  scripts/kernel-doc: Replacing highlights hash by an array
  scripts/kernel-doc: Adding infrastructure for markdown support
  drm/doc: Convert to markdown

 Documentation/DocBook/Makefile     |  68 +++++++++----
 Documentation/DocBook/drm.tmpl     |  86 ----------------
 drivers/gpu/drm/drm_modes.c        |  12 +--
 drivers/gpu/drm/drm_modeset_lock.c |  14 ++-
 drivers/gpu/drm/drm_prime.c        |  16 ++-
 drivers/gpu/drm/i915/i915_reg.h    |  48 ++++-----
 include/drm/drm_vma_manager.h      |  10 +-
 scripts/docproc.c                  |  54 +++++++---
 scripts/kernel-doc                 | 170 ++++++++++++++++++++++---------
 scripts/kernel-doc-xml-ref         | 198 +++++++++++++++++++++++++++++++++++++
 10 files changed, 454 insertions(+), 222 deletions(-)
 create mode 100755 scripts/kernel-doc-xml-ref

-- 
2.4.6

[PATCH v2 1/4] scripts/kernel-doc: Adding cross-reference links to html documentation.

[Danilo Cesar Lemes de Paula]

Functions, Structs and Parameters definitions on kernel documentation
are pure cosmetic, it only highlights the element.

To ease the navigation in the documentation we should use <links> inside
those tags so readers can easily jump between methods directly.

This was discussed in 2014[1] and is implemented by getting a list
of <refentries> from the DocBook XML to generate a database. Then it looks
for <function>,<structnames> and <paramdef> tags that matches the ones in
the database. As it only links existent references, no broken links are
added.

[1] - lists.freedesktop.org/archives/dri-devel/2014-August/065404.html

Signed-off-by: Danilo Cesar Lemes de Paula <danilo.cesar@collabora.co.uk>
Cc: Randy Dunlap <rdunlap@infradead.org>
Cc: Daniel Vetter <daniel.vetter@ffwll.ch>
Cc: Laurent Pinchart <laurent.pinchart@ideasonboard.com>
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: Herbert Xu <herbert@gondor.apana.org.au>
Cc: Stephan Mueller <smueller@chronox.de>
Cc: Michal Marek <mmarek@suse.cz>
Cc: linux-kernel@vger.kernel.org
Cc: linux-doc@vger.kernel.org
Cc: intel-gfx <intel-gfx@lists.freedesktop.org>
Cc: dri-devel <dri-devel@lists.freedesktop.org>
---
 Changelog:
  v1 (missing changlog from v1):
     * Add a "XMLREF FooBar.xml" as build output to reduce build noise.
     * Make kernel-doc-xml-ref ignore invalid arguments.
     * Improve kernel-doc-xml-ref documentation.

 Documentation/DocBook/Makefile |  43 ++++++---
 scripts/kernel-doc-xml-ref     | 198 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 228 insertions(+), 13 deletions(-)
 create mode 100755 scripts/kernel-doc-xml-ref

diff --git a/Documentation/DocBook/Makefile b/Documentation/DocBook/Makefile
index b6a6a2e..322255b 100644
--- a/Documentation/DocBook/Makefile
+++ b/Documentation/DocBook/Makefile
@@ -64,8 +64,9 @@ installmandocs: mandocs
 
 ###
 #External programs used
-KERNELDOC = $(srctree)/scripts/kernel-doc
-DOCPROC   = $(objtree)/scripts/docproc
+KERNELDOCXMLREF = $(srctree)/scripts/kernel-doc-xml-ref
+KERNELDOC       = $(srctree)/scripts/kernel-doc
+DOCPROC         = $(objtree)/scripts/docproc
 
 XMLTOFLAGS = -m $(srctree)/$(src)/stylesheet.xsl
 XMLTOFLAGS += --skip-validation
@@ -89,7 +90,7 @@ define rule_docproc
         ) > $(dir $@).$(notdir $@).cmd
 endef
 
-%.xml: %.tmpl $(KERNELDOC) $(DOCPROC) FORCE
+%.xml: %.tmpl $(KERNELDOC) $(DOCPROC) $(KERNELDOCXMLREF) FORCE
 	$(call if_changed_rule,docproc)
 
 # Tell kbuild to always build the programs
@@ -140,7 +141,20 @@ quiet_cmd_db2html = HTML    $@
 		echo '<a HREF="$(patsubst %.html,%,$(notdir $@))/index.html"> \
 		$(patsubst %.html,%,$(notdir $@))</a><p>' > $@
 
-%.html:	%.xml
+###
+# Rules to create an aux XML and .db, and use them to re-process the DocBook XML
+# to fill internal hyperlinks
+       gen_aux_xml = :
+ quiet_gen_aux_xml = echo '  XMLREF  $@'
+silent_gen_aux_xml = :
+%.aux.xml: %.xml
+	@$($(quiet)gen_aux_xml)
+	@rm -rf $@
+	@(cat $< | egrep "^<refentry id" | egrep -o "\".*\"" | cut -f 2 -d \" > $<.db)
+	@$(KERNELDOCXMLREF) -db $<.db $< > $@
+.PRECIOUS: %.aux.xml
+
+%.html:	%.aux.xml
 	@(which xmlto > /dev/null 2>&1) || \
 	 (echo "*** You need to install xmlto ***"; \
 	  exit 1)
@@ -209,15 +223,18 @@ dochelp:
 ###
 # Temporary files left by various tools
 clean-files := $(DOCBOOKS) \
-	$(patsubst %.xml, %.dvi,  $(DOCBOOKS)) \
-	$(patsubst %.xml, %.aux,  $(DOCBOOKS)) \
-	$(patsubst %.xml, %.tex,  $(DOCBOOKS)) \
-	$(patsubst %.xml, %.log,  $(DOCBOOKS)) \
-	$(patsubst %.xml, %.out,  $(DOCBOOKS)) \
-	$(patsubst %.xml, %.ps,   $(DOCBOOKS)) \
-	$(patsubst %.xml, %.pdf,  $(DOCBOOKS)) \
-	$(patsubst %.xml, %.html, $(DOCBOOKS)) \
-	$(patsubst %.xml, %.9,    $(DOCBOOKS)) \
+	$(patsubst %.xml, %.dvi,     $(DOCBOOKS)) \
+	$(patsubst %.xml, %.aux,     $(DOCBOOKS)) \
+	$(patsubst %.xml, %.tex,     $(DOCBOOKS)) \
+	$(patsubst %.xml, %.log,     $(DOCBOOKS)) \
+	$(patsubst %.xml, %.out,     $(DOCBOOKS)) \
+	$(patsubst %.xml, %.ps,      $(DOCBOOKS)) \
+	$(patsubst %.xml, %.pdf,     $(DOCBOOKS)) \
+	$(patsubst %.xml, %.html,    $(DOCBOOKS)) \
+	$(patsubst %.xml, %.9,       $(DOCBOOKS)) \
+	$(patsubst %.xml, %.aux.xml, $(DOCBOOKS)) \
+	$(patsubst %.xml, %.xml.db,  $(DOCBOOKS)) \
+	$(patsubst %.xml, %.xml,     $(DOCBOOKS)) \
 	$(index)
 
 clean-dirs := $(patsubst %.xml,%,$(DOCBOOKS)) man
diff --git a/scripts/kernel-doc-xml-ref b/scripts/kernel-doc-xml-ref
new file mode 100755
index 0000000..104a5a5
--- /dev/null
+++ b/scripts/kernel-doc-xml-ref
@@ -0,0 +1,198 @@
+#!/usr/bin/perl -w
+
+use strict;
+
+## Copyright (C) 2015  Intel Corporation                         ##
+#                                                                ##
+## This software falls under the GNU General Public License.     ##
+## Please read the COPYING file for more information             ##
+#
+#
+# This software reads a XML file and a list of valid interal
+# references to replace Docbook tags with links.
+#
+# The list of "valid internal references" must be one-per-line in the following format:
+#      API-struct-foo
+#      API-enum-bar
+#      API-my-function
+#
+# The software walks over the XML file looking for xml tags representing possible references
+# to the Document. Each reference will be cross checked against the "Valid Internal Reference" list. If
+# the referece is found it replaces its content by a <link> tag.
+#
+# usage:
+# kernel-doc-xml-ref -db filename
+#		     xml filename > outputfile
+
+# read arguments
+if ($#ARGV != 2) {
+	usage();
+}
+
+#Holds the database filename
+my $databasefile;
+my @database;
+
+#holds the inputfile
+my $inputfile;
+my $errors = 0;
+
+my %highlights = (
+	"<function>(.*?)</function>",
+	    "\"<function>\" . convert_function(\$1, \$line) . \"</function>\"",
+	"<structname>(.*?)</structname>",
+	    "\"<structname>\" . convert_struct(\$1) . \"</structname>\"",
+	"<funcdef>(.*?)<function>(.*?)</function></funcdef>",
+	    "\"<funcdef>\" . convert_param(\$1) . \"<function>\$2</function></funcdef>\"",
+	"<paramdef>(.*?)<parameter>(.*?)</parameter></paramdef>",
+	    "\"<paramdef>\" . convert_param(\$1) . \"<parameter>\$2</parameter></paramdef>\"");
+
+while($ARGV[0] =~ m/^-(.*)/) {
+	my $cmd = shift @ARGV;
+	if ($cmd eq "-db") {
+		$databasefile = shift @ARGV
+	} else {
+		usage();
+	}
+}
+$inputfile = shift @ARGV;
+
+sub open_database {
+	open (my $handle, '<', $databasefile) or die "Cannot open $databasefile";
+	chomp(my @lines = <$handle>);
+	close $handle;
+
+	@database = @lines;
+}
+
+sub process_file {
+	open_database();
+
+	my $dohighlight;
+	foreach my $pattern (keys %highlights) {
+		$dohighlight .=  "\$line =~ s:$pattern:$highlights{$pattern}:eg;\n";
+	}
+
+	open(FILE, $inputfile) or die("Could not open $inputfile") or die ("Cannot open $inputfile");
+	foreach my $line (<FILE>)  {
+		eval $dohighlight;
+		print $line;
+	}
+}
+
+sub trim($_)
+{
+	my $str = $_[0];
+	$str =~ s/^\s+|\s+$//g;
+	return $str
+}
+
+sub has_key_defined($_)
+{
+	if ( grep( /^$_[0]$/, @database)) {
+		return 1;
+	}
+	return 0;
+}
+
+# Gets a <function> content and add it a hyperlink if possible.
+sub convert_function($_)
+{
+	my $arg = $_[0];
+	my $key = $_[0];
+
+	my $line = $_[1];
+
+	$key = trim($key);
+
+	$key =~ s/[^A-Za-z0-9]/-/g;
+	$key = "API-" . $key;
+
+	# We shouldn't add links to <funcdef> prototype
+	if (!has_key_defined($key) || $line =~ m/\s+<funcdef/i) {
+		return $arg;
+	}
+
+	my $head = $arg;
+	my $tail = "";
+	if ($arg =~ /(.*?)( ?)$/) {
+		$head = $1;
+		$tail = $2;
+	}
+	return "<link linkend=\"$key\">$head</link>$tail";
+}
+
+# Converting a struct text to link
+sub convert_struct($_)
+{
+	my $arg = $_[0];
+	my $key = $_[0];
+	$key =~ s/(struct )?(\w)/$2/g;
+	$key =~ s/[^A-Za-z0-9]/-/g;
+	$key = "API-struct-" . $key;
+
+	if (!has_key_defined($key)) {
+		return $arg;
+	}
+
+	my ($head, $tail) = split_pointer($arg);
+	return "<link linkend=\"$key\">$head</link>$tail";
+}
+
+# Identify "object *" elements
+sub split_pointer($_)
+{
+	my $arg = $_[0];
+	if ($arg =~ /(.*?)( ?\* ?)/) {
+		return ($1, $2);
+	}
+	return ($arg, "");
+}
+
+sub convert_param($_)
+{
+	my $type = $_[0];
+	my $keyname = convert_key_name($type);
+
+	if (!has_key_defined($keyname)) {
+		return $type;
+	}
+
+	my ($head, $tail) = split_pointer($type);
+	return "<link linkend=\"$keyname\">$head</link>$tail";
+
+}
+
+# DocBook links are in the API-<TYPE>-<STRUCT-NAME> format
+# This method gets an element and returns a valid DocBook reference for it.
+sub convert_key_name($_)
+{
+	#Pattern $2 is optional and might be uninitialized
+	no warnings 'uninitialized';
+
+	my $str = $_[0];
+	$str =~ s/(const|static)? ?(struct)? ?([a-zA-Z0-9_]+) ?(\*|&)?/$2 $3/g ;
+
+	# trim
+	$str =~ s/^\s+|\s+$//g;
+
+	# spaces and _ to -
+	$str =~ s/[^A-Za-z0-9]/-/g;
+
+	return "API-" . $str;
+}
+
+sub usage {
+	print "Usage: $0 -db database filename\n";
+	print "         xml source file(s) > outputfile\n";
+	exit 1;
+}
+
+# starting point
+process_file();
+
+if ($errors) {
+	print STDERR "$errors errors\n";
+}
+
+exit($errors);
-- 
2.4.6

Re: [PATCH v2 1/4] scripts/kernel-doc: Adding cross-reference links to html documentation.

[Jonathan Corbet]

On Tue, 28 Jul 2015 16:45:15 -0300
Danilo Cesar Lemes de Paula <danilo.cesar@collabora.co.uk> wrote:

> Functions, Structs and Parameters definitions on kernel documentation
> are pure cosmetic, it only highlights the element.
> 
> To ease the navigation in the documentation we should use <links> inside
> those tags so readers can easily jump between methods directly.
> 
> This was discussed in 2014[1] and is implemented by getting a list
> of <refentries> from the DocBook XML to generate a database. Then it looks
> for <function>,<structnames> and <paramdef> tags that matches the ones in
> the database. As it only links existent references, no broken links are
> added.

So I had some airplane time today and was able to mess with this some.  I
can't make it break anymore, and it clearly improves the resulting
documentation, so I've applied it to the docs tree for 4.3.

I want to look at the rest of the stuff a bit more and play with it, but
it's hard to imagine why we wouldn't want that as well.  I'm a bit more
leery just because it adds another dependency to the build, even if it's
an optional dependency.  My thinking at the moment is to apply it shortly
after the merge window so it can have a long soak in linux-next before a
4.4 merge; hope that sounds good.

Thanks for doing this work,

jon

Re: [PATCH v2 1/4] scripts/kernel-doc: Adding cross-reference links to html documentation.

[Danilo Cesar Lemes de Paula]

On 08/17/2015 01:10 AM, Jonathan Corbet wrote:
> On Tue, 28 Jul 2015 16:45:15 -0300
> Danilo Cesar Lemes de Paula <danilo.cesar@collabora.co.uk> wrote:
> 
>> Functions, Structs and Parameters definitions on kernel documentation
>> are pure cosmetic, it only highlights the element.
>>
>> To ease the navigation in the documentation we should use <links> inside
>> those tags so readers can easily jump between methods directly.
>>
>> This was discussed in 2014[1] and is implemented by getting a list
>> of <refentries> from the DocBook XML to generate a database. Then it looks
>> for <function>,<structnames> and <paramdef> tags that matches the ones in
>> the database. As it only links existent references, no broken links are
>> added.
> 
> So I had some airplane time today and was able to mess with this some.  I
> can't make it break anymore, and it clearly improves the resulting
> documentation, so I've applied it to the docs tree for 4.3.
> 
> I want to look at the rest of the stuff a bit more and play with it, but
> it's hard to imagine why we wouldn't want that as well.  I'm a bit more
> leery just because it adds another dependency to the build, even if it's
> an optional dependency.  My thinking at the moment is to apply it shortly

I totally agree about the dependency stuff. I even discussed it with
Daniel Vetter a bit. I started by writing my-very-own-markup-parser to
put alongside kernel-doc to avoid external dependencies, but it gets too
complex too quickly (specially when dealing with tables and multi-line
stuff). It would be a pain to maintain a something like that, and the
world probably doesn't need yet-another-markup-parser, so I decided to
use another tool.

> after the merge window so it can have a long soak in linux-next before a
> 4.4 merge; hope that sounds good.

It does sound good. Thanks!

> 
> Thanks for doing this work,

Glad I could help.


Danilo

[PATCH v2 2/4] scripts/kernel-doc: Replacing highlights hash by an array

[Danilo Cesar Lemes de Paula]

The "highlight" code is very sensible to the order of the hash keys,
but the order of the keys cannot be predicted on Perl. It generates
faulty DocBook entries like:
	- @<function>device_for_each_child</function>

We should use an array for that job, so we can guarantee that the order
of the regex execution on dohighlight won't change.

Signed-off-by: Danilo Cesar Lemes de Paula <danilo.cesar@collabora.co.uk>
Cc: Randy Dunlap <rdunlap@infradead.org>
Cc: Daniel Vetter <daniel.vetter@ffwll.ch>
Cc: Laurent Pinchart <laurent.pinchart@ideasonboard.com>
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: Herbert Xu <herbert@gondor.apana.org.au>
Cc: Stephan Mueller <smueller@chronox.de>
Cc: Michal Marek <mmarek@suse.cz>
Cc: linux-kernel@vger.kernel.org
Cc: linux-doc@vger.kernel.org
Cc: intel-gfx <intel-gfx@lists.freedesktop.org>
Cc: dri-devel <dri-devel@lists.freedesktop.org>
---
 Changelog:
	v2: No changes

 scripts/kernel-doc | 104 ++++++++++++++++++++++++++++++-----------------------
 1 file changed, 60 insertions(+), 44 deletions(-)

diff --git a/scripts/kernel-doc b/scripts/kernel-doc
index 9922e66..a38a69a 100755
--- a/scripts/kernel-doc
+++ b/scripts/kernel-doc
@@ -182,59 +182,73 @@ my $type_env = '(\$\w+)';
 #  One for each output format
 
 # these work fairly well
-my %highlights_html = ( $type_constant, "<i>\$1</i>",
-			$type_func, "<b>\$1</b>",
-			$type_struct_xml, "<i>\$1</i>",
-			$type_env, "<b><i>\$1</i></b>",
-			$type_param, "<tt><b>\$1</b></tt>" );
+my @highlights_html = (
+                       [$type_constant, "<i>\$1</i>"],
+                       [$type_func, "<b>\$1</b>"],
+                       [$type_struct_xml, "<i>\$1</i>"],
+                       [$type_env, "<b><i>\$1</i></b>"],
+                       [$type_param, "<tt><b>\$1</b></tt>"]
+                      );
 my $local_lt = "\\\\\\\\lt:";
 my $local_gt = "\\\\\\\\gt:";
 my $blankline_html = $local_lt . "p" . $local_gt;	# was "<p>"
 
 # html version 5
-my %highlights_html5 = ( $type_constant, "<span class=\"const\">\$1</span>",
-			$type_func, "<span class=\"func\">\$1</span>",
-			$type_struct_xml, "<span class=\"struct\">\$1</span>",
-			$type_env, "<span class=\"env\">\$1</span>",
-			$type_param, "<span class=\"param\">\$1</span>" );
+my @highlights_html5 = (
+                        [$type_constant, "<span class=\"const\">\$1</span>"],
+                        [$type_func, "<span class=\"func\">\$1</span>"],
+                        [$type_struct_xml, "<span class=\"struct\">\$1</span>"],
+                        [$type_env, "<span class=\"env\">\$1</span>"],
+                        [$type_param, "<span class=\"param\">\$1</span>]"]
+		       );
 my $blankline_html5 = $local_lt . "br /" . $local_gt;
 
 # XML, docbook format
-my %highlights_xml = ( "([^=])\\\"([^\\\"<]+)\\\"", "\$1<quote>\$2</quote>",
-			$type_constant, "<constant>\$1</constant>",
-			$type_func, "<function>\$1</function>",
-			$type_struct_xml, "<structname>\$1</structname>",
-			$type_env, "<envar>\$1</envar>",
-			$type_param, "<parameter>\$1</parameter>" );
+my @highlights_xml = (
+                      ["([^=])\\\"([^\\\"<]+)\\\"", "\$1<quote>\$2</quote>"],
+                      [$type_constant, "<constant>\$1</constant>"],
+                      [$type_struct_xml, "<structname>\$1</structname>"],
+                      [$type_param, "<parameter>\$1</parameter>"],
+                      [$type_func, "<function>\$1</function>"],
+                      [$type_env, "<envar>\$1</envar>"]
+		     );
 my $blankline_xml = $local_lt . "/para" . $local_gt . $local_lt . "para" . $local_gt . "\n";
 
 # gnome, docbook format
-my %highlights_gnome = ( $type_constant, "<replaceable class=\"option\">\$1</replaceable>",
-			 $type_func, "<function>\$1</function>",
-			 $type_struct, "<structname>\$1</structname>",
-			 $type_env, "<envar>\$1</envar>",
-			 $type_param, "<parameter>\$1</parameter>" );
+my @highlights_gnome = (
+                        [$type_constant, "<replaceable class=\"option\">\$1</replaceable>"],
+                        [$type_func, "<function>\$1</function>"],
+                        [$type_struct, "<structname>\$1</structname>"],
+                        [$type_env, "<envar>\$1</envar>"],
+                        [$type_param, "<parameter>\$1</parameter>" ]
+		       );
 my $blankline_gnome = "</para><para>\n";
 
 # these are pretty rough
-my %highlights_man = ( $type_constant, "\$1",
-		       $type_func, "\\\\fB\$1\\\\fP",
-		       $type_struct, "\\\\fI\$1\\\\fP",
-		       $type_param, "\\\\fI\$1\\\\fP" );
+my @highlights_man = (
+                      [$type_constant, "\$1"],
+                      [$type_func, "\\\\fB\$1\\\\fP"],
+                      [$type_struct, "\\\\fI\$1\\\\fP"],
+                      [$type_param, "\\\\fI\$1\\\\fP"]
+		     );
 my $blankline_man = "";
 
 # text-mode
-my %highlights_text = ( $type_constant, "\$1",
-			$type_func, "\$1",
-			$type_struct, "\$1",
-			$type_param, "\$1" );
+my @highlights_text = (
+                       [$type_constant, "\$1"],
+                       [$type_func, "\$1"],
+                       [$type_struct, "\$1"],
+                       [$type_param, "\$1"]
+		      );
 my $blankline_text = "";
 
 # list mode
-my %highlights_list = ( $type_constant, "\$1",
-			$type_func, "\$1",
-			$type_struct, "\$1",
-			$type_param, "\$1" );
+my @highlights_list = (
+                       [$type_constant, "\$1"],
+                       [$type_func, "\$1"],
+                       [$type_struct, "\$1"],
+                       [$type_param, "\$1"]
+		      );
 my $blankline_list = "";
 
 # read arguments
@@ -249,7 +263,7 @@ my $verbose = 0;
 my $output_mode = "man";
 my $output_preformatted = 0;
 my $no_doc_sections = 0;
-my %highlights = %highlights_man;
+my @highlights = @highlights_man;
 my $blankline = $blankline_man;
 my $modulename = "Kernel API";
 my $function_only = 0;
@@ -328,31 +342,31 @@ while ($ARGV[0] =~ m/^-(.*)/) {
     my $cmd = shift @ARGV;
     if ($cmd eq "-html") {
 	$output_mode = "html";
-	%highlights = %highlights_html;
+	@highlights = @highlights_html;
 	$blankline = $blankline_html;
     } elsif ($cmd eq "-html5") {
 	$output_mode = "html5";
-	%highlights = %highlights_html5;
+	@highlights = @highlights_html5;
 	$blankline = $blankline_html5;
     } elsif ($cmd eq "-man") {
 	$output_mode = "man";
-	%highlights = %highlights_man;
+	@highlights = @highlights_man;
 	$blankline = $blankline_man;
     } elsif ($cmd eq "-text") {
 	$output_mode = "text";
-	%highlights = %highlights_text;
+	@highlights = @highlights_text;
 	$blankline = $blankline_text;
     } elsif ($cmd eq "-docbook") {
 	$output_mode = "xml";
-	%highlights = %highlights_xml;
+	@highlights = @highlights_xml;
 	$blankline = $blankline_xml;
     } elsif ($cmd eq "-list") {
 	$output_mode = "list";
-	%highlights = %highlights_list;
+	@highlights = @highlights_list;
 	$blankline = $blankline_list;
     } elsif ($cmd eq "-gnome") {
 	$output_mode = "gnome";
-	%highlights = %highlights_gnome;
+	@highlights = @highlights_gnome;
 	$blankline = $blankline_gnome;
     } elsif ($cmd eq "-module") { # not needed for XML, inherits from calling document
 	$modulename = shift @ARGV;
@@ -2587,9 +2601,11 @@ $kernelversion = get_kernel_version();
 
 # generate a sequence of code that will splice in highlighting information
 # using the s// operator.
-foreach my $pattern (keys %highlights) {
-#   print STDERR "scanning pattern:$pattern, highlight:($highlights{$pattern})\n";
-    $dohighlight .=  "\$contents =~ s:$pattern:$highlights{$pattern}:gs;\n";
+foreach my $k (keys @highlights) {
+    my $pattern = $highlights[$k][0];
+    my $result = $highlights[$k][1];
+#   print STDERR "scanning pattern:$pattern, highlight:($result)\n";
+    $dohighlight .=  "\$contents =~ s:$pattern:$result:gs;\n";
 }
 
 # Read the file that maps relative names to absolute names for
-- 
2.4.6

Re: [PATCH v2 2/4] scripts/kernel-doc: Replacing highlights hash by an array

[Mauro Carvalho Chehab]

Hi Danilo,

Em Tue, 28 Jul 2015 16:45:16 -0300
Danilo Cesar Lemes de Paula <danilo.cesar@collabora.co.uk> escreveu:

> The "highlight" code is very sensible to the order of the hash keys,
> but the order of the keys cannot be predicted on Perl. It generates
> faulty DocBook entries like:
> 	- @<function>device_for_each_child</function>
> 
> We should use an array for that job, so we can guarantee that the order
> of the regex execution on dohighlight won't change.

...

> @@ -2587,9 +2601,11 @@ $kernelversion = get_kernel_version();
>  
>  # generate a sequence of code that will splice in highlighting information
>  # using the s// operator.
> -foreach my $pattern (keys %highlights) {
> -#   print STDERR "scanning pattern:$pattern, highlight:($highlights{$pattern})\n";
> -    $dohighlight .=  "\$contents =~ s:$pattern:$highlights{$pattern}:gs;\n";
> +foreach my $k (keys @highlights) {

The above causes some versions of perl to fail, as keys expect a
hash argument:

Execution of .//scripts/kernel-doc aborted due to compilation errors.
Type of arg 1 to keys must be hash (not private array) at .//scripts/kernel-doc line 2714, near "@highlights) "

This is happening at linuxtv.org server, with runs perl version 5.10.1.

I had to revert this patch in order to be able to keep building the
documentation of the media kABI on our server.

Regards,
Mauro

Re: [PATCH v2 2/4] scripts/kernel-doc: Replacing highlights hash by an array

[Jonathan Corbet]

On Tue, 17 Nov 2015 08:40:46 -0200
Mauro Carvalho Chehab <mchehab@osg.samsung.com> wrote:

> The above causes some versions of perl to fail, as keys expect a
> hash argument:
> 
> Execution of .//scripts/kernel-doc aborted due to compilation errors.
> Type of arg 1 to keys must be hash (not private array) at .//scripts/kernel-doc line 2714, near "@highlights) "
> 
> This is happening at linuxtv.org server, with runs perl version 5.10.1.

OK, that's not good.  But I'm not quite sure what to do about it.

Perl 5.10.1 is a little over six years old.  Nobody else has complained
(yet) about this problem.  So it might be best to "fix" this with a
minimum version added to the Changes file.

Or maybe we need to revert the patch.

So I'm far from a Perl expert, so I have no clue what the minimum version
would be if we were to say "5.10.1 is too old."  I don't suppose anybody
out there knows?

Thanks,

jon

Re: [PATCH v2 2/4] scripts/kernel-doc: Replacing highlights hash by an array

[Mauro Carvalho Chehab]

Em Tue, 17 Nov 2015 07:44:31 -0700
Jonathan Corbet <corbet@lwn.net> escreveu:

> On Tue, 17 Nov 2015 08:40:46 -0200
> Mauro Carvalho Chehab <mchehab@osg.samsung.com> wrote:
> 
> > The above causes some versions of perl to fail, as keys expect a
> > hash argument:
> > 
> > Execution of .//scripts/kernel-doc aborted due to compilation errors.
> > Type of arg 1 to keys must be hash (not private array) at .//scripts/kernel-doc line 2714, near "@highlights) "
> > 
> > This is happening at linuxtv.org server, with runs perl version 5.10.1.
> 
> OK, that's not good.  But I'm not quite sure what to do about it.
> 
> Perl 5.10.1 is a little over six years old.  Nobody else has complained
> (yet) about this problem.  So it might be best to "fix" this with a
> minimum version added to the Changes file.
> 
> Or maybe we need to revert the patch.
> 
> So I'm far from a Perl expert, so I have no clue what the minimum version
> would be if we were to say "5.10.1 is too old."  I don't suppose anybody
> out there knows?

I'm also not a Perl expert, and never saw before the usage of "keys" on
an array. Yet, according with:
	http://perldoc.perl.org/functions/keys.html

"in Perl 5.12 or later only, the indices of an array"

If so, then maybe we could replace:
	foreach my $k (keys @highlights)

by a more C style variant, with all versions of perl 5:
	for (my $k = 0; $k < @highlights; $k++) {

The enclosed patch should do the trick. I tested it with perl 5.10 and 
perl 5.22 it worked fine with both versions.

Regards,
Mauro

-

kernel-doc: Make it compatible with Perl versions below 5.12 again

Changeset 4d73270192ec('scripts/kernel-doc: Replacing highlights
hash by an array') broke compatibility of the kernel-doc script with
older versions of perl by using "keys ARRAY" syntax with is available
only on Perl 5.12 or newer, according with:
	http://perldoc.perl.org/functions/keys.html

Restore backward compatibility by replacing "foreach my $k (keys ARRAY)"
by a C-like variant: "for (my $k = 0; $k < !ARRAY; $k++)"

Signed-off-by: Mauro Carvalho Chehab <mchehab@osg.samsung.com>

diff --git a/scripts/kernel-doc b/scripts/kernel-doc
index 125b906..1f61def 100755
--- a/scripts/kernel-doc
+++ b/scripts/kernel-doc
@@ -2711,7 +2711,7 @@ $kernelversion = get_kernel_version();
 
 # generate a sequence of code that will splice in highlighting information
 # using the s// operator.
-foreach my $k (keys @highlights) {
+for (my $k = 0; $k < @highlights; $k++) {
     my $pattern = $highlights[$k][0];
     my $result = $highlights[$k][1];
 #   print STDERR "scanning pattern:$pattern, highlight:($result)\n";

Re: [PATCH v2 2/4] scripts/kernel-doc: Replacing highlights hash by an array

[Danilo Cesar Lemes de Paula]

On 17-11-2015 13:29, Mauro Carvalho Chehab wrote:
> Em Tue, 17 Nov 2015 07:44:31 -0700
> Jonathan Corbet <corbet@lwn.net> escreveu:
> 
>> On Tue, 17 Nov 2015 08:40:46 -0200
>> Mauro Carvalho Chehab <mchehab@osg.samsung.com> wrote:
>>
>>> The above causes some versions of perl to fail, as keys expect a
>>> hash argument:
>>>
>>> Execution of .//scripts/kernel-doc aborted due to compilation errors.
>>> Type of arg 1 to keys must be hash (not private array) at .//scripts/kernel-doc line 2714, near "@highlights) "
>>>
>>> This is happening at linuxtv.org server, with runs perl version 5.10.1.
>>
>> OK, that's not good.  But I'm not quite sure what to do about it.
>>
>> Perl 5.10.1 is a little over six years old.  Nobody else has complained
>> (yet) about this problem.  So it might be best to "fix" this with a
>> minimum version added to the Changes file.
>>
>> Or maybe we need to revert the patch.
>>
>> So I'm far from a Perl expert, so I have no clue what the minimum version
>> would be if we were to say "5.10.1 is too old."  I don't suppose anybody
>> out there knows?
> 
> I'm also not a Perl expert, and never saw before the usage of "keys" on
> an array. Yet, according with:
> 	http://perldoc.perl.org/functions/keys.html
> 
> "in Perl 5.12 or later only, the indices of an array"
> 
> If so, then maybe we could replace:
> 	foreach my $k (keys @highlights)
> 
> by a more C style variant, with all versions of perl 5:
> 	for (my $k = 0; $k < @highlights; $k++) {
> 
> The enclosed patch should do the trick. I tested it with perl 5.10 and 
> perl 5.22 it worked fine with both versions.

I'm Not a perl guru myself either =/.

But thanks for fixing it Mauro!

Danilo

Re: [PATCH v2 2/4] scripts/kernel-doc: Replacing highlights hash by an array

[Jonathan Corbet]

On Tue, 17 Nov 2015 13:29:49 -0200
Mauro Carvalho Chehab <mchehab@osg.samsung.com> wrote:

> The enclosed patch should do the trick. I tested it with perl 5.10 and 
> perl 5.22 it worked fine with both versions.

Indeed it seems to work - thanks!  Applied to the docs tree, I'll get it
upstream before too long.

jon

Re: [PATCH v2 2/4] scripts/kernel-doc: Replacing highlights hash by an array

[Mauro Carvalho Chehab]

Em Tue, 17 Nov 2015 17:21:32 -0700
Jonathan Corbet <corbet@lwn.net> escreveu:

> On Tue, 17 Nov 2015 13:29:49 -0200
> Mauro Carvalho Chehab <mchehab@osg.samsung.com> wrote:
> 
> > The enclosed patch should do the trick. I tested it with perl 5.10 and 
> > perl 5.22 it worked fine with both versions.
> 
> Indeed it seems to work - thanks!  Applied to the docs tree, I'll get it
> upstream before too long.

Thanks, Jon!

Regards,
Mauro

[PATCH v2 3/4] scripts/kernel-doc: Adding infrastructure for markdown support

[Danilo Cesar Lemes de Paula]

Markdown support is given by calling an external tool, pandoc, for all
highlighted text on kernel-doc.

Pandoc converts Markdown text to proper Docbook tags, which will be
later translated to pdf, html or other targets.

This adds the capability of adding human-readle text highlight (bold,
underline, etc), bullet and numbered lists, simple tables, fixed-width
text (including asciiart), requiring minimal changes to current
documentation.

So, text using *must* will be rendered as <emphasis>must</empasis>
inside DocBook and then <strong>must</strong> for HTML.
Bullet lists eg:
	* Element 1
	* Element 2
will also be converted to proper docbook/html lists.
Although it has the support for a good part of the markup language,
pandoc has some limitations and won't render tables with spanning cells
or headings. The use of those features are not recommended.

At this moment, pandoc is totally optional. Docbooks ready for markdown
should be added to the MARKDOWNREADY variable inside the Makefile. In
case the developer doesn't have pandoc installed, Make will throw a
warning and the documentation build will continue, generating
simple Documentation without the features brought by pandoc.

Signed-off-by: Danilo Cesar Lemes de Paula <danilo.cesar@collabora.co.uk>
Cc: Randy Dunlap <rdunlap@infradead.org>
Cc: Daniel Vetter <daniel.vetter@ffwll.ch>
Cc: Laurent Pinchart <laurent.pinchart@ideasonboard.com>
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: Herbert Xu <herbert@gondor.apana.org.au>
Cc: Stephan Mueller <smueller@chronox.de>
Cc: Michal Marek <mmarek@suse.cz>
Cc: linux-kernel@vger.kernel.org
Cc: linux-doc@vger.kernel.org
Cc: intel-gfx <intel-gfx@lists.freedesktop.org>
Cc: dri-devel <dri-devel@lists.freedesktop.org>
---
 Changelog:
  v2:
	* Fix checkpatch issues.
	* Improved commit message.
	* Fixed dependency issue causing "make htmldocs" recalculate without
	  any code change.

 Documentation/DocBook/Makefile | 25 +++++++++++-----
 scripts/docproc.c              | 54 ++++++++++++++++++++++++----------
 scripts/kernel-doc             | 66 ++++++++++++++++++++++++++++++++++++++++--
 3 files changed, 119 insertions(+), 26 deletions(-)

diff --git a/Documentation/DocBook/Makefile b/Documentation/DocBook/Makefile
index 322255b..8e6d022 100644
--- a/Documentation/DocBook/Makefile
+++ b/Documentation/DocBook/Makefile
@@ -17,6 +17,8 @@ DOCBOOKS := z8530book.xml device-drivers.xml \
 	    tracepoint.xml drm.xml media_api.xml w1.xml \
 	    writing_musb_glue_layer.xml crypto-API.xml
 
+MARKDOWNREADY :=
+
 include Documentation/DocBook/media/Makefile
 
 ###
@@ -79,18 +81,23 @@ XMLTOFLAGS += --skip-validation
 # The following rules are used to generate the .xml documentation
 # required to generate the final targets. (ps, pdf, html).
 quiet_cmd_docproc = DOCPROC $@
-      cmd_docproc = SRCTREE=$(srctree)/ $(DOCPROC) doc $< >$@
+      cmd_docproc = SRCTREE=$(srctree)/ $(DOCPROC) doc $<
 define rule_docproc
-	set -e;								\
-        $(if $($(quiet)cmd_$(1)),echo '  $($(quiet)cmd_$(1))';) 	\
-        $(cmd_$(1)); 							\
-        ( 								\
-          echo 'cmd_$@ := $(cmd_$(1))'; 				\
-          echo $@: `SRCTREE=$(srctree) $(DOCPROC) depend $<`; 		\
+	set -e;									\
+	USEMARKDOWN="";								\
+	FILE=`basename $@`;							\
+	[[ "$(MARKDOWNREADY)" =~ "$${FILE}" ]] && USEMARKDOWN="-use-markdown";	\
+        $(if $($(quiet)cmd_$(1)),echo '  $($(quiet)cmd_$(1))';) 		\
+        $(cmd_$(1)) $$USEMARKDOWN >$@;						\
+        ( 									\
+          echo 'cmd_$@ := $(cmd_$(1))';				 		\
+          echo $@: `SRCTREE=$(srctree) $(DOCPROC) depend $<`;			\
         ) > $(dir $@).$(notdir $@).cmd
 endef
 
 %.xml: %.tmpl $(KERNELDOC) $(DOCPROC) $(KERNELDOCXMLREF) FORCE
+	@(which pandoc > /dev/null 2>&1) || \
+	(echo "*** To get propper documentation you need to install pandoc ***";)
 	$(call if_changed_rule,docproc)
 
 # Tell kbuild to always build the programs
@@ -101,6 +108,10 @@ notfoundtemplate = echo "*** You have to install docbook-utils or xmlto ***"; \
 db2xtemplate = db2TYPE -o $(dir $@) $<
 xmltotemplate = xmlto TYPE $(XMLTOFLAGS) -o $(dir $@) $<
 
+ifneq ($(shell which pandoc >/dev/null 2>&1 && echo found),found)
+	MARKDOWNREADY := "";
+endif
+
 # determine which methods are available
 ifeq ($(shell which db2ps >/dev/null 2>&1 && echo found),found)
 	use-db2x = db2x
diff --git a/scripts/docproc.c b/scripts/docproc.c
index e267e621..7c6b225 100644
--- a/scripts/docproc.c
+++ b/scripts/docproc.c
@@ -73,12 +73,15 @@ FILELINE * docsection;
 #define NOFUNCTION    "-nofunction"
 #define NODOCSECTIONS "-no-doc-sections"
 #define SHOWNOTFOUND  "-show-not-found"
+#define USEMARKDOWN   "-use-markdown"
 
 static char *srctree, *kernsrctree;
 
 static char **all_list = NULL;
 static int all_list_len = 0;
 
+static int use_markdown = 0;
+
 static void consume_symbol(const char *sym)
 {
 	int i;
@@ -95,10 +98,11 @@ static void consume_symbol(const char *sym)
 
 static void usage (void)
 {
-	fprintf(stderr, "Usage: docproc {doc|depend} file\n");
+	fprintf(stderr, "Usage: docproc {doc|depend} [-use-markdown] file\n");
 	fprintf(stderr, "Input is read from file.tmpl. Output is sent to stdout\n");
 	fprintf(stderr, "doc: frontend when generating kernel documentation\n");
 	fprintf(stderr, "depend: generate list of files referenced within file\n");
+	fprintf(stderr, "-use-markdown: pass -use-markdown to kernel-doc\n");
 	fprintf(stderr, "Environment variable SRCTREE: absolute path to sources.\n");
 	fprintf(stderr, "                     KBUILD_SRC: absolute path to kernel source tree.\n");
 }
@@ -257,12 +261,15 @@ static void docfunctions(char * filename, char * type)
 
 	for (i=0; i <= symfilecnt; i++)
 		symcnt += symfilelist[i].symbolcnt;
-	vec = malloc((2 + 2 * symcnt + 3) * sizeof(char *));
+	vec = malloc((2 + 2 * symcnt + 4) * sizeof(char *));
 	if (vec == NULL) {
 		perror("docproc: ");
 		exit(1);
 	}
 	vec[idx++] = KERNELDOC;
+	if (use_markdown)
+		vec[idx++] = USEMARKDOWN;
+
 	vec[idx++] = DOCBOOK;
 	vec[idx++] = NODOCSECTIONS;
 	for (i=0; i < symfilecnt; i++) {
@@ -294,6 +301,9 @@ static void singfunc(char * filename, char * line)
 	int i, idx = 0;
 	int startofsym = 1;
 	vec[idx++] = KERNELDOC;
+	if (use_markdown)
+		vec[idx++] = USEMARKDOWN;
+
 	vec[idx++] = DOCBOOK;
 	vec[idx++] = SHOWNOTFOUND;
 
@@ -328,8 +338,9 @@ static void singfunc(char * filename, char * line)
 static void docsect(char *filename, char *line)
 {
 	/* kerneldoc -docbook -show-not-found -function "section" file NULL */
-	char *vec[7];
+	char *vec[8];
 	char *s;
+	int idx = 0;
 
 	for (s = line; *s; s++)
 		if (*s == '\n')
@@ -342,30 +353,37 @@ static void docsect(char *filename, char *line)
 	consume_symbol(s);
 	free(s);
 
-	vec[0] = KERNELDOC;
-	vec[1] = DOCBOOK;
-	vec[2] = SHOWNOTFOUND;
-	vec[3] = FUNCTION;
-	vec[4] = line;
-	vec[5] = filename;
-	vec[6] = NULL;
+	vec[idx++] = KERNELDOC;
+	if (use_markdown)
+		vec[idx++] = USEMARKDOWN;
+
+	vec[idx++] = DOCBOOK;
+	vec[idx++] = SHOWNOTFOUND;
+	vec[idx++] = FUNCTION;
+	vec[idx++] = line;
+	vec[idx++] = filename;
+	vec[idx] = NULL;
 	exec_kernel_doc(vec);
 }
 
 static void find_all_symbols(char *filename)
 {
-	char *vec[4]; /* kerneldoc -list file NULL */
+	char *vec[5]; /* kerneldoc -list file NULL */
 	pid_t pid;
 	int ret, i, count, start;
 	char real_filename[PATH_MAX + 1];
 	int pipefd[2];
 	char *data, *str;
 	size_t data_len = 0;
+	int idx = 0;
+
+	vec[idx++] = KERNELDOC;
+	if (use_markdown)
+		vec[idx++] = USEMARKDOWN;
 
-	vec[0] = KERNELDOC;
-	vec[1] = LIST;
-	vec[2] = filename;
-	vec[3] = NULL;
+	vec[idx++] = LIST;
+	vec[idx++] = filename;
+	vec[idx] = NULL;
 
 	if (pipe(pipefd)) {
 		perror("pipe");
@@ -509,7 +527,7 @@ int main(int argc, char *argv[])
 	kernsrctree = getenv("KBUILD_SRC");
 	if (!kernsrctree || !*kernsrctree)
 		kernsrctree = srctree;
-	if (argc != 3) {
+	if (argc < 3 || argc > 4) {
 		usage();
 		exit(1);
 	}
@@ -521,6 +539,10 @@ int main(int argc, char *argv[])
 		exit(2);
 	}
 
+	if (argc == 4 && strcmp("-use-markdown", argv[3]) == 0) {
+		use_markdown = 1;
+	}
+
 	if (strcmp("doc", argv[1]) == 0) {
 		/* Need to do this in two passes.
 		 * First pass is used to collect all symbols exported
diff --git a/scripts/kernel-doc b/scripts/kernel-doc
index a38a69a..ab2e875 100755
--- a/scripts/kernel-doc
+++ b/scripts/kernel-doc
@@ -1,6 +1,7 @@
 #!/usr/bin/perl -w
 
 use strict;
+use IPC::Open3;
 
 ## Copyright (c) 1998 Michael Zucchi, All Rights Reserved        ##
 ## Copyright (C) 2000, 1  Tim Waugh <twaugh@redhat.com>          ##
@@ -258,6 +259,7 @@ if ($#ARGV == -1) {
 
 my $kernelversion;
 my $dohighlight = "";
+my $use_markdown = 0;
 
 my $verbose = 0;
 my $output_mode = "man";
@@ -378,6 +380,8 @@ while ($ARGV[0] =~ m/^-(.*)/) {
 	$function_only = 2;
 	$function = shift @ARGV;
 	$function_table{$function} = 1;
+    } elsif ($cmd eq "-use-markdown") {
+	$use_markdown = 1;
     } elsif ($cmd eq "-v") {
 	$verbose = 1;
     } elsif (($cmd eq "-h") || ($cmd eq "--help")) {
@@ -396,6 +400,7 @@ sub usage {
     print "         [ -no-doc-sections ]\n";
     print "         [ -function funcname [ -function funcname ...] ]\n";
     print "         [ -nofunction funcname [ -nofunction funcname ...] ]\n";
+    print "         [ -use-markdown ]\n";
     print "         [ -v ]\n";
     print "         c source file(s) > outputfile\n";
     print "         -v : verbose output, more warnings & other info listed\n";
@@ -469,6 +474,49 @@ sub dump_doc_section {
     }
 }
 
+sub markdown_to_docbook {
+	my $orig_content = $_[0];
+
+	my $pid = open3( \*CHLD_IN, \*CHLD_OUT, \*CHLD_ERR, "pandoc  --columns=80 -f markdown -t docbook" );
+
+	print CHLD_IN "$orig_content";
+	close(CHLD_IN);
+
+	waitpid($pid, 0);
+
+	my $content = "";
+	chomp(my @lines = <CHLD_OUT>);
+	foreach my $line (@lines) {
+		$content .= $line . "\n";
+	}
+	close(CHLD_OUT);
+	close(CHLD_ERR);
+
+	# pandoc insists in adding Main <para></para>, we should remove them.
+	$content =~ s:\A\s*<para>\s*\n(.*)\n</para>\Z$:$1:egsm;
+
+	return $content;
+}
+
+# Markdown->Docbook conversion by pandoc requires unescaped text
+# Kernel-doc converts every & to "&amp;", we need to convert it back.
+sub markdown_unescape
+{
+	my $text = shift;
+	my @lines = split /\n/, $text;
+
+	my @result;
+	foreach my $line (@lines) {
+		if ( $line =~ m /^    /s ) {
+			$line =~ s:\&amp;:\&:gs
+		}
+		push @result, $line;
+	}
+
+	return join "\n",@result;
+
+}
+
 ##
 # output function
 #
@@ -495,11 +543,19 @@ sub output_highlight {
 	$contents = local_unescape($contents);
 	# convert data read & converted thru xml_escape() into &xyz; format:
 	$contents =~ s/\\\\\\/\&/g;
+
+	if ($use_markdown) {
+		$contents = markdown_unescape($contents);
+	}
     }
+
 #   print STDERR "contents b4:$contents\n";
     eval $dohighlight;
     die $@ if $@;
 #   print STDERR "contents af:$contents\n";
+    if ($use_markdown) {
+        $contents = markdown_to_docbook($contents);
+    }
 
 #   strip whitespaces when generating html5
     if ($output_mode eq "html5") {
@@ -507,7 +563,8 @@ sub output_highlight {
 	$contents =~ s/\s+$//;
     }
     foreach $line (split "\n", $contents) {
-	if (! $output_preformatted) {
+	if (! $output_preformatted &&
+	    !($use_markdown && $line =~ m /^    /s)) {
 	    $line =~ s/^\s*//;
 	}
 	if ($line eq ""){
@@ -928,7 +985,9 @@ sub output_section_xml(%) {
 	print "<refsect1>\n";
 	print "<title>$section</title>\n";
 	if ($section =~ m/EXAMPLE/i) {
-	    print "<informalexample><programlisting>\n";
+	    print "<informalexample>\n";
+	    # programlisting is already included by pandoc
+	    print "<programlisting>\n" unless $use_markdown;
 	    $output_preformatted = 1;
 	} else {
 	    print "<para>\n";
@@ -936,7 +995,8 @@ sub output_section_xml(%) {
 	output_highlight($args{'sections'}{$section});
 	$output_preformatted = 0;
 	if ($section =~ m/EXAMPLE/i) {
-	    print "</programlisting></informalexample>\n";
+	    print "</programlisting>\n" unless $use_markdown;
+	    print "</informalexample>\n";
 	} else {
 	    print "</para>\n";
 	}
-- 
2.4.6

[PATCH v2 4/4] drm/doc: Convert to markdown

[Danilo Cesar Lemes de Paula]

DRM Docbook is now Markdown ready. This means its doc is able to
use markdown text on it.

* Documentation/DocBook/drm.tmpl: Contains a table duplicated from
  drivers/gpu/drm/i915/i915_reg.h. This is not needed anymore

* drivers/gpu/drm/drm_modeset_lock.c: had a code example that used
  to look pretty bad on html. Fixed by using proper code markup.

* drivers/gpu/drm/drm_prime.c: Remove spaces between lines to make
  a proper markup list.

* drivers/gpu/drm/i915/i915_reg.h: Altought pandoc supports tables,
  it doesn't support table cell spanning. But we can use fixed-width
  for those special cases.

* include/drm/drm_vma_manager.h: Another code example that should be
  proper indented with four spaces.

Signed-off-by: Danilo Cesar Lemes de Paula <danilo.cesar@collabora.co.uk>
Cc: Randy Dunlap <rdunlap@infradead.org>
Cc: Daniel Vetter <daniel.vetter@ffwll.ch>
Cc: Laurent Pinchart <laurent.pinchart@ideasonboard.com>
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: Herbert Xu <herbert@gondor.apana.org.au>
Cc: Stephan Mueller <smueller@chronox.de>
Cc: Michal Marek <mmarek@suse.cz>
Cc: linux-kernel@vger.kernel.org
Cc: linux-doc@vger.kernel.org
Cc: intel-gfx <intel-gfx@lists.freedesktop.org>
Cc: dri-devel <dri-devel@lists.freedesktop.org>
---
 Changelog:
  v2:
	* Improve markup in drivers/gpu/drm/drm_modes.c

 Documentation/DocBook/Makefile     |  2 +-
 Documentation/DocBook/drm.tmpl     | 86 --------------------------------------
 drivers/gpu/drm/drm_modes.c        | 12 +++---
 drivers/gpu/drm/drm_modeset_lock.c | 14 +++----
 drivers/gpu/drm/drm_prime.c        | 16 +++----
 drivers/gpu/drm/i915/i915_reg.h    | 48 ++++++++++-----------
 include/drm/drm_vma_manager.h      | 10 ++---
 7 files changed, 48 insertions(+), 140 deletions(-)

diff --git a/Documentation/DocBook/Makefile b/Documentation/DocBook/Makefile
index 8e6d022..f2408b3 100644
--- a/Documentation/DocBook/Makefile
+++ b/Documentation/DocBook/Makefile
@@ -17,7 +17,7 @@ DOCBOOKS := z8530book.xml device-drivers.xml \
 	    tracepoint.xml drm.xml media_api.xml w1.xml \
 	    writing_musb_glue_layer.xml crypto-API.xml
 
-MARKDOWNREADY :=
+MARKDOWNREADY := drm.xml
 
 include Documentation/DocBook/media/Makefile
 
diff --git a/Documentation/DocBook/drm.tmpl b/Documentation/DocBook/drm.tmpl
index 2fb9a54..4fb4636 100644
--- a/Documentation/DocBook/drm.tmpl
+++ b/Documentation/DocBook/drm.tmpl
@@ -4073,92 +4073,6 @@ int num_ioctls;</synopsis>
       <sect2>
         <title>DPIO</title>
 !Pdrivers/gpu/drm/i915/i915_reg.h DPIO
-	<table id="dpiox2">
-	  <title>Dual channel PHY (VLV/CHV/BXT)</title>
-	  <tgroup cols="8">
-	    <colspec colname="c0" />
-	    <colspec colname="c1" />
-	    <colspec colname="c2" />
-	    <colspec colname="c3" />
-	    <colspec colname="c4" />
-	    <colspec colname="c5" />
-	    <colspec colname="c6" />
-	    <colspec colname="c7" />
-	    <spanspec spanname="ch0" namest="c0" nameend="c3" />
-	    <spanspec spanname="ch1" namest="c4" nameend="c7" />
-	    <spanspec spanname="ch0pcs01" namest="c0" nameend="c1" />
-	    <spanspec spanname="ch0pcs23" namest="c2" nameend="c3" />
-	    <spanspec spanname="ch1pcs01" namest="c4" nameend="c5" />
-	    <spanspec spanname="ch1pcs23" namest="c6" nameend="c7" />
-	    <thead>
-	      <row>
-		<entry spanname="ch0">CH0</entry>
-		<entry spanname="ch1">CH1</entry>
-	      </row>
-	    </thead>
-	    <tbody valign="top" align="center">
-	      <row>
-		<entry spanname="ch0">CMN/PLL/REF</entry>
-		<entry spanname="ch1">CMN/PLL/REF</entry>
-	      </row>
-	      <row>
-		<entry spanname="ch0pcs01">PCS01</entry>
-		<entry spanname="ch0pcs23">PCS23</entry>
-		<entry spanname="ch1pcs01">PCS01</entry>
-		<entry spanname="ch1pcs23">PCS23</entry>
-	      </row>
-	      <row>
-		<entry>TX0</entry>
-		<entry>TX1</entry>
-		<entry>TX2</entry>
-		<entry>TX3</entry>
-		<entry>TX0</entry>
-		<entry>TX1</entry>
-		<entry>TX2</entry>
-		<entry>TX3</entry>
-	      </row>
-	      <row>
-		<entry spanname="ch0">DDI0</entry>
-		<entry spanname="ch1">DDI1</entry>
-	      </row>
-	    </tbody>
-	  </tgroup>
-	</table>
-	<table id="dpiox1">
-	  <title>Single channel PHY (CHV/BXT)</title>
-	  <tgroup cols="4">
-	    <colspec colname="c0" />
-	    <colspec colname="c1" />
-	    <colspec colname="c2" />
-	    <colspec colname="c3" />
-	    <spanspec spanname="ch0" namest="c0" nameend="c3" />
-	    <spanspec spanname="ch0pcs01" namest="c0" nameend="c1" />
-	    <spanspec spanname="ch0pcs23" namest="c2" nameend="c3" />
-	    <thead>
-	      <row>
-		<entry spanname="ch0">CH0</entry>
-	      </row>
-	    </thead>
-	    <tbody valign="top" align="center">
-	      <row>
-		<entry spanname="ch0">CMN/PLL/REF</entry>
-	      </row>
-	      <row>
-		<entry spanname="ch0pcs01">PCS01</entry>
-		<entry spanname="ch0pcs23">PCS23</entry>
-	      </row>
-	      <row>
-		<entry>TX0</entry>
-		<entry>TX1</entry>
-		<entry>TX2</entry>
-		<entry>TX3</entry>
-	      </row>
-	      <row>
-		<entry spanname="ch0">DDI2</entry>
-	      </row>
-	    </tbody>
-	  </tgroup>
-	</table>
       </sect2>
 
       <sect2>
diff --git a/drivers/gpu/drm/drm_modes.c b/drivers/gpu/drm/drm_modes.c
index cd74a09..9480464 100644
--- a/drivers/gpu/drm/drm_modes.c
+++ b/drivers/gpu/drm/drm_modes.c
@@ -553,10 +553,10 @@ EXPORT_SYMBOL(drm_gtf_mode_complex);
  * drivers/video/fbmon.c
  *
  * Standard GTF parameters:
- * M = 600
- * C = 40
- * K = 128
- * J = 20
+ *     M = 600
+ *     C = 40
+ *     K = 128
+ *     J = 20
  *
  * Returns:
  * The modeline based on the GTF algorithm stored in a drm_display_mode object.
@@ -1212,7 +1212,7 @@ EXPORT_SYMBOL(drm_mode_connector_list_update);
  * This uses the same parameters as the fb modedb.c, except for an extra
  * force-enable, force-enable-digital and force-disable bit at the end:
  *
- *	<xres>x<yres>[M][R][-<bpp>][@<refresh>][i][m][eDd]
+ * <xres>x<yres>[M][R][-<bpp>][@<refresh>][i][m][eDd]
  *
  * The intermediate drm_cmdline_mode structure is required to store additional
  * options from the command line modline like the force-enable/disable flag.
@@ -1491,4 +1491,4 @@ int drm_mode_convert_umode(struct drm_display_mode *out,
 
 out:
 	return ret;
-}
\ No newline at end of file
+}
diff --git a/drivers/gpu/drm/drm_modeset_lock.c b/drivers/gpu/drm/drm_modeset_lock.c
index c0a5cd8..de59b0c 100644
--- a/drivers/gpu/drm/drm_modeset_lock.c
+++ b/drivers/gpu/drm/drm_modeset_lock.c
@@ -40,17 +40,15 @@
  * The basic usage pattern is to:
  *
  *     drm_modeset_acquire_init(&ctx)
- *   retry:
+ *     retry:
  *     foreach (lock in random_ordered_set_of_locks) {
- *       ret = drm_modeset_lock(lock, &ctx)
- *       if (ret == -EDEADLK) {
- *          drm_modeset_backoff(&ctx);
- *          goto retry;
- *       }
+ *         ret = drm_modeset_lock(lock, &ctx)
+ *         if (ret == -EDEADLK) {
+ *             drm_modeset_backoff(&ctx);
+ *             goto retry;
+ *         }
  *     }
- *
  *     ... do stuff ...
- *
  *     drm_modeset_drop_locks(&ctx);
  *     drm_modeset_acquire_fini(&ctx);
  */
diff --git a/drivers/gpu/drm/drm_prime.c b/drivers/gpu/drm/drm_prime.c
index 9f935f5..27aa718 100644
--- a/drivers/gpu/drm/drm_prime.c
+++ b/drivers/gpu/drm/drm_prime.c
@@ -313,19 +313,15 @@ static const struct dma_buf_ops drm_gem_prime_dmabuf_ops =  {
  *
  * Export callbacks:
  *
- *  - @gem_prime_pin (optional): prepare a GEM object for exporting
- *
- *  - @gem_prime_get_sg_table: provide a scatter/gather table of pinned pages
- *
- *  - @gem_prime_vmap: vmap a buffer exported by your driver
- *
- *  - @gem_prime_vunmap: vunmap a buffer exported by your driver
- *
- *  - @gem_prime_mmap (optional): mmap a buffer exported by your driver
+ *  * @gem_prime_pin (optional): prepare a GEM object for exporting
+ *  * @gem_prime_get_sg_table: provide a scatter/gather table of pinned pages
+ *  * @gem_prime_vmap: vmap a buffer exported by your driver
+ *  * @gem_prime_vunmap: vunmap a buffer exported by your driver
+ *  * @gem_prime_mmap (optional): mmap a buffer exported by your driver
  *
  * Import callback:
  *
- *  - @gem_prime_import_sg_table (import): produce a GEM object from another
+ *  * @gem_prime_import_sg_table (import): produce a GEM object from another
  *    driver's scatter/gather table
  */
 
diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h
index 2030f60..07c757c 100644
--- a/drivers/gpu/drm/i915/i915_reg.h
+++ b/drivers/gpu/drm/i915/i915_reg.h
@@ -778,31 +778,31 @@ enum skl_disp_power_wells {
  *
  * Note: DDI0 is digital port B, DD1 is digital port C, and DDI2 is
  * digital port D (CHV) or port A (BXT).
- */
-/*
- * Dual channel PHY (VLV/CHV/BXT)
- * ---------------------------------
- * |      CH0      |      CH1      |
- * |  CMN/PLL/REF  |  CMN/PLL/REF  |
- * |---------------|---------------| Display PHY
- * | PCS01 | PCS23 | PCS01 | PCS23 |
- * |-------|-------|-------|-------|
- * |TX0|TX1|TX2|TX3|TX0|TX1|TX2|TX3|
- * ---------------------------------
- * |     DDI0      |     DDI1      | DP/HDMI ports
- * ---------------------------------
  *
- * Single channel PHY (CHV/BXT)
- * -----------------
- * |      CH0      |
- * |  CMN/PLL/REF  |
- * |---------------| Display PHY
- * | PCS01 | PCS23 |
- * |-------|-------|
- * |TX0|TX1|TX2|TX3|
- * -----------------
- * |     DDI2      | DP/HDMI port
- * -----------------
+ *
+ *     Dual channel PHY (VLV/CHV/BXT)
+ *     ---------------------------------
+ *     |      CH0      |      CH1      |
+ *     |  CMN/PLL/REF  |  CMN/PLL/REF  |
+ *     |---------------|---------------| Display PHY
+ *     | PCS01 | PCS23 | PCS01 | PCS23 |
+ *     |-------|-------|-------|-------|
+ *     |TX0|TX1|TX2|TX3|TX0|TX1|TX2|TX3|
+ *     ---------------------------------
+ *     |     DDI0      |     DDI1      | DP/HDMI ports
+ *     ---------------------------------
+ *
+ *     Single channel PHY (CHV/BXT)
+ *     -----------------
+ *     |      CH0      |
+ *     |  CMN/PLL/REF  |
+ *     |---------------| Display PHY
+ *     | PCS01 | PCS23 |
+ *     |-------|-------|
+ *     |TX0|TX1|TX2|TX3|
+ *     -----------------
+ *     |     DDI2      | DP/HDMI port
+ *     -----------------
  */
 #define DPIO_DEVFN			0
 
diff --git a/include/drm/drm_vma_manager.h b/include/drm/drm_vma_manager.h
index 8cd402c..2ca44db 100644
--- a/include/drm/drm_vma_manager.h
+++ b/include/drm/drm_vma_manager.h
@@ -110,11 +110,11 @@ drm_vma_offset_exact_lookup(struct drm_vma_offset_manager *mgr,
  * Note: You're in atomic-context while holding this lock!
  *
  * Example:
- *   drm_vma_offset_lock_lookup(mgr);
- *   node = drm_vma_offset_lookup_locked(mgr);
- *   if (node)
- *       kref_get_unless_zero(container_of(node, sth, entr));
- *   drm_vma_offset_unlock_lookup(mgr);
+ *     drm_vma_offset_lock_lookup(mgr);
+ *     node = drm_vma_offset_lookup_locked(mgr);
+ *     if (node)
+ *         kref_get_unless_zero(container_of(node, sth, entr));
+ *     drm_vma_offset_unlock_lookup(mgr);
  */
 static inline void drm_vma_offset_lock_lookup(struct drm_vma_offset_manager *mgr)
 {
-- 
2.4.6