[PATCH 0/4] docs/html/hcall: Hypercall docs headers

[PATCH 0/4] docs/html/hcall: Hypercall docs headers

[Ian Jackson]

This is my proposed approach to providing hypercall documentation: we
provide an htmlified copy of the xen public headers, with a contents
page and cross-reference hyperlinks added.

This is done by a 350-line Perl script to which we can add features as
we think of them.

An example of the output can be found here:
 http://www.chiark.greenend.org.uk/~ijackson/volatile/2011/xen/hcall/

[PATCH 1/4] docs/html/hcall: Initial cut of header documentation massager

[Ian Jackson]

"xen-headers" generates HTML from header files.  So far this generates
just some type cross-references, if you say
   make -C docs html/hcall/stamp

An index page, proper wiring into the build system, and a few more
annotations in the headers, and will be forthcoming.

Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>
---
 docs/Makefile    |    8 ++
 docs/xen-headers |  281 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 289 insertions(+), 0 deletions(-)
 create mode 100755 docs/xen-headers

diff --git a/docs/Makefile b/docs/Makefile
index 2054541..fc42859 100644
--- a/docs/Makefile
+++ b/docs/Makefile
@@ -129,6 +129,14 @@ html/%.html: %.markdown
 	$(MARKDOWN) $< > $@.tmp ; \
 	$(call move-if-changed,$@.tmp,$@) ; fi
 
+html/hcall/stamp:
+	@$(INSTALL_DIR) $(@D)
+	./xen-headers -O $(@D) \
+		-T 'arch-x86_64 - Xen public headers' \
+		-X arch-ia64 -X arch-x86_32 -X xen-x86_32 \
+		../xen include/public
+	touch $@
+
 txt/%.txt: %.txt
 	$(INSTALL_DIR) $(@D)
 	cp $< $@.tmp
diff --git a/docs/xen-headers b/docs/xen-headers
new file mode 100755
index 0000000..6918380
--- /dev/null
+++ b/docs/xen-headers
@@ -0,0 +1,281 @@
+#!/usr/bin/perl -w
+# usage: xen-headers [ -X GLOB -I GLOB ... [-D...] ] \
+#                    -O HTML-DIR PUBLIC-INCLUDE-DIR
+
+# enum values --> selected function or struct
+# type & function names --> definition
+# function or struct selected by enum ++> ref to enum value
+
+# definitions must start in LH column
+# extra syntax:
+#    /* ` <definition>                          } parse as if <definition>
+#     * ` <definition>                          }  was not commented
+#   enum <name> { // <pattern>* => <func>()     } cross-reference
+#   enum <name> { // <pattern>* => struct <s>   }  enum values
+
+# 1st pass: find where things are defined and what references are wanted
+# 2rd pass: write out output
+
+use strict;
+use warnings;
+
+use Getopt::Long;
+use File::Find;
+use IO::File;
+
+Getopt::Long::Configure('bundling');
+
+our $outdir;
+our $debug=0;
+our $xtitle='';
+our @fglobs;
+
+sub includeexclude {
+    my ($yn, $optobj, $value) = @_;
+    push @fglobs, [ $value, $yn ];
+}
+
+GetOptions("O|output-dir=s" => \$outdir,
+           "D+" => \$debug,
+           "T=s" => \$xtitle,
+	   "I=s" => sub { includeexclude(1, @_); },
+	   "X=s" => sub { includeexclude(0, @_); })
+    or die;
+
+die unless defined $outdir;
+@ARGV>=2 or die;
+
+my ($basedir,@indirs) = @ARGV;
+
+# general globals
+our $pass;
+our %sdef; 
+# $sdef{$type}{$name} => {
+#     DefLocs => { "$leaf_path:$lineno" => $leaf_opath ,... }
+#     Xrefs => { "$leaf_path,$lineno" => "$xref", ... }
+#     Used => 1
+# }
+# $type might be  Func Struct Union Enum EnumVal
+
+# provided by the find() function
+our $leaf;
+our $leaf_opath;
+
+# reset at the start of each file
+our $o;
+our $in_enum;
+our @pending_xrefs;
+
+sub compile_fglobs () {
+    local ($_);
+    my $f = "sub file_wanted (\$) {\n    local (\$_) = \"/\$leaf\";\n";
+    foreach my $fglob (@fglobs) {
+	$_ = $fglob->[0];
+	$_ = "**$_**" unless m/[?*]/;
+	s/\W/\\$&/g;
+	s,\\\*\\\*,.*,g;
+	s,\\\*,[^/]*,g;
+	s,\\\?,[^/],g;
+	$f .= "    return $fglob->[1] if m,$_,o;\n";
+    }
+    $f .= "    return 1;\n}\n1;\n";
+    debug(3, $f);
+    eval $f or die "$@ ";
+}
+
+compile_fglobs();
+
+
+sub warning {
+    print STDERR "$leaf:$.: @_\n";
+}
+
+sub debug {
+    my $msglevel = scalar shift @_;
+    return unless $debug >= $msglevel;
+    print STDERR "DEBUG $pass $msglevel @_\n" or die $!;
+}
+
+sub in_enum ($$$) { $in_enum = [ @_ ]; } # [ $enumvalpfx, RefType, $refnamepfx ]
+
+sub aelem ($$$) {
+    my ($ntext,$ytext,$hparams) = @_;
+    return $ntext unless $hparams =~ m/\S/;
+    return "<a $hparams>$ytext</a>";
+}
+
+sub defn ($$$;$) {
+    my ($text,$type,$name,$hparams) = @_;
+    $hparams='' if !defined $hparams;
+    debug(2,"DEFN $. $type $name $hparams");
+    $sdef{$type}{$name}{DefLocs}{"$leaf:$."} = $leaf_opath;
+    my $xrefs = $sdef{$type}{$name}{Xrefs};
+    push @pending_xrefs, values %$xrefs if $xrefs;
+    $hparams .= " name=\"${type}_$name\"" if $sdef{$type}{$name}{Used};
+    return aelem($text, "<strong>$text</strong>", $hparams);
+}
+
+sub norm ($) {
+    local ($_) = @_;
+    my $no = '';
+    while (length) {
+	if (s/^(?:\s|^\W)+//) {
+	    $no .= $&;
+	} elsif (s/^(struct|union|enum)\s+(\w+)\b//) {
+	    $no .= ahref($&, (ucfirst $1), $2);
+	} elsif (s/^\w+\b//) {
+	    $no .= ahref($&, 'Func', $&);
+	} else {
+	    die "$_ ?";
+	}
+    }
+    return $no;
+}
+
+sub refhref ($$) {
+    my ($type,$name) = @_;
+    $sdef{$type}{$name}{Used} = 1;
+    my $locs = $sdef{$type}{$name}{DefLocs};
+    return '' unless $locs;
+    if ((scalar keys %$locs) != 1 && !$sdef{$type}{$name}{MultiWarned}) {
+	warning("multiple definitions of $type $name: $_")
+	    foreach keys %$locs;
+	$sdef{$type}{$name}{MultiWarned}=1;
+    }
+    my ($loc) = values %$locs;
+    return "href=\"$loc#${type}_$name\"";
+}
+
+sub ahref ($$$) {
+    my ($text,$type,$name) = @_;
+    return aelem($text,$text, refhref($type,$name));
+}
+
+sub defmacro ($) {
+    my ($valname) = @_;
+    if (!$in_enum) {
+	return $valname;
+    } elsif (substr($valname, 0, (length $in_enum->[0])) ne $in_enum->[0]) {
+	warning("in enum expecting $in_enum->[0]* got $valname");
+	return $valname;
+    } else {
+	my $reftype = $in_enum->[1];
+	my $refname = $in_enum->[2].substr($valname, (length $in_enum->[0]));
+	$sdef{$reftype}{$refname}{Xrefs}{$leaf,$.} =
+	    "[see <a href=\"$leaf_opath#EnumVal_$valname\">$valname</a>]";
+	$sdef{EnumVal}{$valname}{Used} = 1;
+	return defn($valname,'EnumVal',$valname, refhref($reftype,$refname));
+    }
+}
+
+sub out_xrefs ($) {
+    my ($linemapfunc) = @_;
+    foreach my $xref (@pending_xrefs) {
+	$o .= $linemapfunc->($xref);
+	$o .= "\n";
+    }
+    @pending_xrefs = ();
+}
+
+sub write_file ($$) {
+    my ($opath, $odata) = @_;
+    my $out = new IO::File "$opath.new", '>' or die "$opath $!";
+    print $out $odata or die $!;
+    rename "$opath.new", "$opath" or die "$opath $!";
+}
+
+sub process_file ($$) {
+    my ($infile, $outfile) = @_;
+    debug(1,"$pass $infile => $outfile");
+    my $in = new IO::File "$infile", '<' or die "$infile $!";
+
+    $o = '';
+    $in_enum = undef;
+    @pending_xrefs = ();
+
+    $o .= "<html><head><title>$leaf - $xtitle</title></head><body><pre>\n";
+    
+    while (<$in>) {
+	s/\&/\&amp;/g;
+	s/\</\&lt;/g;
+	s/\>/\&gt;/g;
+
+	if (m/^(.*\`)[ \t]*$/) {
+	    my $lhs = $1;
+	    out_xrefs(sub { "$1 $_[0]"; });
+	} elsif (m/^\s*$/) {
+	    out_xrefs(sub { sprintf "/* %70s */", $_[0]; });
+	}
+
+	# In case of comments, strip " /* ` " and " * ` ";
+	my $lstripped = s,^ \s* /? \* \s* \` \  ,,x ? $&: '';
+
+	# Strip trailing whitespace and perhaps trailing "*/" or "*"
+	s,(?: \s* \* /? )? \s* $,,x or die;
+	my $rstripped = $&;
+
+	# Now the actual functionality:
+
+	debug(3,"$. $_");
+
+	if (s/^( (?: \w+\  )? ) (\w+[a-z]\w+) ( \( .*)$
+             / $1.defn($2,'Func',$2).norm($3) /xe) {
+	} elsif (s/^((struct|union|enum) \  (\w+)) ( \s+ \{ .* )$
+                  / defn($1,(ucfirst $2),$3).norm($4) /xe) {
+	    if ($2 eq 'enum') {
+		if (m,/[/*] (\w+)\* \=\&gt\; (\w+)\*\(\),) { 
+		    in_enum($1,'Func',$2)
+		} elsif (m,/[/*] (\w+)\* \=\&gt\; (struct) (\w+)\*,) { 
+		    in_enum($1,(ucfirst $2),$3);
+	        }
+	    }
+	} elsif (s/^( \s* \#define \s+ ) (\w+) ( \s+\S )
+                  / $1.defmacro($2).norm($3) /xe) {
+	} else {
+	    if (m/^\s*\}/) {
+		$in_enum = undef;
+	    }
+	    $_ = norm($_);
+	}
+
+	# Write the line out
+
+	if ($pass == 2) {
+	    $o .= $lstripped;
+	    $o .= $_;
+	    $o .= $rstripped;
+	}
+    }
+
+    warning("pending xrefs at end of file") if @pending_xrefs;
+
+    if ($pass == 2) {
+	$o .= "</pre></body></html>";
+	write_file($outfile, $o);
+    }
+}
+
+
+foreach $pass (qw(1 2)) {
+    find({ wanted => 
+	       sub {
+		   return unless m/\.h$/;
+		   lstat $File::Find::name or die "$File::Find::name $!";
+		   -f _ or die "$File::Find::name";
+		   substr($File::Find::name, 0, 1+length $basedir) 
+		       eq "$basedir/"
+		       or die "$File::Find::name $basedir";
+		   $leaf = substr($File::Find::name, 1+length $basedir);
+		   if (!file_wanted()) {
+		       debug(1,"$pass $File::Find::name excluded");
+		       return;
+		   }
+		   $leaf_opath = $leaf;
+		   $leaf_opath =~ s#/#,#g;
+		   $leaf_opath .= ".html";
+		   process_file($File::Find::name, $outdir.'/'.$leaf_opath);
+	   },
+	   no_chdir => 1,
+	 },
+	 map { "$basedir/$_" } @indirs);
+}
-- 
1.7.2.5

Re: [PATCH 1/4] docs/html/hcall: Initial cut of header documentation massager

[Ian Campbell]

On Sun, 2011-11-13 at 19:21 +0000, Ian Jackson wrote:
> "xen-headers" generates HTML from header files.  So far this generates
> just some type cross-references, if you say
>    make -C docs html/hcall/stamp
> 
> An index page, proper wiring into the build system, and a few more
> annotations in the headers, and will be forthcoming.
> 
> Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>
> ---
>  docs/Makefile    |    8 ++
>  docs/xen-headers |  281 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
>  2 files changed, 289 insertions(+), 0 deletions(-)
>  create mode 100755 docs/xen-headers
> 
> diff --git a/docs/Makefile b/docs/Makefile
> index 2054541..fc42859 100644
> --- a/docs/Makefile
> +++ b/docs/Makefile
> @@ -129,6 +129,14 @@ html/%.html: %.markdown
>  	$(MARKDOWN) $< > $@.tmp ; \
>  	$(call move-if-changed,$@.tmp,$@) ; fi
>  
> +html/hcall/stamp:
> +	@$(INSTALL_DIR) $(@D)
> +	./xen-headers -O $(@D) \
> +		-T 'arch-x86_64 - Xen public headers' \
> +		-X arch-ia64 -X arch-x86_32 -X xen-x86_32 \
> +		../xen include/public
> +	touch $@
> +
>  txt/%.txt: %.txt
>  	$(INSTALL_DIR) $(@D)
>  	cp $< $@.tmp
> diff --git a/docs/xen-headers b/docs/xen-headers
> new file mode 100755
> index 0000000..6918380
> --- /dev/null
> +++ b/docs/xen-headers
> @@ -0,0 +1,281 @@
> +#!/usr/bin/perl -w
> +# usage: xen-headers [ -X GLOB -I GLOB ... [-D...] ] \
> +#                    -O HTML-DIR PUBLIC-INCLUDE-DIR

No "-T"? Also for PUBLIC-INCLUDE-DIR you appear to pass "../xen
include/public" so my guess is that it is slightly more complicated?
Looks like "BASE-PUBLIC-INCLUDE-DIR PUBLIC-INCLUDE-FILE
[ PUBLIC-INCLUDE-FILE ... ] 

[...]
> +	s/\&/\&amp;/g;
> +	s/\</\&lt;/g;
> +	s/\>/\&gt;/g;

There must be a perl lib for this?

Ian.

Re: [PATCH 1/4] docs/html/hcall: Initial cut of header documentation massager

[Ian Jackson]

Ian Campbell writes ("Re: [Xen-devel] [PATCH 1/4] docs/html/hcall: Initial cut of header documentation massager"):
> On Sun, 2011-11-13 at 19:21 +0000, Ian Jackson wrote:
> > diff --git a/docs/xen-headers b/docs/xen-headers
> > new file mode 100755
> > index 0000000..6918380
> > --- /dev/null
> > +++ b/docs/xen-headers
> > @@ -0,0 +1,281 @@
> > +#!/usr/bin/perl -w
> > +# usage: xen-headers [ -X GLOB -I GLOB ... [-D...] ] \
> > +#                    -O HTML-DIR PUBLIC-INCLUDE-DIR
> 
> No "-T"? Also for PUBLIC-INCLUDE-DIR you appear to pass "../xen
> include/public" so my guess is that it is slightly more complicated?
> Looks like "BASE-PUBLIC-INCLUDE-DIR PUBLIC-INCLUDE-FILE
> [ PUBLIC-INCLUDE-FILE ... ] 

I seem to have forgotten to update the usage comment there.  I will
fix this.

> [...]
> > +	s/\&/\&/g;
> > +	s/\</\&lt;/g;
> > +	s/\>/\&gt;/g;
> 
> There must be a perl lib for this?

Yes, but it's not in the base perl distribution and I didn't want to
pull in a dependent library for three trivial lines.

Ian.

[PATCH 2/4] docs/html/hcall: Annotations for two hypercalls

[Ian Jackson]

Add annotations for a couple of the hypercalls:
 HYPERVISOR_set_trap_table
 HYPERVISOR_mmu_update

We do this by
 * annotating the list of #defines for hypercall numbers
 * annotating the list of error values
 * providing a function prototype for the systematically-named functions
The header generator does the rest.

This exercise revealed a couple of infelicities:
 * In the actual source code, do_mmu_update is defined to return an int
   and do_set_trap_table a long.  However both functions return either
   -Efoo (on error) or 0 for success.
 * The error numbers are defined only in the private header file
   xen/include/xen/errno.h and then only with names which will
   typically clash with other projects.  It would be nice to include a
   public version of this header which defines XEN_E*.  But for now
   we run xen-headers on errno.h too.

Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>
---
 docs/Makefile                     |    2 +-
 xen/include/public/arch-x86/xen.h |    6 ++++++
 xen/include/public/xen.h          |   11 +++++++++--
 xen/include/xen/errno.h           |    5 +++++
 4 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/docs/Makefile b/docs/Makefile
index fc42859..01e59cb 100644
--- a/docs/Makefile
+++ b/docs/Makefile
@@ -134,7 +134,7 @@ html/hcall/stamp:
 	./xen-headers -O $(@D) \
 		-T 'arch-x86_64 - Xen public headers' \
 		-X arch-ia64 -X arch-x86_32 -X xen-x86_32 \
-		../xen include/public
+		../xen include/public include/xen/errno.h
 	touch $@
 
 txt/%.txt: %.txt
diff --git a/xen/include/public/arch-x86/xen.h b/xen/include/public/arch-x86/xen.h
index 79ec633..d663cef 100644
--- a/xen/include/public/arch-x86/xen.h
+++ b/xen/include/public/arch-x86/xen.h
@@ -82,7 +82,13 @@ typedef unsigned long xen_pfn_t;
 typedef unsigned long xen_ulong_t;
 
 /*
+ * ` enum neg_errnoval
+ * ` HYPERVISOR_set_trap_table(const struct trap_info traps[]);
+ * `
+ */
+/*
  * Send an array of these to HYPERVISOR_set_trap_table().
+ * Terminate the array with a sentinel entry, with traps[].address==0.
  * The privilege level specifies which modes may enter a trap via a software
  * interrupt. On x86/64, since rings 1 and 2 are unavailable, we allocate
  * privilege levels as follows:
diff --git a/xen/include/public/xen.h b/xen/include/public/xen.h
index fde9fa5..f2c9e6f 100644
--- a/xen/include/public/xen.h
+++ b/xen/include/public/xen.h
@@ -55,6 +55,8 @@ DEFINE_XEN_GUEST_HANDLE(xen_pfn_t);
  * HYPERCALLS
  */
 
+/* ` enum hypercall_num { // __HYPERVISOR_* => HYPERVISOR_*() */
+
 #define __HYPERVISOR_set_trap_table        0
 #define __HYPERVISOR_mmu_update            1
 #define __HYPERVISOR_set_gdt               2
@@ -105,6 +107,8 @@ DEFINE_XEN_GUEST_HANDLE(xen_pfn_t);
 #define __HYPERVISOR_arch_6               54
 #define __HYPERVISOR_arch_7               55
 
+/* ` } */
+
 /*
  * HYPERCALL COMPATIBILITY.
  */
@@ -163,8 +167,11 @@ DEFINE_XEN_GUEST_HANDLE(xen_pfn_t);
 #define NR_VIRQS       24
 
 /*
- * HYPERVISOR_mmu_update(reqs, count, pdone, foreigndom)
- * 
+ * ` enum neg_errnoval
+ * ` HYPERVISOR_mmu_update(const struct mmu_update reqs[],
+ * `                       unsigned count, unsigned *done_out,
+ * `                       unsigned foreigndom)
+ * `
  * @reqs is an array of mmu_update_t structures ((ptr, val) pairs).
  * @count is the length of the above array.
  * @pdone is an output parameter indicating number of completed operations
diff --git a/xen/include/xen/errno.h b/xen/include/xen/errno.h
index 7cf599f..c2eb5d3 100644
--- a/xen/include/xen/errno.h
+++ b/xen/include/xen/errno.h
@@ -1,6 +1,9 @@
 #ifndef _I386_ERRNO_H
 #define _I386_ERRNO_H
 
+/* ` enum neg_errnoval { /* -Efoo for each Efoo in the list below } */
+/* ` enum errnoval { */
+
 #define	EPERM		 1	/* Operation not permitted */
 #define	ENOENT		 2	/* No such file or directory */
 #define	ESRCH		 3	/* No such process */
@@ -129,4 +132,6 @@
 #define	ENOMEDIUM	123	/* No medium found */
 #define	EMEDIUMTYPE	124	/* Wrong medium type */
 
+/* } */
+
 #endif
-- 
1.7.2.5

Re: [PATCH 2/4] docs/html/hcall: Annotations for two hypercalls

[Ian Campbell]

On Sun, 2011-11-13 at 19:21 +0000, Ian Jackson wrote:
> diff --git a/xen/include/xen/errno.h b/xen/include/xen/errno.h
> index 7cf599f..c2eb5d3 100644
> --- a/xen/include/xen/errno.h
> +++ b/xen/include/xen/errno.h
> @@ -1,6 +1,9 @@
>  #ifndef _I386_ERRNO_H
>  #define _I386_ERRNO_H
>  
> +/* ` enum neg_errnoval { /* -Efoo for each Efoo in the list below } */

Do we pass -Wcomment (or something which includes it)? If so gcc might
warn:
        /home/ianc/t.c:5:26: warning: "/*" within comment
(and we use -Werror, so...).

Is this syntax trying to encode some meaning or was it just a typo?

> +/* ` enum errnoval { */
> +
>  #define	EPERM		 1	/* Operation not permitted */
>  #define	ENOENT		 2	/* No such file or directory */
>  #define	ESRCH		 3	/* No such process */
> @@ -129,4 +132,6 @@
>  #define	ENOMEDIUM	123	/* No medium found */
>  #define	EMEDIUMTYPE	124	/* Wrong medium type */
>  
> +/* } */

Is the ` supposed to be required here or is omitting it fine?

> +
>  #endif

Re: [PATCH 2/4] docs/html/hcall: Annotations for two hypercalls

[Ian Jackson]

Ian Campbell writes ("Re: [Xen-devel] [PATCH 2/4] docs/html/hcall: Annotations for two hypercalls"):
> On Sun, 2011-11-13 at 19:21 +0000, Ian Jackson wrote:
> > +/* ` enum neg_errnoval { /* -Efoo for each Efoo in the list below } */
> 
> Do we pass -Wcomment (or something which includes it)? If so gcc might
> warn:
>         /home/ianc/t.c:5:26: warning: "/*" within comment
> (and we use -Werror, so...).

Good point.  I don't think we pass that now - I would have noticed.
But we probably should do so I should change the comment there to [ ]
or something.

> Is this syntax trying to encode some meaning or was it just a typo?

Ideally it would have had a matching */ but obv. that wouldn't have
worked :-).

> > +/* ` enum errnoval { */
> > +
> >  #define	EPERM		 1	/* Operation not permitted */
> >  #define	ENOENT		 2	/* No such file or directory */
> >  #define	ESRCH		 3	/* No such process */
> > @@ -129,4 +132,6 @@
> >  #define	ENOMEDIUM	123	/* No medium found */
> >  #define	EMEDIUMTYPE	124	/* Wrong medium type */
> >  
> > +/* } */
> 
> Is the ` supposed to be required here or is omitting it fine?

Omitting the ` means the processor doesn't see that which means the
remaining #defines in that file would count as part of the enum -
except there aren't any.  So it's a bug.

Ian.

Re: [PATCH 2/4] docs/html/hcall: Annotations for two hypercalls

[Konrad Rzeszutek Wilk]

On Sun, Nov 13, 2011 at 07:21:04PM +0000, Ian Jackson wrote:
> Add annotations for a couple of the hypercalls:
>  HYPERVISOR_set_trap_table
>  HYPERVISOR_mmu_update

So I've some comments on the affects of what this hypercall expects
in some details. Where would I put those? In the header file or should
I do it somewhere else?

Re: [PATCH 2/4] docs/html/hcall: Annotations for two hypercalls

[Ian Jackson]

Konrad Rzeszutek Wilk writes ("Re: [Xen-devel] [PATCH 2/4] docs/html/hcall: Annotations for two hypercalls"):
> On Sun, Nov 13, 2011 at 07:21:04PM +0000, Ian Jackson wrote:
> > Add annotations for a couple of the hypercalls:
> >  HYPERVISOR_set_trap_table
> >  HYPERVISOR_mmu_update
> 
> So I've some comments on the affects of what this hypercall expects
> in some details. Where would I put those? In the header file or should
> I do it somewhere else?

In the header file.

Ian.

Re: [PATCH 2/4] docs/html/hcall: Annotations for two hypercalls

[Konrad Rzeszutek Wilk]

On Tue, Nov 15, 2011 at 02:47:09PM +0000, Ian Jackson wrote:
> Konrad Rzeszutek Wilk writes ("Re: [Xen-devel] [PATCH 2/4] docs/html/hcall: Annotations for two hypercalls"):
> > On Sun, Nov 13, 2011 at 07:21:04PM +0000, Ian Jackson wrote:
> > > Add annotations for a couple of the hypercalls:
> > >  HYPERVISOR_set_trap_table
> > >  HYPERVISOR_mmu_update
> > 
> > So I've some comments on the affects of what this hypercall expects
> > in some details. Where would I put those? In the header file or should
> > I do it somewhere else?
> 
> In the header file.

Is it OK if it runs on for pages?

Re: [PATCH 2/4] docs/html/hcall: Annotations for two hypercalls

[Ian Jackson]

Konrad Rzeszutek Wilk writes ("Re: [Xen-devel] [PATCH 2/4] docs/html/hcall: Annotations for two hypercalls"):
> On Tue, Nov 15, 2011 at 02:47:09PM +0000, Ian Jackson wrote:
> > Konrad Rzeszutek Wilk writes ("Re: [Xen-devel] [PATCH 2/4] docs/html/hcall: Annotations for two hypercalls"):
> > > On Sun, Nov 13, 2011 at 07:21:04PM +0000, Ian Jackson wrote:
> > > > Add annotations for a couple of the hypercalls:
> > > >  HYPERVISOR_set_trap_table
> > > >  HYPERVISOR_mmu_update
> > > 
> > > So I've some comments on the affects of what this hypercall expects
> > > in some details. Where would I put those? In the header file or should
> > > I do it somewhere else?
> > 
> > In the header file.
> 
> Is it OK if it runs on for pages?

Yes.  See also the approach we're taking in libxl.h.

If it really gets too long you can take it out and put it in a text or
markdown or pod file in docs/, leaving behind a reference.

Ian.

[PATCH 3/4] docs/html/hcall: Generate an "index.html"

[Ian Jackson]

Generate an "index.html" containing various useful links.

Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>
---
 docs/xen-headers         |   55 ++++++++++++++++++++++++++++++++++++++++++++++
 xen/include/public/xen.h |    4 ++-
 2 files changed, 58 insertions(+), 1 deletions(-)

diff --git a/docs/xen-headers b/docs/xen-headers
index 6918380..3cc9e40 100755
--- a/docs/xen-headers
+++ b/docs/xen-headers
@@ -50,6 +50,8 @@ my ($basedir,@indirs) = @ARGV;
 # general globals
 our $pass;
 our %sdef; 
+our @incontents;
+our @outfiles;
 # $sdef{$type}{$name} => {
 #     DefLocs => { "$leaf_path:$lineno" => $leaf_opath ,... }
 #     Xrefs => { "$leaf_path,$lineno" => "$xref", ... }
@@ -177,6 +179,19 @@ sub out_xrefs ($) {
     @pending_xrefs = ();
 }
 
+sub incontents ($$$) {
+    my ($text, $seq, $anchor) = @_;
+    $anchor = "incontents_$anchor";
+    if ($pass==2) {
+	push @incontents, { 
+	    Seq => $seq,
+	    Href => "$leaf_opath#$anchor",
+	    Title => $text,
+	};
+    }
+    return "<a name=\"$anchor\"><strong>$text</strong></a>";
+}
+
 sub write_file ($$) {
     my ($opath, $odata) = @_;
     my $out = new IO::File "$opath.new", '>' or die "$opath $!";
@@ -231,6 +246,8 @@ sub process_file ($$) {
 	    }
 	} elsif (s/^( \s* \#define \s+ ) (\w+) ( \s+\S )
                   / $1.defmacro($2).norm($3) /xe) {
+	} elsif (s/( \`incontents \s+ (\d+) \s+ (\w+) \s+ )(\S .* \S)
+                 / norm($1).incontents($4, $2, $3) /xe) {
 	} else {
 	    if (m/^\s*\}/) {
 		$in_enum = undef;
@@ -250,11 +267,47 @@ sub process_file ($$) {
     warning("pending xrefs at end of file") if @pending_xrefs;
 
     if ($pass == 2) {
+	push @outfiles, [ $leaf, $leaf_opath ];
 	$o .= "</pre></body></html>";
 	write_file($outfile, $o);
     }
 }
 
+sub output_index () {
+    my $title = "contents - $xtitle";
+    $o = '';
+    $o .= <<END;
+<html><head><title>$title</title></head>
+<body>
+<h1>$title</h1>
+<h2>Starting points</h2>
+<ul>
+END
+    foreach my $ic (sort { $a->{Seq} <=> $b->{Seq} } @incontents) {
+	$o .= "<li><a href=\"$ic->{Href}\">$ic->{Title}</a></li>\n";
+    }
+    $o .= "</ul>\n";
+    my $forkind = sub {
+	my ($type,$desc,$pfx,$sfx) = @_;
+	$o .= "<h2>$desc</h2><ul>\n";
+	foreach my $name (sort keys %{ $sdef{$type} }) {
+	    my $href = refhref($type,$name);
+	    next unless $href =~ m/\S/;
+	    $o .= "<li><a $href>$pfx$name$sfx</a></li>\n";
+	}
+	$o .= "</ul>\n";
+    };
+    $forkind->('Func','Functions','','()');
+    $forkind->('Struct','Structs','struct ','');
+    $forkind->('Enum','Enums and sets of #defines','','');
+    $forkind->('EnumVal','Enum values and individual #defines','','');
+    $o .= "</ul>\n<h2>Files</h2><ul>\n";
+    foreach my $of (sort { $a->[0] cmp $b->[0] } @outfiles) {
+	$o .= "<li><a href=\"$of->[1]\">$of->[0]</a></li>\n";
+    }
+    $o .= "</ul></body></html>\n";
+    write_file("$outdir/index.html", $o);
+}
 
 foreach $pass (qw(1 2)) {
     find({ wanted => 
@@ -279,3 +332,5 @@ foreach $pass (qw(1 2)) {
 	 },
 	 map { "$basedir/$_" } @indirs);
 }
+
+output_index();
diff --git a/xen/include/public/xen.h b/xen/include/public/xen.h
index f2c9e6f..a5b6ad8 100644
--- a/xen/include/public/xen.h
+++ b/xen/include/public/xen.h
@@ -55,7 +55,9 @@ DEFINE_XEN_GUEST_HANDLE(xen_pfn_t);
  * HYPERCALLS
  */
 
-/* ` enum hypercall_num { // __HYPERVISOR_* => HYPERVISOR_*() */
+/* `incontents 100 hcalls List of hypercalls
+ * ` enum hypercall_num { // __HYPERVISOR_* => HYPERVISOR_*()
+ */
 
 #define __HYPERVISOR_set_trap_table        0
 #define __HYPERVISOR_mmu_update            1
-- 
1.7.2.5

Re: [PATCH 3/4] docs/html/hcall: Generate an "index.html"

[Ian Campbell]

On Sun, 2011-11-13 at 19:21 +0000, Ian Jackson wrote:
> @@ -231,6 +246,8 @@ sub process_file ($$) {
>  	    }
>  	} elsif (s/^( \s* \#define \s+ ) (\w+) ( \s+\S )
>                    / $1.defmacro($2).norm($3) /xe) {
> +	} elsif (s/( \`incontents \s+ (\d+) \s+ (\w+) \s+ )(\S .* \S)
> +                 / norm($1).incontents($4, $2, $3) /xe) {

You had a comment describing the syntax which I don't think includes
this bit.

Re: [PATCH 3/4] docs/html/hcall: Generate an "index.html"

[Ian Jackson]

Ian Campbell writes ("Re: [Xen-devel] [PATCH 3/4] docs/html/hcall: Generate an "index.html""):
> On Sun, 2011-11-13 at 19:21 +0000, Ian Jackson wrote:
> > @@ -231,6 +246,8 @@ sub process_file ($$) {
> >  	    }
> >  	} elsif (s/^( \s* \#define \s+ ) (\w+) ( \s+\S )
> >                    / $1.defmacro($2).norm($3) /xe) {
> > +	} elsif (s/( \`incontents \s+ (\d+) \s+ (\w+) \s+ )(\S .* \S)
> > +                 / norm($1).incontents($4, $2, $3) /xe) {
> 
> You had a comment describing the syntax which I don't think includes
> this bit.

Good point.

Ian.

[PATCH 4/4] docs/html/hcall: Arrange for automatic build

[Ian Jackson]

- Use index.html rather than a stamp file.
- Automatically generate dependencies.
- Wire into the docs build system

Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>
---
 docs/Makefile    |    9 ++++++---
 docs/xen-headers |   14 ++++++++++++++
 2 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/docs/Makefile b/docs/Makefile
index 01e59cb..3815170 100644
--- a/docs/Makefile
+++ b/docs/Makefile
@@ -15,7 +15,8 @@ DOC_MARKDOWN	:= $(wildcard misc/*.markdown)
 DOC_PS		:= $(patsubst src/%.tex,ps/%.ps,$(DOC_TEX))
 DOC_PDF		:= $(patsubst src/%.tex,pdf/%.pdf,$(DOC_TEX))
 DOC_HTML	:= $(patsubst src/%.tex,html/%/index.html,$(DOC_TEX)) \
-		   $(patsubst %.markdown,html/%.html,$(DOC_MARKDOWN))
+		   $(patsubst %.markdown,html/%.html,$(DOC_MARKDOWN)) \
+		   html/hcall/index.html
 DOC_TXT         := $(patsubst %.txt,txt/%.txt,$(wildcard misc/*.txt)) \
 		   $(patsubst %.markdown,txt/%.txt,$(DOC_MARKDOWN))
 
@@ -129,13 +130,15 @@ html/%.html: %.markdown
 	$(MARKDOWN) $< > $@.tmp ; \
 	$(call move-if-changed,$@.tmp,$@) ; fi
 
-html/hcall/stamp:
+html/hcall/index.html: ./xen-headers
+	rm -rf $(@D)
 	@$(INSTALL_DIR) $(@D)
 	./xen-headers -O $(@D) \
 		-T 'arch-x86_64 - Xen public headers' \
 		-X arch-ia64 -X arch-x86_32 -X xen-x86_32 \
 		../xen include/public include/xen/errno.h
-	touch $@
+
+-include html/hcall/.deps
 
 txt/%.txt: %.txt
 	$(INSTALL_DIR) $(@D)
diff --git a/docs/xen-headers b/docs/xen-headers
index 3cc9e40..606b824 100755
--- a/docs/xen-headers
+++ b/docs/xen-headers
@@ -310,6 +310,12 @@ END
 }
 
 foreach $pass (qw(1 2)) {
+    my $depspath = "$outdir/.deps";
+    my $depsout;
+    if ($pass==2) {
+	$depsout = new IO::File "$depspath.new", 'w' or die $!;
+    }
+
     find({ wanted => 
 	       sub {
 		   return unless m/\.h$/;
@@ -326,11 +332,19 @@ foreach $pass (qw(1 2)) {
 		   $leaf_opath = $leaf;
 		   $leaf_opath =~ s#/#,#g;
 		   $leaf_opath .= ".html";
+		   print $depsout "$outdir/index.html: $File::Find::name\n"
+		       or die $!
+		       if $pass==2;
 		   process_file($File::Find::name, $outdir.'/'.$leaf_opath);
 	   },
 	   no_chdir => 1,
 	 },
 	 map { "$basedir/$_" } @indirs);
+
+    if ($pass==2) {
+	close $depsout or die $!;
+	rename "$depspath.new", "$depspath" or die $!;
+    }
 }
 
 output_index();
-- 
1.7.2.5

Re: [PATCH 0/4] docs/html/hcall: Hypercall docs headers

[Ian Campbell]

On Sun, 2011-11-13 at 19:21 +0000, Ian Jackson wrote:
> This is my proposed approach to providing hypercall documentation: we
> provide an htmlified copy of the xen public headers, with a contents
> page and cross-reference hyperlinks added.
> 
> This is done by a 350-line Perl script to which we can add features as
> we think of them.
> 
> An example of the output can be found here:
>  http://www.chiark.greenend.org.uk/~ijackson/volatile/2011/xen/hcall/

Excellent stuff.

It seems to consider
        static struct xsd_errors xsd_errors[]
        #if defined(__GNUC__)
        __attribute__((unused))
as defining a function called __attribute__ which is a bit unfortunate!

I had some nitpicks as I looked over the patches but regardless I think
this is a good start and should go in. All:
Acked-by: Ian Campbell <ian.campbell@citrix.com>

Ian.

Re: [PATCH 0/4] docs/html/hcall: Hypercall docs headers

[Ian Jackson]

Ian Campbell writes ("Re: [Xen-devel] [PATCH 0/4] docs/html/hcall: Hypercall doc> Excellent stuff.

Glad you like it.

> It seems to consider
>         static struct xsd_errors xsd_errors[]
>         #if defined(__GNUC__)
>         __attribute__((unused))
> as defining a function called __attribute__ which is a bit unfortunate!

I can fix that easily enough by special-casing keywords.

> I had some nitpicks as I looked over the patches but regardless I think
> this is a good start and should go in. All:
> Acked-by: Ian Campbell <ian.campbell@citrix.com>

Thanks.

Ian.