diff mbox

[08/36] qapi: add qapi2texi script

Message ID 1443189844-20341-9-git-send-email-marcandre.lureau@redhat.com
State New
Headers show

Commit Message

Marc-André Lureau Sept. 25, 2015, 2:03 p.m. UTC
From: Marc-André Lureau <marcandre.lureau@redhat.com>

As the name suggests, the qapi2texi script converts JSON QAPI
description into a standalone texi file suitable for different target
formats.

It parses the following kind of blocks with some little variations:

  ##
  # = Section
  # == Subsection
  #
  # Some text foo with *emphasis*
  # 1. with a list
  # 2. like that
  #
  # And some code:
  # | <- do this
  # | -> get that
  #
  ##

  ##
  # @symbol
  #
  # Symbol body ditto ergo sum. Foo bar
  # baz ding.
  #
  # @arg: foo
  # @arg: #optional foo
  #
  # Returns: returns bla bla
  #
  #          Or bla blah
  #
  # Since: version
  # Notes: notes, comments can have
  #        - itemized list
  #        - like this
  #
  #        and continue...
  #
  # Example:
  #
  # -> { "execute": "quit" }
  # <- { "return": {} }
  #
  ##

Thanks to the json declaration, it's able to give extra information
about the type of arguments and return value expected.

Signed-off-by: Marc-André Lureau <marcandre.lureau@redhat.com>
---
 scripts/qapi.py      |  88 +++++++++++++++-
 scripts/qapi2texi.py | 293 +++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 379 insertions(+), 2 deletions(-)
 create mode 100755 scripts/qapi2texi.py

Comments

Eric Blake Sept. 25, 2015, 3:34 p.m. UTC | #1
On 09/25/2015 08:03 AM, marcandre.lureau@redhat.com wrote:
> From: Marc-André Lureau <marcandre.lureau@redhat.com>
> 
> As the name suggests, the qapi2texi script converts JSON QAPI
> description into a standalone texi file suitable for different target
> formats.
> 
> It parses the following kind of blocks with some little variations:
> 
>   ##
>   # = Section
>   # == Subsection
>   #
>   # Some text foo with *emphasis*
>   # 1. with a list
>   # 2. like that
>   #
>   # And some code:
>   # | <- do this
>   # | -> get that

Backwards; you mean:

# | -> send this
# | <- get that


>   # Example:
>   #
>   # -> { "execute": "quit" }
>   # <- { "return": {} }
>   #

but this one is right.

I'd love to have this formalized semantics of what you plan to parse
documented in docs/qapi-code-gen.txt, as a reference we can point to for
new commands that don't comply.

> Thanks to the json declaration, it's able to give extra information
> about the type of arguments and return value expected.
> 
> Signed-off-by: Marc-André Lureau <marcandre.lureau@redhat.com>
> ---
>  scripts/qapi.py      |  88 +++++++++++++++-
>  scripts/qapi2texi.py | 293 +++++++++++++++++++++++++++++++++++++++++++++++++++
>  2 files changed, 379 insertions(+), 2 deletions(-)
>  create mode 100755 scripts/qapi2texi.py

I will forbear the actual review detailed for now, in part because it
may need another rebase, but the idea seems nice.  But I'll do a quick
read-through review:

> 
> diff --git a/scripts/qapi.py b/scripts/qapi.py
> index 27894c1..2a9b6e5 100644
> --- a/scripts/qapi.py
> +++ b/scripts/qapi.py
> @@ -112,6 +112,67 @@ class QAPIExprError(Exception):
>              "%s:%d: %s" % (self.info['file'], self.info['line'], self.msg)
>  
>  
> +class QAPIDoc:

Use the new-style class inheritance (as in "class QAPIDoc(object):")


> +++ b/scripts/qapi2texi.py
> @@ -0,0 +1,293 @@
> +#!/usr/bin/env python
> +# QAPI texi generator
> +#
> +# This work is licensed under the terms of the GNU GPL, version 2.

Can we please use GPLv2+ for this file? GPLv2-only is a pain (yes,
qapi.py is stuck there unless someone does legwork to see if former
contributors don't mind relaxing it, but that doesn't mean we have to
repeat the mistake)
diff mbox

Patch

diff --git a/scripts/qapi.py b/scripts/qapi.py
index 27894c1..2a9b6e5 100644
--- a/scripts/qapi.py
+++ b/scripts/qapi.py
@@ -112,6 +112,67 @@  class QAPIExprError(Exception):
             "%s:%d: %s" % (self.info['file'], self.info['line'], self.msg)
 
 
+class QAPIDoc:
+    def __init__(self, comment):
+        self.symbol = None
+        self.comment = ""
+        self.args = OrderedDict()
+        self.meta = OrderedDict()
+        self.section = None
+
+        for line in comment.split('\n'):
+            sline = ' '.join(line.split())
+            split = sline.split(' ', 1)
+            key = split[0].rstrip(':')
+
+            if line.startswith(" @"):
+                key = key[1:]
+                sline = split[1] if len(split) > 1 else ""
+                if self.symbol is None:
+                    self.symbol = key
+                else:
+                    self.start_section(self.args, key)
+            elif self.symbol and \
+                    key in ("Since", "Returns", "Notes", "Note", "Example"):
+                sline = split[1] if len(split) > 1 else ""
+                line = None
+                self.start_section(self.meta, key)
+
+            if self.section and self.section[1] == "Example":
+                self.append_comment(line)
+            else:
+                self.append_comment(sline)
+
+        self.end_section()
+
+    def append_comment(self, line):
+        if line is None:
+            return
+        if self.section is not None:
+            if self.section[-1] == "" and line == "":
+                self.end_section()
+            else:
+                self.section.append(line)
+        elif self.comment == "":
+            self.comment = line
+        else:
+            self.comment += "\n" + line
+
+    def end_section(self):
+        if self.section is not None:
+            dic = self.section[0]
+            key = self.section[1]
+            doc = "\n".join(self.section[2:])
+            if key != "Example":
+                doc = doc.strip()
+            dic[key] = doc
+            self.section = None
+
+    def start_section(self, dic, key):
+        self.end_section()
+        self.section = [dic, key]  # .. remaining elems will be the doc
+
+
 class QAPISchemaParser(object):
 
     def __init__(self, fp, previously_included=[], incl_info=None):
@@ -127,11 +188,14 @@  class QAPISchemaParser(object):
         self.line = 1
         self.line_pos = 0
         self.exprs = []
+        self.comment = None
+        self.apidoc = incl_info['doc'] if incl_info else []
         self.accept()
 
         while self.tok is not None:
             expr_info = {'file': fname, 'line': self.line,
-                         'parent': self.incl_info}
+                         'parent': self.incl_info, 'doc': self.apidoc}
+            self.apidoc = []
             expr = self.get_expr(False)
             if isinstance(expr, dict) and "include" in expr:
                 if len(expr) != 1:
@@ -152,6 +216,8 @@  class QAPISchemaParser(object):
                     inf = inf['parent']
                 # skip multiple include of the same file
                 if incl_abs_fname in previously_included:
+                    expr_info['doc'].extend(self.apidoc)
+                    self.apidoc = expr_info['doc']
                     continue
                 try:
                     fobj = open(incl_abs_fname, 'r')
@@ -166,6 +232,12 @@  class QAPISchemaParser(object):
                              'info': expr_info}
                 self.exprs.append(expr_elem)
 
+    def append_doc(self):
+        if self.comment:
+            apidoc = QAPIDoc(self.comment)
+            self.apidoc.append(apidoc)
+            self.comment = None
+
     def accept(self):
         while True:
             self.tok = self.src[self.cursor]
@@ -174,8 +246,20 @@  class QAPISchemaParser(object):
             self.val = None
 
             if self.tok == '#':
-                self.cursor = self.src.find('\n', self.cursor)
+                end = self.src.find('\n', self.cursor)
+                line = self.src[self.cursor:end+1]
+                # start a comment section after ##
+                if line[0] == "#":
+                    if self.comment is None:
+                        self.comment = ""
+                # skip modeline
+                elif line.find("-*") == -1 and self.comment is not None:
+                    self.comment += line
+                if self.src[end] == "\n" and self.src[end+1] == "\n":
+                    self.append_doc()
+                self.cursor = end
             elif self.tok in ['{', '}', ':', ',', '[', ']']:
+                self.append_doc()
                 return
             elif self.tok == "'":
                 string = ''
diff --git a/scripts/qapi2texi.py b/scripts/qapi2texi.py
new file mode 100755
index 0000000..76ade1b
--- /dev/null
+++ b/scripts/qapi2texi.py
@@ -0,0 +1,293 @@ 
+#!/usr/bin/env python
+# QAPI texi generator
+#
+# This work is licensed under the terms of the GNU GPL, version 2.
+# See the COPYING file in the top-level directory.
+"""This script produces the documentation of a qapi schema in texinfo format"""
+import re
+import sys
+
+from qapi import QAPISchemaParser, QAPISchemaError, check_exprs, QAPIExprError
+
+COMMAND_FMT = """
+@deftypefn {type} {{{ret}}} {name} @
+{{{args}}}
+
+{body}
+
+@end deftypefn
+
+""".format
+
+ENUM_FMT = """
+@deftp Enum {name}
+
+{body}
+
+@end deftp
+
+""".format
+
+STRUCT_FMT = """
+@deftp {type} {name} @
+{{{attrs}}}
+
+{body}
+
+@end deftp
+
+""".format
+
+EXAMPLE_FMT = """@example
+{code}
+@end example
+""".format
+
+
+def subst_emph(doc):
+    return re.sub(r'\*(\w*)\*', r'@emph{\1}', doc)
+
+
+def subst_vars(doc):
+    return re.sub(r'@(\w*)', r'@var{\1}', doc)
+
+
+def subst_braces(doc):
+    return doc.replace("{", "@{").replace("}", "@}")
+
+
+def texi_example(doc):
+    return EXAMPLE_FMT(code=subst_braces(doc).strip('\n'))
+
+
+def texi_comment(doc):
+    lines = []
+    doc = subst_vars(doc)
+    doc = subst_emph(doc)
+    inlist = False
+    lastempty = False
+    for line in doc.split('\n'):
+        empty = line == ""
+
+        if line.startswith("| "):
+            line = EXAMPLE_FMT(code=line[1:])
+        elif line.startswith("= "):
+            line = "@section " + line[1:]
+        elif line.startswith("== "):
+            line = "@subsection " + line[2:]
+        elif re.match("^([0-9]*[.) ]) ", line):
+            if not inlist:
+                lines.append("@enumerate")
+                inlist = "enumerate"
+            line = line[line.find(" ")+1:]
+            lines.append("@item")
+        elif re.match("^[o*-] ", line):
+            if not inlist:
+                lines.append("@itemize %s" % {'o': "@bullet",
+                                              '*': "@minus",
+                                              '-': ""}[line[0]])
+                inlist = "itemize"
+            lines.append("@item")
+            line = line[2:]
+        elif lastempty and inlist:
+            lines.append("@end %s\n" % inlist)
+            inlist = False
+
+        lastempty = empty
+        lines.append(line)
+
+    if inlist:
+        lines.append("@end %s\n" % inlist)
+    return "\n".join(lines)
+
+
+def texi_args(expr):
+    data = expr["data"] if "data" in expr else {}
+    if isinstance(data, str):
+        args = data
+    else:
+        args = []
+        for name, typ in data.iteritems():
+            # optional arg
+            if name.startswith("*"):
+                name = name[1:]
+                args.append("['%s': @var{%s}]" % (name, typ))
+            # regular arg
+            else:
+                args.append("'%s': @var{%s}" % (name, typ))
+        args = ", ".join(args)
+    return args
+
+
+def texi_body(doc, arg="@var"):
+    body = "@table %s\n" % arg
+    for arg, desc in doc.args.iteritems():
+        if desc.startswith("#optional"):
+            desc = desc[10:]
+            arg += "*"
+        body += "@item %s\n%s\n" % (arg, texi_comment(desc))
+    body += "@end table\n"
+    body += texi_comment(doc.comment)
+
+    for k in ("Returns", "Note", "Notes", "Since", "Example"):
+        if k not in doc.meta:
+            continue
+        func = texi_comment if k != "Example" else texi_example
+        body += "\n@quotation %s\n%s\n@end quotation" % \
+                (k, func(doc.meta[k]))
+    return body
+
+
+def texi_alternate(expr, doc):
+    args = texi_args(expr)
+    body = texi_body(doc)
+    return STRUCT_FMT(type="Alternate",
+                      name=doc.symbol,
+                      attrs="[ " + args + " ]",
+                      body=body)
+
+
+def texi_union(expr, doc):
+    args = texi_args(expr)
+    body = texi_body(doc)
+    return STRUCT_FMT(type="Union",
+                      name=doc.symbol,
+                      attrs="[ " + args + " ]",
+                      body=body)
+
+
+def texi_enum(_, doc):
+    body = texi_body(doc, "@samp")
+    return ENUM_FMT(name=doc.symbol,
+                    body=body)
+
+
+def texi_struct(expr, doc):
+    args = texi_args(expr)
+    body = texi_body(doc)
+    return STRUCT_FMT(type="Struct",
+                      name=doc.symbol,
+                      attrs="@{ " + args + " @}",
+                      body=body)
+
+
+def texi_command(expr, doc):
+    args = texi_args(expr)
+    ret = expr["returns"] if "returns" in expr else ""
+    body = texi_body(doc)
+    return COMMAND_FMT(type="Command",
+                       name=doc.symbol,
+                       ret=ret,
+                       args="(" + args + ")",
+                       body=body)
+
+
+def texi_event(expr, doc):
+    args = texi_args(expr)
+    body = texi_body(doc)
+    return COMMAND_FMT(type="Event",
+                       name=doc.symbol,
+                       ret="",
+                       args="(" + args + ")",
+                       body=body)
+
+
+def parse_schema(fname):
+    try:
+        schema = QAPISchemaParser(open(fname, "r"))
+        check_exprs(schema.exprs)
+        return schema.exprs
+    except (QAPISchemaError, QAPIExprError), err:
+        print >>sys.stderr, err
+        exit(1)
+
+def main(argv):
+    if len(argv) != 5:
+        print >>sys.stderr, "%s: need exactly 4 arguments" % argv[0]
+        sys.exit(1)
+
+    exprs = parse_schema(argv[4])
+
+    print r"""
+\input texinfo
+@setfilename {filename}
+@documentlanguage en
+@exampleindent 0
+@paragraphindent 0
+
+@settitle {title}
+
+@ifinfo
+@direntry
+* QEMU: (qemu-doc).    {title}
+@end direntry
+@end ifinfo
+
+@titlepage
+@title {title} {version}
+@end titlepage
+
+@ifnottex
+@node Top
+@top
+
+This is the API reference for QEMU {version}.
+
+@menu
+* API Reference::
+* Commands and Events Index::
+* Data Types Index::
+@end menu
+
+@end ifnottex
+
+@contents
+
+@node API Reference
+@chapter API Reference
+
+@c man begin DESCRIPTION
+""".format(title=argv[1], version=argv[2], filename=argv[3])
+
+    for cmd in exprs:
+        try:
+            expr = cmd['expr']
+            docs = cmd['info']['doc']
+
+            (kind, _) = expr.items()[0]
+
+            for doc in docs[0:-1]:
+                print texi_body(doc)
+
+            texi = {"command": texi_command,
+                    "struct": texi_struct,
+                    "enum": texi_enum,
+                    "union": texi_union,
+                    "alternate": texi_alternate,
+                    "event": texi_event}
+            try:
+                print texi[kind](expr, docs[-1])
+            except KeyError:
+                raise ValueError("Unknown expression kind '%s'" % kind)
+        except:
+            print >>sys.stderr, "error at @%s" % cmd
+            raise
+
+    print """
+@c man end
+
+@c man begin SEEALSO
+The HTML documentation of QEMU for more precise information.
+@c man end
+
+@node Commands and Events Index
+@unnumbered Commands and Events Index
+@printindex fn
+@node Data Types Index
+@unnumbered Data Types Index
+@printindex tp
+@bye
+"""
+
+if __name__ == "__main__":
+    main(sys.argv)