[docs] Update djangodocs extension and conf.py to build against current Django and sphinx verions

docs/_ext/djangodocs.py

@@ -6,15 +6,18 @@ import os
 import re
 
 from docutils import nodes
-from docutils.parsers.rst import directives
+from docutils.parsers.rst import Directive
+from docutils.statemachine import ViewList
 from sphinx import addnodes
 from sphinx.builders.html import StandaloneHTMLBuilder
+from sphinx.directives.code import CodeBlock
 from sphinx.domains.std import Cmdoption
-from sphinx.util.compat import Directive
+from sphinx.errors import ExtensionError
+from sphinx.util import logging
 from sphinx.util.console import bold
-from sphinx.util.nodes import set_source_info
+from sphinx.writers.html import HTMLTranslator
-from sphinx.writers.html import SmartyPantsHTMLTranslator
 
+logger = logging.getLogger(__name__)
 # RE for option descriptions without a '--' prefix
 simple_option_desc_re = re.compile(
     r'([-_a-zA-Z0-9]+)(\s*.*?)(?=,\s+(?:/|-|--)|$)')
@@ -52,147 +55,22 @@ def setup(app):
     app.add_directive('versionadded', VersionDirective)
     app.add_directive('versionchanged', VersionDirective)
     app.add_builder(DjangoStandaloneHTMLBuilder)
-
+    app.set_translator('djangohtml', DjangoHTMLTranslator)
-    # register the snippet directive
+    app.set_translator('json', DjangoHTMLTranslator)
-    app.add_directive('snippet', SnippetWithFilename)
+    app.add_node(
-    # register a node for snippet directive so that the xml parser
+        ConsoleNode,
-    # knows how to handle the enter/exit parsing event
+        html=(visit_console_html, None),
-    app.add_node(snippet_with_filename,
+        latex=(visit_console_dummy, depart_console_dummy),
-                 html=(visit_snippet, depart_snippet_literal),
+        man=(visit_console_dummy, depart_console_dummy),
-                 latex=(visit_snippet_latex, depart_snippet_latex),
+        text=(visit_console_dummy, depart_console_dummy),
-                 man=(visit_snippet_literal, depart_snippet_literal),
+        texinfo=(visit_console_dummy, depart_console_dummy),
-                 text=(visit_snippet_literal, depart_snippet_literal),
+    )
-                 texinfo=(visit_snippet_literal, depart_snippet_literal))
+    app.add_directive('console', ConsoleDirective)
+    app.connect('html-page-context', html_page_context_hook)
+    app.add_role('default-role-error', default_role_error)
     return {'parallel_read_safe': True}
 
 
-class snippet_with_filename(nodes.literal_block):
-    """
-    Subclass the literal_block to override the visit/depart event handlers
-    """
-    pass
-
-
-def visit_snippet_literal(self, node):
-    """
-    default literal block handler
-    """
-    self.visit_literal_block(node)
-
-
-def depart_snippet_literal(self, node):
-    """
-    default literal block handler
-    """
-    self.depart_literal_block(node)
-
-
-def visit_snippet(self, node):
-    """
-    HTML document generator visit handler
-    """
-    lang = self.highlightlang
-    linenos = node.rawsource.count('\n') >= self.highlightlinenothreshold - 1
-    fname = node['filename']
-    highlight_args = node.get('highlight_args', {})
-    if 'language' in node:
-        # code-block directives
-        lang = node['language']
-        highlight_args['force'] = True
-    if 'linenos' in node:
-        linenos = node['linenos']
-
-    def warner(msg):
-        self.builder.warn(msg, (self.builder.current_docname, node.line))
-
-    highlighted = self.highlighter.highlight_block(node.rawsource, lang,
-                                                   warn=warner,
-                                                   linenos=linenos,
-                                                   **highlight_args)
-    starttag = self.starttag(node, 'div', suffix='',
-                             CLASS='highlight-%s snippet' % lang)
-    self.body.append(starttag)
-    self.body.append('<div class="snippet-filename">%s</div>\n''' % (fname,))
-    self.body.append(highlighted)
-    self.body.append('</div>\n')
-    raise nodes.SkipNode
-
-
-def visit_snippet_latex(self, node):
-    """
-    Latex document generator visit handler
-    """
-    code = node.rawsource.rstrip('\n')
-
-    lang = self.hlsettingstack[-1][0]
-    linenos = code.count('\n') >= self.hlsettingstack[-1][1] - 1
-    fname = node['filename']
-    highlight_args = node.get('highlight_args', {})
-    if 'language' in node:
-        # code-block directives
-        lang = node['language']
-        highlight_args['force'] = True
-    if 'linenos' in node:
-        linenos = node['linenos']
-
-    def warner(msg):
-        self.builder.warn(msg, (self.curfilestack[-1], node.line))
-
-    hlcode = self.highlighter.highlight_block(code, lang, warn=warner,
-                                              linenos=linenos,
-                                              **highlight_args)
-
-    self.body.append(
-        '\n{\\colorbox[rgb]{0.9,0.9,0.9}'
-        '{\\makebox[\\textwidth][l]'
-        '{\\small\\texttt{%s}}}}\n' % (
-            # Some filenames have '_', which is special in latex.
-            fname.replace('_', r'\_'),
-        )
-    )
-
-    if self.table:
-        hlcode = hlcode.replace('\\begin{Verbatim}',
-                                '\\begin{OriginalVerbatim}')
-        self.table.has_problematic = True
-        self.table.has_verbatim = True
-
-    hlcode = hlcode.rstrip()[:-14]  # strip \end{Verbatim}
-    hlcode = hlcode.rstrip() + '\n'
-    self.body.append('\n' + hlcode + '\\end{%sVerbatim}\n' %
-                     (self.table and 'Original' or ''))
-
-    # Prevent rawsource from appearing in output a second time.
-    raise nodes.SkipNode
-
-
-def depart_snippet_latex(self, node):
-    """
-    Latex document generator depart handler.
-    """
-    pass
-
-
-class SnippetWithFilename(Directive):
-    """
-    The 'snippet' directive that allows to add the filename (optional)
-    of a code snippet in the document. This is modeled after CodeBlock.
-    """
-    has_content = True
-    optional_arguments = 1
-    option_spec = {'filename': directives.unchanged_required}
-
-    def run(self):
-        code = '\n'.join(self.content)
-
-        literal = snippet_with_filename(code, code)
-        if self.arguments:
-            literal['language'] = self.arguments[0]
-        literal['filename'] = self.options['filename']
-        set_source_info(self, literal)
-        return [literal]
-
-
 class VersionDirective(Directive):
     has_content = True
     required_arguments = 1
@@ -220,11 +98,15 @@ class VersionDirective(Directive):
         node['type'] = self.name
         if self.content:
             self.state.nested_parse(self.content, self.content_offset, node)
+        try:
+            env.get_domain('changeset').note_changeset(node)
+        except ExtensionError:
+            # Sphinx < 1.8: Domain 'changeset' is not registered
             env.note_versionchange(node['type'], node['version'], node, self.lineno)
         return ret
 
 
-class DjangoHTMLTranslator(SmartyPantsHTMLTranslator):
+class DjangoHTMLTranslator(HTMLTranslator):
     """
     Django-specific reST to HTML tweaks.
     """
@@ -245,8 +127,7 @@ class DjangoHTMLTranslator(SmartyPantsHTMLTranslator):
         self.first_param = 1
         self.optional_param_level = 0
         self.param_separator = node.child_text_separator
-        self.required_params_left = sum([isinstance(c, addnodes.desc_parameter)
+        self.required_params_left = sum(isinstance(c, addnodes.desc_parameter) for c in node.children)
-                                         for c in node.children])
 
     def depart_desc_parameterlist(self, node):
         self.body.append(')')
@@ -285,7 +166,7 @@ class DjangoHTMLTranslator(SmartyPantsHTMLTranslator):
         old_ids = node.get('ids', [])
         node['ids'] = ['s-' + i for i in old_ids]
         node['ids'].extend(old_ids)
-        SmartyPantsHTMLTranslator.visit_section(self, node)
+        super().visit_section(node)
         node['ids'] = old_ids
 
 
@@ -305,17 +186,201 @@ class DjangoStandaloneHTMLBuilder(StandaloneHTMLBuilder):
     name = 'djangohtml'
 
     def finish(self):
-        super(DjangoStandaloneHTMLBuilder, self).finish()
+        super().finish()
-        self.info(bold("writing templatebuiltins.js..."))
+        logger.info(bold("writing templatebuiltins.js..."))
         xrefs = self.env.domaindata["std"]["objects"]
         templatebuiltins = {
-            "ttags": [n for ((t, n), (l, a)) in xrefs.items()
+            "ttags": [
-                      if t == "templatetag" and l == "ref/templates/builtins"],
+                n for ((t, n), (k, a)) in xrefs.items()
-            "tfilters": [n for ((t, n), (l, a)) in xrefs.items()
+                if t == "templatetag" and k == "ref/templates/builtins"
-                         if t == "templatefilter" and l == "ref/templates/builtins"],
+            ],
+            "tfilters": [
+                n for ((t, n), (k, a)) in xrefs.items()
+                if t == "templatefilter" and k == "ref/templates/builtins"
+            ],
         }
         outfilename = os.path.join(self.outdir, "templatebuiltins.js")
         with open(outfilename, 'w') as fp:
             fp.write('var django_template_builtins = ')
             json.dump(templatebuiltins, fp)
             fp.write(';\n')
+
+
+class ConsoleNode(nodes.literal_block):
+    """
+    Custom node to override the visit/depart event handlers at registration
+    time. Wrap a literal_block object and defer to it.
+    """
+    tagname = 'ConsoleNode'
+
+    def __init__(self, litblk_obj):
+        self.wrapped = litblk_obj
+
+    def __getattr__(self, attr):
+        if attr == 'wrapped':
+            return self.__dict__.wrapped
+        return getattr(self.wrapped, attr)
+
+
+def visit_console_dummy(self, node):
+    """Defer to the corresponding parent's handler."""
+    self.visit_literal_block(node)
+
+
+def depart_console_dummy(self, node):
+    """Defer to the corresponding parent's handler."""
+    self.depart_literal_block(node)
+
+
+def visit_console_html(self, node):
+    """Generate HTML for the console directive."""
+    if self.builder.name in ('djangohtml', 'json') and node['win_console_text']:
+        # Put a mark on the document object signaling the fact the directive
+        # has been used on it.
+        self.document._console_directive_used_flag = True
+        uid = node['uid']
+        self.body.append('''\
+<div class="console-block" id="console-block-%(id)s">
+<input class="c-tab-unix" id="c-tab-%(id)s-unix" type="radio" name="console-%(id)s" checked>
+<label for="c-tab-%(id)s-unix" title="Linux/macOS">&#xf17c/&#xf179</label>
+<input class="c-tab-win" id="c-tab-%(id)s-win" type="radio" name="console-%(id)s">
+<label for="c-tab-%(id)s-win" title="Windows">&#xf17a</label>
+<section class="c-content-unix" id="c-content-%(id)s-unix">\n''' % {'id': uid})
+        try:
+            self.visit_literal_block(node)
+        except nodes.SkipNode:
+            pass
+        self.body.append('</section>\n')
+
+        self.body.append('<section class="c-content-win" id="c-content-%(id)s-win">\n' % {'id': uid})
+        win_text = node['win_console_text']
+        highlight_args = {'force': True}
+        linenos = node.get('linenos', False)
+
+        def warner(msg):
+            self.builder.warn(msg, (self.builder.current_docname, node.line))
+
+        highlighted = self.highlighter.highlight_block(
+            win_text, 'doscon', warn=warner, linenos=linenos, **highlight_args
+        )
+        self.body.append(highlighted)
+        self.body.append('</section>\n')
+        self.body.append('</div>\n')
+        raise nodes.SkipNode
+    else:
+        self.visit_literal_block(node)
+
+
+class ConsoleDirective(CodeBlock):
+    """
+    A reStructuredText directive which renders a two-tab code block in which
+    the second tab shows a Windows command line equivalent of the usual
+    Unix-oriented examples.
+    """
+    required_arguments = 0
+    # The 'doscon' Pygments formatter needs a prompt like this. '>' alone
+    # won't do it because then it simply paints the whole command line as a
+    # grey comment with no highlighting at all.
+    WIN_PROMPT = r'...\> '
+
+    def run(self):
+
+        def args_to_win(cmdline):
+            changed = False
+            out = []
+            for token in cmdline.split():
+                if token[:2] == './':
+                    token = token[2:]
+                    changed = True
+                elif token[:2] == '~/':
+                    token = '%HOMEPATH%\\' + token[2:]
+                    changed = True
+                elif token == 'make':
+                    token = 'make.bat'
+                    changed = True
+                if '://' not in token and 'git' not in cmdline:
+                    out.append(token.replace('/', '\\'))
+                    changed = True
+                else:
+                    out.append(token)
+            if changed:
+                return ' '.join(out)
+            return cmdline
+
+        def cmdline_to_win(line):
+            if line.startswith('# '):
+                return 'REM ' + args_to_win(line[2:])
+            if line.startswith('$ # '):
+                return 'REM ' + args_to_win(line[4:])
+            if line.startswith('$ ./manage.py'):
+                return 'manage.py ' + args_to_win(line[13:])
+            if line.startswith('$ manage.py'):
+                return 'manage.py ' + args_to_win(line[11:])
+            if line.startswith('$ ./runtests.py'):
+                return 'runtests.py ' + args_to_win(line[15:])
+            if line.startswith('$ ./'):
+                return args_to_win(line[4:])
+            if line.startswith('$ python3'):
+                return 'py ' + args_to_win(line[9:])
+            if line.startswith('$ python'):
+                return 'py ' + args_to_win(line[8:])
+            if line.startswith('$ '):
+                return args_to_win(line[2:])
+            return None
+
+        def code_block_to_win(content):
+            bchanged = False
+            lines = []
+            for line in content:
+                modline = cmdline_to_win(line)
+                if modline is None:
+                    lines.append(line)
+                else:
+                    lines.append(self.WIN_PROMPT + modline)
+                    bchanged = True
+            if bchanged:
+                return ViewList(lines)
+            return None
+
+        env = self.state.document.settings.env
+        self.arguments = ['console']
+        lit_blk_obj = super().run()[0]
+
+        # Only do work when the djangohtml HTML Sphinx builder is being used,
+        # invoke the default behavior for the rest.
+        if env.app.builder.name not in ('djangohtml', 'json'):
+            return [lit_blk_obj]
+
+        lit_blk_obj['uid'] = str(env.new_serialno('console'))
+        # Only add the tabbed UI if there is actually a Windows-specific
+        # version of the CLI example.
+        win_content = code_block_to_win(self.content)
+        if win_content is None:
+            lit_blk_obj['win_console_text'] = None
+        else:
+            self.content = win_content
+            lit_blk_obj['win_console_text'] = super().run()[0].rawsource
+
+        # Replace the literal_node object returned by Sphinx's CodeBlock with
+        # the ConsoleNode wrapper.
+        return [ConsoleNode(lit_blk_obj)]
+
+
+def html_page_context_hook(app, pagename, templatename, context, doctree):
+    # Put a bool on the context used to render the template. It's used to
+    # control inclusion of console-tabs.css and activation of the JavaScript.
+    # This way it's include only from HTML files rendered from reST files where
+    # the ConsoleDirective is used.
+    context['include_console_assets'] = getattr(doctree, '_console_directive_used_flag', False)
+
+
+def default_role_error(
+    name, rawtext, text, lineno, inliner, options=None, content=None
+):
+    msg = (
+        "Default role used (`single backticks`): %s. Did you mean to use two "
+        "backticks for ``code``, or miss an underscore for a `link`_ ?"
+        % rawtext
+    )
+    logger.warning(msg, location=(inliner.document.current_source, lineno))
+    return [nodes.Text(text)], []

docs/conf.py

@@ -350,7 +350,7 @@ texinfo_documents = [
 # Example configuration for intersphinx: refer to the Python standard library.
 intersphinx_mapping = {
     "python": ('https://docs.python.org/', None),
-    "django": ('https://docs.djangoproject.com/en/1.9/', 'django.inv'),
+    "django": ('https://docs.djangoproject.com/en/3.2/', 'django.inv'),
 }
 
 autodoc_member_order = 'bysource'
@@ -360,8 +360,8 @@ locale_dirs = ['../test_venv/lib/python2.7/site-packages/django/conf/locale/']
 
 def _download_django_inv():
     import requests
-    with open(_download_django_inv.path, 'w') as f:
+    with open(_download_django_inv.path, 'wb') as f:
-        r = requests.get("https://docs.djangoproject.com/en/1.9/_objects")
+        r = requests.get("https://docs.djangoproject.com/en/3.2/_objects")
         f.write(r.content)
 _download_django_inv.path = os.path.abspath(os.path.join(os.path.dirname(__file__), "django.inv"))
 

docs/index.rst

@@ -23,6 +23,7 @@ Indices and tables
 ==================
 
 * :ref:`genindex`
+
 .. * :ref:`modindex`
 .. * :ref:`search`