# -*- coding: utf-8 -*- """ sphinx.ext.intersphinx ~~~~~~~~~~~~~~~~~~~~~~ Insert links to objects documented in remote Sphinx documentation. This works as follows: * Each Sphinx HTML build creates a file named "objects.inv" that contains a mapping from object names to URIs relative to the HTML set's root. * Projects using the Intersphinx extension can specify links to such mapping files in the `intersphinx_mapping` config value. The mapping will then be used to resolve otherwise missing references to objects into links to the other documentation. * By default, the mapping file is assumed to be at the same location as the rest of the documentation; however, the location of the mapping file can also be specified individually, e.g. if the docs should be buildable without Internet access. :copyright: Copyright 2007-2011 by the Sphinx team, see AUTHORS. :license: BSD, see LICENSE for details. """ import time import zlib import codecs import urllib2 import posixpath from os import path import re from docutils import nodes from sphinx.locale import _ from sphinx.builders.html import INVENTORY_FILENAME from sphinx.util.pycompat import b handlers = [urllib2.ProxyHandler(), urllib2.HTTPRedirectHandler(), urllib2.HTTPHandler()] try: handlers.append(urllib2.HTTPSHandler) except AttributeError: pass urllib2.install_opener(urllib2.build_opener(*handlers)) UTF8StreamReader = codecs.lookup('utf-8')[2] def read_inventory_v1(f, uri, join): f = UTF8StreamReader(f) invdata = {} line = f.next() projname = line.rstrip()[11:] line = f.next() version = line.rstrip()[11:] for line in f: name, type, location = line.rstrip().split(None, 2) location = join(uri, location) # version 1 did not add anchors to the location if type == 'mod': type = 'py:module' location += '#module-' + name else: type = 'py:' + type location += '#' + name invdata.setdefault(type, {})[name] = (projname, version, location, '-') return invdata def read_inventory_v2(f, uri, join, bufsize=16*1024): invdata = {} line = f.readline() projname = line.rstrip()[11:].decode('utf-8') line = f.readline() version = line.rstrip()[11:].decode('utf-8') line = f.readline().decode('utf-8') if 'zlib' not in line: raise ValueError def read_chunks(): decompressor = zlib.decompressobj() for chunk in iter(lambda: f.read(bufsize), b('')): yield decompressor.decompress(chunk) yield decompressor.flush() def split_lines(iter): buf = b('') for chunk in iter: buf += chunk lineend = buf.find(b('\n')) while lineend != -1: yield buf[:lineend].decode('utf-8') buf = buf[lineend+1:] lineend = buf.find(b('\n')) assert not buf for line in split_lines(read_chunks()): # be careful to handle names with embedded spaces correctly m = re.match(r'(?x)(.+?)\s+(\S*:\S*)\s+(\S+)\s+(\S+)\s+(.*)', line.rstrip()) if not m: continue name, type, prio, location, dispname = m.groups() if location.endswith(u'$'): location = location[:-1] + name location = join(uri, location) invdata.setdefault(type, {})[name] = (projname, version, location, dispname) return invdata def fetch_inventory(app, uri, inv): """Fetch, parse and return an intersphinx inventory file.""" # both *uri* (base URI of the links to generate) and *inv* (actual # location of the inventory file) can be local or remote URIs localuri = uri.find('://') == -1 join = localuri and path.join or posixpath.join try: if inv.find('://') != -1: f = urllib2.urlopen(inv) else: f = open(path.join(app.srcdir, inv), 'rb') except Exception, err: app.warn('intersphinx inventory %r not fetchable due to ' '%s: %s' % (inv, err.__class__, err)) return try: line = f.readline().rstrip().decode('utf-8') try: if line == '# Sphinx inventory version 1': invdata = read_inventory_v1(f, uri, join) elif line == '# Sphinx inventory version 2': invdata = read_inventory_v2(f, uri, join) else: raise ValueError f.close() except ValueError: f.close() raise ValueError('unknown or unsupported inventory version') except Exception, err: app.warn('intersphinx inventory %r not readable due to ' '%s: %s' % (inv, err.__class__.__name__, err)) else: return invdata def load_mappings(app): """Load all intersphinx mappings into the environment.""" now = int(time.time()) cache_time = now - app.config.intersphinx_cache_limit * 86400 env = app.builder.env if not hasattr(env, 'intersphinx_cache'): env.intersphinx_cache = {} env.intersphinx_inventory = {} env.intersphinx_named_inventory = {} cache = env.intersphinx_cache update = False for key, value in app.config.intersphinx_mapping.iteritems(): if isinstance(value, tuple): # new format name, (uri, inv) = key, value if not name.isalnum(): app.warn('intersphinx identifier %r is not alphanumeric' % name) else: # old format, no name name, uri, inv = None, key, value # we can safely assume that the uri<->inv mapping is not changed # during partial rebuilds since a changed intersphinx_mapping # setting will cause a full environment reread if not inv: inv = posixpath.join(uri, INVENTORY_FILENAME) # decide whether the inventory must be read: always read local # files; remote ones only if the cache time is expired if '://' not in inv or uri not in cache \ or cache[uri][1] < cache_time: app.info('loading intersphinx inventory from %s...' % inv) invdata = fetch_inventory(app, uri, inv) if invdata: cache[uri] = (name, now, invdata) else: cache.pop(uri, None) update = True if update: env.intersphinx_inventory = {} env.intersphinx_named_inventory = {} for name, _, invdata in cache.itervalues(): if name: env.intersphinx_named_inventory[name] = invdata for type, objects in invdata.iteritems(): env.intersphinx_inventory.setdefault( type, {}).update(objects) def missing_reference(app, env, node, contnode): """Attempt to resolve a missing reference via intersphinx references.""" domain = node.get('refdomain') if not domain: # only objects in domains are in the inventory return target = node['reftarget'] objtypes = env.domains[domain].objtypes_for_role(node['reftype']) if not objtypes: return objtypes = ['%s:%s' % (domain, objtype) for objtype in objtypes] to_try = [(env.intersphinx_inventory, target)] in_set = None if ':' in target: # first part may be the foreign doc set name setname, newtarget = target.split(':', 1) if setname in env.intersphinx_named_inventory: in_set = setname to_try.append((env.intersphinx_named_inventory[setname], newtarget)) for inventory, target in to_try: for objtype in objtypes: if objtype not in inventory or target not in inventory[objtype]: continue proj, version, uri, dispname = inventory[objtype][target] newnode = nodes.reference('', '', internal=False, refuri=uri, reftitle=_('(in %s v%s)') % (proj, version)) if node.get('refexplicit'): # use whatever title was given newnode.append(contnode) elif dispname == '-' or \ (domain == 'std' and node['reftype'] == 'keyword'): # use whatever title was given, but strip prefix title = contnode.astext() if in_set and title.startswith(in_set+':'): newnode.append(contnode.__class__(title[len(in_set)+1:], title[len(in_set)+1:])) else: newnode.append(contnode) else: # else use the given display name (used for :ref:) newnode.append(contnode.__class__(dispname, dispname)) return newnode # at least get rid of the ':' in the target if no explicit title given if in_set is not None and not node.get('refexplicit', True): if len(contnode) and isinstance(contnode[0], nodes.Text): contnode[0] = nodes.Text(newtarget, contnode[0].rawsource) def setup(app): app.add_config_value('intersphinx_mapping', {}, True) app.add_config_value('intersphinx_cache_limit', 5, False) app.connect('missing-reference', missing_reference) app.connect('builder-inited', load_mappings)