docs/doxygen/gmxtree.py

   1 #!/usr/bin/python
   2 #
   3 # This file is part of the GROMACS molecular simulation package.
   4 #
   5 # Copyright (c) 2014, by the GROMACS development team, led by
   6 # Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
   7 # and including many others, as listed in the AUTHORS file in the
   8 # top-level source directory and at http://www.gromacs.org.
   9 #
  10 # GROMACS is free software; you can redistribute it and/or
  11 # modify it under the terms of the GNU Lesser General Public License
  12 # as published by the Free Software Foundation; either version 2.1
  13 # of the License, or (at your option) any later version.
  14 #
  15 # GROMACS is distributed in the hope that it will be useful,
  16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18 # Lesser General Public License for more details.
  19 #
  20 # You should have received a copy of the GNU Lesser General Public
  21 # License along with GROMACS; if not, see
  22 # http://www.gnu.org/licenses, or write to the Free Software Foundation,
  23 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
  24 #
  25 # If you want to redistribute modifications to GROMACS, please
  26 # consider that scientific software is very special. Version
  27 # control is crucial - bugs must be traceable. We will be happy to
  28 # consider code for inclusion in the official distribution, but
  29 # derived work must not be called official GROMACS. Details are found
  30 # in the README & COPYING files - if they are missing, get the
  31 # official version at http://www.gromacs.org.
  32 #
  33 # To help us fund GROMACS development, we humbly ask that you cite
  34 # the research papers on the package. Check out http://www.gromacs.org.
  35
  36 """GROMACS-specific representation for source tree and documentation.
  37
  38 This module provides classes that construct a GROMACS-specific representation
  39 of the source tree and associate the Doxygen XML output with it.  It constructs
  40 an initial representation by walking the source tree in the file system, and
  41 then associates information from the Doxygen XML output into this.
  42 It also adds some additional knowledge from how the GROMACS source tree is
  43 organized to construct a representation that is easy to process and check as
  44 the top-level scripts expect.
  45
  46 The object model is rooted at a GromacsTree object.  Currently, it constructs a
  47 representation of the source tree from the file system, but is otherwise mostly
  48 a thin wrapper around the Doxygen XML tree.  It already adds some relations and
  49 rules that come from GROMACS-specific knowledge.  In the future, more such
  50 customizations will be added.
  51 """
  52
  53 import collections
  54 import os
  55 import os.path
  56 import re
  57
  58 import doxygenxml as xml
  59 import reporter
  60 # We import DocType directly so that it is exposed from this module as well.
  61 from doxygenxml import DocType
  62
  63 def _get_api_type_for_compound(grouplist):
  64     """Helper function to deduce API type from Doxygen group membership."""
  65     result = DocType.internal
  66     for group in grouplist:
  67         if isinstance(group, xml.Group):
  68             if group.get_name() == 'group_publicapi':
  69                 result = DocType.public
  70             elif group.get_name() == 'group_libraryapi':
  71                 result = DocType.library
  72             # TODO: Check for multiple group membership
  73     return result
  74
  75 class IncludedFile(object):
  76
  77     """Information about an #include directive in a file."""
  78
  79     def __init__(self, including_file, lineno, included_file, included_path, is_relative, is_system):
  80         self._including_file = including_file
  81         self._line_number = lineno
  82         self._included_file = included_file
  83         self._included_path = included_path
  84         #self._used_include_path = used_include_path
  85         self._is_relative = is_relative
  86         self._is_system = is_system
  87
  88     def __str__(self):
  89         if self._is_system:
  90             return '<{0}>'.format(self._included_path)
  91         else:
  92             return '"{0}"'.format(self._included_path)
  93
  94     def is_system(self):
  95         return self._is_system
  96
  97     def is_relative(self):
  98         return self._is_relative
  99
 100     def get_included_path(self):
 101         return self._included_path
 102
 103     def get_including_file(self):
 104         return self._including_file
 105
 106     def get_file(self):
 107         return self._included_file
 108
 109     def get_line_number(self):
 110         return self._line_number
 111
 112     def get_reporter_location(self):
 113         return reporter.Location(self._including_file.get_abspath(), self._line_number)
 114
 115 class IncludeBlock(object):
 116
 117     """Block of consequent #include directives in a file."""
 118
 119     def __init__(self, first_included_file):
 120         self._first_line = first_included_file.get_line_number()
 121         self._last_line = self._first_line
 122         self._files = []
 123         self.add_file(first_included_file)
 124
 125     def add_file(self, included_file):
 126         self._files.append(included_file)
 127         self._last_line = included_file.get_line_number()
 128
 129     def get_includes(self):
 130         return self._files
 131
 132     def get_first_line(self):
 133         return self._first_line
 134
 135     def get_last_line(self):
 136         return self._last_line
 137
 138 class File(object):
 139
 140     """Source/header file in the GROMACS tree."""
 141
 142     def __init__(self, abspath, relpath, directory):
 143         """Initialize a file representation with basic information."""
 144         self._abspath = abspath
 145         self._relpath = relpath
 146         self._dir = directory
 147         self._rawdoc = None
 148         self._installed = False
 149         extension = os.path.splitext(abspath)[1]
 150         self._sourcefile = (extension in ('.c', '.cc', '.cpp', '.cu'))
 151         self._apitype = DocType.none
 152         self._modules = set()
 153         self._includes = []
 154         self._include_blocks = []
 155         self._main_header = None
 156         self._lines = None
 157         directory.add_file(self)
 158
 159     def set_doc_xml(self, rawdoc, sourcetree):
 160         """Assiociate Doxygen documentation entity with the file."""
 161         assert self._rawdoc is None
 162         assert rawdoc.is_source_file() == self._sourcefile
 163         self._rawdoc = rawdoc
 164         if self._rawdoc.is_documented():
 165             grouplist = self._rawdoc.get_groups()
 166             self._apitype = _get_api_type_for_compound(grouplist)
 167             for group in grouplist:
 168                 module = sourcetree.get_object(group)
 169                 if module:
 170                     self._modules.add(module)
 171
 172     def set_installed(self):
 173         """Mark the file installed."""
 174         self._installed = True
 175
 176     def set_main_header(self, included_file):
 177         """Set the main header file for a source file."""
 178         assert self.is_source_file()
 179         self._main_header = included_file
 180
 181     def _process_include(self, lineno, is_system, includedpath, sourcetree):
 182         """Process #include directive during scan()."""
 183         is_relative = False
 184         if is_system:
 185             fileobj = sourcetree.find_include_file(includedpath)
 186         else:
 187             fullpath = os.path.join(self._dir.get_abspath(), includedpath)
 188             fullpath = os.path.abspath(fullpath)
 189             if os.path.exists(fullpath):
 190                 is_relative = True
 191                 fileobj = sourcetree.get_file(fullpath)
 192             else:
 193                 fileobj = sourcetree.find_include_file(includedpath)
 194         included_file = IncludedFile(self, lineno, fileobj, includedpath,
 195             is_relative, is_system)
 196         self._includes.append(included_file)
 197         return included_file
 198
 199     def scan_contents(self, sourcetree, keep_contents):
 200         """Scan the file contents and initialize information based on it."""
 201         # TODO: Consider a more robust regex.
 202         include_re = r'^\s*#\s*include\s+(?P<quote>["<])(?P<path>[^">]*)[">]'
 203         current_block = None
 204         # TODO: Consider reading directly into this list, and iterate that.
 205         lines = []
 206         with open(self._abspath, 'r') as scanfile:
 207             for lineno, line in enumerate(scanfile, 1):
 208                 lines.append(line)
 209                 match = re.match(include_re, line)
 210                 if match:
 211                     is_system = (match.group('quote') == '<')
 212                     includedpath = match.group('path')
 213                     included_file = self._process_include(lineno, is_system,
 214                             includedpath, sourcetree)
 215                     if current_block is None:
 216                         current_block = IncludeBlock(included_file)
 217                         self._include_blocks.append(current_block)
 218                     else:
 219                         current_block.add_file(included_file)
 220                 elif line and not line.isspace():
 221                     current_block = None
 222         if keep_contents:
 223             self._lines = lines
 224
 225     def get_reporter_location(self):
 226         return reporter.Location(self._abspath, None)
 227
 228     def is_installed(self):
 229         return self._installed
 230
 231     def is_external(self):
 232         return self._dir.is_external()
 233
 234     def is_source_file(self):
 235         return self._sourcefile
 236
 237     def is_test_file(self):
 238         return self._dir.is_test_directory()
 239
 240     def is_documented(self):
 241         return self._rawdoc and self._rawdoc.is_documented()
 242
 243     def has_brief_description(self):
 244         return self._rawdoc and self._rawdoc.has_brief_description()
 245
 246     def get_abspath(self):
 247         return self._abspath
 248
 249     def get_relpath(self):
 250         return self._relpath
 251
 252     def get_name(self):
 253         return os.path.basename(self._abspath)
 254
 255     def get_directory(self):
 256         return self._dir
 257
 258     def get_doc_type(self):
 259         if not self._rawdoc:
 260             return DocType.none
 261         return self._rawdoc.get_visibility()
 262
 263     def get_api_type(self):
 264         return self._apitype
 265
 266     def api_type_is_reliable(self):
 267         if self._apitype in (DocType.internal, DocType.library):
 268             return True
 269         module = self.get_module()
 270         return module and module.is_documented()
 271
 272     def is_public(self):
 273         if self.api_type_is_reliable():
 274             return self.get_api_type() == DocType.public
 275         return self.get_api_type() == DocType.public or self.is_installed()
 276
 277     def is_module_internal(self):
 278         if self.is_source_file():
 279             return True
 280         return not self.is_installed() and self.get_api_type() <= DocType.internal
 281
 282     def get_expected_module(self):
 283         return self._dir.get_module()
 284
 285     def get_doc_modules(self):
 286         return self._modules
 287
 288     def get_module(self):
 289         module = self.get_expected_module()
 290         if not module and len(self._modules) == 1:
 291             module = list(self._modules)[0]
 292         return module
 293
 294     def get_includes(self):
 295         return self._includes
 296
 297     def get_include_blocks(self):
 298         return self._include_blocks
 299
 300     def get_main_header(self):
 301         return self._main_header
 302
 303     def get_contents(self):
 304         return self._lines
 305
 306 class GeneratedFile(File):
 307     def __init__(self, abspath, relpath, directory):
 308         File.__init__(self, abspath, relpath, directory)
 309         self._generator_source_file = None
 310
 311     def set_generator_source(self, sourcefile):
 312         self._generator_source_file = sourcefile
 313
 314     def get_reporter_location(self):
 315         if self._generator_source_file:
 316             return self._generator_source_file.get_reporter_location()
 317         return File.get_reporter_location(self)
 318
 319 class GeneratorSourceFile(File):
 320     pass
 321
 322 class Directory(object):
 323
 324     """(Sub)directory in the GROMACS tree."""
 325
 326     def __init__(self, abspath, relpath, parent):
 327         """Initialize a file representation with basic information."""
 328         self._abspath = abspath
 329         self._relpath = relpath
 330         self._name = os.path.basename(abspath)
 331         self._parent = parent
 332         self._rawdoc = None
 333         self._module = None
 334         self._is_test_dir = False
 335         if parent and parent.is_test_directory() or \
 336                 self._name in ('tests', 'legacytests'):
 337             self._is_test_dir = True
 338         self._is_external = False
 339         if parent and parent.is_external() or self._name == 'external':
 340             self._is_external = True
 341         self._subdirs = set()
 342         if parent:
 343             parent._subdirs.add(self)
 344         self._files = set()
 345         self._has_installed_files = None
 346
 347     def set_doc_xml(self, rawdoc, sourcetree):
 348         """Assiociate Doxygen documentation entity with the directory."""
 349         assert self._rawdoc is None
 350         assert self._abspath == rawdoc.get_path().rstrip('/')
 351         self._rawdoc = rawdoc
 352
 353     def set_module(self, module):
 354         assert self._module is None
 355         self._module = module
 356
 357     def add_file(self, fileobj):
 358         self._files.add(fileobj)
 359
 360     def get_name(self):
 361         return self._name
 362
 363     def get_reporter_location(self):
 364         return reporter.Location(self._abspath, None)
 365
 366     def get_abspath(self):
 367         return self._abspath
 368
 369     def get_relpath(self):
 370         return self._relpath
 371
 372     def is_test_directory(self):
 373         return self._is_test_dir
 374
 375     def is_external(self):
 376         return self._is_external
 377
 378     def has_installed_files(self):
 379         if self._has_installed_files is None:
 380             self._has_installed_files = False
 381             for subdir in self._subdirs:
 382                 if subdir.has_installed_files():
 383                     self._has_installed_files = True
 384                     return True
 385             for fileobj in self._files:
 386                 if fileobj.is_installed():
 387                     self._has_installed_files = True
 388                     return True
 389         return self._has_installed_files
 390
 391     def get_module(self):
 392         if self._module:
 393             return self._module
 394         if self._parent:
 395             return self._parent.get_module()
 396         return None
 397
 398     def get_subdirectories(self):
 399         return self._subdirs
 400
 401     def get_files(self):
 402         for subdir in self._subdirs:
 403             for fileobj in subdir.get_files():
 404                 yield fileobj
 405         for fileobj in self._files:
 406             yield fileobj
 407
 408     def contains(self, fileobj):
 409         """Check whether file is within the directory or its subdirectories."""
 410         dirobj = fileobj.get_directory()
 411         while dirobj:
 412             if dirobj == self:
 413                 return True
 414             dirobj = dirobj._parent
 415         return False
 416
 417 class ModuleDependency(object):
 418
 419     """Dependency between modules."""
 420
 421     def __init__(self, othermodule):
 422         """Initialize empty dependency object with given module as dependency."""
 423         self._othermodule = othermodule
 424         self._includedfiles = []
 425         self._cyclesuppression = None
 426
 427     def add_included_file(self, includedfile):
 428         """Add IncludedFile that is part of this dependency."""
 429         assert includedfile.get_file().get_module() == self._othermodule
 430         self._includedfiles.append(includedfile)
 431
 432     def set_cycle_suppression(self):
 433         """Set suppression on cycles containing this dependency."""
 434         self._cyclesuppression = True
 435
 436     def is_cycle_suppressed(self):
 437         """Return whether cycles containing this dependency are suppressed."""
 438         return self._cyclesuppression is not None
 439
 440     def get_other_module(self):
 441         """Get module that this dependency is to."""
 442         return self._othermodule
 443
 444     def get_included_files(self):
 445         """Get IncludedFile objects for the individual include dependencies."""
 446         return self._includedfiles
 447
 448 class Module(object):
 449
 450     """Code module in the GROMACS source tree.
 451
 452     Modules are specific subdirectories that host a more or less coherent
 453     set of routines.  Simplified, every subdirectory under src/gromacs/ is
 454     a different module.  This object provides that abstraction and also links
 455     the subdirectory to the module documentation (documented as a group in
 456     Doxygen) if that exists.
 457     """
 458
 459     def __init__(self, name, rootdir):
 460         self._name = name
 461         self._rawdoc = None
 462         self._rootdir = rootdir
 463         self._group = None
 464         self._dependencies = dict()
 465
 466     def set_doc_xml(self, rawdoc, sourcetree):
 467         """Assiociate Doxygen documentation entity with the module."""
 468         assert self._rawdoc is None
 469         self._rawdoc = rawdoc
 470         if self._rawdoc.is_documented():
 471             groups = list(self._rawdoc.get_groups())
 472             if len(groups) == 1:
 473                 groupname = groups[0].get_name()
 474                 if groupname.startswith('group_'):
 475                     self._group = groupname[6:]
 476
 477     def add_dependency(self, othermodule, includedfile):
 478         """Add #include dependency from a file in this module."""
 479         assert includedfile.get_file().get_module() == othermodule
 480         if othermodule not in self._dependencies:
 481             self._dependencies[othermodule] = ModuleDependency(othermodule)
 482         self._dependencies[othermodule].add_included_file(includedfile)
 483
 484     def is_documented(self):
 485         return self._rawdoc is not None
 486
 487     def get_name(self):
 488         return self._name
 489
 490     def get_root_dir(self):
 491         return self._rootdir
 492
 493     def get_files(self):
 494         # TODO: Include public API convenience headers?
 495         return self._rootdir.get_files()
 496
 497     def get_group(self):
 498         return self._group
 499
 500     def get_dependencies(self):
 501         return self._dependencies.itervalues()
 502
 503 class Namespace(object):
 504
 505     """Namespace in the GROMACS source code."""
 506
 507     def __init__(self, rawdoc):
 508         self._rawdoc = rawdoc
 509
 510     def is_anonymous(self):
 511         return self._rawdoc.is_anonymous()
 512
 513 class Class(object):
 514
 515     """Class/struct/union in the GROMACS source code."""
 516
 517     def __init__(self, rawdoc, files):
 518         self._rawdoc = rawdoc
 519         self._files = set(files)
 520
 521     def get_name(self):
 522         return self._rawdoc.get_name()
 523
 524     def get_reporter_location(self):
 525         return self._rawdoc.get_reporter_location()
 526
 527     def get_files(self):
 528         return self._files
 529
 530     def is_documented(self):
 531         return self._rawdoc.is_documented()
 532
 533     def has_brief_description(self):
 534         return self._rawdoc.has_brief_description()
 535
 536     def get_doc_type(self):
 537         """Return documentation type (visibility) for the class.
 538
 539         In addition to the actual code, this encodes GROMACS-specific logic
 540         of setting EXTRACT_LOCAL_CLASSES=YES only for the full documentation.
 541         Local classes never appear outside the full documentation, no matter
 542         what is their visibility.
 543         """
 544         if not self.is_documented():
 545             return DocType.none
 546         if self._rawdoc.is_local():
 547             return DocType.internal
 548         return self._rawdoc.get_visibility()
 549
 550     def get_file_doc_type(self):
 551         return max([fileobj.get_doc_type() for fileobj in self._files])
 552
 553     def is_in_installed_file(self):
 554         return any([fileobj.is_installed() for fileobj in self._files])
 555
 556 class Member(object):
 557
 558     """Member (in Doxygen terminology) in the GROMACS source tree.
 559
 560     Currently, modeling is limited to the minimal set of properties that the
 561     checker uses.
 562     """
 563
 564     def __init__(self, rawdoc, namespace):
 565         self._rawdoc = rawdoc
 566         self._namespace = namespace
 567
 568     def get_name(self):
 569         return self._rawdoc.get_name()
 570
 571     def get_reporter_location(self):
 572         return self._rawdoc.get_reporter_location()
 573
 574     def is_documented(self):
 575         return self._rawdoc.is_documented()
 576
 577     def has_brief_description(self):
 578         return self._rawdoc.has_brief_description()
 579
 580     def has_inbody_description(self):
 581         return self._rawdoc.has_inbody_description()
 582
 583     def is_visible(self):
 584         """Return whether the member is visible in Doxygen documentation.
 585
 586         Doxygen ignores members whose parent compounds are not documented.
 587         However, when EXTRACT_ANON_NPACES=ON (which is set for our full
 588         documentation), members of anonymous namespaces are extracted even if
 589         the namespace is the only parent and is not documented.
 590         """
 591         if self._namespace and self._namespace.is_anonymous():
 592             return True
 593         return self._rawdoc.get_inherited_visibility() != DocType.none
 594
 595
 596 class GromacsTree(object):
 597
 598     """Root object for navigating the GROMACS source tree.
 599
 600     On initialization, the list of files and directories is initialized by
 601     walking the source tree, and modules are created for top-level
 602     subdirectories.  At this point, only information that is accessible from
 603     file names and paths only is available.
 604
 605     load_installed_file_list() can be called to load the list of installed
 606     files from the build tree (generated by the find-installed-headers target).
 607
 608     scan_files() can be called to read all the files and initialize #include
 609     dependencies between the files based on the information.  This is done like
 610     this instead of relying on Doxygen-extracted include files to make the
 611     dependency graph independent from preprocessor macro definitions
 612     (Doxygen only sees those #includes that the preprocessor sees, which
 613     depends on what #defines it has seen).
 614
 615     load_xml() can be called to load information from Doxygen XML data in
 616     the build tree (the Doxygen XML data must have been built separately).
 617     """
 618
 619     def __init__(self, source_root, build_root, reporter):
 620         """Initialize the tree object by walking the source tree."""
 621         self._source_root = os.path.abspath(source_root)
 622         self._build_root = os.path.abspath(build_root)
 623         self._reporter = reporter
 624         self._docset = None
 625         self._docmap = dict()
 626         self._dirs = dict()
 627         self._files = dict()
 628         self._modules = dict()
 629         self._classes = set()
 630         self._namespaces = set()
 631         self._members = set()
 632         self._walk_dir(os.path.join(self._source_root, 'src'))
 633         for fileobj in self.get_files():
 634             if fileobj and fileobj.is_source_file() and not fileobj.is_external():
 635                 (basedir, name) = os.path.split(fileobj.get_abspath())
 636                 (basename, ext) = os.path.splitext(name)
 637                 header = self.get_file(os.path.join(basedir, basename + '.h'))
 638                 if not header and ext == '.cu':
 639                     header = self.get_file(os.path.join(basedir, basename + '.cuh'))
 640                 if not header and fileobj.is_test_file():
 641                     basedir = os.path.dirname(basedir)
 642                     header = self.get_file(os.path.join(basedir, basename + '.h'))
 643                     if not header:
 644                         # Somewhat of a hack; currently, the tests for
 645                         # analysisdata/modules/ and trajectoryanalysis/modules/
 646                         # is at the top-level tests directory.
 647                         # TODO: It could be clearer to split the tests so that
 648                         # there would be a separate modules/tests/.
 649                         header = self.get_file(os.path.join(basedir, 'modules', basename + '.h'))
 650                     if not header and basename.endswith('_tests'):
 651                         header = self.get_file(os.path.join(basedir, basename[:-6] + '.h'))
 652                 if not header and fileobj.get_relpath().startswith('src/gromacs'):
 653                     header = self._files.get(os.path.join('src/gromacs/legacyheaders', basename + '.h'))
 654                 if header:
 655                     fileobj.set_main_header(header)
 656         rootdir = self._get_dir(os.path.join('src', 'gromacs'))
 657         for subdir in rootdir.get_subdirectories():
 658             self._create_module(subdir)
 659         rootdir = self._get_dir(os.path.join('src', 'testutils'))
 660         self._create_module(rootdir)
 661
 662     def _get_rel_path(self, path):
 663         assert os.path.isabs(path)
 664         if path.startswith(self._build_root):
 665             return os.path.relpath(path, self._build_root)
 666         if path.startswith(self._source_root):
 667             return os.path.relpath(path, self._source_root)
 668         raise ValueError("path not under build nor source tree: {0}".format(path))
 669
 670     def _walk_dir(self, rootpath):
 671         """Construct representation of the source tree by walking the file system."""
 672         assert os.path.isabs(rootpath)
 673         assert rootpath not in self._dirs
 674         relpath = self._get_rel_path(rootpath)
 675         self._dirs[relpath] = Directory(rootpath, relpath, None)
 676         for dirpath, dirnames, filenames in os.walk(rootpath):
 677             if 'contrib' in dirnames:
 678                 dirnames.remove('contrib')
 679             if 'refdata' in dirnames:
 680                 dirnames.remove('refdata')
 681             currentdir = self._dirs[self._get_rel_path(dirpath)]
 682             # Loop through a copy so that we can modify dirnames.
 683             for dirname in list(dirnames):
 684                 fullpath = os.path.join(dirpath, dirname)
 685                 if fullpath == self._build_root:
 686                     dirnames.remove(dirname)
 687                     continue
 688                 relpath = self._get_rel_path(fullpath)
 689                 self._dirs[relpath] = Directory(fullpath, relpath, currentdir)
 690             extensions = ('.h', '.cuh', '.hpp', '.c', '.cc', '.cpp', '.cu', '.bm')
 691             for filename in filenames:
 692                 basename, extension = os.path.splitext(filename)
 693                 if extension in extensions:
 694                     fullpath = os.path.join(dirpath, filename)
 695                     relpath = self._get_rel_path(fullpath)
 696                     self._files[relpath] = File(fullpath, relpath, currentdir)
 697                 elif extension == '.cmakein':
 698                     extension = os.path.splitext(basename)[1]
 699                     if extension in extensions:
 700                         fullpath = os.path.join(dirpath, filename)
 701                         relpath = self._get_rel_path(fullpath)
 702                         sourcefile = GeneratorSourceFile(fullpath, relpath, currentdir)
 703                         self._files[relpath] = sourcefile
 704                         fullpath = os.path.join(dirpath, basename)
 705                         relpath = self._get_rel_path(fullpath)
 706                         fullpath = os.path.join(self._build_root, relpath)
 707                         generatedfile = GeneratedFile(fullpath, relpath, currentdir)
 708                         self._files[relpath] = generatedfile
 709                         generatedfile.set_generator_source(sourcefile)
 710                 elif extension in ('.l', '.y', '.pre'):
 711                     fullpath = os.path.join(dirpath, filename)
 712                     relpath = self._get_rel_path(fullpath)
 713                     self._files[relpath] = GeneratorSourceFile(fullpath, relpath, currentdir)
 714
 715     def _create_module(self, rootdir):
 716         """Create module for a subdirectory."""
 717         name = 'module_' + rootdir.get_name()
 718         moduleobj = Module(name, rootdir)
 719         rootdir.set_module(moduleobj)
 720         self._modules[name] = moduleobj
 721
 722     def scan_files(self, only_files=None, keep_contents=False):
 723         """Read source files to initialize #include dependencies."""
 724         if only_files:
 725             filelist = only_files
 726         else:
 727             filelist = self._files.itervalues()
 728         for fileobj in filelist:
 729             if not fileobj.is_external():
 730                 fileobj.scan_contents(self, keep_contents)
 731                 module = fileobj.get_module()
 732                 if module:
 733                     for includedfile in fileobj.get_includes():
 734                         otherfile = includedfile.get_file()
 735                         if otherfile:
 736                             othermodule = otherfile.get_module()
 737                             if othermodule and othermodule != module:
 738                                 module.add_dependency(othermodule, includedfile)
 739
 740     def load_xml(self, only_files=None):
 741         """Load Doxygen XML information.
 742
 743         If only_files is True, XML data is not loaded for code constructs, but
 744         only for files, directories, and their potential parents.
 745         """
 746         xmldir = os.path.join(self._build_root, 'docs', 'html', 'doxygen', 'xml')
 747         self._docset = xml.DocumentationSet(xmldir, self._reporter)
 748         if only_files:
 749             if isinstance(only_files, collections.Iterable):
 750                 filelist = [x.get_abspath() for x in only_files]
 751                 self._docset.load_file_details(filelist)
 752             else:
 753                 self._docset.load_file_details()
 754         else:
 755             self._docset.load_details()
 756             self._docset.merge_duplicates()
 757         self._load_dirs()
 758         self._load_modules()
 759         self._load_files()
 760         if not only_files:
 761             self._load_namespaces()
 762             self._load_classes()
 763             self._load_members()
 764
 765     def _load_dirs(self):
 766         """Load Doxygen XML directory information."""
 767         rootdirs = self._docset.get_compounds(xml.Directory,
 768                 lambda x: x.get_parent() is None)
 769         for dirdoc in rootdirs:
 770             self._load_dir(dirdoc, None)
 771
 772     def _load_dir(self, dirdoc, parent):
 773         """Load Doxygen XML directory information for a single directory."""
 774         path = dirdoc.get_path().rstrip('/')
 775         if not os.path.isabs(path):
 776             self._reporter.xml_assert(dirdoc.get_xml_path(),
 777                     "expected absolute path in Doxygen-produced XML file")
 778             return
 779         relpath = self._get_rel_path(path)
 780         dirobj = self._dirs.get(relpath)
 781         if not dirobj:
 782             dirobj = Directory(path, relpath, parent)
 783             self._dirs[relpath] = dirobj
 784         dirobj.set_doc_xml(dirdoc, self)
 785         self._docmap[dirdoc] = dirobj
 786         for subdirdoc in dirdoc.get_subdirectories():
 787             self._load_dir(subdirdoc, dirobj)
 788
 789     def _load_modules(self):
 790         """Load Doxygen XML module (group) information."""
 791         moduledocs = self._docset.get_compounds(xml.Group,
 792                 lambda x: x.get_name().startswith('module_'))
 793         for moduledoc in moduledocs:
 794             moduleobj = self._modules.get(moduledoc.get_name())
 795             if not moduleobj:
 796                 self._reporter.input_error(
 797                         "no matching directory for module: {0}".format(moduledoc))
 798                 continue
 799             moduleobj.set_doc_xml(moduledoc, self)
 800             self._docmap[moduledoc] = moduleobj
 801
 802     def _load_files(self):
 803         """Load Doxygen XML file information."""
 804         for filedoc in self._docset.get_files():
 805             path = filedoc.get_path()
 806             if not path:
 807                 # In case of only partially loaded file information,
 808                 # the path information is not set for unloaded files.
 809                 continue
 810             if not os.path.isabs(path):
 811                 self._reporter.xml_assert(filedoc.get_xml_path(),
 812                         "expected absolute path in Doxygen-produced XML file")
 813                 continue
 814             extension = os.path.splitext(path)[1]
 815             # We don't care about Markdown files that only produce pages
 816             # (and fail the directory check below).
 817             if extension == '.md':
 818                 continue
 819             dirdoc = filedoc.get_directory()
 820             if not dirdoc:
 821                 self._reporter.xml_assert(filedoc.get_xml_path(),
 822                         "file is not in any directory in Doxygen")
 823                 continue
 824             relpath = self._get_rel_path(path)
 825             fileobj = self._files.get(relpath)
 826             if not fileobj:
 827                 fileobj = File(path, relpath, self._docmap[dirdoc])
 828                 self._files[relpath] = fileobj
 829             fileobj.set_doc_xml(filedoc, self)
 830             self._docmap[filedoc] = fileobj
 831
 832     def _load_namespaces(self):
 833         """Load Doxygen XML namespace information."""
 834         nsdocs = self._docset.get_namespaces()
 835         for nsdoc in nsdocs:
 836             nsobj = Namespace(nsdoc)
 837             self._docmap[nsdoc] = nsobj
 838             self._namespaces.add(nsobj)
 839
 840     def _load_classes(self):
 841         """Load Doxygen XML class information."""
 842         classdocs = self._docset.get_classes()
 843         for classdoc in classdocs:
 844             files = [self._docmap[filedoc] for filedoc in classdoc.get_files()]
 845             classobj = Class(classdoc, files)
 846             self._docmap[classdoc] = classobj
 847             self._classes.add(classobj)
 848
 849     def _load_members(self):
 850         """Load Doxygen XML member information."""
 851         memberdocs = self._docset.get_members()
 852         for memberdoc in memberdocs:
 853             nsdoc = memberdoc.get_namespace()
 854             nsobj = self.get_object(nsdoc)
 855             memberobj = Member(memberdoc, nsobj)
 856             self._docmap[memberdoc] = memberobj
 857             self._members.add(memberobj)
 858
 859     def _get_dir(self, relpath):
 860         """Get directory object for a path relative to source tree root."""
 861         return self._dirs.get(relpath)
 862
 863     def get_file(self, path):
 864         """Get file object for a path relative to source tree root."""
 865         return self._files.get(self._get_rel_path(path))
 866
 867     def find_include_file(self, includedpath):
 868         """Find a file object corresponding to an include path."""
 869         for testdir in ('src', 'src/external/thread_mpi/include',
 870                 'src/external/tng_io/include'):
 871             testpath = os.path.join(testdir, includedpath)
 872             if testpath in self._files:
 873                 return self._files[testpath]
 874
 875     def load_installed_file_list(self):
 876         """Load list of installed files from the build tree."""
 877         listpath = os.path.join(self._build_root, 'docs', 'doxygen', 'installed-headers.txt')
 878         with open(listpath, 'r') as installedfp:
 879             for line in installedfp:
 880                 path = line.strip()
 881                 if not os.path.isabs(path):
 882                     self._reporter.input_error(
 883                             "installed file not specified with absolute path: {0}"
 884                             .format(path))
 885                     continue
 886                 relpath = self._get_rel_path(path)
 887                 if relpath not in self._files:
 888                     self._reporter.input_error(
 889                             "installed file not in source tree: {0}".format(path))
 890                     continue
 891                 self._files[relpath].set_installed()
 892
 893     def load_cycle_suppression_list(self, filename):
 894         """Load a list of edges to suppress in cycles.
 895
 896         These edges between modules, if present, will be marked in the
 897         corresponding ModuleDependency objects.
 898         """
 899         with open(filename, 'r') as fp:
 900             for line in fp:
 901                 line = line.strip()
 902                 if not line or line.startswith('#'):
 903                     continue
 904                 modulenames = ['module_' + x.strip() for x in line.split('->')]
 905                 if len(modulenames) != 2:
 906                     self._reporter.input_error(
 907                             "invalid cycle suppression line: {0}".format(line))
 908                     continue
 909                 firstmodule = self._modules.get(modulenames[0])
 910                 secondmodule = self._modules.get(modulenames[1])
 911                 if not firstmodule or not secondmodule:
 912                     self._reporter.input_error(
 913                             "unknown modules mentioned on cycle suppression line: {0}".format(line))
 914                     continue
 915                 for dep in firstmodule.get_dependencies():
 916                     if dep.get_other_module() == secondmodule:
 917                         # TODO: Check that each suppression is actually part of
 918                         # a cycle.
 919                         dep.set_cycle_suppression()
 920
 921     def get_object(self, docobj):
 922         """Get tree object for a Doxygen XML object."""
 923         if docobj is None:
 924             return None
 925         return self._docmap.get(docobj)
 926
 927     def get_files(self):
 928         """Get iterable for all files in the source tree."""
 929         return self._files.itervalues()
 930
 931     def get_modules(self):
 932         """Get iterable for all modules in the source tree."""
 933         return self._modules.itervalues()
 934
 935     def get_classes(self):
 936         """Get iterable for all classes in the source tree."""
 937         return self._classes
 938
 939     def get_members(self):
 940         """Get iterable for all members (in Doxygen terms) in the source tree."""
 941         return self._members