docs/doxygen/gmxtree.py

   1 #!/usr/bin/python
   2 #
   3 # This file is part of the GROMACS molecular simulation package.
   4 #
   5 # Copyright (c) 2014, by the GROMACS development team, led by
   6 # Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
   7 # and including many others, as listed in the AUTHORS file in the
   8 # top-level source directory and at http://www.gromacs.org.
   9 #
  10 # GROMACS is free software; you can redistribute it and/or
  11 # modify it under the terms of the GNU Lesser General Public License
  12 # as published by the Free Software Foundation; either version 2.1
  13 # of the License, or (at your option) any later version.
  14 #
  15 # GROMACS is distributed in the hope that it will be useful,
  16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18 # Lesser General Public License for more details.
  19 #
  20 # You should have received a copy of the GNU Lesser General Public
  21 # License along with GROMACS; if not, see
  22 # http://www.gnu.org/licenses, or write to the Free Software Foundation,
  23 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
  24 #
  25 # If you want to redistribute modifications to GROMACS, please
  26 # consider that scientific software is very special. Version
  27 # control is crucial - bugs must be traceable. We will be happy to
  28 # consider code for inclusion in the official distribution, but
  29 # derived work must not be called official GROMACS. Details are found
  30 # in the README & COPYING files - if they are missing, get the
  31 # official version at http://www.gromacs.org.
  32 #
  33 # To help us fund GROMACS development, we humbly ask that you cite
  34 # the research papers on the package. Check out http://www.gromacs.org.
  35
  36 """GROMACS-specific representation for source tree and documentation.
  37
  38 This module provides classes that construct a GROMACS-specific representation
  39 of the source tree and associate the Doxygen XML output with it.  It constructs
  40 an initial representation by walking the source tree in the file system, and
  41 then associates information from the Doxygen XML output into this.
  42 It also adds some additional knowledge from how the GROMACS source tree is
  43 organized to construct a representation that is easy to process and check as
  44 the top-level scripts expect.
  45
  46 The object model is rooted at a GromacsTree object.  Currently, it constructs a
  47 representation of the source tree from the file system, but is otherwise mostly
  48 a thin wrapper around the Doxygen XML tree.  It already adds some relations and
  49 rules that come from GROMACS-specific knowledge.  In the future, more such
  50 customizations will be added.
  51 """
  52
  53 import collections
  54 import os
  55 import os.path
  56 import re
  57 import subprocess
  58
  59 import doxygenxml as xml
  60 import reporter
  61 # We import DocType directly so that it is exposed from this module as well.
  62 from doxygenxml import DocType
  63
  64 def _get_api_type_for_compound(grouplist):
  65     """Helper function to deduce API type from Doxygen group membership."""
  66     result = DocType.internal
  67     for group in grouplist:
  68         if isinstance(group, xml.Group):
  69             if group.get_name() == 'group_publicapi':
  70                 result = DocType.public
  71             elif group.get_name() == 'group_libraryapi':
  72                 result = DocType.library
  73             # TODO: Check for multiple group membership
  74     return result
  75
  76 class IncludedFile(object):
  77
  78     """Information about an #include directive in a file."""
  79
  80     def __init__(self, including_file, lineno, included_file, included_path, is_relative, is_system, line):
  81         self._including_file = including_file
  82         self._line_number = lineno
  83         self._included_file = included_file
  84         self._included_path = included_path
  85         #self._used_include_path = used_include_path
  86         self._is_relative = is_relative
  87         self._is_system = is_system
  88         self._line = line
  89
  90     def __str__(self):
  91         if self._is_system:
  92             return '<{0}>'.format(self._included_path)
  93         else:
  94             return '"{0}"'.format(self._included_path)
  95
  96     def is_system(self):
  97         return self._is_system
  98
  99     def is_relative(self):
 100         return self._is_relative
 101
 102     def get_included_path(self):
 103         return self._included_path
 104
 105     def get_including_file(self):
 106         return self._including_file
 107
 108     def get_file(self):
 109         return self._included_file
 110
 111     def get_line_number(self):
 112         return self._line_number
 113
 114     def get_full_line(self):
 115         """Return the full source line on which this include appears.
 116
 117         Trailing newline is included."""
 118         return self._line
 119
 120     def get_reporter_location(self):
 121         return reporter.Location(self._including_file.get_abspath(), self._line_number)
 122
 123 class IncludeBlock(object):
 124
 125     """Block of consequent #include directives in a file."""
 126
 127     def __init__(self, first_included_file):
 128         self._first_line = first_included_file.get_line_number()
 129         self._last_line = self._first_line
 130         self._files = []
 131         self.add_file(first_included_file)
 132
 133     def add_file(self, included_file):
 134         self._files.append(included_file)
 135         self._last_line = included_file.get_line_number()
 136
 137     def get_includes(self):
 138         return self._files
 139
 140     def get_first_line(self):
 141         return self._first_line
 142
 143     def get_last_line(self):
 144         return self._last_line
 145
 146 class File(object):
 147
 148     """Source/header file in the GROMACS tree."""
 149
 150     def __init__(self, abspath, relpath, directory):
 151         """Initialize a file representation with basic information."""
 152         self._abspath = abspath
 153         self._relpath = relpath
 154         self._dir = directory
 155         self._rawdoc = None
 156         self._installed = False
 157         extension = os.path.splitext(abspath)[1]
 158         self._sourcefile = (extension in ('.c', '.cc', '.cpp', '.cu'))
 159         self._apitype = DocType.none
 160         self._modules = set()
 161         self._includes = []
 162         self._include_blocks = []
 163         self._main_header = None
 164         self._lines = None
 165         self._filter = None
 166         directory.add_file(self)
 167
 168     def set_doc_xml(self, rawdoc, sourcetree):
 169         """Assiociate Doxygen documentation entity with the file."""
 170         assert self._rawdoc is None
 171         assert rawdoc.is_source_file() == self._sourcefile
 172         self._rawdoc = rawdoc
 173         if self._rawdoc.is_documented():
 174             grouplist = self._rawdoc.get_groups()
 175             self._apitype = _get_api_type_for_compound(grouplist)
 176             for group in grouplist:
 177                 module = sourcetree.get_object(group)
 178                 if module:
 179                     self._modules.add(module)
 180
 181     def set_installed(self):
 182         """Mark the file installed."""
 183         self._installed = True
 184
 185     def set_git_filter_attribute(self, filtername):
 186         """Set the git filter attribute associated with the file."""
 187         self._filter = filtername
 188
 189     def set_main_header(self, included_file):
 190         """Set the main header file for a source file."""
 191         assert self.is_source_file()
 192         self._main_header = included_file
 193
 194     def _process_include(self, lineno, is_system, includedpath, line, sourcetree):
 195         """Process #include directive during scan()."""
 196         is_relative = False
 197         if is_system:
 198             fileobj = sourcetree.find_include_file(includedpath)
 199         else:
 200             fullpath = os.path.join(self._dir.get_abspath(), includedpath)
 201             fullpath = os.path.abspath(fullpath)
 202             if os.path.exists(fullpath):
 203                 is_relative = True
 204                 fileobj = sourcetree.get_file(fullpath)
 205             else:
 206                 fileobj = sourcetree.find_include_file(includedpath)
 207         included_file = IncludedFile(self, lineno, fileobj, includedpath,
 208             is_relative, is_system, line)
 209         self._includes.append(included_file)
 210         return included_file
 211
 212     def scan_contents(self, sourcetree, keep_contents):
 213         """Scan the file contents and initialize information based on it."""
 214         # TODO: Consider a more robust regex.
 215         include_re = r'^\s*#\s*include\s+(?P<quote>["<])(?P<path>[^">]*)[">]'
 216         current_block = None
 217         # TODO: Consider reading directly into this list, and iterate that.
 218         lines = []
 219         with open(self._abspath, 'r') as scanfile:
 220             for lineno, line in enumerate(scanfile, 1):
 221                 lines.append(line)
 222                 match = re.match(include_re, line)
 223                 if match:
 224                     is_system = (match.group('quote') == '<')
 225                     includedpath = match.group('path')
 226                     included_file = self._process_include(lineno, is_system,
 227                             includedpath, line, sourcetree)
 228                     if current_block is None:
 229                         current_block = IncludeBlock(included_file)
 230                         self._include_blocks.append(current_block)
 231                     else:
 232                         current_block.add_file(included_file)
 233                 elif line and not line.isspace():
 234                     current_block = None
 235         if keep_contents:
 236             self._lines = lines
 237
 238     def get_reporter_location(self):
 239         return reporter.Location(self._abspath, None)
 240
 241     def is_installed(self):
 242         return self._installed
 243
 244     def is_external(self):
 245         return self._dir.is_external()
 246
 247     def is_source_file(self):
 248         return self._sourcefile
 249
 250     def is_test_file(self):
 251         return self._dir.is_test_directory()
 252
 253     def should_includes_be_sorted(self):
 254         """Return whether the include directives in the file should be sorted."""
 255         return self._filter in ('includesort', 'uncrustify')
 256
 257     def is_documented(self):
 258         return self._rawdoc and self._rawdoc.is_documented()
 259
 260     def has_brief_description(self):
 261         return self._rawdoc and self._rawdoc.has_brief_description()
 262
 263     def get_abspath(self):
 264         return self._abspath
 265
 266     def get_relpath(self):
 267         return self._relpath
 268
 269     def get_name(self):
 270         return os.path.basename(self._abspath)
 271
 272     def get_directory(self):
 273         return self._dir
 274
 275     def get_doc_type(self):
 276         if not self._rawdoc:
 277             return DocType.none
 278         return self._rawdoc.get_visibility()
 279
 280     def get_api_type(self):
 281         return self._apitype
 282
 283     def api_type_is_reliable(self):
 284         if self._apitype in (DocType.internal, DocType.library):
 285             return True
 286         module = self.get_module()
 287         return module and module.is_documented()
 288
 289     def is_public(self):
 290         if self.api_type_is_reliable():
 291             return self.get_api_type() == DocType.public
 292         return self.get_api_type() == DocType.public or self.is_installed()
 293
 294     def is_module_internal(self):
 295         if self.is_source_file():
 296             return True
 297         return not self.is_installed() and self.get_api_type() <= DocType.internal
 298
 299     def get_expected_module(self):
 300         return self._dir.get_module()
 301
 302     def get_doc_modules(self):
 303         return self._modules
 304
 305     def get_module(self):
 306         module = self.get_expected_module()
 307         if not module and len(self._modules) == 1:
 308             module = list(self._modules)[0]
 309         return module
 310
 311     def get_includes(self):
 312         return self._includes
 313
 314     def get_include_blocks(self):
 315         return self._include_blocks
 316
 317     def get_main_header(self):
 318         return self._main_header
 319
 320     def get_contents(self):
 321         return self._lines
 322
 323 class GeneratedFile(File):
 324     def __init__(self, abspath, relpath, directory):
 325         File.__init__(self, abspath, relpath, directory)
 326         self._generator_source_file = None
 327
 328     def scan_contents(self, sourcetree, keep_contents):
 329         if os.path.exists(self.get_abspath()):
 330             File.scan_contents(self, sourcetree, keep_contents)
 331
 332     def set_generator_source(self, sourcefile):
 333         self._generator_source_file = sourcefile
 334
 335     def get_reporter_location(self):
 336         if self._generator_source_file:
 337             return self._generator_source_file.get_reporter_location()
 338         return File.get_reporter_location(self)
 339
 340 class GeneratorSourceFile(File):
 341     pass
 342
 343 class Directory(object):
 344
 345     """(Sub)directory in the GROMACS tree."""
 346
 347     def __init__(self, abspath, relpath, parent):
 348         """Initialize a file representation with basic information."""
 349         self._abspath = abspath
 350         self._relpath = relpath
 351         self._name = os.path.basename(abspath)
 352         self._parent = parent
 353         self._rawdoc = None
 354         self._module = None
 355         self._is_test_dir = False
 356         if parent and parent.is_test_directory() or \
 357                 self._name in ('tests', 'legacytests'):
 358             self._is_test_dir = True
 359         self._is_external = False
 360         if parent and parent.is_external() or self._name == 'external':
 361             self._is_external = True
 362         self._subdirs = set()
 363         if parent:
 364             parent._subdirs.add(self)
 365         self._files = set()
 366         self._has_installed_files = None
 367
 368     def set_doc_xml(self, rawdoc, sourcetree):
 369         """Assiociate Doxygen documentation entity with the directory."""
 370         assert self._rawdoc is None
 371         assert self._abspath == rawdoc.get_path().rstrip('/')
 372         self._rawdoc = rawdoc
 373
 374     def set_module(self, module):
 375         assert self._module is None
 376         self._module = module
 377
 378     def add_file(self, fileobj):
 379         self._files.add(fileobj)
 380
 381     def get_name(self):
 382         return self._name
 383
 384     def get_reporter_location(self):
 385         return reporter.Location(self._abspath, None)
 386
 387     def get_abspath(self):
 388         return self._abspath
 389
 390     def get_relpath(self):
 391         return self._relpath
 392
 393     def is_test_directory(self):
 394         return self._is_test_dir
 395
 396     def is_external(self):
 397         return self._is_external
 398
 399     def has_installed_files(self):
 400         if self._has_installed_files is None:
 401             self._has_installed_files = False
 402             for subdir in self._subdirs:
 403                 if subdir.has_installed_files():
 404                     self._has_installed_files = True
 405                     return True
 406             for fileobj in self._files:
 407                 if fileobj.is_installed():
 408                     self._has_installed_files = True
 409                     return True
 410         return self._has_installed_files
 411
 412     def get_module(self):
 413         if self._module:
 414             return self._module
 415         if self._parent:
 416             return self._parent.get_module()
 417         return None
 418
 419     def get_subdirectories(self):
 420         return self._subdirs
 421
 422     def get_files(self):
 423         for subdir in self._subdirs:
 424             for fileobj in subdir.get_files():
 425                 yield fileobj
 426         for fileobj in self._files:
 427             yield fileobj
 428
 429     def contains(self, fileobj):
 430         """Check whether file is within the directory or its subdirectories."""
 431         dirobj = fileobj.get_directory()
 432         while dirobj:
 433             if dirobj == self:
 434                 return True
 435             dirobj = dirobj._parent
 436         return False
 437
 438 class ModuleDependency(object):
 439
 440     """Dependency between modules."""
 441
 442     def __init__(self, othermodule):
 443         """Initialize empty dependency object with given module as dependency."""
 444         self._othermodule = othermodule
 445         self._includedfiles = []
 446         self._cyclesuppression = None
 447
 448     def add_included_file(self, includedfile):
 449         """Add IncludedFile that is part of this dependency."""
 450         assert includedfile.get_file().get_module() == self._othermodule
 451         self._includedfiles.append(includedfile)
 452
 453     def set_cycle_suppression(self):
 454         """Set suppression on cycles containing this dependency."""
 455         self._cyclesuppression = True
 456
 457     def is_cycle_suppressed(self):
 458         """Return whether cycles containing this dependency are suppressed."""
 459         return self._cyclesuppression is not None
 460
 461     def get_other_module(self):
 462         """Get module that this dependency is to."""
 463         return self._othermodule
 464
 465     def get_included_files(self):
 466         """Get IncludedFile objects for the individual include dependencies."""
 467         return self._includedfiles
 468
 469 class Module(object):
 470
 471     """Code module in the GROMACS source tree.
 472
 473     Modules are specific subdirectories that host a more or less coherent
 474     set of routines.  Simplified, every subdirectory under src/gromacs/ is
 475     a different module.  This object provides that abstraction and also links
 476     the subdirectory to the module documentation (documented as a group in
 477     Doxygen) if that exists.
 478     """
 479
 480     def __init__(self, name, rootdir):
 481         self._name = name
 482         self._rawdoc = None
 483         self._rootdir = rootdir
 484         self._group = None
 485         self._dependencies = dict()
 486
 487     def set_doc_xml(self, rawdoc, sourcetree):
 488         """Assiociate Doxygen documentation entity with the module."""
 489         assert self._rawdoc is None
 490         self._rawdoc = rawdoc
 491         if self._rawdoc.is_documented():
 492             groups = list(self._rawdoc.get_groups())
 493             if len(groups) == 1:
 494                 groupname = groups[0].get_name()
 495                 if groupname.startswith('group_'):
 496                     self._group = groupname[6:]
 497
 498     def add_dependency(self, othermodule, includedfile):
 499         """Add #include dependency from a file in this module."""
 500         assert includedfile.get_file().get_module() == othermodule
 501         if othermodule not in self._dependencies:
 502             self._dependencies[othermodule] = ModuleDependency(othermodule)
 503         self._dependencies[othermodule].add_included_file(includedfile)
 504
 505     def is_documented(self):
 506         return self._rawdoc is not None
 507
 508     def get_name(self):
 509         return self._name
 510
 511     def get_root_dir(self):
 512         return self._rootdir
 513
 514     def get_files(self):
 515         # TODO: Include public API convenience headers?
 516         return self._rootdir.get_files()
 517
 518     def get_group(self):
 519         return self._group
 520
 521     def get_dependencies(self):
 522         return self._dependencies.itervalues()
 523
 524 class Namespace(object):
 525
 526     """Namespace in the GROMACS source code."""
 527
 528     def __init__(self, rawdoc):
 529         self._rawdoc = rawdoc
 530
 531     def is_anonymous(self):
 532         return self._rawdoc.is_anonymous()
 533
 534 class Class(object):
 535
 536     """Class/struct/union in the GROMACS source code."""
 537
 538     def __init__(self, rawdoc, files):
 539         self._rawdoc = rawdoc
 540         self._files = set(files)
 541
 542     def get_name(self):
 543         return self._rawdoc.get_name()
 544
 545     def get_reporter_location(self):
 546         return self._rawdoc.get_reporter_location()
 547
 548     def get_files(self):
 549         return self._files
 550
 551     def is_documented(self):
 552         return self._rawdoc.is_documented()
 553
 554     def has_brief_description(self):
 555         return self._rawdoc.has_brief_description()
 556
 557     def get_doc_type(self):
 558         """Return documentation type (visibility) for the class.
 559
 560         In addition to the actual code, this encodes GROMACS-specific logic
 561         of setting EXTRACT_LOCAL_CLASSES=YES only for the full documentation.
 562         Local classes never appear outside the full documentation, no matter
 563         what is their visibility.
 564         """
 565         if not self.is_documented():
 566             return DocType.none
 567         if self._rawdoc.is_local():
 568             return DocType.internal
 569         return self._rawdoc.get_visibility()
 570
 571     def get_file_doc_type(self):
 572         return max([fileobj.get_doc_type() for fileobj in self._files])
 573
 574     def is_in_installed_file(self):
 575         return any([fileobj.is_installed() for fileobj in self._files])
 576
 577 class Member(object):
 578
 579     """Member (in Doxygen terminology) in the GROMACS source tree.
 580
 581     Currently, modeling is limited to the minimal set of properties that the
 582     checker uses.
 583     """
 584
 585     def __init__(self, rawdoc, namespace):
 586         self._rawdoc = rawdoc
 587         self._namespace = namespace
 588
 589     def get_name(self):
 590         return self._rawdoc.get_name()
 591
 592     def get_reporter_location(self):
 593         return self._rawdoc.get_reporter_location()
 594
 595     def is_documented(self):
 596         return self._rawdoc.is_documented()
 597
 598     def has_brief_description(self):
 599         return self._rawdoc.has_brief_description()
 600
 601     def has_inbody_description(self):
 602         return self._rawdoc.has_inbody_description()
 603
 604     def is_visible(self):
 605         """Return whether the member is visible in Doxygen documentation.
 606
 607         Doxygen ignores members whose parent compounds are not documented.
 608         However, when EXTRACT_ANON_NPACES=ON (which is set for our full
 609         documentation), members of anonymous namespaces are extracted even if
 610         the namespace is the only parent and is not documented.
 611         """
 612         if self._namespace and self._namespace.is_anonymous():
 613             return True
 614         return self._rawdoc.get_inherited_visibility() != DocType.none
 615
 616
 617 class GromacsTree(object):
 618
 619     """Root object for navigating the GROMACS source tree.
 620
 621     On initialization, the list of files and directories is initialized by
 622     walking the source tree, and modules are created for top-level
 623     subdirectories.  At this point, only information that is accessible from
 624     file names and paths only is available.
 625
 626     load_git_attributes() can be called to load attribute information from
 627     .gitattributes for all the files.
 628
 629     load_installed_file_list() can be called to load the list of installed
 630     files from the build tree (generated by the find-installed-headers target).
 631
 632     scan_files() can be called to read all the files and initialize #include
 633     dependencies between the files based on the information.  This is done like
 634     this instead of relying on Doxygen-extracted include files to make the
 635     dependency graph independent from preprocessor macro definitions
 636     (Doxygen only sees those #includes that the preprocessor sees, which
 637     depends on what #defines it has seen).
 638
 639     load_xml() can be called to load information from Doxygen XML data in
 640     the build tree (the Doxygen XML data must have been built separately).
 641     """
 642
 643     def __init__(self, source_root, build_root, reporter):
 644         """Initialize the tree object by walking the source tree."""
 645         self._source_root = os.path.abspath(source_root)
 646         self._build_root = os.path.abspath(build_root)
 647         self._reporter = reporter
 648         self._docset = None
 649         self._docmap = dict()
 650         self._dirs = dict()
 651         self._files = dict()
 652         self._modules = dict()
 653         self._classes = set()
 654         self._namespaces = set()
 655         self._members = set()
 656         self._walk_dir(os.path.join(self._source_root, 'src'))
 657         for fileobj in self.get_files():
 658             if fileobj and fileobj.is_source_file() and not fileobj.is_external():
 659                 (basedir, name) = os.path.split(fileobj.get_abspath())
 660                 (basename, ext) = os.path.splitext(name)
 661                 header = self.get_file(os.path.join(basedir, basename + '.h'))
 662                 if not header and ext == '.cu':
 663                     header = self.get_file(os.path.join(basedir, basename + '.cuh'))
 664                 if not header and fileobj.is_test_file():
 665                     basedir = os.path.dirname(basedir)
 666                     header = self.get_file(os.path.join(basedir, basename + '.h'))
 667                     if not header:
 668                         # Somewhat of a hack; currently, the tests for
 669                         # analysisdata/modules/ and trajectoryanalysis/modules/
 670                         # is at the top-level tests directory.
 671                         # TODO: It could be clearer to split the tests so that
 672                         # there would be a separate modules/tests/.
 673                         header = self.get_file(os.path.join(basedir, 'modules', basename + '.h'))
 674                     if not header and basename.endswith('_tests'):
 675                         header = self.get_file(os.path.join(basedir, basename[:-6] + '.h'))
 676                 if not header and fileobj.get_relpath().startswith('src/gromacs'):
 677                     header = self._files.get(os.path.join('src/gromacs/legacyheaders', basename + '.h'))
 678                 if header:
 679                     fileobj.set_main_header(header)
 680         rootdir = self._get_dir(os.path.join('src', 'gromacs'))
 681         for subdir in rootdir.get_subdirectories():
 682             self._create_module(subdir)
 683         rootdir = self._get_dir(os.path.join('src', 'testutils'))
 684         self._create_module(rootdir)
 685
 686     def _get_rel_path(self, path):
 687         assert os.path.isabs(path)
 688         if path.startswith(self._build_root):
 689             return os.path.relpath(path, self._build_root)
 690         if path.startswith(self._source_root):
 691             return os.path.relpath(path, self._source_root)
 692         raise ValueError("path not under build nor source tree: {0}".format(path))
 693
 694     def _walk_dir(self, rootpath):
 695         """Construct representation of the source tree by walking the file system."""
 696         assert os.path.isabs(rootpath)
 697         assert rootpath not in self._dirs
 698         relpath = self._get_rel_path(rootpath)
 699         self._dirs[relpath] = Directory(rootpath, relpath, None)
 700         for dirpath, dirnames, filenames in os.walk(rootpath):
 701             if 'contrib' in dirnames:
 702                 dirnames.remove('contrib')
 703             if 'refdata' in dirnames:
 704                 dirnames.remove('refdata')
 705             currentdir = self._dirs[self._get_rel_path(dirpath)]
 706             # Loop through a copy so that we can modify dirnames.
 707             for dirname in list(dirnames):
 708                 fullpath = os.path.join(dirpath, dirname)
 709                 if fullpath == self._build_root:
 710                     dirnames.remove(dirname)
 711                     continue
 712                 relpath = self._get_rel_path(fullpath)
 713                 self._dirs[relpath] = Directory(fullpath, relpath, currentdir)
 714             extensions = ('.h', '.cuh', '.hpp', '.c', '.cc', '.cpp', '.cu', '.bm')
 715             for filename in filenames:
 716                 basename, extension = os.path.splitext(filename)
 717                 if extension in extensions:
 718                     fullpath = os.path.join(dirpath, filename)
 719                     relpath = self._get_rel_path(fullpath)
 720                     self._files[relpath] = File(fullpath, relpath, currentdir)
 721                 elif extension == '.cmakein':
 722                     extension = os.path.splitext(basename)[1]
 723                     if extension in extensions:
 724                         fullpath = os.path.join(dirpath, filename)
 725                         relpath = self._get_rel_path(fullpath)
 726                         sourcefile = GeneratorSourceFile(fullpath, relpath, currentdir)
 727                         self._files[relpath] = sourcefile
 728                         fullpath = os.path.join(dirpath, basename)
 729                         relpath = self._get_rel_path(fullpath)
 730                         fullpath = os.path.join(self._build_root, relpath)
 731                         generatedfile = GeneratedFile(fullpath, relpath, currentdir)
 732                         self._files[relpath] = generatedfile
 733                         generatedfile.set_generator_source(sourcefile)
 734                 elif extension in ('.l', '.y', '.pre'):
 735                     fullpath = os.path.join(dirpath, filename)
 736                     relpath = self._get_rel_path(fullpath)
 737                     self._files[relpath] = GeneratorSourceFile(fullpath, relpath, currentdir)
 738
 739     def _create_module(self, rootdir):
 740         """Create module for a subdirectory."""
 741         name = 'module_' + rootdir.get_name()
 742         moduleobj = Module(name, rootdir)
 743         rootdir.set_module(moduleobj)
 744         self._modules[name] = moduleobj
 745
 746     def scan_files(self, only_files=None, keep_contents=False):
 747         """Read source files to initialize #include dependencies."""
 748         if only_files:
 749             filelist = only_files
 750         else:
 751             filelist = self._files.itervalues()
 752         for fileobj in filelist:
 753             if not fileobj.is_external():
 754                 fileobj.scan_contents(self, keep_contents)
 755                 module = fileobj.get_module()
 756                 if module:
 757                     for includedfile in fileobj.get_includes():
 758                         otherfile = includedfile.get_file()
 759                         if otherfile:
 760                             othermodule = otherfile.get_module()
 761                             if othermodule and othermodule != module:
 762                                 module.add_dependency(othermodule, includedfile)
 763
 764     def load_xml(self, only_files=None):
 765         """Load Doxygen XML information.
 766
 767         If only_files is True, XML data is not loaded for code constructs, but
 768         only for files, directories, and their potential parents.
 769         """
 770         xmldir = os.path.join(self._build_root, 'docs', 'html', 'doxygen', 'xml')
 771         self._docset = xml.DocumentationSet(xmldir, self._reporter)
 772         if only_files:
 773             if isinstance(only_files, collections.Iterable):
 774                 filelist = [x.get_abspath() for x in only_files]
 775                 self._docset.load_file_details(filelist)
 776             else:
 777                 self._docset.load_file_details()
 778         else:
 779             self._docset.load_details()
 780             self._docset.merge_duplicates()
 781         self._load_dirs()
 782         self._load_modules()
 783         self._load_files()
 784         if not only_files:
 785             self._load_namespaces()
 786             self._load_classes()
 787             self._load_members()
 788
 789     def _load_dirs(self):
 790         """Load Doxygen XML directory information."""
 791         rootdirs = self._docset.get_compounds(xml.Directory,
 792                 lambda x: x.get_parent() is None)
 793         for dirdoc in rootdirs:
 794             self._load_dir(dirdoc, None)
 795
 796     def _load_dir(self, dirdoc, parent):
 797         """Load Doxygen XML directory information for a single directory."""
 798         path = dirdoc.get_path().rstrip('/')
 799         if not os.path.isabs(path):
 800             self._reporter.xml_assert(dirdoc.get_xml_path(),
 801                     "expected absolute path in Doxygen-produced XML file")
 802             return
 803         relpath = self._get_rel_path(path)
 804         dirobj = self._dirs.get(relpath)
 805         if not dirobj:
 806             dirobj = Directory(path, relpath, parent)
 807             self._dirs[relpath] = dirobj
 808         dirobj.set_doc_xml(dirdoc, self)
 809         self._docmap[dirdoc] = dirobj
 810         for subdirdoc in dirdoc.get_subdirectories():
 811             self._load_dir(subdirdoc, dirobj)
 812
 813     def _load_modules(self):
 814         """Load Doxygen XML module (group) information."""
 815         moduledocs = self._docset.get_compounds(xml.Group,
 816                 lambda x: x.get_name().startswith('module_'))
 817         for moduledoc in moduledocs:
 818             moduleobj = self._modules.get(moduledoc.get_name())
 819             if not moduleobj:
 820                 self._reporter.input_error(
 821                         "no matching directory for module: {0}".format(moduledoc))
 822                 continue
 823             moduleobj.set_doc_xml(moduledoc, self)
 824             self._docmap[moduledoc] = moduleobj
 825
 826     def _load_files(self):
 827         """Load Doxygen XML file information."""
 828         for filedoc in self._docset.get_files():
 829             path = filedoc.get_path()
 830             if not path:
 831                 # In case of only partially loaded file information,
 832                 # the path information is not set for unloaded files.
 833                 continue
 834             if not os.path.isabs(path):
 835                 self._reporter.xml_assert(filedoc.get_xml_path(),
 836                         "expected absolute path in Doxygen-produced XML file")
 837                 continue
 838             extension = os.path.splitext(path)[1]
 839             # We don't care about Markdown files that only produce pages
 840             # (and fail the directory check below).
 841             if extension == '.md':
 842                 continue
 843             dirdoc = filedoc.get_directory()
 844             if not dirdoc:
 845                 self._reporter.xml_assert(filedoc.get_xml_path(),
 846                         "file is not in any directory in Doxygen")
 847                 continue
 848             relpath = self._get_rel_path(path)
 849             fileobj = self._files.get(relpath)
 850             if not fileobj:
 851                 fileobj = File(path, relpath, self._docmap[dirdoc])
 852                 self._files[relpath] = fileobj
 853             fileobj.set_doc_xml(filedoc, self)
 854             self._docmap[filedoc] = fileobj
 855
 856     def _load_namespaces(self):
 857         """Load Doxygen XML namespace information."""
 858         nsdocs = self._docset.get_namespaces()
 859         for nsdoc in nsdocs:
 860             nsobj = Namespace(nsdoc)
 861             self._docmap[nsdoc] = nsobj
 862             self._namespaces.add(nsobj)
 863
 864     def _load_classes(self):
 865         """Load Doxygen XML class information."""
 866         classdocs = self._docset.get_classes()
 867         for classdoc in classdocs:
 868             files = [self._docmap[filedoc] for filedoc in classdoc.get_files()]
 869             classobj = Class(classdoc, files)
 870             self._docmap[classdoc] = classobj
 871             self._classes.add(classobj)
 872
 873     def _load_members(self):
 874         """Load Doxygen XML member information."""
 875         memberdocs = self._docset.get_members()
 876         for memberdoc in memberdocs:
 877             nsdoc = memberdoc.get_namespace()
 878             nsobj = self.get_object(nsdoc)
 879             memberobj = Member(memberdoc, nsobj)
 880             self._docmap[memberdoc] = memberobj
 881             self._members.add(memberobj)
 882
 883     def _get_dir(self, relpath):
 884         """Get directory object for a path relative to source tree root."""
 885         return self._dirs.get(relpath)
 886
 887     def get_file(self, path):
 888         """Get file object for a path relative to source tree root."""
 889         return self._files.get(self._get_rel_path(path))
 890
 891     def find_include_file(self, includedpath):
 892         """Find a file object corresponding to an include path."""
 893         for testdir in ('src', 'src/external/thread_mpi/include',
 894                 'src/external/tng_io/include'):
 895             testpath = os.path.join(testdir, includedpath)
 896             if testpath in self._files:
 897                 return self._files[testpath]
 898
 899     def load_git_attributes(self):
 900         """Load git attribute information for files."""
 901         args = ['git', 'check-attr', '--stdin', 'filter']
 902         git_check_attr = subprocess.Popen(args, stdin=subprocess.PIPE,
 903                 stdout=subprocess.PIPE, cwd=self._source_root)
 904         filelist = '\n'.join(map(File.get_relpath, self._files.itervalues()))
 905         filters = git_check_attr.communicate(filelist)[0]
 906         for fileinfo in filters.splitlines():
 907             path, dummy, value = fileinfo.split(': ')
 908             fileobj = self._files.get(path)
 909             assert fileobj is not None
 910             fileobj.set_git_filter_attribute(value)
 911
 912     def load_installed_file_list(self):
 913         """Load list of installed files from the build tree."""
 914         listpath = os.path.join(self._build_root, 'docs', 'doxygen', 'installed-headers.txt')
 915         with open(listpath, 'r') as installedfp:
 916             for line in installedfp:
 917                 path = line.strip()
 918                 if not os.path.isabs(path):
 919                     self._reporter.input_error(
 920                             "installed file not specified with absolute path: {0}"
 921                             .format(path))
 922                     continue
 923                 relpath = self._get_rel_path(path)
 924                 if relpath not in self._files:
 925                     self._reporter.input_error(
 926                             "installed file not in source tree: {0}".format(path))
 927                     continue
 928                 self._files[relpath].set_installed()
 929
 930     def load_cycle_suppression_list(self, filename):
 931         """Load a list of edges to suppress in cycles.
 932
 933         These edges between modules, if present, will be marked in the
 934         corresponding ModuleDependency objects.
 935         """
 936         with open(filename, 'r') as fp:
 937             for line in fp:
 938                 line = line.strip()
 939                 if not line or line.startswith('#'):
 940                     continue
 941                 modulenames = ['module_' + x.strip() for x in line.split('->')]
 942                 if len(modulenames) != 2:
 943                     self._reporter.input_error(
 944                             "invalid cycle suppression line: {0}".format(line))
 945                     continue
 946                 firstmodule = self._modules.get(modulenames[0])
 947                 secondmodule = self._modules.get(modulenames[1])
 948                 if not firstmodule or not secondmodule:
 949                     self._reporter.input_error(
 950                             "unknown modules mentioned on cycle suppression line: {0}".format(line))
 951                     continue
 952                 for dep in firstmodule.get_dependencies():
 953                     if dep.get_other_module() == secondmodule:
 954                         # TODO: Check that each suppression is actually part of
 955                         # a cycle.
 956                         dep.set_cycle_suppression()
 957
 958     def get_object(self, docobj):
 959         """Get tree object for a Doxygen XML object."""
 960         if docobj is None:
 961             return None
 962         return self._docmap.get(docobj)
 963
 964     def get_files(self):
 965         """Get iterable for all files in the source tree."""
 966         return self._files.itervalues()
 967
 968     def get_modules(self):
 969         """Get iterable for all modules in the source tree."""
 970         return self._modules.itervalues()
 971
 972     def get_classes(self):
 973         """Get iterable for all classes in the source tree."""
 974         return self._classes
 975
 976     def get_members(self):
 977         """Get iterable for all members (in Doxygen terms) in the source tree."""
 978         return self._members