docs/doxygen/gmxtree.py

   1 #!/usr/bin/python
   2 #
   3 # This file is part of the GROMACS molecular simulation package.
   4 #
   5 # Copyright (c) 2014, by the GROMACS development team, led by
   6 # Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
   7 # and including many others, as listed in the AUTHORS file in the
   8 # top-level source directory and at http://www.gromacs.org.
   9 #
  10 # GROMACS is free software; you can redistribute it and/or
  11 # modify it under the terms of the GNU Lesser General Public License
  12 # as published by the Free Software Foundation; either version 2.1
  13 # of the License, or (at your option) any later version.
  14 #
  15 # GROMACS is distributed in the hope that it will be useful,
  16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18 # Lesser General Public License for more details.
  19 #
  20 # You should have received a copy of the GNU Lesser General Public
  21 # License along with GROMACS; if not, see
  22 # http://www.gnu.org/licenses, or write to the Free Software Foundation,
  23 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
  24 #
  25 # If you want to redistribute modifications to GROMACS, please
  26 # consider that scientific software is very special. Version
  27 # control is crucial - bugs must be traceable. We will be happy to
  28 # consider code for inclusion in the official distribution, but
  29 # derived work must not be called official GROMACS. Details are found
  30 # in the README & COPYING files - if they are missing, get the
  31 # official version at http://www.gromacs.org.
  32 #
  33 # To help us fund GROMACS development, we humbly ask that you cite
  34 # the research papers on the package. Check out http://www.gromacs.org.
  35
  36 """GROMACS-specific representation for source tree and documentation.
  37
  38 This module provides classes that construct a GROMACS-specific representation
  39 of the source tree and associate the Doxygen XML output with it.  It constructs
  40 an initial representation by walking the source tree in the file system, and
  41 then associates information from the Doxygen XML output into this.
  42 It also adds some additional knowledge from how the GROMACS source tree is
  43 organized to construct a representation that is easy to process and check as
  44 the top-level scripts expect.
  45
  46 The object model is rooted at a GromacsTree object.  Currently, it constructs a
  47 representation of the source tree from the file system, but is otherwise mostly
  48 a thin wrapper around the Doxygen XML tree.  It already adds some relations and
  49 rules that come from GROMACS-specific knowledge.  In the future, more such
  50 customizations will be added.
  51 """
  52
  53 import collections
  54 import os
  55 import os.path
  56 import re
  57 import subprocess
  58
  59 import doxygenxml as xml
  60 import reporter
  61 # We import DocType directly so that it is exposed from this module as well.
  62 from doxygenxml import DocType
  63
  64 def _get_api_type_for_compound(grouplist):
  65     """Helper function to deduce API type from Doxygen group membership."""
  66     result = DocType.internal
  67     for group in grouplist:
  68         if isinstance(group, xml.Group):
  69             if group.get_name() == 'group_publicapi':
  70                 result = DocType.public
  71             elif group.get_name() == 'group_libraryapi':
  72                 result = DocType.library
  73             # TODO: Check for multiple group membership
  74     return result
  75
  76 class IncludedFile(object):
  77
  78     """Information about an #include directive in a file."""
  79
  80     def __init__(self, including_file, lineno, included_file, included_path, is_relative, is_system, line):
  81         self._including_file = including_file
  82         self._line_number = lineno
  83         self._included_file = included_file
  84         self._included_path = included_path
  85         #self._used_include_path = used_include_path
  86         self._is_relative = is_relative
  87         self._is_system = is_system
  88         self._line = line
  89
  90     def __str__(self):
  91         if self._is_system:
  92             return '<{0}>'.format(self._included_path)
  93         else:
  94             return '"{0}"'.format(self._included_path)
  95
  96     def is_system(self):
  97         return self._is_system
  98
  99     def is_relative(self):
 100         return self._is_relative
 101
 102     def get_included_path(self):
 103         return self._included_path
 104
 105     def get_including_file(self):
 106         return self._including_file
 107
 108     def get_file(self):
 109         return self._included_file
 110
 111     def get_line_number(self):
 112         return self._line_number
 113
 114     def get_full_line(self):
 115         """Return the full source line on which this include appears.
 116
 117         Trailing newline is included."""
 118         return self._line
 119
 120     def get_reporter_location(self):
 121         return reporter.Location(self._including_file.get_abspath(), self._line_number)
 122
 123 class IncludeBlock(object):
 124
 125     """Block of consequent #include directives in a file."""
 126
 127     def __init__(self, first_included_file):
 128         self._first_line = first_included_file.get_line_number()
 129         self._last_line = self._first_line
 130         self._files = []
 131         self.add_file(first_included_file)
 132
 133     def add_file(self, included_file):
 134         self._files.append(included_file)
 135         self._last_line = included_file.get_line_number()
 136
 137     def get_includes(self):
 138         return self._files
 139
 140     def get_first_line(self):
 141         return self._first_line
 142
 143     def get_last_line(self):
 144         return self._last_line
 145
 146 class File(object):
 147
 148     """Source/header file in the GROMACS tree."""
 149
 150     def __init__(self, abspath, relpath, directory):
 151         """Initialize a file representation with basic information."""
 152         self._abspath = abspath
 153         self._relpath = relpath
 154         self._dir = directory
 155         self._rawdoc = None
 156         self._installed = False
 157         extension = os.path.splitext(abspath)[1]
 158         self._sourcefile = (extension in ('.c', '.cc', '.cpp', '.cu'))
 159         self._apitype = DocType.none
 160         self._modules = set()
 161         self._includes = []
 162         self._include_blocks = []
 163         self._main_header = None
 164         self._lines = None
 165         self._filter = None
 166         directory.add_file(self)
 167
 168     def set_doc_xml(self, rawdoc, sourcetree):
 169         """Assiociate Doxygen documentation entity with the file."""
 170         assert self._rawdoc is None
 171         assert rawdoc.is_source_file() == self._sourcefile
 172         self._rawdoc = rawdoc
 173         if self._rawdoc.is_documented():
 174             grouplist = self._rawdoc.get_groups()
 175             self._apitype = _get_api_type_for_compound(grouplist)
 176             for group in grouplist:
 177                 module = sourcetree.get_object(group)
 178                 if module:
 179                     self._modules.add(module)
 180
 181     def set_installed(self):
 182         """Mark the file installed."""
 183         self._installed = True
 184
 185     def set_git_filter_attribute(self, filtername):
 186         """Set the git filter attribute associated with the file."""
 187         self._filter = filtername
 188
 189     def set_main_header(self, included_file):
 190         """Set the main header file for a source file."""
 191         assert self.is_source_file()
 192         self._main_header = included_file
 193
 194     def _process_include(self, lineno, is_system, includedpath, line, sourcetree):
 195         """Process #include directive during scan()."""
 196         is_relative = False
 197         if is_system:
 198             fileobj = sourcetree.find_include_file(includedpath)
 199         else:
 200             fullpath = os.path.join(self._dir.get_abspath(), includedpath)
 201             fullpath = os.path.abspath(fullpath)
 202             if os.path.exists(fullpath):
 203                 is_relative = True
 204                 fileobj = sourcetree.get_file(fullpath)
 205             else:
 206                 fileobj = sourcetree.find_include_file(includedpath)
 207         included_file = IncludedFile(self, lineno, fileobj, includedpath,
 208             is_relative, is_system, line)
 209         self._includes.append(included_file)
 210         return included_file
 211
 212     def scan_contents(self, sourcetree, keep_contents):
 213         """Scan the file contents and initialize information based on it."""
 214         # TODO: Consider a more robust regex.
 215         include_re = r'^\s*#\s*include\s+(?P<quote>["<])(?P<path>[^">]*)[">]'
 216         current_block = None
 217         # TODO: Consider reading directly into this list, and iterate that.
 218         lines = []
 219         with open(self._abspath, 'r') as scanfile:
 220             for lineno, line in enumerate(scanfile, 1):
 221                 lines.append(line)
 222                 match = re.match(include_re, line)
 223                 if match:
 224                     is_system = (match.group('quote') == '<')
 225                     includedpath = match.group('path')
 226                     included_file = self._process_include(lineno, is_system,
 227                             includedpath, line, sourcetree)
 228                     if current_block is None:
 229                         current_block = IncludeBlock(included_file)
 230                         self._include_blocks.append(current_block)
 231                     else:
 232                         current_block.add_file(included_file)
 233                 elif line and not line.isspace():
 234                     current_block = None
 235         if keep_contents:
 236             self._lines = lines
 237
 238     def get_reporter_location(self):
 239         return reporter.Location(self._abspath, None)
 240
 241     def is_installed(self):
 242         return self._installed
 243
 244     def is_external(self):
 245         return self._dir.is_external()
 246
 247     def is_source_file(self):
 248         return self._sourcefile
 249
 250     def is_test_file(self):
 251         return self._dir.is_test_directory()
 252
 253     def should_includes_be_sorted(self):
 254         """Return whether the include directives in the file should be sorted."""
 255         return self._filter in ('includesort', 'uncrustify')
 256
 257     def is_documented(self):
 258         return self._rawdoc and self._rawdoc.is_documented()
 259
 260     def has_brief_description(self):
 261         return self._rawdoc and self._rawdoc.has_brief_description()
 262
 263     def get_abspath(self):
 264         return self._abspath
 265
 266     def get_relpath(self):
 267         return self._relpath
 268
 269     def get_name(self):
 270         return os.path.basename(self._abspath)
 271
 272     def get_directory(self):
 273         return self._dir
 274
 275     def get_doc_type(self):
 276         if not self._rawdoc:
 277             return DocType.none
 278         return self._rawdoc.get_visibility()
 279
 280     def get_api_type(self):
 281         return self._apitype
 282
 283     def api_type_is_reliable(self):
 284         if self._apitype in (DocType.internal, DocType.library):
 285             return True
 286         module = self.get_module()
 287         return module and module.is_documented()
 288
 289     def is_public(self):
 290         if self.api_type_is_reliable():
 291             return self.get_api_type() == DocType.public
 292         return self.get_api_type() == DocType.public or self.is_installed()
 293
 294     def is_module_internal(self):
 295         if self.is_source_file():
 296             return True
 297         return not self.is_installed() and self.get_api_type() <= DocType.internal
 298
 299     def get_expected_module(self):
 300         return self._dir.get_module()
 301
 302     def get_doc_modules(self):
 303         return self._modules
 304
 305     def get_module(self):
 306         module = self.get_expected_module()
 307         if not module and len(self._modules) == 1:
 308             module = list(self._modules)[0]
 309         return module
 310
 311     def get_includes(self):
 312         return self._includes
 313
 314     def get_include_blocks(self):
 315         return self._include_blocks
 316
 317     def get_main_header(self):
 318         return self._main_header
 319
 320     def get_contents(self):
 321         return self._lines
 322
 323 class GeneratedFile(File):
 324     def __init__(self, abspath, relpath, directory):
 325         File.__init__(self, abspath, relpath, directory)
 326         self._generator_source_file = None
 327
 328     def set_generator_source(self, sourcefile):
 329         self._generator_source_file = sourcefile
 330
 331     def get_reporter_location(self):
 332         if self._generator_source_file:
 333             return self._generator_source_file.get_reporter_location()
 334         return File.get_reporter_location(self)
 335
 336 class GeneratorSourceFile(File):
 337     pass
 338
 339 class Directory(object):
 340
 341     """(Sub)directory in the GROMACS tree."""
 342
 343     def __init__(self, abspath, relpath, parent):
 344         """Initialize a file representation with basic information."""
 345         self._abspath = abspath
 346         self._relpath = relpath
 347         self._name = os.path.basename(abspath)
 348         self._parent = parent
 349         self._rawdoc = None
 350         self._module = None
 351         self._is_test_dir = False
 352         if parent and parent.is_test_directory() or \
 353                 self._name in ('tests', 'legacytests'):
 354             self._is_test_dir = True
 355         self._is_external = False
 356         if parent and parent.is_external() or self._name == 'external':
 357             self._is_external = True
 358         self._subdirs = set()
 359         if parent:
 360             parent._subdirs.add(self)
 361         self._files = set()
 362         self._has_installed_files = None
 363
 364     def set_doc_xml(self, rawdoc, sourcetree):
 365         """Assiociate Doxygen documentation entity with the directory."""
 366         assert self._rawdoc is None
 367         assert self._abspath == rawdoc.get_path().rstrip('/')
 368         self._rawdoc = rawdoc
 369
 370     def set_module(self, module):
 371         assert self._module is None
 372         self._module = module
 373
 374     def add_file(self, fileobj):
 375         self._files.add(fileobj)
 376
 377     def get_name(self):
 378         return self._name
 379
 380     def get_reporter_location(self):
 381         return reporter.Location(self._abspath, None)
 382
 383     def get_abspath(self):
 384         return self._abspath
 385
 386     def get_relpath(self):
 387         return self._relpath
 388
 389     def is_test_directory(self):
 390         return self._is_test_dir
 391
 392     def is_external(self):
 393         return self._is_external
 394
 395     def has_installed_files(self):
 396         if self._has_installed_files is None:
 397             self._has_installed_files = False
 398             for subdir in self._subdirs:
 399                 if subdir.has_installed_files():
 400                     self._has_installed_files = True
 401                     return True
 402             for fileobj in self._files:
 403                 if fileobj.is_installed():
 404                     self._has_installed_files = True
 405                     return True
 406         return self._has_installed_files
 407
 408     def get_module(self):
 409         if self._module:
 410             return self._module
 411         if self._parent:
 412             return self._parent.get_module()
 413         return None
 414
 415     def get_subdirectories(self):
 416         return self._subdirs
 417
 418     def get_files(self):
 419         for subdir in self._subdirs:
 420             for fileobj in subdir.get_files():
 421                 yield fileobj
 422         for fileobj in self._files:
 423             yield fileobj
 424
 425     def contains(self, fileobj):
 426         """Check whether file is within the directory or its subdirectories."""
 427         dirobj = fileobj.get_directory()
 428         while dirobj:
 429             if dirobj == self:
 430                 return True
 431             dirobj = dirobj._parent
 432         return False
 433
 434 class ModuleDependency(object):
 435
 436     """Dependency between modules."""
 437
 438     def __init__(self, othermodule):
 439         """Initialize empty dependency object with given module as dependency."""
 440         self._othermodule = othermodule
 441         self._includedfiles = []
 442         self._cyclesuppression = None
 443
 444     def add_included_file(self, includedfile):
 445         """Add IncludedFile that is part of this dependency."""
 446         assert includedfile.get_file().get_module() == self._othermodule
 447         self._includedfiles.append(includedfile)
 448
 449     def set_cycle_suppression(self):
 450         """Set suppression on cycles containing this dependency."""
 451         self._cyclesuppression = True
 452
 453     def is_cycle_suppressed(self):
 454         """Return whether cycles containing this dependency are suppressed."""
 455         return self._cyclesuppression is not None
 456
 457     def get_other_module(self):
 458         """Get module that this dependency is to."""
 459         return self._othermodule
 460
 461     def get_included_files(self):
 462         """Get IncludedFile objects for the individual include dependencies."""
 463         return self._includedfiles
 464
 465 class Module(object):
 466
 467     """Code module in the GROMACS source tree.
 468
 469     Modules are specific subdirectories that host a more or less coherent
 470     set of routines.  Simplified, every subdirectory under src/gromacs/ is
 471     a different module.  This object provides that abstraction and also links
 472     the subdirectory to the module documentation (documented as a group in
 473     Doxygen) if that exists.
 474     """
 475
 476     def __init__(self, name, rootdir):
 477         self._name = name
 478         self._rawdoc = None
 479         self._rootdir = rootdir
 480         self._group = None
 481         self._dependencies = dict()
 482
 483     def set_doc_xml(self, rawdoc, sourcetree):
 484         """Assiociate Doxygen documentation entity with the module."""
 485         assert self._rawdoc is None
 486         self._rawdoc = rawdoc
 487         if self._rawdoc.is_documented():
 488             groups = list(self._rawdoc.get_groups())
 489             if len(groups) == 1:
 490                 groupname = groups[0].get_name()
 491                 if groupname.startswith('group_'):
 492                     self._group = groupname[6:]
 493
 494     def add_dependency(self, othermodule, includedfile):
 495         """Add #include dependency from a file in this module."""
 496         assert includedfile.get_file().get_module() == othermodule
 497         if othermodule not in self._dependencies:
 498             self._dependencies[othermodule] = ModuleDependency(othermodule)
 499         self._dependencies[othermodule].add_included_file(includedfile)
 500
 501     def is_documented(self):
 502         return self._rawdoc is not None
 503
 504     def get_name(self):
 505         return self._name
 506
 507     def get_root_dir(self):
 508         return self._rootdir
 509
 510     def get_files(self):
 511         # TODO: Include public API convenience headers?
 512         return self._rootdir.get_files()
 513
 514     def get_group(self):
 515         return self._group
 516
 517     def get_dependencies(self):
 518         return self._dependencies.itervalues()
 519
 520 class Namespace(object):
 521
 522     """Namespace in the GROMACS source code."""
 523
 524     def __init__(self, rawdoc):
 525         self._rawdoc = rawdoc
 526
 527     def is_anonymous(self):
 528         return self._rawdoc.is_anonymous()
 529
 530 class Class(object):
 531
 532     """Class/struct/union in the GROMACS source code."""
 533
 534     def __init__(self, rawdoc, files):
 535         self._rawdoc = rawdoc
 536         self._files = set(files)
 537
 538     def get_name(self):
 539         return self._rawdoc.get_name()
 540
 541     def get_reporter_location(self):
 542         return self._rawdoc.get_reporter_location()
 543
 544     def get_files(self):
 545         return self._files
 546
 547     def is_documented(self):
 548         return self._rawdoc.is_documented()
 549
 550     def has_brief_description(self):
 551         return self._rawdoc.has_brief_description()
 552
 553     def get_doc_type(self):
 554         """Return documentation type (visibility) for the class.
 555
 556         In addition to the actual code, this encodes GROMACS-specific logic
 557         of setting EXTRACT_LOCAL_CLASSES=YES only for the full documentation.
 558         Local classes never appear outside the full documentation, no matter
 559         what is their visibility.
 560         """
 561         if not self.is_documented():
 562             return DocType.none
 563         if self._rawdoc.is_local():
 564             return DocType.internal
 565         return self._rawdoc.get_visibility()
 566
 567     def get_file_doc_type(self):
 568         return max([fileobj.get_doc_type() for fileobj in self._files])
 569
 570     def is_in_installed_file(self):
 571         return any([fileobj.is_installed() for fileobj in self._files])
 572
 573 class Member(object):
 574
 575     """Member (in Doxygen terminology) in the GROMACS source tree.
 576
 577     Currently, modeling is limited to the minimal set of properties that the
 578     checker uses.
 579     """
 580
 581     def __init__(self, rawdoc, namespace):
 582         self._rawdoc = rawdoc
 583         self._namespace = namespace
 584
 585     def get_name(self):
 586         return self._rawdoc.get_name()
 587
 588     def get_reporter_location(self):
 589         return self._rawdoc.get_reporter_location()
 590
 591     def is_documented(self):
 592         return self._rawdoc.is_documented()
 593
 594     def has_brief_description(self):
 595         return self._rawdoc.has_brief_description()
 596
 597     def has_inbody_description(self):
 598         return self._rawdoc.has_inbody_description()
 599
 600     def is_visible(self):
 601         """Return whether the member is visible in Doxygen documentation.
 602
 603         Doxygen ignores members whose parent compounds are not documented.
 604         However, when EXTRACT_ANON_NPACES=ON (which is set for our full
 605         documentation), members of anonymous namespaces are extracted even if
 606         the namespace is the only parent and is not documented.
 607         """
 608         if self._namespace and self._namespace.is_anonymous():
 609             return True
 610         return self._rawdoc.get_inherited_visibility() != DocType.none
 611
 612
 613 class GromacsTree(object):
 614
 615     """Root object for navigating the GROMACS source tree.
 616
 617     On initialization, the list of files and directories is initialized by
 618     walking the source tree, and modules are created for top-level
 619     subdirectories.  At this point, only information that is accessible from
 620     file names and paths only is available.
 621
 622     load_git_attributes() can be called to load attribute information from
 623     .gitattributes for all the files.
 624
 625     load_installed_file_list() can be called to load the list of installed
 626     files from the build tree (generated by the find-installed-headers target).
 627
 628     scan_files() can be called to read all the files and initialize #include
 629     dependencies between the files based on the information.  This is done like
 630     this instead of relying on Doxygen-extracted include files to make the
 631     dependency graph independent from preprocessor macro definitions
 632     (Doxygen only sees those #includes that the preprocessor sees, which
 633     depends on what #defines it has seen).
 634
 635     load_xml() can be called to load information from Doxygen XML data in
 636     the build tree (the Doxygen XML data must have been built separately).
 637     """
 638
 639     def __init__(self, source_root, build_root, reporter):
 640         """Initialize the tree object by walking the source tree."""
 641         self._source_root = os.path.abspath(source_root)
 642         self._build_root = os.path.abspath(build_root)
 643         self._reporter = reporter
 644         self._docset = None
 645         self._docmap = dict()
 646         self._dirs = dict()
 647         self._files = dict()
 648         self._modules = dict()
 649         self._classes = set()
 650         self._namespaces = set()
 651         self._members = set()
 652         self._walk_dir(os.path.join(self._source_root, 'src'))
 653         for fileobj in self.get_files():
 654             if fileobj and fileobj.is_source_file() and not fileobj.is_external():
 655                 (basedir, name) = os.path.split(fileobj.get_abspath())
 656                 (basename, ext) = os.path.splitext(name)
 657                 header = self.get_file(os.path.join(basedir, basename + '.h'))
 658                 if not header and ext == '.cu':
 659                     header = self.get_file(os.path.join(basedir, basename + '.cuh'))
 660                 if not header and fileobj.is_test_file():
 661                     basedir = os.path.dirname(basedir)
 662                     header = self.get_file(os.path.join(basedir, basename + '.h'))
 663                     if not header:
 664                         # Somewhat of a hack; currently, the tests for
 665                         # analysisdata/modules/ and trajectoryanalysis/modules/
 666                         # is at the top-level tests directory.
 667                         # TODO: It could be clearer to split the tests so that
 668                         # there would be a separate modules/tests/.
 669                         header = self.get_file(os.path.join(basedir, 'modules', basename + '.h'))
 670                     if not header and basename.endswith('_tests'):
 671                         header = self.get_file(os.path.join(basedir, basename[:-6] + '.h'))
 672                 if not header and fileobj.get_relpath().startswith('src/gromacs'):
 673                     header = self._files.get(os.path.join('src/gromacs/legacyheaders', basename + '.h'))
 674                 if header:
 675                     fileobj.set_main_header(header)
 676         rootdir = self._get_dir(os.path.join('src', 'gromacs'))
 677         for subdir in rootdir.get_subdirectories():
 678             self._create_module(subdir)
 679         rootdir = self._get_dir(os.path.join('src', 'testutils'))
 680         self._create_module(rootdir)
 681
 682     def _get_rel_path(self, path):
 683         assert os.path.isabs(path)
 684         if path.startswith(self._build_root):
 685             return os.path.relpath(path, self._build_root)
 686         if path.startswith(self._source_root):
 687             return os.path.relpath(path, self._source_root)
 688         raise ValueError("path not under build nor source tree: {0}".format(path))
 689
 690     def _walk_dir(self, rootpath):
 691         """Construct representation of the source tree by walking the file system."""
 692         assert os.path.isabs(rootpath)
 693         assert rootpath not in self._dirs
 694         relpath = self._get_rel_path(rootpath)
 695         self._dirs[relpath] = Directory(rootpath, relpath, None)
 696         for dirpath, dirnames, filenames in os.walk(rootpath):
 697             if 'contrib' in dirnames:
 698                 dirnames.remove('contrib')
 699             if 'refdata' in dirnames:
 700                 dirnames.remove('refdata')
 701             currentdir = self._dirs[self._get_rel_path(dirpath)]
 702             # Loop through a copy so that we can modify dirnames.
 703             for dirname in list(dirnames):
 704                 fullpath = os.path.join(dirpath, dirname)
 705                 if fullpath == self._build_root:
 706                     dirnames.remove(dirname)
 707                     continue
 708                 relpath = self._get_rel_path(fullpath)
 709                 self._dirs[relpath] = Directory(fullpath, relpath, currentdir)
 710             extensions = ('.h', '.cuh', '.hpp', '.c', '.cc', '.cpp', '.cu', '.bm')
 711             for filename in filenames:
 712                 basename, extension = os.path.splitext(filename)
 713                 if extension in extensions:
 714                     fullpath = os.path.join(dirpath, filename)
 715                     relpath = self._get_rel_path(fullpath)
 716                     self._files[relpath] = File(fullpath, relpath, currentdir)
 717                 elif extension == '.cmakein':
 718                     extension = os.path.splitext(basename)[1]
 719                     if extension in extensions:
 720                         fullpath = os.path.join(dirpath, filename)
 721                         relpath = self._get_rel_path(fullpath)
 722                         sourcefile = GeneratorSourceFile(fullpath, relpath, currentdir)
 723                         self._files[relpath] = sourcefile
 724                         fullpath = os.path.join(dirpath, basename)
 725                         relpath = self._get_rel_path(fullpath)
 726                         fullpath = os.path.join(self._build_root, relpath)
 727                         generatedfile = GeneratedFile(fullpath, relpath, currentdir)
 728                         self._files[relpath] = generatedfile
 729                         generatedfile.set_generator_source(sourcefile)
 730                 elif extension in ('.l', '.y', '.pre'):
 731                     fullpath = os.path.join(dirpath, filename)
 732                     relpath = self._get_rel_path(fullpath)
 733                     self._files[relpath] = GeneratorSourceFile(fullpath, relpath, currentdir)
 734
 735     def _create_module(self, rootdir):
 736         """Create module for a subdirectory."""
 737         name = 'module_' + rootdir.get_name()
 738         moduleobj = Module(name, rootdir)
 739         rootdir.set_module(moduleobj)
 740         self._modules[name] = moduleobj
 741
 742     def scan_files(self, only_files=None, keep_contents=False):
 743         """Read source files to initialize #include dependencies."""
 744         if only_files:
 745             filelist = only_files
 746         else:
 747             filelist = self._files.itervalues()
 748         for fileobj in filelist:
 749             if not fileobj.is_external():
 750                 fileobj.scan_contents(self, keep_contents)
 751                 module = fileobj.get_module()
 752                 if module:
 753                     for includedfile in fileobj.get_includes():
 754                         otherfile = includedfile.get_file()
 755                         if otherfile:
 756                             othermodule = otherfile.get_module()
 757                             if othermodule and othermodule != module:
 758                                 module.add_dependency(othermodule, includedfile)
 759
 760     def load_xml(self, only_files=None):
 761         """Load Doxygen XML information.
 762
 763         If only_files is True, XML data is not loaded for code constructs, but
 764         only for files, directories, and their potential parents.
 765         """
 766         xmldir = os.path.join(self._build_root, 'docs', 'html', 'doxygen', 'xml')
 767         self._docset = xml.DocumentationSet(xmldir, self._reporter)
 768         if only_files:
 769             if isinstance(only_files, collections.Iterable):
 770                 filelist = [x.get_abspath() for x in only_files]
 771                 self._docset.load_file_details(filelist)
 772             else:
 773                 self._docset.load_file_details()
 774         else:
 775             self._docset.load_details()
 776             self._docset.merge_duplicates()
 777         self._load_dirs()
 778         self._load_modules()
 779         self._load_files()
 780         if not only_files:
 781             self._load_namespaces()
 782             self._load_classes()
 783             self._load_members()
 784
 785     def _load_dirs(self):
 786         """Load Doxygen XML directory information."""
 787         rootdirs = self._docset.get_compounds(xml.Directory,
 788                 lambda x: x.get_parent() is None)
 789         for dirdoc in rootdirs:
 790             self._load_dir(dirdoc, None)
 791
 792     def _load_dir(self, dirdoc, parent):
 793         """Load Doxygen XML directory information for a single directory."""
 794         path = dirdoc.get_path().rstrip('/')
 795         if not os.path.isabs(path):
 796             self._reporter.xml_assert(dirdoc.get_xml_path(),
 797                     "expected absolute path in Doxygen-produced XML file")
 798             return
 799         relpath = self._get_rel_path(path)
 800         dirobj = self._dirs.get(relpath)
 801         if not dirobj:
 802             dirobj = Directory(path, relpath, parent)
 803             self._dirs[relpath] = dirobj
 804         dirobj.set_doc_xml(dirdoc, self)
 805         self._docmap[dirdoc] = dirobj
 806         for subdirdoc in dirdoc.get_subdirectories():
 807             self._load_dir(subdirdoc, dirobj)
 808
 809     def _load_modules(self):
 810         """Load Doxygen XML module (group) information."""
 811         moduledocs = self._docset.get_compounds(xml.Group,
 812                 lambda x: x.get_name().startswith('module_'))
 813         for moduledoc in moduledocs:
 814             moduleobj = self._modules.get(moduledoc.get_name())
 815             if not moduleobj:
 816                 self._reporter.input_error(
 817                         "no matching directory for module: {0}".format(moduledoc))
 818                 continue
 819             moduleobj.set_doc_xml(moduledoc, self)
 820             self._docmap[moduledoc] = moduleobj
 821
 822     def _load_files(self):
 823         """Load Doxygen XML file information."""
 824         for filedoc in self._docset.get_files():
 825             path = filedoc.get_path()
 826             if not path:
 827                 # In case of only partially loaded file information,
 828                 # the path information is not set for unloaded files.
 829                 continue
 830             if not os.path.isabs(path):
 831                 self._reporter.xml_assert(filedoc.get_xml_path(),
 832                         "expected absolute path in Doxygen-produced XML file")
 833                 continue
 834             extension = os.path.splitext(path)[1]
 835             # We don't care about Markdown files that only produce pages
 836             # (and fail the directory check below).
 837             if extension == '.md':
 838                 continue
 839             dirdoc = filedoc.get_directory()
 840             if not dirdoc:
 841                 self._reporter.xml_assert(filedoc.get_xml_path(),
 842                         "file is not in any directory in Doxygen")
 843                 continue
 844             relpath = self._get_rel_path(path)
 845             fileobj = self._files.get(relpath)
 846             if not fileobj:
 847                 fileobj = File(path, relpath, self._docmap[dirdoc])
 848                 self._files[relpath] = fileobj
 849             fileobj.set_doc_xml(filedoc, self)
 850             self._docmap[filedoc] = fileobj
 851
 852     def _load_namespaces(self):
 853         """Load Doxygen XML namespace information."""
 854         nsdocs = self._docset.get_namespaces()
 855         for nsdoc in nsdocs:
 856             nsobj = Namespace(nsdoc)
 857             self._docmap[nsdoc] = nsobj
 858             self._namespaces.add(nsobj)
 859
 860     def _load_classes(self):
 861         """Load Doxygen XML class information."""
 862         classdocs = self._docset.get_classes()
 863         for classdoc in classdocs:
 864             files = [self._docmap[filedoc] for filedoc in classdoc.get_files()]
 865             classobj = Class(classdoc, files)
 866             self._docmap[classdoc] = classobj
 867             self._classes.add(classobj)
 868
 869     def _load_members(self):
 870         """Load Doxygen XML member information."""
 871         memberdocs = self._docset.get_members()
 872         for memberdoc in memberdocs:
 873             nsdoc = memberdoc.get_namespace()
 874             nsobj = self.get_object(nsdoc)
 875             memberobj = Member(memberdoc, nsobj)
 876             self._docmap[memberdoc] = memberobj
 877             self._members.add(memberobj)
 878
 879     def _get_dir(self, relpath):
 880         """Get directory object for a path relative to source tree root."""
 881         return self._dirs.get(relpath)
 882
 883     def get_file(self, path):
 884         """Get file object for a path relative to source tree root."""
 885         return self._files.get(self._get_rel_path(path))
 886
 887     def find_include_file(self, includedpath):
 888         """Find a file object corresponding to an include path."""
 889         for testdir in ('src', 'src/external/thread_mpi/include',
 890                 'src/external/tng_io/include'):
 891             testpath = os.path.join(testdir, includedpath)
 892             if testpath in self._files:
 893                 return self._files[testpath]
 894
 895     def load_git_attributes(self):
 896         """Load git attribute information for files."""
 897         args = ['git', 'check-attr', '--stdin', 'filter']
 898         git_check_attr = subprocess.Popen(args, stdin=subprocess.PIPE,
 899                 stdout=subprocess.PIPE, cwd=self._source_root)
 900         filelist = '\n'.join(map(File.get_relpath, self._files.itervalues()))
 901         filters = git_check_attr.communicate(filelist)[0]
 902         for fileinfo in filters.splitlines():
 903             path, dummy, value = fileinfo.split(': ')
 904             fileobj = self._files.get(path)
 905             assert fileobj is not None
 906             fileobj.set_git_filter_attribute(value)
 907
 908     def load_installed_file_list(self):
 909         """Load list of installed files from the build tree."""
 910         listpath = os.path.join(self._build_root, 'docs', 'doxygen', 'installed-headers.txt')
 911         with open(listpath, 'r') as installedfp:
 912             for line in installedfp:
 913                 path = line.strip()
 914                 if not os.path.isabs(path):
 915                     self._reporter.input_error(
 916                             "installed file not specified with absolute path: {0}"
 917                             .format(path))
 918                     continue
 919                 relpath = self._get_rel_path(path)
 920                 if relpath not in self._files:
 921                     self._reporter.input_error(
 922                             "installed file not in source tree: {0}".format(path))
 923                     continue
 924                 self._files[relpath].set_installed()
 925
 926     def load_cycle_suppression_list(self, filename):
 927         """Load a list of edges to suppress in cycles.
 928
 929         These edges between modules, if present, will be marked in the
 930         corresponding ModuleDependency objects.
 931         """
 932         with open(filename, 'r') as fp:
 933             for line in fp:
 934                 line = line.strip()
 935                 if not line or line.startswith('#'):
 936                     continue
 937                 modulenames = ['module_' + x.strip() for x in line.split('->')]
 938                 if len(modulenames) != 2:
 939                     self._reporter.input_error(
 940                             "invalid cycle suppression line: {0}".format(line))
 941                     continue
 942                 firstmodule = self._modules.get(modulenames[0])
 943                 secondmodule = self._modules.get(modulenames[1])
 944                 if not firstmodule or not secondmodule:
 945                     self._reporter.input_error(
 946                             "unknown modules mentioned on cycle suppression line: {0}".format(line))
 947                     continue
 948                 for dep in firstmodule.get_dependencies():
 949                     if dep.get_other_module() == secondmodule:
 950                         # TODO: Check that each suppression is actually part of
 951                         # a cycle.
 952                         dep.set_cycle_suppression()
 953
 954     def get_object(self, docobj):
 955         """Get tree object for a Doxygen XML object."""
 956         if docobj is None:
 957             return None
 958         return self._docmap.get(docobj)
 959
 960     def get_files(self):
 961         """Get iterable for all files in the source tree."""
 962         return self._files.itervalues()
 963
 964     def get_modules(self):
 965         """Get iterable for all modules in the source tree."""
 966         return self._modules.itervalues()
 967
 968     def get_classes(self):
 969         """Get iterable for all classes in the source tree."""
 970         return self._classes
 971
 972     def get_members(self):
 973         """Get iterable for all members (in Doxygen terms) in the source tree."""
 974         return self._members