docs/doxygen/gmxtree.py

   1 #!/usr/bin/env python3
   2 #
   3 # This file is part of the GROMACS molecular simulation package.
   4 #
   5 # Copyright (c) 2014,2015,2016,2017,2018,2019, by the GROMACS development team, led by
   6 # Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
   7 # and including many others, as listed in the AUTHORS file in the
   8 # top-level source directory and at http://www.gromacs.org.
   9 #
  10 # GROMACS is free software; you can redistribute it and/or
  11 # modify it under the terms of the GNU Lesser General Public License
  12 # as published by the Free Software Foundation; either version 2.1
  13 # of the License, or (at your option) any later version.
  14 #
  15 # GROMACS is distributed in the hope that it will be useful,
  16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18 # Lesser General Public License for more details.
  19 #
  20 # You should have received a copy of the GNU Lesser General Public
  21 # License along with GROMACS; if not, see
  22 # http://www.gnu.org/licenses, or write to the Free Software Foundation,
  23 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
  24 #
  25 # If you want to redistribute modifications to GROMACS, please
  26 # consider that scientific software is very special. Version
  27 # control is crucial - bugs must be traceable. We will be happy to
  28 # consider code for inclusion in the official distribution, but
  29 # derived work must not be called official GROMACS. Details are found
  30 # in the README & COPYING files - if they are missing, get the
  31 # official version at http://www.gromacs.org.
  32 #
  33 # To help us fund GROMACS development, we humbly ask that you cite
  34 # the research papers on the package. Check out http://www.gromacs.org.
  35
  36 """GROMACS-specific representation for source tree and documentation.
  37
  38 This module provides classes that construct a GROMACS-specific representation
  39 of the source tree and associate the Doxygen XML output with it.  It constructs
  40 an initial representation by walking the source tree in the file system, and
  41 then associates information from the Doxygen XML output into this.
  42 It also adds some additional knowledge from how the GROMACS source tree is
  43 organized to construct a representation that is easy to process and check as
  44 the top-level scripts expect.
  45
  46 The object model is rooted at a GromacsTree object.  Currently, it constructs a
  47 representation of the source tree from the file system, but is otherwise mostly
  48 a thin wrapper around the Doxygen XML tree.  It already adds some relations and
  49 rules that come from GROMACS-specific knowledge.  In the future, more such
  50 customizations will be added.
  51 """
  52
  53 import collections
  54 import os
  55 import os.path
  56 import re
  57 import subprocess
  58
  59 import doxygenxml as xml
  60 import reporter
  61 # We import DocType directly so that it is exposed from this module as well.
  62 from doxygenxml import DocType
  63
  64 def _get_api_type_for_compound(grouplist):
  65     """Helper function to deduce API type from Doxygen group membership."""
  66     result = DocType.internal
  67     for group in grouplist:
  68         if isinstance(group, xml.Group):
  69             if group.get_name() == 'group_publicapi':
  70                 result = DocType.public
  71             elif group.get_name() == 'group_libraryapi':
  72                 result = DocType.library
  73             # TODO: Check for multiple group membership
  74     return result
  75
  76 class IncludedFile(object):
  77
  78     """Information about an #include directive in a file."""
  79
  80     def __init__(self, including_file, lineno, included_file, included_path, is_relative, is_system, line):
  81         self._including_file = including_file
  82         self._line_number = lineno
  83         self._included_file = included_file
  84         self._included_path = included_path
  85         #self._used_include_path = used_include_path
  86         self._is_relative = is_relative
  87         self._is_system = is_system
  88         self._line = line
  89
  90     def __str__(self):
  91         if self._is_system:
  92             return '<{0}>'.format(self._included_path)
  93         else:
  94             return '"{0}"'.format(self._included_path)
  95
  96     def __lt__(self, other):
  97         return str(self) < str(other)
  98
  99     def is_system(self):
 100         return self._is_system
 101
 102     def is_relative(self):
 103         return self._is_relative
 104
 105     def get_included_path(self):
 106         return self._included_path
 107
 108     def get_including_file(self):
 109         return self._including_file
 110
 111     def get_file(self):
 112         return self._included_file
 113
 114     def get_line_number(self):
 115         return self._line_number
 116
 117     def get_full_line(self):
 118         """Return the full source line on which this include appears.
 119
 120         Trailing newline is included."""
 121         return self._line
 122
 123     def get_reporter_location(self):
 124         return reporter.Location(self._including_file.get_abspath(), self._line_number)
 125
 126 class IncludeBlock(object):
 127
 128     """Block of consequent #include directives in a file."""
 129
 130     def __init__(self, first_included_file):
 131         self._first_line = first_included_file.get_line_number()
 132         self._last_line = self._first_line
 133         self._files = []
 134         self.add_file(first_included_file)
 135
 136     def add_file(self, included_file):
 137         self._files.append(included_file)
 138         self._last_line = included_file.get_line_number()
 139
 140     def get_includes(self):
 141         return self._files
 142
 143     def get_first_line(self):
 144         return self._first_line
 145
 146     def get_last_line(self):
 147         return self._last_line
 148
 149 class File(object):
 150
 151     """Source/header file in the GROMACS tree."""
 152
 153     def __init__(self, abspath, relpath, directory):
 154         """Initialize a file representation with basic information."""
 155         self._abspath = abspath
 156         self._relpath = relpath
 157         self._dir = directory
 158         self._rawdoc = None
 159         extension = os.path.splitext(abspath)[1]
 160         self._sourcefile = (extension in ('.c', '.cc', '.cpp', '.cu'))
 161         self._apitype = DocType.none
 162         self._modules = set()
 163         self._includes = []
 164         self._include_blocks = []
 165         self._main_header = None
 166         self._lines = None
 167         self._filter = None
 168         self._declared_defines = None
 169         self._used_defines = dict()
 170         directory.add_file(self)
 171
 172     def set_doc_xml(self, rawdoc, sourcetree):
 173         """Assiociate Doxygen documentation entity with the file."""
 174         assert self._rawdoc is None
 175         assert rawdoc.is_source_file() == self._sourcefile
 176         self._rawdoc = rawdoc
 177         if self._rawdoc.is_documented():
 178             grouplist = self._rawdoc.get_groups()
 179             self._apitype = _get_api_type_for_compound(grouplist)
 180             for group in grouplist:
 181                 module = sourcetree.get_object(group)
 182                 if module:
 183                     self._modules.add(module)
 184
 185     def set_git_filter_attribute(self, filtername):
 186         """Set the git filter attribute associated with the file."""
 187         self._filter = filtername
 188
 189     def set_main_header(self, included_file):
 190         """Set the main header file for a source file."""
 191         assert self.is_source_file()
 192         self._main_header = included_file
 193
 194     def _process_include(self, lineno, is_system, includedpath, line, sourcetree):
 195         """Process #include directive during scan()."""
 196         is_relative = False
 197         if is_system:
 198             fileobj = sourcetree.find_include_file(includedpath)
 199         else:
 200             fullpath = os.path.join(self._dir.get_abspath(), includedpath)
 201             fullpath = os.path.abspath(fullpath)
 202             if os.path.exists(fullpath):
 203                 is_relative = True
 204                 fileobj = sourcetree.get_file(fullpath)
 205             else:
 206                 fileobj = sourcetree.find_include_file(includedpath)
 207         included_file = IncludedFile(self, lineno, fileobj, includedpath,
 208             is_relative, is_system, line)
 209         self._includes.append(included_file)
 210         return included_file
 211
 212     def scan_contents(self, sourcetree, keep_contents, detect_defines):
 213         """Scan the file contents and initialize information based on it."""
 214         # TODO: Consider a more robust regex.
 215         include_re = r'^\s*#\s*include\s+(?P<quote>["<])(?P<path>[^">]*)[">]'
 216         define_re = r'^\s*#.*define(?:01)?\s+(\w*)'
 217         current_block = None
 218         with open(self._abspath, 'r', encoding='utf8') as scanfile:
 219             contents = scanfile.read()
 220         lines = contents.splitlines(True)
 221         for lineno, line in enumerate(lines, 1):
 222             match = re.match(include_re, line)
 223             if match:
 224                 is_system = (match.group('quote') == '<')
 225                 includedpath = match.group('path')
 226                 included_file = self._process_include(lineno, is_system,
 227                         includedpath, line, sourcetree)
 228                 if current_block is None:
 229                     current_block = IncludeBlock(included_file)
 230                     self._include_blocks.append(current_block)
 231                 else:
 232                     current_block.add_file(included_file)
 233             elif line and not line.isspace():
 234                 current_block = None
 235         if detect_defines:
 236             self._declared_defines = []
 237             for line in lines:
 238                 match = re.match(define_re, line)
 239                 if match:
 240                     self._declared_defines.append(match.group(1))
 241         if keep_contents:
 242             self._lines = lines
 243
 244     def add_used_defines(self, define_file, defines):
 245         """Add defines used in this file.
 246
 247         Used internally by find_define_file_uses()."""
 248         if define_file not in self._used_defines:
 249             self._used_defines[define_file] = set()
 250         self._used_defines[define_file].update(defines)
 251
 252     def get_reporter_location(self):
 253         return reporter.Location(self._abspath, None)
 254
 255     def is_external(self):
 256         return self._dir.is_external()
 257
 258     def is_source_file(self):
 259         return self._sourcefile
 260
 261     def is_test_file(self):
 262         return self._dir.is_test_directory()
 263
 264     def should_includes_be_sorted(self):
 265         """Return whether the include directives in the file should be sorted."""
 266         return self._filter in ('includesort', 'complete_formatting')
 267
 268     def is_documented(self):
 269         return self._rawdoc and self._rawdoc.is_documented()
 270
 271     def has_brief_description(self):
 272         return self._rawdoc and self._rawdoc.has_brief_description()
 273
 274     def get_abspath(self):
 275         return self._abspath
 276
 277     def get_relpath(self):
 278         return self._relpath
 279
 280     def get_name(self):
 281         return os.path.basename(self._abspath)
 282
 283     def get_directory(self):
 284         return self._dir
 285
 286     def get_doc_type(self):
 287         if not self._rawdoc:
 288             return DocType.none
 289         return self._rawdoc.get_visibility()
 290
 291     def get_api_type(self):
 292         return self._apitype
 293
 294     def api_type_is_reliable(self):
 295         if self._apitype in (DocType.internal, DocType.library):
 296             return True
 297         module = self.get_module()
 298         return module and module.is_documented()
 299
 300     def is_public(self):
 301         return self.get_api_type() == DocType.public
 302
 303     def is_module_internal(self):
 304         if self.is_source_file():
 305             return True
 306         return self.get_api_type() <= DocType.internal
 307
 308     def get_expected_module(self):
 309         return self._dir.get_module()
 310
 311     def get_doc_modules(self):
 312         return self._modules
 313
 314     def get_module(self):
 315         module = self.get_expected_module()
 316         if not module and len(self._modules) == 1:
 317             module = list(self._modules)[0]
 318         return module
 319
 320     def get_includes(self):
 321         return self._includes
 322
 323     def get_include_blocks(self):
 324         return self._include_blocks
 325
 326     def _get_included_files_recurse(self, result):
 327         for include in self._includes:
 328             included_file = include.get_file()
 329             if included_file is not None and not included_file in result:
 330                 result.add(included_file)
 331                 included_file._get_included_files_recurse(result)
 332
 333     def get_included_files(self, recursive=False):
 334         if recursive:
 335             result = set()
 336             self._get_included_files_recurse(result)
 337             return result
 338         return set([x.get_file() for x in self._includes])
 339
 340     def get_main_header(self):
 341         return self._main_header
 342
 343     def get_contents(self):
 344         return self._lines
 345
 346     def get_declared_defines(self):
 347         """Return set of defines declared in this file.
 348
 349         The information is only populated for selected files."""
 350         return self._declared_defines
 351
 352     def get_used_define_files(self):
 353         """Return files like config.h whose defines are used in this file.
 354
 355         The return value is empty if find_define_file_uses() has not been called,
 356         as well as for headers that declare these defines."""
 357         return set(self._used_defines.keys())
 358
 359     def get_used_defines(self, define_file):
 360         """Return set of defines used in this file for a given file like config.h.
 361         """
 362         return self._used_defines.get(define_file, set())
 363
 364 class GeneratedFile(File):
 365     def __init__(self, abspath, relpath, directory):
 366         File.__init__(self, abspath, relpath, directory)
 367         self._generator_source_file = None
 368
 369     def scan_contents(self, sourcetree, keep_contents, detect_defines):
 370         if os.path.exists(self.get_abspath()):
 371             File.scan_contents(self, sourcetree, keep_contents, False)
 372
 373     def set_generator_source(self, sourcefile):
 374         self._generator_source_file = sourcefile
 375
 376     def get_generator_source(self):
 377         return self._generator_source_file
 378
 379     def get_reporter_location(self):
 380         if self._generator_source_file:
 381             return self._generator_source_file.get_reporter_location()
 382         return File.get_reporter_location(self)
 383
 384     def get_declared_defines(self):
 385         if self._generator_source_file:
 386             return self._generator_source_file.get_declared_defines()
 387         return File.get_declared_defines(self)
 388
 389 class GeneratorSourceFile(File):
 390     pass
 391
 392 class Directory(object):
 393
 394     """(Sub)directory in the GROMACS tree."""
 395
 396     def __init__(self, abspath, relpath, parent):
 397         """Initialize a file representation with basic information."""
 398         self._abspath = abspath
 399         self._relpath = relpath
 400         self._name = os.path.basename(abspath)
 401         self._parent = parent
 402         self._rawdoc = None
 403         self._module = None
 404         self._is_test_dir = False
 405         if parent and parent.is_test_directory() or \
 406                 self._name == 'tests':
 407             self._is_test_dir = True
 408         self._is_external = False
 409         if parent and parent.is_external() or self._name == 'external':
 410             self._is_external = True
 411         self._subdirs = set()
 412         if parent:
 413             parent._subdirs.add(self)
 414         self._files = set()
 415
 416     def set_doc_xml(self, rawdoc, sourcetree):
 417         """Assiociate Doxygen documentation entity with the directory."""
 418         assert self._rawdoc is None
 419         assert rawdoc.get_path().rstrip('/') in (self._abspath, self._relpath)
 420         self._rawdoc = rawdoc
 421
 422     def set_module(self, module):
 423         assert self._module is None
 424         self._module = module
 425
 426     def add_file(self, fileobj):
 427         self._files.add(fileobj)
 428
 429     def get_name(self):
 430         return self._name
 431
 432     def get_reporter_location(self):
 433         return reporter.Location(self._abspath, None)
 434
 435     def get_abspath(self):
 436         return self._abspath
 437
 438     def get_relpath(self):
 439         return self._relpath
 440
 441     def is_test_directory(self):
 442         return self._is_test_dir
 443
 444     def is_external(self):
 445         return self._is_external
 446
 447     def get_module(self):
 448         if self._module:
 449             return self._module
 450         if self._parent:
 451             return self._parent.get_module()
 452         return None
 453
 454     def get_subdirectories(self):
 455         return self._subdirs
 456
 457     def get_files(self):
 458         for subdir in self._subdirs:
 459             for fileobj in subdir.get_files():
 460                 yield fileobj
 461         for fileobj in self._files:
 462             yield fileobj
 463
 464     def contains(self, fileobj):
 465         """Check whether file is within the directory or its subdirectories."""
 466         dirobj = fileobj.get_directory()
 467         while dirobj:
 468             if dirobj == self:
 469                 return True
 470             dirobj = dirobj._parent
 471         return False
 472
 473 class ModuleDependency(object):
 474
 475     """Dependency between modules."""
 476
 477     def __init__(self, othermodule):
 478         """Initialize empty dependency object with given module as dependency."""
 479         self._othermodule = othermodule
 480         self._includedfiles = []
 481         self._cyclesuppression = None
 482         self._is_test_only_dependency = True
 483         self.suppression_used = True
 484
 485     def add_included_file(self, includedfile):
 486         """Add IncludedFile that is part of this dependency."""
 487         assert includedfile.get_file().get_module() == self._othermodule
 488         if not includedfile.get_including_file().is_test_file():
 489             self._is_test_only_dependency = False
 490         self._includedfiles.append(includedfile)
 491
 492     def set_cycle_suppression(self):
 493         """Set suppression on cycles containing this dependency."""
 494         self._cyclesuppression = True
 495         self.suppression_used = False
 496
 497     def is_cycle_suppressed(self):
 498         """Return whether cycles containing this dependency are suppressed."""
 499         self.suppression_used = True
 500         return self._cyclesuppression is not None
 501
 502     def is_test_only_dependency(self):
 503         """Return whether this dependency is only from test code."""
 504         return self._is_test_only_dependency
 505
 506     def get_other_module(self):
 507         """Get module that this dependency is to."""
 508         return self._othermodule
 509
 510     def get_included_files(self):
 511         """Get IncludedFile objects for the individual include dependencies."""
 512         return self._includedfiles
 513
 514 class Module(object):
 515
 516     """Code module in the GROMACS source tree.
 517
 518     Modules are specific subdirectories that host a more or less coherent
 519     set of routines.  Simplified, every subdirectory under src/gromacs/ is
 520     a different module.  This object provides that abstraction and also links
 521     the subdirectory to the module documentation (documented as a group in
 522     Doxygen) if that exists.
 523     """
 524
 525     def __init__(self, name, rootdir):
 526         self._name = name
 527         self._rawdoc = None
 528         self._rootdir = rootdir
 529         self._group = None
 530         self._dependencies = dict()
 531
 532     def set_doc_xml(self, rawdoc, sourcetree):
 533         """Assiociate Doxygen documentation entity with the module."""
 534         assert self._rawdoc is None
 535         self._rawdoc = rawdoc
 536         if self._rawdoc.is_documented():
 537             groups = list(self._rawdoc.get_groups())
 538             if len(groups) == 1:
 539                 groupname = groups[0].get_name()
 540                 if groupname.startswith('group_'):
 541                     self._group = groupname[6:]
 542
 543     def add_dependency(self, othermodule, includedfile):
 544         """Add #include dependency from a file in this module."""
 545         assert includedfile.get_file().get_module() == othermodule
 546         if othermodule not in self._dependencies:
 547             self._dependencies[othermodule] = ModuleDependency(othermodule)
 548         self._dependencies[othermodule].add_included_file(includedfile)
 549
 550     def is_documented(self):
 551         return self._rawdoc is not None
 552
 553     def get_name(self):
 554         return self._name
 555
 556     def get_root_dir(self):
 557         return self._rootdir
 558
 559     def get_files(self):
 560         # TODO: Include public API convenience headers?
 561         return self._rootdir.get_files()
 562
 563     def get_group(self):
 564         return self._group
 565
 566     def get_dependencies(self):
 567         return self._dependencies.values()
 568
 569 class Namespace(object):
 570
 571     """Namespace in the GROMACS source code."""
 572
 573     def __init__(self, rawdoc):
 574         self._rawdoc = rawdoc
 575
 576     def is_anonymous(self):
 577         return self._rawdoc.is_anonymous()
 578
 579 class Class(object):
 580
 581     """Class/struct/union in the GROMACS source code."""
 582
 583     def __init__(self, rawdoc, files):
 584         self._rawdoc = rawdoc
 585         self._files = set(files)
 586
 587     def get_name(self):
 588         return self._rawdoc.get_name()
 589
 590     def get_reporter_location(self):
 591         return self._rawdoc.get_reporter_location()
 592
 593     def get_files(self):
 594         return self._files
 595
 596     def is_documented(self):
 597         return self._rawdoc.is_documented()
 598
 599     def has_brief_description(self):
 600         return self._rawdoc.has_brief_description()
 601
 602     def get_doc_type(self):
 603         """Return documentation type (visibility) for the class.
 604
 605         In addition to the actual code, this encodes GROMACS-specific logic
 606         of setting EXTRACT_LOCAL_CLASSES=YES only for the full documentation.
 607         Local classes never appear outside the full documentation, no matter
 608         what is their visibility.
 609         """
 610         if not self.is_documented():
 611             return DocType.none
 612         if self._rawdoc.is_local():
 613             return DocType.internal
 614         return self._rawdoc.get_visibility()
 615
 616     def get_file_doc_type(self):
 617         return max([fileobj.get_doc_type() for fileobj in self._files])
 618
 619 class Member(object):
 620
 621     """Member (in Doxygen terminology) in the GROMACS source tree.
 622
 623     Currently, modeling is limited to the minimal set of properties that the
 624     checker uses.
 625     """
 626
 627     def __init__(self, rawdoc, namespace):
 628         self._rawdoc = rawdoc
 629         self._namespace = namespace
 630
 631     def get_name(self):
 632         return self._rawdoc.get_name()
 633
 634     def get_reporter_location(self):
 635         return self._rawdoc.get_reporter_location()
 636
 637     def is_documented(self):
 638         return self._rawdoc.is_documented()
 639
 640     def has_brief_description(self):
 641         return self._rawdoc.has_brief_description()
 642
 643     def has_inbody_description(self):
 644         return self._rawdoc.has_inbody_description()
 645
 646     def is_visible(self):
 647         """Return whether the member is visible in Doxygen documentation.
 648
 649         Doxygen ignores members whose parent compounds are not documented.
 650         However, when EXTRACT_ANON_NPACES=ON (which is set for our full
 651         documentation), members of anonymous namespaces are extracted even if
 652         the namespace is the only parent and is not documented.
 653         """
 654         if self._namespace and self._namespace.is_anonymous():
 655             return True
 656         return self._rawdoc.get_inherited_visibility() != DocType.none
 657
 658
 659 class GromacsTree(object):
 660
 661     """Root object for navigating the GROMACS source tree.
 662
 663     On initialization, the list of files and directories is initialized by
 664     walking the source tree, and modules are created for top-level
 665     subdirectories.  At this point, only information that is accessible from
 666     file names and paths only is available.
 667
 668     load_git_attributes() can be called to load attribute information from
 669     .gitattributes for all the files.
 670
 671     scan_files() can be called to read all the files and initialize #include
 672     dependencies between the files based on the information.  This is done like
 673     this instead of relying on Doxygen-extracted include files to make the
 674     dependency graph independent from preprocessor macro definitions
 675     (Doxygen only sees those #includes that the preprocessor sees, which
 676     depends on what #defines it has seen).
 677
 678     find_define_file_uses() can be called to find all uses of defines
 679     declared in config.h and some other macro headers. In the current
 680     implementation, scan_files() must have been called earlier.
 681
 682     load_xml() can be called to load information from Doxygen XML data in
 683     the build tree (the Doxygen XML data must have been built separately).
 684     """
 685
 686     def __init__(self, source_root, build_root, reporter):
 687         """Initialize the tree object by walking the source tree."""
 688         self._source_root = os.path.abspath(source_root)
 689         self._build_root = os.path.abspath(build_root)
 690         self._reporter = reporter
 691         self._docset = None
 692         self._docmap = dict()
 693         self._dirs = dict()
 694         self._files = dict()
 695         self._modules = dict()
 696         self._classes = set()
 697         self._namespaces = set()
 698         self._members = set()
 699         self._walk_dir(os.path.join(self._source_root, 'src'))
 700         for fileobj in self.get_files():
 701             if fileobj and fileobj.is_source_file() and not fileobj.is_external():
 702                 (basedir, name) = os.path.split(fileobj.get_abspath())
 703                 (basename, ext) = os.path.splitext(name)
 704                 header = self.get_file(os.path.join(basedir, basename + '.h'))
 705                 if not header and ext == '.cu':
 706                     header = self.get_file(os.path.join(basedir, basename + '.cuh'))
 707                 if not header and fileobj.is_test_file():
 708                     basedir = os.path.dirname(basedir)
 709                     header = self.get_file(os.path.join(basedir, basename + '.h'))
 710                     if not header:
 711                         # Somewhat of a hack; currently, the tests for
 712                         # analysisdata/modules/ and trajectoryanalysis/modules/
 713                         # is at the top-level tests directory.
 714                         # TODO: It could be clearer to split the tests so that
 715                         # there would be a separate modules/tests/.
 716                         header = self.get_file(os.path.join(basedir, 'modules', basename + '.h'))
 717                     if not header and basename.endswith('_tests'):
 718                         header = self.get_file(os.path.join(basedir, basename[:-6] + '.h'))
 719                 if header:
 720                     fileobj.set_main_header(header)
 721         rootdir = self._get_dir(os.path.join('src', 'gromacs'))
 722         for subdir in rootdir.get_subdirectories():
 723             self._create_module(subdir)
 724         rootdir = self._get_dir(os.path.join('src', 'testutils'))
 725         self._create_module(rootdir)
 726
 727     def _get_rel_path(self, path):
 728         assert os.path.isabs(path)
 729         if path.startswith(self._build_root):
 730             return os.path.relpath(path, self._build_root)
 731         if path.startswith(self._source_root):
 732             return os.path.relpath(path, self._source_root)
 733         raise ValueError("path not under build nor source tree: {0}".format(path))
 734
 735     def _walk_dir(self, rootpath):
 736         """Construct representation of the source tree by walking the file system."""
 737         assert os.path.isabs(rootpath)
 738         assert rootpath not in self._dirs
 739         relpath = self._get_rel_path(rootpath)
 740         self._dirs[relpath] = Directory(rootpath, relpath, None)
 741         for dirpath, dirnames, filenames in os.walk(rootpath):
 742             if 'refdata' in dirnames:
 743                 dirnames.remove('refdata')
 744             currentdir = self._dirs[self._get_rel_path(dirpath)]
 745             # Loop through a copy so that we can modify dirnames.
 746             for dirname in list(dirnames):
 747                 fullpath = os.path.join(dirpath, dirname)
 748                 if fullpath == self._build_root:
 749                     dirnames.remove(dirname)
 750                     continue
 751                 relpath = self._get_rel_path(fullpath)
 752                 self._dirs[relpath] = Directory(fullpath, relpath, currentdir)
 753             extensions = ('.h', '.cuh', '.hpp', '.c', '.cc', '.cpp', '.cu', '.bm')
 754             for filename in filenames:
 755                 basename, extension = os.path.splitext(filename)
 756                 if extension in extensions:
 757                     fullpath = os.path.join(dirpath, filename)
 758                     relpath = self._get_rel_path(fullpath)
 759                     self._files[relpath] = File(fullpath, relpath, currentdir)
 760                 elif extension == '.cmakein':
 761                     extension = os.path.splitext(basename)[1]
 762                     if extension in extensions:
 763                         fullpath = os.path.join(dirpath, filename)
 764                         relpath = self._get_rel_path(fullpath)
 765                         sourcefile = GeneratorSourceFile(fullpath, relpath, currentdir)
 766                         self._files[relpath] = sourcefile
 767                         fullpath = os.path.join(dirpath, basename)
 768                         relpath = self._get_rel_path(fullpath)
 769                         fullpath = os.path.join(self._build_root, relpath)
 770                         generatedfile = GeneratedFile(fullpath, relpath, currentdir)
 771                         self._files[relpath] = generatedfile
 772                         generatedfile.set_generator_source(sourcefile)
 773                 elif extension in ('.l', '.y', '.pre'):
 774                     fullpath = os.path.join(dirpath, filename)
 775                     relpath = self._get_rel_path(fullpath)
 776                     self._files[relpath] = GeneratorSourceFile(fullpath, relpath, currentdir)
 777
 778     def _create_module(self, rootdir):
 779         """Create module for a subdirectory."""
 780         name = 'module_' + rootdir.get_name()
 781         moduleobj = Module(name, rootdir)
 782         rootdir.set_module(moduleobj)
 783         self._modules[name] = moduleobj
 784
 785     def scan_files(self, only_files=None, keep_contents=False):
 786         """Read source files to initialize #include dependencies."""
 787         if only_files:
 788             filelist = only_files
 789         else:
 790             filelist = self._files.values()
 791         define_files = list(self.get_checked_define_files())
 792         for define_file in list(define_files):
 793             if isinstance(define_file, GeneratedFile) and \
 794                     define_file.get_generator_source() is not None:
 795                 define_files.append(define_file.get_generator_source())
 796         for fileobj in filelist:
 797             if not fileobj.is_external():
 798                 detect_defines = fileobj in define_files
 799                 fileobj.scan_contents(self, keep_contents, detect_defines)
 800                 module = fileobj.get_module()
 801                 if module:
 802                     for includedfile in fileobj.get_includes():
 803                         otherfile = includedfile.get_file()
 804                         if otherfile:
 805                             othermodule = otherfile.get_module()
 806                             if othermodule and othermodule != module:
 807                                 module.add_dependency(othermodule, includedfile)
 808
 809     def load_xml(self, only_files=None):
 810         """Load Doxygen XML information.
 811
 812         If only_files is True, XML data is not loaded for code constructs, but
 813         only for files, directories, and their potential parents.
 814         """
 815         xmldir = os.path.join(self._build_root, 'docs', 'html', 'doxygen', 'xml')
 816         self._docset = xml.DocumentationSet(xmldir, self._reporter)
 817         if only_files:
 818             if isinstance(only_files, collections.Iterable):
 819                 filelist = [x.get_relpath() for x in only_files]
 820                 self._docset.load_file_details(filelist)
 821             else:
 822                 self._docset.load_file_details()
 823         else:
 824             self._docset.load_details()
 825             self._docset.merge_duplicates()
 826         self._load_dirs()
 827         self._load_modules()
 828         self._load_files()
 829         if not only_files:
 830             self._load_namespaces()
 831             self._load_classes()
 832             self._load_members()
 833
 834     def _load_dirs(self):
 835         """Load Doxygen XML directory information."""
 836         rootdirs = self._docset.get_compounds(xml.Directory,
 837                 lambda x: x.get_parent() is None)
 838         for dirdoc in rootdirs:
 839             self._load_dir(dirdoc, None)
 840
 841     def _load_dir(self, dirdoc, parent):
 842         """Load Doxygen XML directory information for a single directory."""
 843         path = dirdoc.get_path().rstrip('/')
 844         if not os.path.isabs(path):
 845             path = os.path.join(self._source_root, path)
 846         relpath = self._get_rel_path(path)
 847         dirobj = self._dirs.get(relpath)
 848         if not dirobj:
 849             dirobj = Directory(path, relpath, parent)
 850             self._dirs[relpath] = dirobj
 851         dirobj.set_doc_xml(dirdoc, self)
 852         self._docmap[dirdoc] = dirobj
 853         for subdirdoc in dirdoc.get_subdirectories():
 854             self._load_dir(subdirdoc, dirobj)
 855
 856     def _load_modules(self):
 857         """Load Doxygen XML module (group) information."""
 858         moduledocs = self._docset.get_compounds(xml.Group,
 859                 lambda x: x.get_name().startswith('module_'))
 860         for moduledoc in moduledocs:
 861             moduleobj = self._modules.get(moduledoc.get_name())
 862             if not moduleobj:
 863                 self._reporter.input_error(
 864                         "no matching directory for module: {0}".format(moduledoc))
 865                 continue
 866             moduleobj.set_doc_xml(moduledoc, self)
 867             self._docmap[moduledoc] = moduleobj
 868
 869     def _load_files(self):
 870         """Load Doxygen XML file information."""
 871         for filedoc in self._docset.get_files():
 872             path = filedoc.get_path()
 873             if not path:
 874                 # In case of only partially loaded file information,
 875                 # the path information is not set for unloaded files.
 876                 continue
 877             if not os.path.isabs(path):
 878                 path = os.path.join(self._source_root, path)
 879             extension = os.path.splitext(path)[1]
 880             # We don't care about Markdown files that only produce pages
 881             # (and fail the directory check below).
 882             if extension == '.md':
 883                 continue
 884             dirdoc = filedoc.get_directory()
 885             if not dirdoc:
 886                 self._reporter.xml_assert(filedoc.get_xml_path(),
 887                         "file is not in any directory in Doxygen")
 888                 continue
 889             relpath = self._get_rel_path(path)
 890             fileobj = self._files.get(relpath)
 891             if not fileobj:
 892                 fileobj = File(path, relpath, self._docmap[dirdoc])
 893                 self._files[relpath] = fileobj
 894             fileobj.set_doc_xml(filedoc, self)
 895             self._docmap[filedoc] = fileobj
 896
 897     def _load_namespaces(self):
 898         """Load Doxygen XML namespace information."""
 899         nsdocs = self._docset.get_namespaces()
 900         for nsdoc in nsdocs:
 901             nsobj = Namespace(nsdoc)
 902             self._docmap[nsdoc] = nsobj
 903             self._namespaces.add(nsobj)
 904
 905     def _load_classes(self):
 906         """Load Doxygen XML class information."""
 907         classdocs = self._docset.get_classes()
 908         for classdoc in classdocs:
 909             files = [self._docmap[filedoc] for filedoc in classdoc.get_files()]
 910             classobj = Class(classdoc, files)
 911             self._docmap[classdoc] = classobj
 912             self._classes.add(classobj)
 913
 914     def _load_members(self):
 915         """Load Doxygen XML member information."""
 916         memberdocs = self._docset.get_members()
 917         for memberdoc in memberdocs:
 918             nsdoc = memberdoc.get_namespace()
 919             nsobj = self.get_object(nsdoc)
 920             memberobj = Member(memberdoc, nsobj)
 921             self._docmap[memberdoc] = memberobj
 922             self._members.add(memberobj)
 923
 924     def _get_dir(self, relpath):
 925         """Get directory object for a path relative to source tree root."""
 926         return self._dirs.get(relpath)
 927
 928     def get_file(self, path):
 929         """Get file object for a path relative to source tree root."""
 930         return self._files.get(self._get_rel_path(path))
 931
 932     def find_include_file(self, includedpath):
 933         """Find a file object corresponding to an include path."""
 934         for testdir in ('src', 'src/external/thread_mpi/include',
 935                 'src/external/tng_io/include'):
 936             testpath = os.path.join(testdir, includedpath)
 937             if testpath in self._files:
 938                 return self._files[testpath]
 939
 940     def load_git_attributes(self):
 941         """Load git attribute information for files."""
 942         args = ['git', 'check-attr', '--stdin', 'filter']
 943         git_check_attr = subprocess.Popen(args, stdin=subprocess.PIPE,
 944                 stdout=subprocess.PIPE, cwd=self._source_root)
 945         filelist = '\n'.join(map(File.get_relpath, self._files.values()))
 946         filters = git_check_attr.communicate(filelist.encode())[0].decode()
 947         for fileinfo in filters.splitlines():
 948             path, dummy, value = fileinfo.split(': ')
 949             fileobj = self._files.get(path)
 950             assert fileobj is not None
 951             fileobj.set_git_filter_attribute(value)
 952
 953     def find_define_file_uses(self):
 954         """Find files that use defines from config.h."""
 955         # Executing git grep is substantially faster than using the define_re
 956         # directly on the contents of the file in Python.
 957         for define_file in self.get_checked_define_files():
 958             excluded_files = set([define_file])
 959             excluded_files.update(define_file.get_included_files(recursive=True))
 960             all_defines = define_file.get_declared_defines()
 961             args = ['git', 'grep', '-zwIF']
 962             for define in all_defines:
 963                 args.extend(['-e', define])
 964             args.extend(['--', '*.cpp', '*.c', '*.cu', '*.h', '*.cuh'])
 965             define_re = r'\b(?:' + '|'.join(all_defines)+ r')\b'
 966             output = subprocess.check_output(args, cwd=self._source_root).decode()
 967             for line in output.splitlines():
 968                 (filename, text) = line.split('\0')
 969                 fileobj = self._files.get(filename)
 970                 if fileobj is not None and fileobj not in excluded_files:
 971                     defines = re.findall(define_re, text)
 972                     fileobj.add_used_defines(define_file, defines)
 973
 974     def load_cycle_suppression_list(self, filename):
 975         """Load a list of edges to suppress in cycles.
 976
 977         These edges between modules, if present, will be marked in the
 978         corresponding ModuleDependency objects.
 979         """
 980         with open(filename, 'r') as fp:
 981             for line in fp:
 982                 line = line.strip()
 983                 if not line or line.startswith('#'):
 984                     continue
 985                 modulenames = ['module_' + x.strip() for x in line.split('->')]
 986                 if len(modulenames) != 2:
 987                     self._reporter.input_error(
 988                             "invalid cycle suppression line: {0}".format(line))
 989                     continue
 990                 firstmodule = self._modules.get(modulenames[0])
 991                 secondmodule = self._modules.get(modulenames[1])
 992                 if not firstmodule or not secondmodule:
 993                     self._reporter.input_error(
 994                             "unknown modules mentioned on cycle suppression line: {0}".format(line))
 995                     continue
 996                 for dep in firstmodule.get_dependencies():
 997                     if dep.get_other_module() == secondmodule:
 998                         dep.set_cycle_suppression()
 999                         break
1000                 else:
1001                     self._reporter.cyclic_issue("unused cycle suppression: {0}".format(line))
1002
1003     def report_unused_cycle_suppressions(self, reporter):
1004         """Reports unused cycle suppressions."""
1005         for module in self.get_modules():
1006             for dep in module.get_dependencies():
1007                 if not dep.suppression_used:
1008                     reporter.cyclic_issue("unused cycle suppression: {0} -> {1}".format(module.get_name()[7:], dep.get_other_module().get_name()[7:]))
1009
1010     def get_object(self, docobj):
1011         """Get tree object for a Doxygen XML object."""
1012         if docobj is None:
1013             return None
1014         return self._docmap.get(docobj)
1015
1016     def get_files(self):
1017         """Get iterable for all files in the source tree."""
1018         return self._files.values()
1019
1020     def get_modules(self):
1021         """Get iterable for all modules in the source tree."""
1022         return self._modules.values()
1023
1024     def get_classes(self):
1025         """Get iterable for all classes in the source tree."""
1026         return self._classes
1027
1028     def get_members(self):
1029         """Get iterable for all members (in Doxygen terms) in the source tree."""
1030         return self._members
1031
1032     def get_checked_define_files(self):
1033         """Get list of files that contain #define macros whose usage needs to
1034         be checked."""
1035         return (self._files['src/config.h'],
1036                 self._files['src/gromacs/simd/simd.h'],
1037                 self._files['src/gromacs/ewald/pme_simd.h'],
1038                 self._files['src/gromacs/nbnxm/nbnxm_simd.h'])