3 # This file is part of the GROMACS molecular simulation package.
5 # Copyright (c) 2014, by the GROMACS development team, led by
6 # Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
7 # and including many others, as listed in the AUTHORS file in the
8 # top-level source directory and at http://www.gromacs.org.
10 # GROMACS is free software; you can redistribute it and/or
11 # modify it under the terms of the GNU Lesser General Public License
12 # as published by the Free Software Foundation; either version 2.1
13 # of the License, or (at your option) any later version.
15 # GROMACS is distributed in the hope that it will be useful,
16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
18 # Lesser General Public License for more details.
20 # You should have received a copy of the GNU Lesser General Public
21 # License along with GROMACS; if not, see
22 # http://www.gnu.org/licenses, or write to the Free Software Foundation,
23 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
25 # If you want to redistribute modifications to GROMACS, please
26 # consider that scientific software is very special. Version
27 # control is crucial - bugs must be traceable. We will be happy to
28 # consider code for inclusion in the official distribution, but
29 # derived work must not be called official GROMACS. Details are found
30 # in the README & COPYING files - if they are missing, get the
31 # official version at http://www.gromacs.org.
33 # To help us fund GROMACS development, we humbly ask that you cite
34 # the research papers on the package. Check out http://www.gromacs.org.
36 """GROMACS-specific representation for source tree and documentation.
38 This module provides classes that construct a GROMACS-specific representation
39 of the source tree and associate the Doxygen XML output with it. It constructs
40 an initial representation by walking the source tree in the file system, and
41 then associates information from the Doxygen XML output into this.
42 It also adds some additional knowledge from how the GROMACS source tree is
43 organized to construct a representation that is easy to process and check as
44 the top-level scripts expect.
46 The object model is rooted at a GromacsTree object. Currently, it constructs a
47 representation of the source tree from the file system, but is otherwise mostly
48 a thin wrapper around the Doxygen XML tree. It already adds some relations and
49 rules that come from GROMACS-specific knowledge. In the future, more such
50 customizations will be added.
57 import doxygenxml as xml
59 # We import DocType directly so that it is exposed from this module as well.
60 from doxygenxml import DocType
62 def _get_api_type_for_compound(grouplist):
63 """Helper function to deduce API type from Doxygen group membership."""
64 result = DocType.internal
65 for group in grouplist:
66 if isinstance(group, xml.Group):
67 if group.get_name() == 'group_publicapi':
68 result = DocType.public
69 elif group.get_name() == 'group_libraryapi':
70 result = DocType.library
71 # TODO: Check for multiple group membership
74 class IncludedFile(object):
76 """Information about an #include directive in a file."""
78 def __init__(self, including_file, lineno, included_file, included_path, is_relative, is_system):
79 self._including_file = including_file
80 self._line_number = lineno
81 self._included_file = included_file
82 self._included_path = included_path
83 #self._used_include_path = used_include_path
84 self._is_relative = is_relative
85 self._is_system = is_system
89 return '<{0}>'.format(self._included_path)
91 return '"{0}"'.format(self._included_path)
94 return self._is_system
96 def is_relative(self):
97 return self._is_relative
99 def get_including_file(self):
100 return self._including_file
103 return self._included_file
105 def get_reporter_location(self):
106 return reporter.Location(self._including_file.get_abspath(), self._line_number)
110 """Source/header file in the GROMACS tree."""
112 def __init__(self, abspath, relpath, directory):
113 """Initialize a file representation with basic information."""
114 self._abspath = abspath
115 self._relpath = relpath
116 self._dir = directory
118 self._installed = False
119 extension = os.path.splitext(abspath)[1]
120 self._sourcefile = (extension in ('.c', '.cc', '.cpp', '.cu'))
121 self._apitype = DocType.none
122 self._modules = set()
124 directory.add_file(self)
126 def set_doc_xml(self, rawdoc, sourcetree):
127 """Assiociate Doxygen documentation entity with the file."""
128 assert self._rawdoc is None
129 assert rawdoc.is_source_file() == self._sourcefile
130 self._rawdoc = rawdoc
131 if self._rawdoc.is_documented():
132 grouplist = self._rawdoc.get_groups()
133 self._apitype = _get_api_type_for_compound(grouplist)
134 for group in grouplist:
135 module = sourcetree.get_object(group)
137 self._modules.add(module)
139 def set_installed(self):
140 """Mark the file installed."""
141 self._installed = True
143 def _process_include(self, lineno, is_system, includedpath, sourcetree):
144 """Process #include directive during scan()."""
147 fileobj = sourcetree.find_include_file(includedpath)
149 fullpath = os.path.join(self._dir.get_abspath(), includedpath)
150 fullpath = os.path.abspath(fullpath)
151 if os.path.exists(fullpath):
153 fileobj = sourcetree.get_file(fullpath)
155 fileobj = sourcetree.find_include_file(includedpath)
156 self._includes.append(IncludedFile(self, lineno, fileobj, includedpath,
157 is_relative, is_system))
159 def scan_contents(self, sourcetree):
160 """Scan the file contents and initialize information based on it."""
161 # TODO: Consider a more robust regex.
162 include_re = r'^#\s*include\s+(?P<quote>["<])(?P<path>[^">]*)[">]'
163 with open(self._abspath, 'r') as scanfile:
164 for lineno, line in enumerate(scanfile, 1):
165 match = re.match(include_re, line)
167 is_system = (match.group('quote') == '<')
168 includedpath = match.group('path')
169 self._process_include(lineno, is_system, includedpath,
172 def get_reporter_location(self):
173 return reporter.Location(self._abspath, None)
175 def is_installed(self):
176 return self._installed
178 def is_external(self):
179 return self._dir.is_external()
181 def is_source_file(self):
182 return self._sourcefile
184 def is_test_file(self):
185 return self._dir.is_test_directory()
187 def is_documented(self):
188 return self._rawdoc and self._rawdoc.is_documented()
190 def has_brief_description(self):
191 return self._rawdoc and self._rawdoc.has_brief_description()
193 def get_abspath(self):
196 def get_relpath(self):
200 return os.path.basename(self._abspath)
202 def get_doc_type(self):
205 return self._rawdoc.get_visibility()
207 def get_api_type(self):
210 def api_type_is_reliable(self):
211 if self._apitype > DocType.internal:
213 module = self.get_module()
214 return module and module.is_documented()
217 if self.api_type_is_reliable():
218 return self.get_api_type() == DocType.public
219 return self.get_api_type() == DocType.public or self.is_installed()
221 def is_module_internal(self):
222 if self.is_source_file():
224 return not self.is_installed() and self.get_api_type() <= DocType.internal
226 def get_expected_module(self):
227 return self._dir.get_module()
229 def get_doc_modules(self):
232 def get_module(self):
233 module = self.get_expected_module()
234 if not module and len(self._modules) == 1:
235 module = list(self._modules)[0]
238 def get_includes(self):
239 return self._includes
241 class GeneratedFile(File):
244 class Directory(object):
246 """(Sub)directory in the GROMACS tree."""
248 def __init__(self, abspath, relpath, parent):
249 """Initialize a file representation with basic information."""
250 self._abspath = abspath
251 self._relpath = relpath
252 self._name = os.path.basename(abspath)
253 self._parent = parent
256 self._is_test_dir = False
257 if parent and parent.is_test_directory() or \
258 self._name in ('tests', 'legacytests'):
259 self._is_test_dir = True
260 self._is_external = False
261 if parent and parent.is_external() or self._name == 'external':
262 self._is_external = True
263 self._subdirs = set()
265 parent._subdirs.add(self)
267 self._has_installed_files = None
269 def set_doc_xml(self, rawdoc, sourcetree):
270 """Assiociate Doxygen documentation entity with the directory."""
271 assert self._rawdoc is None
272 assert self._abspath == rawdoc.get_path().rstrip('/')
273 self._rawdoc = rawdoc
275 def set_module(self, module):
276 assert self._module is None
277 self._module = module
279 def add_file(self, fileobj):
280 self._files.add(fileobj)
285 def get_reporter_location(self):
286 return reporter.Location(self._abspath, None)
288 def get_abspath(self):
291 def get_relpath(self):
294 def is_test_directory(self):
295 return self._is_test_dir
297 def is_external(self):
298 return self._is_external
300 def has_installed_files(self):
301 if self._has_installed_files is None:
302 self._has_installed_files = False
303 for subdir in self._subdirs:
304 if subdir.has_installed_files():
305 self._has_installed_files = True
307 for fileobj in self._files:
308 if fileobj.is_installed():
309 self._has_installed_files = True
311 return self._has_installed_files
313 def get_module(self):
317 return self._parent.get_module()
320 def get_subdirectories(self):
324 for subdir in self._subdirs:
325 for fileobj in subdir.get_files():
327 for fileobj in self._files:
330 class ModuleDependency(object):
332 """Dependency between modules."""
334 def __init__(self, othermodule):
335 """Initialize empty dependency object with given module as dependency."""
336 self._othermodule = othermodule
337 self._includedfiles = []
338 self._cyclesuppression = None
340 def add_included_file(self, includedfile):
341 """Add IncludedFile that is part of this dependency."""
342 assert includedfile.get_file().get_module() == self._othermodule
343 self._includedfiles.append(includedfile)
345 def set_cycle_suppression(self):
346 """Set suppression on cycles containing this dependency."""
347 self._cyclesuppression = True
349 def is_cycle_suppressed(self):
350 """Return whether cycles containing this dependency are suppressed."""
351 return self._cyclesuppression is not None
353 def get_other_module(self):
354 """Get module that this dependency is to."""
355 return self._othermodule
357 def get_included_files(self):
358 """Get IncludedFile objects for the individual include dependencies."""
359 return self._includedfiles
361 class Module(object):
363 """Code module in the GROMACS source tree.
365 Modules are specific subdirectories that host a more or less coherent
366 set of routines. Simplified, every subdirectory under src/gromacs/ is
367 a different module. This object provides that abstraction and also links
368 the subdirectory to the module documentation (documented as a group in
369 Doxygen) if that exists.
372 def __init__(self, name, rootdir):
375 self._rootdir = rootdir
377 self._dependencies = dict()
379 def set_doc_xml(self, rawdoc, sourcetree):
380 """Assiociate Doxygen documentation entity with the module."""
381 assert self._rawdoc is None
382 self._rawdoc = rawdoc
383 if self._rawdoc.is_documented():
384 groups = list(self._rawdoc.get_groups())
386 groupname = groups[0].get_name()
387 if groupname.startswith('group_'):
388 self._group = groupname[6:]
390 def add_dependency(self, othermodule, includedfile):
391 """Add #include dependency from a file in this module."""
392 assert includedfile.get_file().get_module() == othermodule
393 if othermodule not in self._dependencies:
394 self._dependencies[othermodule] = ModuleDependency(othermodule)
395 self._dependencies[othermodule].add_included_file(includedfile)
397 def is_documented(self):
398 return self._rawdoc is not None
403 def get_root_dir(self):
407 # TODO: Include public API convenience headers?
408 return self._rootdir.get_files()
413 def get_dependencies(self):
414 return self._dependencies.itervalues()
416 class Namespace(object):
418 """Namespace in the GROMACS source code."""
420 def __init__(self, rawdoc):
421 self._rawdoc = rawdoc
423 def is_anonymous(self):
424 return self._rawdoc.is_anonymous()
428 """Class/struct/union in the GROMACS source code."""
430 def __init__(self, rawdoc, files):
431 self._rawdoc = rawdoc
432 self._files = set(files)
435 return self._rawdoc.get_name()
437 def get_reporter_location(self):
438 return self._rawdoc.get_reporter_location()
443 def is_documented(self):
444 return self._rawdoc.is_documented()
446 def has_brief_description(self):
447 return self._rawdoc.has_brief_description()
449 def get_doc_type(self):
450 """Return documentation type (visibility) for the class.
452 In addition to the actual code, this encodes GROMACS-specific logic
453 of setting EXTRACT_LOCAL_CLASSES=YES only for the full documentation.
454 Local classes never appear outside the full documentation, no matter
455 what is their visibility.
457 if not self.is_documented():
459 if self._rawdoc.is_local():
460 return DocType.internal
461 return self._rawdoc.get_visibility()
463 def get_file_doc_type(self):
464 return max([fileobj.get_doc_type() for fileobj in self._files])
466 def is_in_installed_file(self):
467 return any([fileobj.is_installed() for fileobj in self._files])
469 class Member(object):
471 """Member (in Doxygen terminology) in the GROMACS source tree.
473 Currently, modeling is limited to the minimal set of properties that the
477 def __init__(self, rawdoc, namespace):
478 self._rawdoc = rawdoc
479 self._namespace = namespace
482 return self._rawdoc.get_name()
484 def get_reporter_location(self):
485 return self._rawdoc.get_reporter_location()
487 def is_documented(self):
488 return self._rawdoc.is_documented()
490 def has_brief_description(self):
491 return self._rawdoc.has_brief_description()
493 def has_inbody_description(self):
494 return self._rawdoc.has_inbody_description()
496 def is_visible(self):
497 """Return whether the member is visible in Doxygen documentation.
499 Doxygen ignores members whose parent compounds are not documented.
500 However, when EXTRACT_ANON_NPACES=ON (which is set for our full
501 documentation), members of anonymous namespaces are extracted even if
502 the namespace is the only parent and is not documented.
504 if self._namespace and self._namespace.is_anonymous():
506 return self._rawdoc.get_inherited_visibility() != DocType.none
509 class GromacsTree(object):
511 """Root object for navigating the GROMACS source tree.
513 On initialization, the list of files and directories is initialized by
514 walking the source tree, and modules are created for top-level
515 subdirectories. At this point, only information that is accessible from
516 file names and paths only is available.
518 set_installed_file_list() can be called to set the list of installed
521 scan_files() can be called to read all the files and initialize #include
522 dependencies between the files based on the information. This is done like
523 this instead of relying on Doxygen-extracted include files to make the
524 dependency graph independent from preprocessor macro definitions
525 (Doxygen only sees those #includes that the preprocessor sees, which
526 depends on what #defines it has seen).
528 load_xml() can be called to load information from Doxygen XML data in
529 the build tree (the Doxygen XML data must have been built separately).
532 def __init__(self, source_root, build_root, reporter):
533 """Initialize the tree object by walking the source tree."""
534 self._source_root = os.path.abspath(source_root)
535 self._build_root = os.path.abspath(build_root)
536 self._reporter = reporter
538 self._docmap = dict()
541 self._modules = dict()
542 self._classes = set()
543 self._namespaces = set()
544 self._members = set()
545 self._walk_dir(os.path.join(self._source_root, 'src'))
546 rootdir = self._get_dir(os.path.join('src', 'gromacs'))
547 for subdir in rootdir.get_subdirectories():
548 self._create_module(subdir)
549 rootdir = self._get_dir(os.path.join('src', 'testutils'))
550 self._create_module(rootdir)
552 def _get_rel_path(self, path):
553 assert os.path.isabs(path)
554 if path.startswith(self._build_root):
555 return os.path.relpath(path, self._build_root)
556 if path.startswith(self._source_root):
557 return os.path.relpath(path, self._source_root)
558 raise ValueError("path not under build nor source tree: {0}".format(path))
560 def _walk_dir(self, rootpath):
561 """Construct representation of the source tree by walking the file system."""
562 assert os.path.isabs(rootpath)
563 assert rootpath not in self._dirs
564 relpath = self._get_rel_path(rootpath)
565 self._dirs[relpath] = Directory(rootpath, relpath, None)
566 for dirpath, dirnames, filenames in os.walk(rootpath):
567 if 'contrib' in dirnames:
568 dirnames.remove('contrib')
569 if 'refdata' in dirnames:
570 dirnames.remove('refdata')
571 currentdir = self._dirs[self._get_rel_path(dirpath)]
572 # Loop through a copy so that we can modify dirnames.
573 for dirname in list(dirnames):
574 fullpath = os.path.join(dirpath, dirname)
575 if fullpath == self._build_root:
576 dirnames.remove(dirname)
578 relpath = self._get_rel_path(fullpath)
579 self._dirs[relpath] = Directory(fullpath, relpath, currentdir)
580 extensions = ('.h', '.cuh', '.hpp', '.c', '.cc', '.cpp', '.cu', '.bm')
581 for filename in filenames:
582 basename, extension = os.path.splitext(filename)
583 if extension in extensions:
584 fullpath = os.path.join(dirpath, filename)
585 relpath = self._get_rel_path(fullpath)
586 self._files[relpath] = File(fullpath, relpath, currentdir)
587 elif extension == '.cmakein':
588 extension = os.path.splitext(basename)[1]
589 if extension in extensions:
590 fullpath = os.path.join(dirpath, basename)
591 relpath = self._get_rel_path(fullpath)
592 fullpath = os.path.join(dirpath, filename)
593 self._files[relpath] = GeneratedFile(fullpath, relpath, currentdir)
595 def _create_module(self, rootdir):
596 """Create module for a subdirectory."""
597 name = 'module_' + rootdir.get_name()
598 moduleobj = Module(name, rootdir)
599 rootdir.set_module(moduleobj)
600 self._modules[name] = moduleobj
602 def scan_files(self):
603 """Read source files to initialize #include dependencies."""
604 for fileobj in self._files.itervalues():
605 if not fileobj.is_external():
606 fileobj.scan_contents(self)
607 module = fileobj.get_module()
609 for includedfile in fileobj.get_includes():
610 otherfile = includedfile.get_file()
612 othermodule = otherfile.get_module()
613 if othermodule and othermodule != module:
614 module.add_dependency(othermodule, includedfile)
616 def load_xml(self, only_files=False):
617 """Load Doxygen XML information.
619 If only_files is True, XML data is not loaded for code constructs, but
620 only for files, directories, and their potential parents.
622 xmldir = os.path.join(self._build_root, 'doxygen', 'xml')
623 self._docset = xml.DocumentationSet(xmldir, self._reporter)
625 self._docset.load_file_details()
627 self._docset.load_details()
628 self._docset.merge_duplicates()
633 self._load_namespaces()
637 def _load_dirs(self):
638 """Load Doxygen XML directory information."""
639 rootdirs = self._docset.get_compounds(xml.Directory,
640 lambda x: x.get_parent() is None)
641 for dirdoc in rootdirs:
642 self._load_dir(dirdoc, None)
644 def _load_dir(self, dirdoc, parent):
645 """Load Doxygen XML directory information for a single directory."""
646 path = dirdoc.get_path().rstrip('/')
647 if not os.path.isabs(path):
648 self._reporter.xml_assert(dirdoc.get_xml_path(),
649 "expected absolute path in Doxygen-produced XML file")
651 relpath = self._get_rel_path(path)
652 dirobj = self._dirs.get(relpath)
654 dirobj = Directory(path, relpath, parent)
655 self._dirs[relpath] = dirobj
656 dirobj.set_doc_xml(dirdoc, self)
657 self._docmap[dirdoc] = dirobj
658 for subdirdoc in dirdoc.get_subdirectories():
659 self._load_dir(subdirdoc, dirobj)
661 def _load_modules(self):
662 """Load Doxygen XML module (group) information."""
663 moduledocs = self._docset.get_compounds(xml.Group,
664 lambda x: x.get_name().startswith('module_'))
665 for moduledoc in moduledocs:
666 moduleobj = self._modules.get(moduledoc.get_name())
668 self._reporter.input_error(
669 "no matching directory for module: {0}".format(moduledoc))
671 moduleobj.set_doc_xml(moduledoc, self)
672 self._docmap[moduledoc] = moduleobj
674 def _load_files(self):
675 """Load Doxygen XML file information."""
676 for filedoc in self._docset.get_files():
677 path = filedoc.get_path()
678 if not os.path.isabs(path):
679 self._reporter.xml_assert(filedoc.get_xml_path(),
680 "expected absolute path in Doxygen-produced XML file")
682 extension = os.path.splitext(filedoc.get_path())[1]
683 # We don't care about Markdown files that only produce pages
684 # (and fail the directory check below).
685 if extension == '.md':
687 dirdoc = filedoc.get_directory()
689 self._reporter.xml_assert(filedoc.get_xml_path(),
690 "file is not in any directory in Doxygen")
692 relpath = self._get_rel_path(path)
693 fileobj = self._files.get(relpath)
695 fileobj = File(path, relpath, self._docmap[dirdoc])
696 self._files[relpath] = fileobj
697 fileobj.set_doc_xml(filedoc, self)
698 self._docmap[filedoc] = fileobj
700 def _load_namespaces(self):
701 """Load Doxygen XML namespace information."""
702 nsdocs = self._docset.get_namespaces()
704 nsobj = Namespace(nsdoc)
705 self._docmap[nsdoc] = nsobj
706 self._namespaces.add(nsobj)
708 def _load_classes(self):
709 """Load Doxygen XML class information."""
710 classdocs = self._docset.get_classes()
711 for classdoc in classdocs:
712 files = [self._docmap[filedoc] for filedoc in classdoc.get_files()]
713 classobj = Class(classdoc, files)
714 self._docmap[classdoc] = classobj
715 self._classes.add(classobj)
717 def _load_members(self):
718 """Load Doxygen XML member information."""
719 memberdocs = self._docset.get_members()
720 for memberdoc in memberdocs:
721 nsdoc = memberdoc.get_namespace()
722 nsobj = self.get_object(nsdoc)
723 memberobj = Member(memberdoc, nsobj)
724 self._docmap[memberdoc] = memberobj
725 self._members.add(memberobj)
727 def _get_dir(self, relpath):
728 """Get directory object for a path relative to source tree root."""
729 return self._dirs.get(relpath)
731 def get_file(self, path):
732 """Get file object for a path relative to source tree root."""
733 return self._files.get(self._get_rel_path(path))
735 def find_include_file(self, includedpath):
736 """Find a file object corresponding to an include path."""
737 for testdir in ('src', 'src/gromacs/legacyheaders', 'src/external/thread_mpi/include',
738 'src/external/tng_io/include'):
739 testpath = os.path.join(testdir, includedpath)
740 if testpath in self._files:
741 return self._files[testpath]
743 def set_installed_file_list(self, installedfiles):
744 """Set list of installed files."""
745 for path in installedfiles:
746 if not os.path.isabs(path):
747 self._reporter.input_error(
748 "installed file not specified with absolute path: {0}"
751 relpath = self._get_rel_path(path)
752 if relpath not in self._files:
753 self._reporter.input_error(
754 "installed file not in source tree: {0}".format(path))
756 self._files[relpath].set_installed()
758 def load_cycle_suppression_list(self, filename):
759 """Load a list of edges to suppress in cycles.
761 These edges between modules, if present, will be marked in the
762 corresponding ModuleDependency objects.
764 with open(filename, 'r') as fp:
767 if not line or line.startswith('#'):
769 modulenames = ['module_' + x.strip() for x in line.split('->')]
770 if len(modulenames) != 2:
771 self._reporter.input_error(
772 "invalid cycle suppression line: {0}".format(line))
774 firstmodule = self._modules.get(modulenames[0])
775 secondmodule = self._modules.get(modulenames[1])
776 if not firstmodule or not secondmodule:
777 self._reporter.input_error(
778 "unknown modules mentioned on cycle suppression line: {0}".format(line))
780 for dep in firstmodule.get_dependencies():
781 if dep.get_other_module() == secondmodule:
782 # TODO: Check that each suppression is actually part of
784 dep.set_cycle_suppression()
786 def get_object(self, docobj):
787 """Get tree object for a Doxygen XML object."""
790 return self._docmap.get(docobj)
793 """Get iterable for all files in the source tree."""
794 return self._files.itervalues()
796 def get_modules(self):
797 """Get iterable for all modules in the source tree."""
798 return self._modules.itervalues()
800 def get_classes(self):
801 """Get iterable for all classes in the source tree."""
804 def get_members(self):
805 """Get iterable for all members (in Doxygen terms) in the source tree."""