3 # This file is part of the GROMACS molecular simulation package.
5 # Copyright (c) 2014, by the GROMACS development team, led by
6 # Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
7 # and including many others, as listed in the AUTHORS file in the
8 # top-level source directory and at http://www.gromacs.org.
10 # GROMACS is free software; you can redistribute it and/or
11 # modify it under the terms of the GNU Lesser General Public License
12 # as published by the Free Software Foundation; either version 2.1
13 # of the License, or (at your option) any later version.
15 # GROMACS is distributed in the hope that it will be useful,
16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
18 # Lesser General Public License for more details.
20 # You should have received a copy of the GNU Lesser General Public
21 # License along with GROMACS; if not, see
22 # http://www.gnu.org/licenses, or write to the Free Software Foundation,
23 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
25 # If you want to redistribute modifications to GROMACS, please
26 # consider that scientific software is very special. Version
27 # control is crucial - bugs must be traceable. We will be happy to
28 # consider code for inclusion in the official distribution, but
29 # derived work must not be called official GROMACS. Details are found
30 # in the README & COPYING files - if they are missing, get the
31 # official version at http://www.gromacs.org.
33 # To help us fund GROMACS development, we humbly ask that you cite
34 # the research papers on the package. Check out http://www.gromacs.org.
36 """GROMACS-specific representation for source tree and documentation.
38 This module provides classes that construct a GROMACS-specific representation
39 of the source tree and associate the Doxygen XML output with it. It constructs
40 an initial representation by walking the source tree in the file system, and
41 then associates information from the Doxygen XML output into this.
42 It also adds some additional knowledge from how the GROMACS source tree is
43 organized to construct a representation that is easy to process and check as
44 the top-level scripts expect.
46 The object model is rooted at a GromacsTree object. Currently, it constructs a
47 representation of the source tree from the file system, but is otherwise mostly
48 a thin wrapper around the Doxygen XML tree. It already adds some relations and
49 rules that come from GROMACS-specific knowledge. In the future, more such
50 customizations will be added.
57 import doxygenxml as xml
59 # We import DocType directly so that it is exposed from this module as well.
60 from doxygenxml import DocType
62 def _get_api_type_for_compound(grouplist):
63 """Helper function to deduce API type from Doxygen group membership."""
64 result = DocType.internal
65 for group in grouplist:
66 if isinstance(group, xml.Group):
67 if group.get_name() == 'group_publicapi':
68 result = DocType.public
69 elif group.get_name() == 'group_libraryapi':
70 result = DocType.library
71 # TODO: Check for multiple group membership
74 class IncludedFile(object):
76 """Information about an #include directive in a file."""
78 def __init__(self, abspath, lineno, included_file, included_path, is_relative, is_system):
79 self._abspath = abspath
80 self._line_number = lineno
81 self._included_file = included_file
82 self._included_path = included_path
83 #self._used_include_path = used_include_path
84 self._is_relative = is_relative
85 self._is_system = is_system
89 return '<{0}>'.format(self._included_path)
91 return '"{0}"'.format(self._included_path)
94 return self._is_system
96 def is_relative(self):
97 return self._is_relative
100 return self._included_file
102 def get_reporter_location(self):
103 return reporter.Location(self._abspath, self._line_number)
107 """Source/header file in the GROMACS tree."""
109 def __init__(self, abspath, relpath, directory):
110 """Initialize a file representation with basic information."""
111 self._abspath = abspath
112 self._relpath = relpath
113 self._dir = directory
115 self._installed = False
116 extension = os.path.splitext(abspath)[1]
117 self._sourcefile = (extension in ('.c', '.cc', '.cpp', '.cu'))
118 self._apitype = DocType.none
119 self._modules = set()
121 directory.add_file(self)
123 def set_doc_xml(self, rawdoc, sourcetree):
124 """Assiociate Doxygen documentation entity with the file."""
125 assert self._rawdoc is None
126 assert rawdoc.is_source_file() == self._sourcefile
127 self._rawdoc = rawdoc
128 if self._rawdoc.is_documented():
129 grouplist = self._rawdoc.get_groups()
130 self._apitype = _get_api_type_for_compound(grouplist)
131 for group in grouplist:
132 module = sourcetree.get_object(group)
134 self._modules.add(module)
136 def set_installed(self):
137 """Mark the file installed."""
138 self._installed = True
140 def _process_include(self, lineno, is_system, includedpath, sourcetree):
141 """Process #include directive during scan()."""
144 fileobj = sourcetree.find_include_file(includedpath)
146 fullpath = os.path.join(self._dir.get_abspath(), includedpath)
147 fullpath = os.path.abspath(fullpath)
148 if os.path.exists(fullpath):
150 fileobj = sourcetree.get_file(fullpath)
152 fileobj = sourcetree.find_include_file(includedpath)
153 self._includes.append(IncludedFile(self.get_abspath(), lineno, fileobj, includedpath,
154 is_relative, is_system))
156 def scan_contents(self, sourcetree):
157 """Scan the file contents and initialize information based on it."""
158 # TODO: Consider a more robust regex.
159 include_re = r'^#\s*include\s+(?P<quote>["<])(?P<path>[^">]*)[">]'
160 with open(self._abspath, 'r') as scanfile:
161 for lineno, line in enumerate(scanfile, 1):
162 match = re.match(include_re, line)
164 is_system = (match.group('quote') == '<')
165 includedpath = match.group('path')
166 self._process_include(lineno, is_system, includedpath,
169 def get_reporter_location(self):
170 return reporter.Location(self._abspath, None)
172 def is_installed(self):
173 return self._installed
175 def is_external(self):
176 return self._dir.is_external()
178 def is_source_file(self):
179 return self._sourcefile
181 def is_test_file(self):
182 return self._dir.is_test_directory()
184 def is_documented(self):
185 return self._rawdoc and self._rawdoc.is_documented()
187 def has_brief_description(self):
188 return self._rawdoc and self._rawdoc.has_brief_description()
190 def get_abspath(self):
193 def get_relpath(self):
197 return os.path.basename(self._abspath)
199 def get_doc_type(self):
202 return self._rawdoc.get_visibility()
204 def get_api_type(self):
207 def api_type_is_reliable(self):
208 if self._apitype > DocType.internal:
210 module = self.get_module()
211 return module and module.is_documented()
214 if self.api_type_is_reliable():
215 return self.get_api_type() == DocType.public
216 return self.get_api_type() == DocType.public or self.is_installed()
218 def is_module_internal(self):
219 if self.is_source_file():
221 return not self.is_installed() and self.get_api_type() <= DocType.internal
223 def get_expected_module(self):
224 return self._dir.get_module()
226 def get_doc_modules(self):
229 def get_module(self):
230 module = self.get_expected_module()
231 if not module and len(self._modules) == 1:
232 module = list(self._modules)[0]
235 def get_includes(self):
236 return self._includes
238 class GeneratedFile(File):
241 class Directory(object):
243 """(Sub)directory in the GROMACS tree."""
245 def __init__(self, abspath, relpath, parent):
246 """Initialize a file representation with basic information."""
247 self._abspath = abspath
248 self._relpath = relpath
249 self._name = os.path.basename(abspath)
250 self._parent = parent
253 self._is_test_dir = False
254 if parent and parent.is_test_directory() or \
255 self._name in ('tests', 'legacytests'):
256 self._is_test_dir = True
257 self._is_external = False
258 if parent and parent.is_external() or self._name == 'external':
259 self._is_external = True
260 self._subdirs = set()
262 parent._subdirs.add(self)
264 self._has_installed_files = None
266 def set_doc_xml(self, rawdoc, sourcetree):
267 """Assiociate Doxygen documentation entity with the directory."""
268 assert self._rawdoc is None
269 assert self._abspath == rawdoc.get_path().rstrip('/')
270 self._rawdoc = rawdoc
272 def set_module(self, module):
273 assert self._module is None
274 self._module = module
276 def add_file(self, fileobj):
277 self._files.add(fileobj)
282 def get_reporter_location(self):
283 return reporter.Location(self._abspath, None)
285 def get_abspath(self):
288 def get_relpath(self):
291 def is_test_directory(self):
292 return self._is_test_dir
294 def is_external(self):
295 return self._is_external
297 def has_installed_files(self):
298 if self._has_installed_files is None:
299 self._has_installed_files = False
300 for subdir in self._subdirs:
301 if subdir.has_installed_files():
302 self._has_installed_files = True
304 for fileobj in self._files:
305 if fileobj.is_installed():
306 self._has_installed_files = True
308 return self._has_installed_files
310 def get_module(self):
314 return self._parent.get_module()
317 def get_subdirectories(self):
321 for subdir in self._subdirs:
322 for fileobj in subdir.get_files():
324 for fileobj in self._files:
327 class Module(object):
329 """Code module in the GROMACS source tree.
331 Modules are specific subdirectories that host a more or less coherent
332 set of routines. Simplified, every subdirectory under src/gromacs/ is
333 a different module. This object provides that abstraction and also links
334 the subdirectory to the module documentation (documented as a group in
335 Doxygen) if that exists.
338 def __init__(self, name, rootdir):
341 self._rootdir = rootdir
344 def set_doc_xml(self, rawdoc, sourcetree):
345 """Assiociate Doxygen documentation entity with the module."""
346 assert self._rawdoc is None
347 self._rawdoc = rawdoc
348 if self._rawdoc.is_documented():
349 groups = list(self._rawdoc.get_groups())
351 groupname = groups[0].get_name()
352 if groupname.startswith('group_'):
353 self._group = groupname[6:]
355 def is_documented(self):
356 return self._rawdoc is not None
361 def get_root_dir(self):
365 # TODO: Include public API convenience headers?
366 return self._rootdir.get_files()
371 class Namespace(object):
373 """Namespace in the GROMACS source code."""
375 def __init__(self, rawdoc):
376 self._rawdoc = rawdoc
378 def is_anonymous(self):
379 return self._rawdoc.is_anonymous()
383 """Class/struct/union in the GROMACS source code."""
385 def __init__(self, rawdoc, files):
386 self._rawdoc = rawdoc
387 self._files = set(files)
390 return self._rawdoc.get_name()
392 def get_reporter_location(self):
393 return self._rawdoc.get_reporter_location()
398 def is_documented(self):
399 return self._rawdoc.is_documented()
401 def has_brief_description(self):
402 return self._rawdoc.has_brief_description()
404 def get_doc_type(self):
405 """Return documentation type (visibility) for the class.
407 In addition to the actual code, this encodes GROMACS-specific logic
408 of setting EXTRACT_LOCAL_CLASSES=YES only for the full documentation.
409 Local classes never appear outside the full documentation, no matter
410 what is their visibility.
412 if not self.is_documented():
414 if self._rawdoc.is_local():
415 return DocType.internal
416 return self._rawdoc.get_visibility()
418 def get_file_doc_type(self):
419 return max([fileobj.get_doc_type() for fileobj in self._files])
421 def is_in_installed_file(self):
422 return any([fileobj.is_installed() for fileobj in self._files])
424 class Member(object):
426 """Member (in Doxygen terminology) in the GROMACS source tree.
428 Currently, modeling is limited to the minimal set of properties that the
432 def __init__(self, rawdoc, namespace):
433 self._rawdoc = rawdoc
434 self._namespace = namespace
437 return self._rawdoc.get_name()
439 def get_reporter_location(self):
440 return self._rawdoc.get_reporter_location()
442 def is_documented(self):
443 return self._rawdoc.is_documented()
445 def has_brief_description(self):
446 return self._rawdoc.has_brief_description()
448 def has_inbody_description(self):
449 return self._rawdoc.has_inbody_description()
451 def is_visible(self):
452 """Return whether the member is visible in Doxygen documentation.
454 Doxygen ignores members whose parent compounds are not documented.
455 However, when EXTRACT_ANON_NPACES=ON (which is set for our full
456 documentation), members of anonymous namespaces are extracted even if
457 the namespace is the only parent and is not documented.
459 if self._namespace and self._namespace.is_anonymous():
461 return self._rawdoc.get_inherited_visibility() != DocType.none
464 class GromacsTree(object):
466 """Root object for navigating the GROMACS source tree.
468 On initialization, the list of files and directories is initialized by
469 walking the source tree, and modules are created for top-level
470 subdirectories. At this point, only information that is accessible from
471 file names and paths only is available.
473 set_installed_file_list() can be called to set the list of installed
476 scan_files() can be called to read all the files and initialize #include
477 dependencies between the files based on the information. This is done like
478 this instead of relying on Doxygen-extracted include files to make the
479 dependency graph independent from preprocessor macro definitions
480 (Doxygen only sees those #includes that the preprocessor sees, which
481 depends on what #defines it has seen).
483 load_xml() can be called to load information from Doxygen XML data in
484 the build tree (the Doxygen XML data must have been built separately).
487 def __init__(self, source_root, build_root, reporter):
488 """Initialize the tree object by walking the source tree."""
489 self._source_root = os.path.abspath(source_root)
490 self._build_root = os.path.abspath(build_root)
491 self._reporter = reporter
493 self._docmap = dict()
496 self._modules = dict()
497 self._classes = set()
498 self._namespaces = set()
499 self._members = set()
500 self._walk_dir(os.path.join(self._source_root, 'src'))
501 rootdir = self._get_dir(os.path.join('src', 'gromacs'))
502 for subdir in rootdir.get_subdirectories():
503 self._create_module(subdir)
504 rootdir = self._get_dir(os.path.join('src', 'testutils'))
505 self._create_module(rootdir)
507 def _get_rel_path(self, path):
508 assert os.path.isabs(path)
509 if path.startswith(self._build_root):
510 return os.path.relpath(path, self._build_root)
511 if path.startswith(self._source_root):
512 return os.path.relpath(path, self._source_root)
513 raise ValueError("path not under build nor source tree: {0}".format(path))
515 def _walk_dir(self, rootpath):
516 """Construct representation of the source tree by walking the file system."""
517 assert os.path.isabs(rootpath)
518 assert rootpath not in self._dirs
519 relpath = self._get_rel_path(rootpath)
520 self._dirs[relpath] = Directory(rootpath, relpath, None)
521 for dirpath, dirnames, filenames in os.walk(rootpath):
522 if 'contrib' in dirnames:
523 dirnames.remove('contrib')
524 if 'refdata' in dirnames:
525 dirnames.remove('refdata')
526 currentdir = self._dirs[self._get_rel_path(dirpath)]
527 # Loop through a copy so that we can modify dirnames.
528 for dirname in list(dirnames):
529 fullpath = os.path.join(dirpath, dirname)
530 if fullpath == self._build_root:
531 dirnames.remove(dirname)
533 relpath = self._get_rel_path(fullpath)
534 self._dirs[relpath] = Directory(fullpath, relpath, currentdir)
535 extensions = ('.h', '.cuh', '.hpp', '.c', '.cc', '.cpp', '.cu', '.bm')
536 for filename in filenames:
537 basename, extension = os.path.splitext(filename)
538 if extension in extensions:
539 fullpath = os.path.join(dirpath, filename)
540 relpath = self._get_rel_path(fullpath)
541 self._files[relpath] = File(fullpath, relpath, currentdir)
542 elif extension == '.cmakein':
543 extension = os.path.splitext(basename)[1]
544 if extension in extensions:
545 fullpath = os.path.join(dirpath, basename)
546 relpath = self._get_rel_path(fullpath)
547 fullpath = os.path.join(dirpath, filename)
548 self._files[relpath] = GeneratedFile(fullpath, relpath, currentdir)
550 def _create_module(self, rootdir):
551 """Create module for a subdirectory."""
552 name = 'module_' + rootdir.get_name()
553 moduleobj = Module(name, rootdir)
554 rootdir.set_module(moduleobj)
555 self._modules[name] = moduleobj
557 def scan_files(self):
558 """Read source files to initialize #include dependencies."""
559 for fileobj in self._files.itervalues():
560 if not fileobj.is_external():
561 fileobj.scan_contents(self)
563 def load_xml(self, only_files=False):
564 """Load Doxygen XML information.
566 If only_files is True, XML data is not loaded for code constructs, but
567 only for files, directories, and their potential parents.
569 xmldir = os.path.join(self._build_root, 'doxygen', 'xml')
570 self._docset = xml.DocumentationSet(xmldir, self._reporter)
572 self._docset.load_file_details()
574 self._docset.load_details()
575 self._docset.merge_duplicates()
580 self._load_namespaces()
584 def _load_dirs(self):
585 """Load Doxygen XML directory information."""
586 rootdirs = self._docset.get_compounds(xml.Directory,
587 lambda x: x.get_parent() is None)
588 for dirdoc in rootdirs:
589 self._load_dir(dirdoc, None)
591 def _load_dir(self, dirdoc, parent):
592 """Load Doxygen XML directory information for a single directory."""
593 path = dirdoc.get_path().rstrip('/')
594 if not os.path.isabs(path):
595 self._reporter.xml_assert(dirdoc.get_xml_path(),
596 "expected absolute path in Doxygen-produced XML file")
598 relpath = self._get_rel_path(path)
599 dirobj = self._dirs.get(relpath)
601 dirobj = Directory(path, relpath, parent)
602 self._dirs[relpath] = dirobj
603 dirobj.set_doc_xml(dirdoc, self)
604 self._docmap[dirdoc] = dirobj
605 for subdirdoc in dirdoc.get_subdirectories():
606 self._load_dir(subdirdoc, dirobj)
608 def _load_modules(self):
609 """Load Doxygen XML module (group) information."""
610 moduledocs = self._docset.get_compounds(xml.Group,
611 lambda x: x.get_name().startswith('module_'))
612 for moduledoc in moduledocs:
613 moduleobj = self._modules.get(moduledoc.get_name())
615 self._reporter.input_error(
616 "no matching directory for module: {0}".format(moduledoc))
618 moduleobj.set_doc_xml(moduledoc, self)
619 self._docmap[moduledoc] = moduleobj
621 def _load_files(self):
622 """Load Doxygen XML file information."""
623 for filedoc in self._docset.get_files():
624 path = filedoc.get_path()
625 if not os.path.isabs(path):
626 self._reporter.xml_assert(filedoc.get_xml_path(),
627 "expected absolute path in Doxygen-produced XML file")
629 extension = os.path.splitext(filedoc.get_path())[1]
630 # We don't care about Markdown files that only produce pages
631 # (and fail the directory check below).
632 if extension == '.md':
634 dirdoc = filedoc.get_directory()
636 self._reporter.xml_assert(filedoc.get_xml_path(),
637 "file is not in any directory in Doxygen")
639 relpath = self._get_rel_path(path)
640 fileobj = self._files.get(relpath)
642 fileobj = File(path, relpath, self._docmap[dirdoc])
643 self._files[relpath] = fileobj
644 fileobj.set_doc_xml(filedoc, self)
645 self._docmap[filedoc] = fileobj
647 def _load_namespaces(self):
648 """Load Doxygen XML namespace information."""
649 nsdocs = self._docset.get_namespaces()
651 nsobj = Namespace(nsdoc)
652 self._docmap[nsdoc] = nsobj
653 self._namespaces.add(nsobj)
655 def _load_classes(self):
656 """Load Doxygen XML class information."""
657 classdocs = self._docset.get_classes()
658 for classdoc in classdocs:
659 files = [self._docmap[filedoc] for filedoc in classdoc.get_files()]
660 classobj = Class(classdoc, files)
661 self._docmap[classdoc] = classobj
662 self._classes.add(classobj)
664 def _load_members(self):
665 """Load Doxygen XML member information."""
666 memberdocs = self._docset.get_members()
667 for memberdoc in memberdocs:
668 nsdoc = memberdoc.get_namespace()
669 nsobj = self.get_object(nsdoc)
670 memberobj = Member(memberdoc, nsobj)
671 self._docmap[memberdoc] = memberobj
672 self._members.add(memberobj)
674 def _get_dir(self, relpath):
675 """Get directory object for a path relative to source tree root."""
676 return self._dirs.get(relpath)
678 def get_file(self, path):
679 """Get file object for a path relative to source tree root."""
680 return self._files.get(self._get_rel_path(path))
682 def find_include_file(self, includedpath):
683 """Find a file object corresponding to an include path."""
684 for testdir in ('src', 'src/gromacs/legacyheaders', 'src/external/thread_mpi/include'):
685 testpath = os.path.join(testdir, includedpath)
686 if testpath in self._files:
687 return self._files[testpath]
689 def set_installed_file_list(self, installedfiles):
690 """Set list of installed files."""
691 for path in installedfiles:
692 if not os.path.isabs(path):
693 self._reporter.input_error(
694 "installed file not specified with absolute path: {0}"
697 relpath = self._get_rel_path(path)
698 if relpath not in self._files:
699 self._reporter.input_error(
700 "installed file not in source tree: {0}".format(path))
702 self._files[relpath].set_installed()
704 def get_object(self, docobj):
705 """Get tree object for a Doxygen XML object."""
708 return self._docmap.get(docobj)
711 """Get iterable for all files in the source tree."""
712 return self._files.itervalues()
714 def get_modules(self):
715 """Get iterable for all modules in the source tree."""
716 return self._modules.itervalues()
718 def get_classes(self):
719 """Get iterable for all classes in the source tree."""
722 def get_members(self):
723 """Get iterable for all members (in Doxygen terms) in the source tree."""