3 # This file is part of the GROMACS molecular simulation package.
5 # Copyright (c) 2012,2013, by the GROMACS development team, led by
6 # Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
7 # and including many others, as listed in the AUTHORS file in the
8 # top-level source directory and at http://www.gromacs.org.
10 # GROMACS is free software; you can redistribute it and/or
11 # modify it under the terms of the GNU Lesser General Public License
12 # as published by the Free Software Foundation; either version 2.1
13 # of the License, or (at your option) any later version.
15 # GROMACS is distributed in the hope that it will be useful,
16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
18 # Lesser General Public License for more details.
20 # You should have received a copy of the GNU Lesser General Public
21 # License along with GROMACS; if not, see
22 # http://www.gnu.org/licenses, or write to the Free Software Foundation,
23 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
25 # If you want to redistribute modifications to GROMACS, please
26 # consider that scientific software is very special. Version
27 # control is crucial - bugs must be traceable. We will be happy to
28 # consider code for inclusion in the official distribution, but
29 # derived work must not be called official GROMACS. Details are found
30 # in the README & COPYING files - if they are missing, get the
31 # official version at http://www.gromacs.org.
33 # To help us fund GROMACS development, we humbly ask that you cite
34 # the research papers on the package. Check out http://www.gromacs.org.
36 """Generate include dependency graphs for Gromacs.
38 This script can generate two types of include dependency graphs: per-file or
39 per-module (where module is equivalent to a subdirectory).
40 It is intended to be run on a subset of files under the src/ directory.
41 Output format is suitable for processing with 'dot'.
45 The legend for per-file graph nodex:
47 light blue: public headers
48 dark blue: library headers
49 no background: other files
53 Module graph will contain one node for each top-level subdirectory under src/,
54 except that the src/gromacs/ directory will be expanded one level further.
56 The legend for per-module graph links (a link with a certain color indicates
57 that types above it in the list are not present):
58 red: invalid dependency (e.g., undocumented file)
59 dark blue: library header depends on the other module
60 light blue: public header depends on the other module
61 dashed black: source file depends on a library header in the other module
62 solid black: source file depends on a public header in the other module
63 dotted grey: test files depend on the other module
70 from optparse import OptionParser
72 class ErrorReporter(object):
73 def input_warning(self, file_path, msg):
74 sys.stderr.write('warning: {0}: {1}\n'.format(file_path, msg))
76 def error(self, file_path, msg):
77 sys.stderr.write('error: {0}: {1}\n'.format(file_path, msg))
81 """Link between two node objects.
83 Signifies an include dependency between the two nodes, and manages types
84 associated with the dependencies.
96 def __init__(self, fromnode, tonode, link_type=None):
97 self.fromnode = fromnode
99 self.link_type = link_type
103 def refresh_type(self):
104 """Initialize type of a link between two file nodes.
106 Both endpoints of the link must be file objects when this method is
109 fromfile = self.fromnode.obj
110 tofile = self.tonode.obj
112 (fromfile.module.get_top_level_module() == \
113 tofile.module.get_top_level_module())
114 if tofile.type != 'publicheader' and tofile.type != 'libheader':
116 link_type = 'intramodule'
118 link_type = 'undocumented'
119 elif fromfile.type == 'test':
121 elif fromfile.type in ('source', 'header', 'implheader'):
122 if tofile.type == 'publicheader':
123 link_type = 'pubimpl'
124 elif tofile.type == 'libheader':
125 link_type = 'libimpl'
127 raise ValueError('Unknown link type between {0} and {1}'
128 .format(fromfile.path, tofile.path))
129 elif fromfile.type == 'libheader':
130 link_type = 'library'
131 elif fromfile.type == 'publicheader':
132 if tofile.type == 'publicheader' or tofile.doctype == 'public':
135 link_type = 'undocumented'
137 raise ValueError('Unknown link type between {0} and {1}'
138 .format(fromfile.path, tofile.path))
139 self.link_type = link_type
141 def merge_link(self, other):
142 """Merge another link into this one and choose an appropriate type.
144 Updates the type of this link based on the types of the merged links.
146 if Link.priorities[other.link_type] < Link.priorities[self.link_type]:
147 self.link_type = other.link_type
150 """Format this link for 'dot'."""
151 if isinstance(self.fromnode.obj, File) and \
152 isinstance(self.tonode.obj, File):
154 elif self.link_type == 'intramodule':
156 elif self.link_type == 'test':
157 properties = 'color=grey75, style=dotted'
158 elif self.link_type == 'libimpl':
159 properties = 'color=".66 .5 1"'
160 elif self.link_type == 'pubimpl':
161 properties = 'color=".66 .2 1"'
162 elif self.link_type == 'library':
163 properties = 'color=black, style=dashed'
164 elif self.link_type == 'public':
165 properties = 'color=black'
167 properties = 'color=red'
168 return '{0} -> {1} [{2}]'.format(self.fromnode.nodename,
169 self.tonode.nodename,
173 def __init__(self, obj, nodename, label):
174 self.nodename = nodename
183 def add_child(self, child):
184 self.children.append(child)
186 def clear_children(self):
189 def get_children(self, recursive=False):
191 result = list(self.children)
192 for child in self.children:
193 result.extend(child.get_children(recursive=True))
199 """Format this node for 'dot'."""
203 result += ' subgraph cluster_{0} {{\n' \
204 .format(self.nodename)
205 result += ' label = "{0}"\n'.format(self.label)
206 for child in self.children:
207 result += child.format()
211 result += ' {0} [{1}]\n'.format(
212 self.nodename, self.obj.node_properties())
217 def __init__(self, nodes, links):
218 self.nodes = set(nodes)
220 self.left_to_right = False
221 self.concentrate = True
223 def set_options(self, left_to_right=None, concentrate=None):
224 if left_to_right != None:
225 self.left_to_right = left_to_right
226 if concentrate != None:
227 self.concentrate = concentrate
229 def prune_links(self):
231 for node in self.nodes:
232 nodes.update(node.get_children(recursive=True))
234 for link in self.links:
235 if link.fromnode in nodes and link.tonode in nodes:
236 newlinks.append(link)
237 self.links = newlinks
239 def merge_nodes(self, nodes, target):
245 for link in self.links:
246 isfrom = (link.fromnode in nodes)
247 isto = (link.tonode in nodes)
251 if not link.tonode in linksfrom:
252 linksfrom[link.tonode] = \
253 Link(target, link.tonode, link.link_type)
255 linksfrom[link.tonode].merge_link(link)
257 if not link.fromnode in linksto:
258 linksto[link.fromnode] = \
259 Link(link.fromnode, target, link.link_type)
261 linksto[link.fromnode].merge_link(link)
263 newlinks.append(link)
264 newlinks.extend(linksfrom.values())
265 newlinks.extend(linksto.values())
266 self.links = newlinks
268 def collapse_node(self, node):
269 nodes = node.get_children(recursive=True)
270 self.merge_nodes(nodes, node)
271 node.clear_children()
273 def write(self, outfile):
274 outfile.write('digraph includedeps {\n')
275 if self.left_to_right:
276 outfile.write(' rankdir = LR\n')
278 outfile.write(' concentrate = true\n')
279 outfile.write(' node [fontname="FreeSans",fontsize=10,height=.2,'
281 for link in self.links:
282 outfile.write(' ' + link.format() + '\n')
283 for node in self.nodes:
284 outfile.write(node.format())
288 def find_include_file(filename, includedirs):
289 """Find full path to filename, looking in a set of directories."""
290 for includedir in includedirs:
291 fullpath = os.path.abspath(os.path.join(includedir, filename))
292 if os.path.exists(fullpath):
297 class IncludedFile(object):
298 def __init__(self, included_file, included_path, is_relative, is_system):
299 self._included_file = included_file
300 self._included_path = included_path
301 #self._used_include_path = used_include_path
302 self._is_relative = is_relative
303 self._is_system = is_system
307 def __init__(self, path, module):
309 self.name = os.path.basename(path)
311 if module.name == 'tests':
313 elif re.search(r'\.c(pp)?$', self.name) != None:
317 self.doctype = 'none'
318 #headername = re.sub(r'\.cpp$', '.h', self.name)
319 #implheadername = re.sub(r'\.cpp$', '-impl.h', self.name)
321 self.installed = False
323 def is_documented(self):
324 return self.doctype != 'none'
326 def set_installed(self, reporter):
327 if self.type != 'header':
328 reporter.input_warning(self.path,
329 'installing {0} file'.format(self.type))
331 self.installed = True
333 def get_included_files(self):
334 return self._included
336 def node_properties(self):
339 properties.append('label="{0}"'.format(self.name))
340 properties.append('URL="\\ref {0}"'.format(self.name))
343 properties.append('color=red')
344 if self.type == 'source':
345 style.append('filled')
346 properties.append('fillcolor=grey75')
347 elif self.type == 'publicheader':
348 style.append('filled')
349 properties.append('fillcolor=".66 .2 1"')
350 elif self.type == 'libheader':
351 style.append('filled')
352 properties.append('fillcolor=".66 .5 1"')
354 properties.append('style="{0}"'.format(','.join(style)))
355 return ', '.join(properties)
357 def scan_include_file(self, line, allfiles, selfdir, includedirs,
358 ignorelist, reporter):
359 """Process #include directive during scan().
361 Searches for the included file in given directories, does some checks,
362 and adds the dependency link to the other file if applicable.
369 match = re.match(r'#include *<([^>]*)>', line)
371 includedpath = match.group(1)
373 fullpath = find_include_file(includedpath, includedirs)
375 match = re.match(r'#include *"([^"]*)"', line)
377 includedpath = match.group(1)
378 fullpath = os.path.join(selfdir, includedpath)
379 #if os.path.abspath(fullpath) in ignorelist:
381 if os.path.exists(fullpath):
384 fullpath = find_include_file(includedpath, includedirs)
386 reporter.input_warning(self.path,
387 'included file "{0}" not found'
388 .format(includedpath))
390 reporter.input_warning(self.path, 'line "{0}" could not be parsed'
393 if fullpath and fullpath in allfiles:
394 includedfile = allfiles[fullpath]
395 #elif not dep in ignorelist:
396 # depfile = File(dep, None)
397 # files[dep] = depfile
398 # file.add_dependency(depfile)
399 # extrafiles.append(dep)
400 self._included.append(IncludedFile(includedfile, includedpath,
401 is_relative, is_system))
403 def scan(self, filename, allfiles, includedirs, ignorelist, reporter):
404 selfdir = os.path.dirname(filename)
406 foundfileblock = False
407 self.docmodule = None
408 with open(filename, 'r') as scanfile:
409 for line in scanfile:
410 if line.startswith('#include'):
411 self.scan_include_file(line, allfiles, selfdir,
412 includedirs, ignorelist, reporter)
414 if not foundfileblock:
418 foundfileblock = True
420 if self.type == 'implheader':
421 if line.startswith(r' * \inpublicapi'):
422 self.type = 'publicheader'
423 elif line.startswith(r' * \inlibraryapi'):
424 self.type = 'libheader'
425 match = re.match(r' \* \\ingroup module_([a-z_]*)', line)
428 reporter.error(self.path,
429 'file documented in multiple modules')
430 self.docmodule = match.group(1)
432 match = re.match(r'/\*! *(\\[a-z]*internal)? *\\file', line)
434 docspec = match.group(1)
436 self.doctype = 'public'
437 elif docspec == r'\libinternal':
438 self.doctype = 'library'
439 elif docspec == r'\internal':
440 self.doctype = 'implementation'
442 reporter.input_warning(self.path,
443 'unknown specifier "{0}"'.format(docspec))
444 self.doctype = 'unknown'
446 if self.type == 'header':
447 # Default type if no other found
448 self.type = 'implheader'
451 class Module(object):
452 def __init__(self, name, parent = None):
456 self.fullname = parent.fullname + '_' + name
458 self.fullname = 'module'
460 self.children = dict()
461 self.is_top_level = (not parent or parent.name in ('', 'gromacs'))
463 def is_child(self, module):
464 parent = module.parent
468 parent = parent.parent
471 def get_top_level_module(self):
472 if self.is_top_level or not self.parent:
474 return self.parent.get_top_level_module()
476 def add_nested_file(self, modules, path):
477 if len(modules) == 1:
478 newfile = File(path, self)
479 self.files.append(newfile)
481 if not modules[0] in self.children:
482 module = Module(modules[0], self)
483 self.children[modules[0]] = module
485 module = self.children[modules[0]]
486 newfile = module.add_nested_file(modules[1:], path)
489 def node_properties(self):
490 properties = 'label="{0}", shape=ellipse'.format(self.name)
491 properties += ', URL="\\ref module_{0}"'.format(self.name)
495 class Dependencies(object):
496 def __init__(self, rootdir, includedirs, installedfiles):
498 self.root = Module("")
501 self.rootpath.append(os.path.abspath(root))
503 self.includedirs = self.rootpath + includedirs
505 self.includedirs = self.rootpath
506 self.installedfiles = installedfiles
508 def add_file(self, filename, reporter):
509 fullpath = os.path.abspath(filename)
510 for root in self.rootpath:
511 if fullpath.startswith(root):
512 relpath = fullpath[len(root)+1:]
515 reporter.input_warning(filename,
516 'input file not under root path, skipped')
518 modules = relpath.split(os.sep)
519 newfile = self.root.add_nested_file(modules, relpath)
520 if fullpath in self.installedfiles:
521 newfile.set_installed(reporter)
522 self.files[os.path.abspath(filename)] = newfile
524 def scan_files(self, ignorelist, reporter):
525 for (filename, scanfile) in self.files.iteritems():
526 scanfile.scan(filename, self.files, self.includedirs, ignorelist,
529 def get_toplevel_modules(self):
531 for module in self.root.children.itervalues():
532 if module.name == 'gromacs':
533 result.extend(module.children.itervalues())
535 result.append(module)
539 class IncludeFileChecker(object):
540 def __init__(self, deps, options):
542 self._options = options
544 def _check_file(self, checkfile, reporter):
545 if not self._options.check_doc:
547 if not checkfile.is_documented():
548 if self._options.warn_undoc:
549 reporter.error(checkfile.path, 'file not documented')
550 elif checkfile.doctype == 'implementation' and \
551 checkfile.type in ('publicheader', 'libheader'):
552 reporter.error(checkfile.path,
553 'file documentation visibility incorrect')
554 elif checkfile.doctype == 'library' and checkfile.type == 'publicheader':
555 reporter.error(checkfile.path,
556 'file documentation visibility incorrect')
557 elif checkfile.installed and checkfile.doctype not in ('public', 'unknown'):
558 reporter.error(checkfile.path,
559 'installed header has no public documentation')
560 elif not checkfile.installed and checkfile.doctype == 'public':
561 reporter.error(checkfile.path,
562 'non-installed file has public documentation')
563 selfmodfullname = checkfile.module.fullname
564 docmodule = checkfile.docmodule
566 not selfmodfullname.startswith('module_' + docmodule) and \
567 not selfmodfullname.startswith('module_gromacs_' + docmodule):
568 reporter.error(checkfile.path,
569 'file documented in incorrect module "{0}"'
572 def _check_included_file(self, checkfile, includedfile, reporter):
573 otherfile = includedfile._included_file
574 if includedfile._is_system:
576 reporter.error(checkfile.path,
577 'local file included as <{0}>'
578 .format(includedfile._included_path))
579 elif not includedfile._is_relative and checkfile.installed:
580 reporter.error(checkfile.path,
581 'installed header includes "{0}", '
582 'which is not found using relative path'
583 .format(includedfile._included_path))
586 if checkfile.installed and not otherfile.installed:
587 reporter.error(checkfile.path,
588 'installed header includes '
589 'non-installed header "{0}"'
590 .format(includedfile._included_path))
591 if not otherfile.is_documented():
593 if not self._options.check_doc:
596 (checkfile.module.get_top_level_module() == \
597 otherfile.module.get_top_level_module())
598 if otherfile.type not in ('publicheader', 'libheader'):
600 reporter.error(checkfile.path,
601 'included file "{0}" is missing API definition'
602 .format(otherfile.path))
603 elif checkfile.type == 'publicheader':
604 if not otherfile.type == 'publicheader' and not otherfile.doctype == 'public':
605 reporter.error(checkfile.path,
606 'public API file includes non-public header "{0}"'
607 .format(otherfile.path))
609 def check_all(self, reporter):
610 for checkfile in self._deps.files.itervalues():
611 self._check_file(checkfile, reporter)
612 for includedfile in checkfile.get_included_files():
613 self._check_included_file(checkfile, includedfile, reporter)
616 class GraphBuilder(object):
617 def __init__(self, deps):
620 def create_file_node(self, fileobj, filenodes):
621 nodename = re.subn(r'[-./]', '_', fileobj.path)[0]
622 node = Node(fileobj, nodename, fileobj.name)
623 filenodes[fileobj] = node
626 def create_file_edges(self, fileobj, filenodes):
628 if fileobj in filenodes:
629 for includedfile in fileobj.get_included_files():
630 otherfile = includedfile._included_file
631 if otherfile and otherfile in filenodes:
632 link = Link(filenodes[fileobj], filenodes[otherfile])
636 def create_module_node(self, module, filenodes):
637 node = Node(module, module.fullname, module.name)
638 for childfile in module.files:
639 node.add_child(self.create_file_node(childfile, filenodes))
640 for childmodule in module.children.itervalues():
641 node.add_child(self.create_module_node(childmodule, filenodes))
644 def create_file_graph(self):
646 rootnode = self.create_module_node(self._deps.root, filenodes)
649 for scanfile in self._deps.files.itervalues():
650 links.extend(self.create_file_edges(scanfile, filenodes))
651 graph = Graph([rootnode], links)
654 def create_modules_graph(self):
656 rootnode = self.create_module_node(self._deps.root, filenodes)
659 for scanfile in self._deps.files.itervalues():
660 links.extend(self.create_file_edges(scanfile, filenodes))
661 graph = Graph([rootnode], links)
662 for node in rootnode.get_children():
663 if node.label == 'gromacs':
664 for child in node.get_children():
665 graph.collapse_node(child)
667 graph.collapse_node(node)
668 graph.set_options(concentrate=False)
671 def create_module_file_graph(self, module):
673 rootnode = self.create_module_node(module, filenodes)
676 for scanfile in self._deps.files.itervalues():
677 links.extend(self.create_file_edges(scanfile, filenodes))
678 graph = Graph([rootnode], links)
683 def print_module_graph(outfile, graphbuilder, options):
684 graph = graphbuilder.create_modules_graph()
687 def print_file_graph(outfile, graphbuilder, options):
688 graph = graphbuilder.create_file_graph()
689 graph.set_options(left_to_right=options.left_to_right)
691 #if options.source_at_top:
693 # for file in deps.files.itervalues():
694 # if file.sourcefile:
695 # sourcenodes.append(file.nodename)
697 # outfile.write(' { rank = min; ' + '; '.join(sourcenodes) + '}\n')
698 #if options.with_external and options.external_at_bottom:
700 # for file in deps.files.itervalues():
701 # if not file.module:
702 # extnodes.append(file.nodename)
704 # outfile.write(' { rank = max; ' + '; '.join(extnodes) + '}\n')
706 def print_module_file_graph(outfile, graphbuilder, module, options):
707 graph = graphbuilder.create_module_file_graph(module)
708 graph.set_options(left_to_right=options.left_to_right)
712 parser = OptionParser()
713 parser.add_option('-f', '--files',
714 help='Read list of input files from given file')
715 parser.add_option('--installed',
716 help='Read list of installed files from given file')
717 parser.add_option('-R', '--rootdir', action='append',
718 help='Remove this prefix from all files')
719 parser.add_option('-I', '--includedir', action='append',
720 help='Specify additional directories to search for '
722 parser.add_option('-o', '--outdir', default='.',
723 help='Specify output directory for graphs')
724 #parser.add_option('--source-at-top', action='store_true',
725 # help='Force source files at the top of the graph')
726 #parser.add_option('--with-external', action='store_true',
727 # help='Include external dependencies in the graph')
728 #parser.add_option('--external-at-bottom', action='store_true',
729 # help='Force external dependencies files at the bottom '
731 parser.add_option('--check', action='store_true',
732 help='Check for problems in include file dependencies')
733 parser.add_option('--check-doc', action='store_true',
734 help='Check for problems in Doxygen documentation')
735 parser.add_option('--warn-undoc', action='store_true',
736 help='Warn for files that do not have Doxygen documentation')
737 parser.add_option('--left-to-right', action='store_true',
738 help='Lay out from left to right')
739 parser.add_option('--file-graph',
740 help='Write graph for individual files')
741 parser.add_option('--module-graph',
742 help='Write graph for modules')
743 parser.add_option('--module-file-graphs', action='store_true',
744 help='Write file graphs for each module')
745 options, args = parser.parse_args()
747 if not options.file_graph and not options.module_graph and \
748 not options.module_file_graphs:
751 # Constructs lists of files
756 with open(options.files, 'r') as outfile:
758 if line.startswith('!'):
759 ignorelist.append(os.path.abspath(line[1:].strip()))
761 filelist.append(line.strip())
762 filelist.extend(args)
763 if options.installed:
764 with open(options.installed, 'r') as outfile:
766 installedlist.append(line.strip())
768 # Creates objects for all files and modules
769 reporter = ErrorReporter()
770 deps = Dependencies(options.rootdir, options.includedir, installedlist)
771 for filename in filelist:
772 deps.add_file(filename, reporter)
774 deps.scan_files(ignorelist, reporter)
776 if options.check or options.check_doc:
777 checker = IncludeFileChecker(deps, options)
778 checker.check_all(reporter)
780 #if options.with_external:
781 # for filename in extrafiles:
782 # file = files[filename]
783 # if os.path.exists(filename):
784 # with open(filename, 'r') as outfile:
785 # for line in outfile:
787 # if line.startswith(' * \inpublicapi'):
788 # file.api = "public"
789 # elif line.startswith(' * \inlibraryapi'):
790 # file.api = "library"
792 # Prints out the graph
793 graphbuilder = GraphBuilder(deps)
794 if options.module_graph:
795 graphpath = os.path.join(options.outdir, options.module_graph)
796 with open(graphpath, 'w') as outfile:
797 print_module_graph(outfile, graphbuilder, options)
798 if options.file_graph:
799 graphpath = os.path.join(options.outdir, options.file_graph)
800 with open(graphpath, 'w') as outfile:
801 print_file_graph(outfile, graphbuilder, options)
802 if options.module_file_graphs:
803 options.left_to_right = True
804 for module in deps.get_toplevel_modules():
805 filename = 'module_{0}-deps.dot'.format(module.name)
806 with open(os.path.join(options.outdir, filename), 'w') as outfile:
807 print_module_file_graph(outfile, graphbuilder, module, options)