docs/doxygen/includesorter.py

   1 #!/usr/bin/env python3
   2 #
   3 # This file is part of the GROMACS molecular simulation package.
   4 #
   5 # Copyright (c) 2012,2013,2014,2015,2016 by the GROMACS development team.
   6 # Copyright (c) 2017,2018,2019,2020, by the GROMACS development team, led by
   7 # Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
   8 # and including many others, as listed in the AUTHORS file in the
   9 # top-level source directory and at http://www.gromacs.org.
  10 #
  11 # GROMACS is free software; you can redistribute it and/or
  12 # modify it under the terms of the GNU Lesser General Public License
  13 # as published by the Free Software Foundation; either version 2.1
  14 # of the License, or (at your option) any later version.
  15 #
  16 # GROMACS is distributed in the hope that it will be useful,
  17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  19 # Lesser General Public License for more details.
  20 #
  21 # You should have received a copy of the GNU Lesser General Public
  22 # License along with GROMACS; if not, see
  23 # http://www.gnu.org/licenses, or write to the Free Software Foundation,
  24 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
  25 #
  26 # If you want to redistribute modifications to GROMACS, please
  27 # consider that scientific software is very special. Version
  28 # control is crucial - bugs must be traceable. We will be happy to
  29 # consider code for inclusion in the official distribution, but
  30 # derived work must not be called official GROMACS. Details are found
  31 # in the README & COPYING files - if they are missing, get the
  32 # official version at http://www.gromacs.org.
  33 #
  34 # To help us fund GROMACS development, we humbly ask that you cite
  35 # the research papers on the package. Check out http://www.gromacs.org.
  36
  37 """Include directive sorter for GROMACS.
  38
  39 This module implements an #include directive sorter for GROMACS C/C++ files.
  40 It allows (in most cases) automatically sorting includes and formatting
  41 the paths to use either relative paths or paths relative to src/.
  42 It groups includes in groups of related headers, sorts the headers
  43 alphabetically within each block, and inserts empty lines in between.
  44 It can be run as a standalone script, in which case it requires an up-to-date
  45 Doxygen XML documentation to be present in the
  46 build tree.  It can also be imported as a module to be embedded in other
  47 scripts.  In the latter case, the IncludeSorter provides the main interface.
  48
  49 The sorting assumes some conventions (e.g., that system headers are included
  50 with angle brackets instead of quotes).  Generally, these conventions are
  51 checked by the check-source.py script.
  52
  53 A more detailed description can be found in the developer manual.
  54 """
  55
  56 import os.path
  57 import re
  58 import sys
  59 import functools
  60
  61 @functools.total_ordering
  62 class IncludeGroup(object):
  63
  64     """Enumeration type for grouping includes."""
  65
  66     def __init__(self, value):
  67         """Initialize a IncludeGroup instance.
  68
  69         IncludeGroup.{main,system_c,...} should be used outside the
  70         class instead of calling the constructor.
  71         """
  72         self._value = value
  73
  74     def __eq__(self, other):
  75         """Order include groups in the desired order."""
  76         return self._value == other._value
  77
  78     def __lt__(self, other):
  79         """Order include groups in the desired order."""
  80         return self._value < other._value
  81
  82 # gmxpre.h is always first
  83 IncludeGroup.pre = IncludeGroup(0)
  84 # "main" include file for the source file is next
  85 IncludeGroup.main = IncludeGroup(1)
  86 # config.h is next, if present, to keep its location consistent
  87 IncludeGroup.config = IncludeGroup(2)
  88 # Followed by system headers, with C first and C++ following
  89 IncludeGroup.system_c = IncludeGroup(3)
  90 IncludeGroup.system_c_cpp = IncludeGroup(4)
  91 IncludeGroup.system_cpp = IncludeGroup(5)
  92 # System headers not in standard C/C++ are in a separate block
  93 IncludeGroup.system_other = IncludeGroup(6)
  94 # src/external/ contents that are included with quotes go here
  95 IncludeGroup.nonsystem_other = IncludeGroup(7)
  96 # Other GROMACS headers
  97 IncludeGroup.gmx_general = IncludeGroup(8)
  98 # This group is for shared (unit) testing utilities
  99 IncludeGroup.gmx_test = IncludeGroup(9)
 100 # This group is for headers local to the including file/module
 101 IncludeGroup.gmx_local = IncludeGroup(10)
 102
 103 class GroupedSorter(object):
 104
 105     """Grouping and formatting logic for #include directives.
 106
 107     This class implements the actual logic that decides how includes are
 108     grouped and sorted, and how they are formatted."""
 109
 110     # These variables contain the list of system headers for various blocks
 111     _std_c_headers = ['assert.h', 'ctype.h', 'errno.h', 'float.h',
 112             'inttypes.h', 'limits.h', 'math.h', 'signal.h', 'stdarg.h',
 113             'stddef.h', 'stdint.h', 'stdio.h', 'stdlib.h', 'string.h',
 114             'time.h']
 115     _std_c_cpp_headers = ['c' + x[:-2] for x in _std_c_headers]
 116     _std_cpp_headers = ['algorithm',
 117                         'any',
 118                         'array',
 119                         'atomic',
 120                         'charconv',
 121                         'chrono',
 122                         'cinttypes',
 123                         'cfenv',
 124                         'complex',
 125                         'cstdint',
 126                         'deque',
 127                         'exception',
 128                         'execution',
 129                         'filesystem',
 130                         'forward_list',
 131                         'fstream',
 132                         'functional',
 133                         'future',
 134                         'initializer_list',
 135                         'iomanip',
 136                         'ios',
 137                         'iosfwd',
 138                         'iostream',
 139                         'istream',
 140                         'iterator',
 141                         'limits',
 142                         'list',
 143                         'locale',
 144                         'map',
 145                         'memory',
 146                         'memory_resource',
 147                         'mutex',
 148                         'new',
 149                         'numeric',
 150                         'optional',
 151                         'ostream',
 152                         'queue',
 153                         'random',
 154                         'ratio',
 155                         'regex',
 156                         'scoped_allocator',
 157                         'set',
 158                         'sstream',
 159                         'stack',
 160                         'stdexcept',
 161                         'streambuf',
 162                         'string',
 163                         'string_view',
 164                         'strstream',
 165                         'system_error',
 166                         'thread',
 167                         'tuple',
 168                         'type_traits',
 169                         'typeindex',
 170                         'typeinfo',
 171                         'unordered_map',
 172                         'unordered_set',
 173                         'valarray',
 174                         'variant',
 175                         'vector',
 176                         'unordered_map',
 177                         'utility']
 178
 179     def __init__(self, style='pub-priv', absolute=False):
 180         """Initialize a sorted with the given style."""
 181         if style == 'single-group':
 182             self._local_group = 'none'
 183         elif style == 'pub-priv':
 184             self._local_group = 'private'
 185         else:
 186             self._local_group = 'local'
 187         if absolute:
 188             self._abspath_main = True
 189             self._abspath_local = True
 190         else:
 191             self._abspath_main = False
 192             self._abspath_local = False
 193
 194     def _get_path(self, included_file, group, including_file):
 195         """Compute include path to use for an #include.
 196
 197         The path is made either absolute (i.e., relative to src/), or
 198         relative to the location of the including file, depending on the group
 199         the file is in.
 200         """
 201         use_abspath = including_file is None or group is None
 202         if not use_abspath:
 203             if group in (IncludeGroup.gmx_general, IncludeGroup.gmx_test):
 204                 use_abspath = True
 205             elif group == IncludeGroup.main and self._abspath_main:
 206                 use_abspath = True
 207             elif group == IncludeGroup.gmx_local and self._abspath_local:
 208                 use_abspath = True
 209         if not use_abspath:
 210             fromdir = os.path.dirname(including_file.get_abspath())
 211             relpath = os.path.relpath(included_file.get_abspath(), fromdir)
 212             if not relpath.startswith('..'):
 213                 return relpath
 214         path = included_file.get_relpath()
 215         assert path.startswith('src/')
 216         return path[4:]
 217
 218     def _get_gmx_group(self, including_file, included_file):
 219         """Determine group for GROMACS headers.
 220
 221         Helper function to determine the group for an #include directive
 222         when the #include is in one of the gmx_* groups (or in the main group).
 223         """
 224         main_header = including_file.get_main_header()
 225         if main_header and main_header == included_file:
 226             return IncludeGroup.main
 227         if included_file.get_directory().get_name() == 'testutils':
 228             return IncludeGroup.gmx_test
 229         if including_file.get_directory().contains(included_file):
 230             if self._local_group == 'local':
 231                 return IncludeGroup.gmx_local
 232             if self._local_group == 'private':
 233                 if included_file.api_type_is_reliable() \
 234                         and included_file.is_module_internal():
 235                     return IncludeGroup.gmx_local
 236                 if not included_file.api_type_is_reliable() \
 237                         and including_file.get_relpath().startswith('src/programs'):
 238                     return IncludeGroup.gmx_local
 239         if included_file.is_test_file():
 240             return IncludeGroup.gmx_test
 241         return IncludeGroup.gmx_general
 242
 243     def _split_path(self, path):
 244         """Split include path into sortable compoments.
 245
 246         Plain string on the full path in the #include directive causes some
 247         unintuitive behavior, so this splits the path into a tuple at
 248         points that allow more natural sorting: primary sort criterion is the
 249         directory name, followed by the basename (without extension) of the
 250         included file.
 251         """
 252         path_components = list(os.path.split(path))
 253         path_components[1] = os.path.splitext(path_components[1])
 254         return tuple(path_components)
 255
 256     def _join_path(self, path_components):
 257         """Reconstruct path from the return value of _split_path."""
 258         return os.path.join(path_components[0], ''.join(path_components[1]))
 259
 260     def get_sortable_object(self, include):
 261         """Produce a sortable, opaque object for an include.
 262
 263         Includes are sorted by calling this function for each #include object,
 264         and sorting the list made up of these objects (using the default
 265         comparison operators).  Each element from the sorted list is then
 266         passed to format_include(), which extracts information from the opaque
 267         object and formats the #include directive for output.
 268         """
 269         included_file = include.get_file()
 270         if not included_file:
 271             path = include.get_included_path()
 272             if path in self._std_c_headers:
 273                 group = IncludeGroup.system_c
 274             elif path in self._std_c_cpp_headers:
 275                 group = IncludeGroup.system_c_cpp
 276             elif path in self._std_cpp_headers:
 277                 group = IncludeGroup.system_cpp
 278             else:
 279                 group = IncludeGroup.system_other
 280         elif included_file.is_external():
 281             group = IncludeGroup.nonsystem_other
 282             if 'external/' in include.get_included_path():
 283                 path = self._get_path(included_file, group, None)
 284             else:
 285                 path = include.get_included_path()
 286         elif included_file.get_name() == 'gmxpre.h':
 287             group = IncludeGroup.pre
 288             path = self._get_path(included_file, group, None)
 289         elif included_file.get_name() == 'config.h':
 290             group = IncludeGroup.config
 291             path = self._get_path(included_file, group, None)
 292         else:
 293             including_file = include.get_including_file()
 294             group = self._get_gmx_group(including_file, included_file)
 295             path = self._get_path(included_file, group, including_file)
 296         return (group, self._split_path(path), include)
 297
 298     def format_include(self, obj, prev):
 299         """Format an #include directive after sorting."""
 300         result = []
 301         if prev:
 302             if prev[0] != obj[0]:
 303                 # Print empty line between groups
 304                 result.append('\n')
 305             elif prev[1] == obj[1]:
 306                 # Skip duplicates
 307                 return result
 308         include = obj[2]
 309         line = include.get_full_line()
 310         include_re = r'^(?P<head>\s*#\s*include\s+)["<][^">]*[">](?P<tail>.*)$'
 311         match = re.match(include_re, line)
 312         assert match
 313         if include.is_system():
 314             path = '<{0}>'.format(self._join_path(obj[1]))
 315         else:
 316             path = '"{0}"'.format(self._join_path(obj[1]))
 317         result.append('{0}{1}{2}\n'.format(match.group('head'), path, match.group('tail')))
 318         return result
 319
 320 class IncludeSorter(object):
 321
 322     """High-level logic for sorting includes.
 323
 324     This class contains the high-level logic for sorting include statements.
 325     The actual ordering and formatting the includes is delegated to a sort method
 326     (see GroupedSorter) to keep things separated.
 327     """
 328
 329     def __init__(self, sortmethod=None, quiet=True):
 330         """Initialize the include sorter with the given sorter and options."""
 331         if not sortmethod:
 332             sortmethod = GroupedSorter()
 333         self._sortmethod = sortmethod
 334         self._quiet = quiet
 335         self._changed = False
 336
 337     def _sort_include_block(self, block, lines):
 338         """Sort a single include block.
 339
 340         Returns a new list of lines for the block.
 341         If anything is changed, self._changed is set to True, and the caller
 342         can check that."""
 343         includes = sorted(map(self._sortmethod.get_sortable_object, block.get_includes()))
 344         result = []
 345         prev = None
 346         current_line_number = block.get_first_line()-1
 347         for include in includes:
 348             newlines = self._sortmethod.format_include(include, prev)
 349             result.extend(newlines)
 350             if not self._changed:
 351                 for offset, newline in enumerate(newlines):
 352                     if lines[current_line_number + offset] != newline:
 353                         self._changed = True
 354                         break
 355                 current_line_number += len(newlines)
 356             prev = include
 357         return result
 358
 359     def sort_includes(self, fileobj):
 360         """Sort all includes in a file."""
 361         lines = fileobj.get_contents()
 362         # Format into a list first:
 363         #  - avoid bugs or issues in the script truncating the file
 364         #  - can check whether anything was changed before touching the file
 365         newlines = []
 366         prev = 0
 367         self._changed = False
 368         for block in fileobj.get_include_blocks():
 369             newlines.extend(lines[prev:block.get_first_line()-1])
 370             newlines.extend(self._sort_include_block(block, lines))
 371             # The returned values are 1-based, but indexing here is 0-based,
 372             # so an explicit +1 is not needed.
 373             prev = block.get_last_line()
 374         if self._changed:
 375             if not self._quiet:
 376                 sys.stderr.write('{0}: includes reformatted\n'.format(fileobj.get_relpath()))
 377             newlines.extend(lines[prev:])
 378             with open(fileobj.get_abspath(), 'w') as fp:
 379                 fp.write(''.join(newlines))
 380
 381     def check_sorted(self, fileobj):
 382         """Check that includes within a file are sorted."""
 383         # TODO: Make the checking work without full contents of the file
 384         lines = fileobj.get_contents()
 385         is_sorted = True
 386         details = None
 387         for block in fileobj.get_include_blocks():
 388             self._changed = False
 389             sorted_lines = self._sort_include_block(block, lines)
 390             if self._changed:
 391                 is_sorted = False
 392                 # TODO: Do a proper diff to show the actual changes.
 393                 if details is None:
 394                     details = ["Correct order/style is:"]
 395                 else:
 396                     details.append("    ...")
 397                 details.extend(["    " + x.rstrip() for x in sorted_lines])
 398         return (is_sorted, details)
 399
 400 def main():
 401     """Run the include sorter script."""
 402     import os
 403     import sys
 404
 405     from optparse import OptionParser
 406
 407     from gmxtree import GromacsTree
 408     from reporter import Reporter
 409
 410     parser = OptionParser()
 411     parser.add_option('-S', '--source-root',
 412                       help='Source tree root directory')
 413     parser.add_option('-B', '--build-root',
 414                       help='Build tree root directory')
 415     parser.add_option('-F', '--files',
 416                       help='Specify files to sort')
 417     parser.add_option('-q', '--quiet', action='store_true',
 418                       help='Do not write status messages')
 419     # This is for evaluating different options; can be removed from the final
 420     # version.
 421     parser.add_option('-s', '--style', type='choice', default='pub-priv',
 422                       choices=('single-group', 'pub-priv', 'pub-local'),
 423                       help='Style for GROMACS includes')
 424     parser.add_option('--absolute', action='store_true',
 425                       help='Write all include paths relative to src/')
 426     options, args = parser.parse_args()
 427
 428     filelist = args
 429     if options.files:
 430         if options.files == '-':
 431             lines = sys.stdin.readlines()
 432         else:
 433             with open(options.files, 'r') as fp:
 434                 lines = fp.readlines()
 435         filelist.extend([x.strip() for x in lines])
 436
 437     reporter = Reporter(quiet=True)
 438
 439     if not options.quiet:
 440         sys.stderr.write('Scanning source tree...\n')
 441     if not options.source_root:
 442         sys.stderr.write('Source root required not specified.\n')
 443         sys.exit(2)
 444     if not options.build_root:
 445         sys.stderr.write('Build root required not specified.\n')
 446         sys.exit(2)
 447     tree = GromacsTree(options.source_root, options.build_root, reporter)
 448     files = []
 449     for filename in filelist:
 450         fileobj = tree.get_file(os.path.abspath(filename))
 451         if not fileobj:
 452             sys.stderr.write('warning: ignoring unknown file {0}\n'.format(filename))
 453             continue
 454         files.append(fileobj)
 455     if not options.quiet:
 456         sys.stderr.write('Reading source files...\n')
 457     tree.scan_files(only_files=files, keep_contents=True)
 458     extfiles = set(files)
 459     for fileobj in files:
 460         for included_file in fileobj.get_includes():
 461             other_file = included_file.get_file()
 462             if other_file:
 463                 extfiles.add(other_file)
 464     if not options.quiet:
 465         sys.stderr.write('Reading Doxygen XML files...\n')
 466     tree.load_xml(only_files=extfiles)
 467
 468     if not options.quiet:
 469         sys.stderr.write('Sorting includes...\n')
 470
 471     sorter = IncludeSorter(GroupedSorter(options.style, options.absolute), options.quiet)
 472
 473     for fileobj in files:
 474         sorter.sort_includes(fileobj)
 475
 476 if __name__ == '__main__':
 477     main()