3 # Syntax: mkdoc.py [-I<path> ..] [.. a list of header files ..]
5 # Extract documentation from C++ header files to use it in Python bindings
14 from clang import cindex
15 from clang.cindex import CursorKind
16 from collections import OrderedDict
17 from threading import Thread, Semaphore
18 from multiprocessing import cpu_count
21 CursorKind.TRANSLATION_UNIT,
23 CursorKind.CLASS_DECL,
24 CursorKind.STRUCT_DECL,
26 CursorKind.CLASS_TEMPLATE
30 CursorKind.CLASS_DECL,
31 CursorKind.STRUCT_DECL,
33 CursorKind.ENUM_CONSTANT_DECL,
34 CursorKind.CLASS_TEMPLATE,
35 CursorKind.FUNCTION_DECL,
36 CursorKind.FUNCTION_TEMPLATE,
37 CursorKind.CONVERSION_FUNCTION,
38 CursorKind.CXX_METHOD,
39 CursorKind.CONSTRUCTOR,
44 '<=': 'le', '>=': 'ge', '==': 'eq', '!=': 'ne', '[]': 'array',
45 '+=': 'iadd', '-=': 'isub', '*=': 'imul', '/=': 'idiv', '%=':
46 'imod', '&=': 'iand', '|=': 'ior', '^=': 'ixor', '<<=': 'ilshift',
47 '>>=': 'irshift', '++': 'inc', '--': 'dec', '<<': 'lshift', '>>':
48 'rshift', '&&': 'land', '||': 'lor', '!': 'lnot', '~': 'bnot',
49 '&': 'band', '|': 'bor', '+': 'add', '-': 'sub', '*': 'mul', '/':
50 'div', '%': 'mod', '<': 'lt', '>': 'gt', '=': 'assign', '()': 'call'
53 CPP_OPERATORS = OrderedDict(
54 sorted(CPP_OPERATORS.items(), key=lambda t: -len(t[0])))
56 job_count = cpu_count()
57 job_semaphore = Semaphore(job_count)
62 return s.decode('utf8')
65 def sanitize_name(name):
66 name = re.sub(r'type-parameter-0-([0-9]+)', r'T\1', name)
67 for k, v in CPP_OPERATORS.items():
68 name = name.replace('operator%s' % k, 'operator_%s' % v)
69 name = re.sub('<.*>', '', name)
70 name = ''.join([ch if ch.isalnum() else '_' for ch in name])
71 name = re.sub('_$', '', re.sub('_+', '_', name))
72 return '__doc_' + name
75 def process_comment(comment):
78 # Remove C++ comment syntax
79 leading_spaces = float('inf')
80 for s in comment.expandtabs(tabsize=4).splitlines():
82 if s.startswith('/*'):
84 elif s.endswith('*/'):
85 s = s[:-2].rstrip('*')
86 elif s.startswith('///'):
91 leading_spaces = min(leading_spaces, len(s) - len(s.lstrip()))
94 if leading_spaces != float('inf'):
96 for s in result.splitlines():
97 result2 += s[leading_spaces:] + '\n'
101 cpp_group = '([\w:]+)'
102 param_group = '([\[\w:\]]+)'
105 s = re.sub(r'\\c\s+%s' % cpp_group, r'``\1``', s)
106 s = re.sub(r'\\a\s+%s' % cpp_group, r'*\1*', s)
107 s = re.sub(r'\\e\s+%s' % cpp_group, r'*\1*', s)
108 s = re.sub(r'\\em\s+%s' % cpp_group, r'*\1*', s)
109 s = re.sub(r'\\b\s+%s' % cpp_group, r'**\1**', s)
110 s = re.sub(r'\\ingroup\s+%s' % cpp_group, r'', s)
111 s = re.sub(r'\\param%s?\s+%s' % (param_group, cpp_group),
112 r'\n\n$Parameter ``\2``:\n\n', s)
113 s = re.sub(r'\\tparam%s?\s+%s' % (param_group, cpp_group),
114 r'\n\n$Template parameter ``\2``:\n\n', s)
119 'authors': 'Authors',
120 'copyright': 'Copyright',
125 'extends': 'Extends',
129 s = re.sub(r'\\%s\s*' % in_, r'\n\n$%s:\n\n' % out_, s)
131 s = re.sub(r'\\details\s*', r'\n\n', s)
132 s = re.sub(r'\\brief\s*', r'', s)
133 s = re.sub(r'\\short\s*', r'', s)
134 s = re.sub(r'\\ref\s*', r'', s)
136 s = re.sub(r'\\code\s?(.*?)\s?\\endcode',
137 r"```\n\1\n```\n", s, flags=re.DOTALL)
140 s = re.sub(r'<tt>(.*?)</tt>', r'``\1``', s, flags=re.DOTALL)
141 s = re.sub(r'<pre>(.*?)</pre>', r"```\n\1\n```\n", s, flags=re.DOTALL)
142 s = re.sub(r'<em>(.*?)</em>', r'*\1*', s, flags=re.DOTALL)
143 s = re.sub(r'<b>(.*?)</b>', r'**\1**', s, flags=re.DOTALL)
144 s = re.sub(r'\\f\$(.*?)\\f\$', r'$\1$', s, flags=re.DOTALL)
145 s = re.sub(r'<li>', r'\n\n* ', s)
146 s = re.sub(r'</?ul>', r'', s)
147 s = re.sub(r'</li>', r'\n\n', s)
149 s = s.replace('``true``', '``True``')
150 s = s.replace('``false``', '``False``')
153 wrapper = textwrap.TextWrapper()
154 wrapper.expand_tabs = True
155 wrapper.replace_whitespace = True
156 wrapper.drop_whitespace = True
158 wrapper.initial_indent = wrapper.subsequent_indent = ''
161 in_code_segment = False
162 for x in re.split(r'(```)', s):
164 if not in_code_segment:
167 result += '\n```\n\n'
168 in_code_segment = not in_code_segment
169 elif in_code_segment:
172 for y in re.split(r'(?: *\n *){2,}', x):
173 wrapped = wrapper.fill(re.sub(r'\s+', ' ', y).strip())
174 if len(wrapped) > 0 and wrapped[0] == '$':
175 result += wrapped[1:] + '\n'
176 wrapper.initial_indent = \
177 wrapper.subsequent_indent = ' ' * 4
180 result += wrapped + '\n\n'
181 wrapper.initial_indent = wrapper.subsequent_indent = ''
182 return result.rstrip().lstrip('\n')
185 def extract(filename, node, prefix):
186 if not (node.location.file is None or
187 os.path.samefile(d(node.location.file.name), filename)):
189 if node.kind in RECURSE_LIST:
191 if node.kind != CursorKind.TRANSLATION_UNIT:
192 if len(sub_prefix) > 0:
194 sub_prefix += d(node.spelling)
195 for i in node.get_children():
196 extract(filename, i, sub_prefix)
197 if node.kind in PRINT_LIST:
198 comment = d(node.raw_comment) if node.raw_comment is not None else ''
199 comment = process_comment(comment)
201 if len(sub_prefix) > 0:
203 if len(node.spelling) > 0:
204 name = sanitize_name(sub_prefix + d(node.spelling))
206 output.append((name, filename, comment))
209 class ExtractionThread(Thread):
210 def __init__(self, filename, parameters):
211 Thread.__init__(self)
212 self.filename = filename
213 self.parameters = parameters
214 job_semaphore.acquire()
217 print('Processing "%s" ..' % self.filename, file=sys.stderr)
219 index = cindex.Index(
220 cindex.conf.lib.clang_createIndex(False, True))
221 tu = index.parse(self.filename, self.parameters)
222 extract(self.filename, tu.cursor, '')
224 job_semaphore.release()
226 if __name__ == '__main__':
227 parameters = ['-x', 'c++', '-std=c++11']
230 if platform.system() == 'Darwin':
231 dev_path = '/Applications/Xcode.app/Contents/Developer/'
232 lib_dir = dev_path + 'Toolchains/XcodeDefault.xctoolchain/usr/lib/'
233 sdk_dir = dev_path + 'Platforms/MacOSX.platform/Developer/SDKs'
234 libclang = lib_dir + 'libclang.dylib'
236 if os.path.exists(libclang):
237 cindex.Config.set_library_path(os.path.dirname(libclang))
239 if os.path.exists(sdk_dir):
240 sysroot_dir = os.path.join(sdk_dir, next(os.walk(sdk_dir))[1][0])
241 parameters.append('-isysroot')
242 parameters.append(sysroot_dir)
244 for item in sys.argv[1:]:
245 if item.startswith('-'):
246 parameters.append(item)
248 filenames.append(item)
250 if len(filenames) == 0:
251 print('Syntax: %s [.. a list of header files ..]' % sys.argv[0])
255 This file contains docstrings for the Python bindings.
256 Do not edit! These were automatically extracted by mkdoc.py
259 #define __EXPAND(x) x
260 #define __COUNT(_1, _2, _3, _4, _5, _6, _7, COUNT, ...) COUNT
261 #define __VA_SIZE(...) __EXPAND(__COUNT(__VA_ARGS__, 7, 6, 5, 4, 3, 2, 1))
262 #define __CAT1(a, b) a ## b
263 #define __CAT2(a, b) __CAT1(a, b)
264 #define __DOC1(n1) __doc_##n1
265 #define __DOC2(n1, n2) __doc_##n1##_##n2
266 #define __DOC3(n1, n2, n3) __doc_##n1##_##n2##_##n3
267 #define __DOC4(n1, n2, n3, n4) __doc_##n1##_##n2##_##n3##_##n4
268 #define __DOC5(n1, n2, n3, n4, n5) __doc_##n1##_##n2##_##n3##_##n4##_##n5
269 #define __DOC6(n1, n2, n3, n4, n5, n6) __doc_##n1##_##n2##_##n3##_##n4##_##n5##_##n6
270 #define __DOC7(n1, n2, n3, n4, n5, n6, n7) __doc_##n1##_##n2##_##n3##_##n4##_##n5##_##n6##_##n7
271 #define DOC(...) __EXPAND(__EXPAND(__CAT2(__DOC, __VA_SIZE(__VA_ARGS__)))(__VA_ARGS__))
273 #if defined(__GNUG__)
274 #pragma GCC diagnostic push
275 #pragma GCC diagnostic ignored "-Wunused-variable"
280 for filename in filenames:
281 thr = ExtractionThread(filename, parameters)
284 print('Waiting for jobs to finish ..', file=sys.stderr)
285 for i in range(job_count):
286 job_semaphore.acquire()
290 for name, _, comment in list(sorted(output, key=lambda x: (x[0], x[1]))):
291 if name == name_prev:
293 name = name + "_%i" % name_ctr
297 print('\nstatic const char *%s =%sR"doc(%s)doc";' %
298 (name, '\n' if '\n' in comment else ' ', comment))
301 #if defined(__GNUG__)
302 #pragma GCC diagnostic pop