diff --git a/CHANGELOG.md b/CHANGELOG.md index 05ed17620fb9e00b917975991e8125801d156a50..436d5b7f0a87e832f22bb077caf2a3d7297a049d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # v1.2.3 -* _TBA_ +* `Multibuild` will pass given kwargs to `Extension` object +* added an option to monkey-patch `distutils` to compile multicore # v1.2.2 diff --git a/example/setup.py b/example/setup.py index bcec7bf58991c736715dda1b703eb26aa30bd804..b0832cd52d13aa9a62f71e117b28cbcfc8766c64 100644 --- a/example/setup.py +++ b/example/setup.py @@ -1,8 +1,10 @@ from setuptools import setup -from snakehouse import Multibuild, build +from snakehouse import Multibuild, build, monkey_patch_parallel_compilation from setuptools import Extension +monkey_patch_parallel_compilation() + # note that you can include standard Extension classes in this list, those won't be touched # and will be directed directly to Cython.Build.cythonize() cython_multibuilds = [ @@ -11,7 +13,8 @@ cython_multibuilds = [ Multibuild('example_module', ['example_module/test.pyx', 'example_module/test2.pyx', 'example_module/test3/test3.pyx', 'example_module/test3/test2.pyx', - 'example_module/test_n.c']), + 'example_module/test_n.c'], + define_macros=[("CYTHON_TRACE_NOGIL", "1")]), Extension('example2.example', ['example2/example.pyx']), Multibuild('example3.example3.example3', ['example3/example3/example3/test.pyx']) ] diff --git a/snakehouse/__init__.py b/snakehouse/__init__.py index d504b6da292ccf33d2523adfa53b3a5359beb93c..849c48b8cb35f9ad0d3d3bd3a7e7187aec8a80a4 100644 --- a/snakehouse/__init__.py +++ b/snakehouse/__init__.py @@ -1,4 +1,5 @@ from .build import build from .multibuild import Multibuild +from .faster_builds import monkey_patch_parallel_compilation -__version__ = '1.2.3_a1' +__version__ = '1.2.3' diff --git a/snakehouse/faster_builds.py b/snakehouse/faster_builds.py new file mode 100644 index 0000000000000000000000000000000000000000..13c9cc3759211853c2bb3f8b438d60df34950954 --- /dev/null +++ b/snakehouse/faster_builds.py @@ -0,0 +1,43 @@ +import multiprocessing + +__all__ = ['monkey_patch_parallel_compilation'] + + +def monkey_patch_parallel_compilation(cores: tp.Optional[int] = None) -> None: + """ + This monkey-patches distutils to provide parallel compilation, even if you have + a single extension built from multiple .c files. + + Invoke in your setup.py file + + :param cores: amount of cores. Leave at default (None) for autodetection. + """ + if cores is None: + cores = multiprocessing.cpu_count() + + # monkey-patch for parallel compilation + def parallelCCompile(self, sources, output_dir=None, macros=None, include_dirs=None, debug=0, + extra_preargs=None, extra_postargs=None, depends=None): + # those lines are copied from distutils.ccompiler.CCompiler directly + macros, objects, extra_postargs, pp_opts, build = self._setup_compile(output_dir, macros, + include_dirs, sources, + depends, + extra_postargs) + cc_args = self._get_cc_args(pp_opts, debug, extra_preargs) + # parallel code + import multiprocessing.pool + + def single_compile(obj): + try: + src, ext = build[obj] + except KeyError: + return + self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts) + + # evaluate everything + for _ in multiprocessing.pool.ThreadPool(cores).imap(single_compile, objects): + pass + return objects + + import distutils.ccompiler + distutils.ccompiler.CCompiler.compile = parallelCCompile diff --git a/snakehouse/multibuild.py b/snakehouse/multibuild.py index f17b06afcb7dd7a2f406821c483df295ad2bd202..baba3d9e1939dd2991a679d1c6ee47737699d42c 100644 --- a/snakehouse/multibuild.py +++ b/snakehouse/multibuild.py @@ -51,6 +51,7 @@ class Multibuild: All kwargs will be sent straight to Cython's Extension :param extension_name: the module name :param files: list of pyx and c files + :param kwargs: extra arguments to be passed to Extension() object """ def __init__(self, extension_name: str, files: tp.Iterator[str], **kwargs): # sanitize path separators so that Linux-style paths are supported on Windows