Skip to content
Snippets Groups Projects
Commit cfbc6232 authored by Piotr Maślanka's avatar Piotr Maślanka
Browse files

v1.2.3

parent 3a4bafb3
No related branches found
No related tags found
No related merge requests found
# v1.2.3
* _TBA_
* `Multibuild` will pass given kwargs to `Extension` object
* added an option to monkey-patch `distutils` to compile multicore
# v1.2.2
......
from setuptools import setup
from snakehouse import Multibuild, build
from snakehouse import Multibuild, build, monkey_patch_parallel_compilation
from setuptools import Extension
monkey_patch_parallel_compilation()
# note that you can include standard Extension classes in this list, those won't be touched
# and will be directed directly to Cython.Build.cythonize()
cython_multibuilds = [
......@@ -11,7 +13,8 @@ cython_multibuilds = [
Multibuild('example_module', ['example_module/test.pyx', 'example_module/test2.pyx',
'example_module/test3/test3.pyx',
'example_module/test3/test2.pyx',
'example_module/test_n.c']),
'example_module/test_n.c'],
define_macros=[("CYTHON_TRACE_NOGIL", "1")]),
Extension('example2.example', ['example2/example.pyx']),
Multibuild('example3.example3.example3', ['example3/example3/example3/test.pyx'])
]
......
from .build import build
from .multibuild import Multibuild
from .faster_builds import monkey_patch_parallel_compilation
__version__ = '1.2.3_a1'
__version__ = '1.2.3'
import multiprocessing
__all__ = ['monkey_patch_parallel_compilation']
def monkey_patch_parallel_compilation(cores: tp.Optional[int] = None) -> None:
"""
This monkey-patches distutils to provide parallel compilation, even if you have
a single extension built from multiple .c files.
Invoke in your setup.py file
:param cores: amount of cores. Leave at default (None) for autodetection.
"""
if cores is None:
cores = multiprocessing.cpu_count()
# monkey-patch for parallel compilation
def parallelCCompile(self, sources, output_dir=None, macros=None, include_dirs=None, debug=0,
extra_preargs=None, extra_postargs=None, depends=None):
# those lines are copied from distutils.ccompiler.CCompiler directly
macros, objects, extra_postargs, pp_opts, build = self._setup_compile(output_dir, macros,
include_dirs, sources,
depends,
extra_postargs)
cc_args = self._get_cc_args(pp_opts, debug, extra_preargs)
# parallel code
import multiprocessing.pool
def single_compile(obj):
try:
src, ext = build[obj]
except KeyError:
return
self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts)
# evaluate everything
for _ in multiprocessing.pool.ThreadPool(cores).imap(single_compile, objects):
pass
return objects
import distutils.ccompiler
distutils.ccompiler.CCompiler.compile = parallelCCompile
......@@ -51,6 +51,7 @@ class Multibuild:
All kwargs will be sent straight to Cython's Extension
:param extension_name: the module name
:param files: list of pyx and c files
:param kwargs: extra arguments to be passed to Extension() object
"""
def __init__(self, extension_name: str, files: tp.Iterator[str], **kwargs):
# sanitize path separators so that Linux-style paths are supported on Windows
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment