From cfbc62329d9c1aa502d0d6be8bd451851df77e6b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Piotr=20Ma=C5=9Blanka?= <piotr.maslanka@henrietta.com.pl>
Date: Mon, 14 Dec 2020 18:39:07 +0100
Subject: [PATCH] v1.2.3
---
CHANGELOG.md | 3 ++-
example/setup.py | 7 ++++--
snakehouse/__init__.py | 3 ++-
snakehouse/faster_builds.py | 43 +++++++++++++++++++++++++++++++++++++
snakehouse/multibuild.py | 1 +
5 files changed, 53 insertions(+), 4 deletions(-)
create mode 100644 snakehouse/faster_builds.py
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 05ed176..436d5b7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,7 @@
# v1.2.3
-* _TBA_
+* `Multibuild` will pass given kwargs to `Extension` object
+* added an option to monkey-patch `distutils` to compile multicore
# v1.2.2
diff --git a/example/setup.py b/example/setup.py
index bcec7bf..b0832cd 100644
--- a/example/setup.py
+++ b/example/setup.py
@@ -1,8 +1,10 @@
from setuptools import setup
-from snakehouse import Multibuild, build
+from snakehouse import Multibuild, build, monkey_patch_parallel_compilation
from setuptools import Extension
+monkey_patch_parallel_compilation()
+
# note that you can include standard Extension classes in this list, those won't be touched
# and will be directed directly to Cython.Build.cythonize()
cython_multibuilds = [
@@ -11,7 +13,8 @@ cython_multibuilds = [
Multibuild('example_module', ['example_module/test.pyx', 'example_module/test2.pyx',
'example_module/test3/test3.pyx',
'example_module/test3/test2.pyx',
- 'example_module/test_n.c']),
+ 'example_module/test_n.c'],
+ define_macros=[("CYTHON_TRACE_NOGIL", "1")]),
Extension('example2.example', ['example2/example.pyx']),
Multibuild('example3.example3.example3', ['example3/example3/example3/test.pyx'])
]
diff --git a/snakehouse/__init__.py b/snakehouse/__init__.py
index d504b6d..849c48b 100644
--- a/snakehouse/__init__.py
+++ b/snakehouse/__init__.py
@@ -1,4 +1,5 @@
from .build import build
from .multibuild import Multibuild
+from .faster_builds import monkey_patch_parallel_compilation
-__version__ = '1.2.3_a1'
+__version__ = '1.2.3'
diff --git a/snakehouse/faster_builds.py b/snakehouse/faster_builds.py
new file mode 100644
index 0000000..13c9cc3
--- /dev/null
+++ b/snakehouse/faster_builds.py
@@ -0,0 +1,43 @@
+import multiprocessing
+
+__all__ = ['monkey_patch_parallel_compilation']
+
+
+def monkey_patch_parallel_compilation(cores: tp.Optional[int] = None) -> None:
+ """
+ This monkey-patches distutils to provide parallel compilation, even if you have
+ a single extension built from multiple .c files.
+
+ Invoke in your setup.py file
+
+ :param cores: amount of cores. Leave at default (None) for autodetection.
+ """
+ if cores is None:
+ cores = multiprocessing.cpu_count()
+
+ # monkey-patch for parallel compilation
+ def parallelCCompile(self, sources, output_dir=None, macros=None, include_dirs=None, debug=0,
+ extra_preargs=None, extra_postargs=None, depends=None):
+ # those lines are copied from distutils.ccompiler.CCompiler directly
+ macros, objects, extra_postargs, pp_opts, build = self._setup_compile(output_dir, macros,
+ include_dirs, sources,
+ depends,
+ extra_postargs)
+ cc_args = self._get_cc_args(pp_opts, debug, extra_preargs)
+ # parallel code
+ import multiprocessing.pool
+
+ def single_compile(obj):
+ try:
+ src, ext = build[obj]
+ except KeyError:
+ return
+ self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts)
+
+ # evaluate everything
+ for _ in multiprocessing.pool.ThreadPool(cores).imap(single_compile, objects):
+ pass
+ return objects
+
+ import distutils.ccompiler
+ distutils.ccompiler.CCompiler.compile = parallelCCompile
diff --git a/snakehouse/multibuild.py b/snakehouse/multibuild.py
index f17b06a..baba3d9 100644
--- a/snakehouse/multibuild.py
+++ b/snakehouse/multibuild.py
@@ -51,6 +51,7 @@ class Multibuild:
All kwargs will be sent straight to Cython's Extension
:param extension_name: the module name
:param files: list of pyx and c files
+ :param kwargs: extra arguments to be passed to Extension() object
"""
def __init__(self, extension_name: str, files: tp.Iterator[str], **kwargs):
# sanitize path separators so that Linux-style paths are supported on Windows
--
GitLab