From d67590f4368b034b521b17a8de4a01a6c33ebf61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Ma=C5=9Blanka?= <piotr.maslanka@henrietta.com.pl> Date: Wed, 22 Jul 2020 19:08:40 +0200 Subject: [PATCH] 1.2.2 --- CHANGELOG.md | 7 +- .../example3/example3/example3/__init__.py | 2 - example/example_module/__init__.py | 3 - snakehouse/__init__.py | 2 +- snakehouse/multibuild.py | 77 ++++++++++++------- snakehouse/templates/bootstrap.mako | 1 - snakehouse/templates/hfile.mako | 2 +- 7 files changed, 59 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a707250..b85f23a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,11 @@ +# v1.2.2 + +* snakehouse will pass the remaining arguments to Multibuild to Cython's Extension + # v1.2.1 -* _TBA_ +* snakehouse won't complain anymore if installing + from a source wheel # v1.2 diff --git a/example/example3/example3/example3/__init__.py b/example/example3/example3/example3/__init__.py index 7d924a0..fa84e47 100644 --- a/example/example3/example3/example3/__init__.py +++ b/example/example3/example3/example3/__init__.py @@ -1,5 +1,3 @@ -from example3.example3.example3.__bootstrap__ import bootstrap_cython_submodules -bootstrap_cython_submodules() import logging import typing as tp diff --git a/example/example_module/__init__.py b/example/example_module/__init__.py index a29ecf4..e69de29 100644 --- a/example/example_module/__init__.py +++ b/example/example_module/__init__.py @@ -1,3 +0,0 @@ - -from example_module.__bootstrap__ import bootstrap_cython_submodules -bootstrap_cython_submodules() diff --git a/snakehouse/__init__.py b/snakehouse/__init__.py index f0e1638..49ccbca 100644 --- a/snakehouse/__init__.py +++ b/snakehouse/__init__.py @@ -1,4 +1,4 @@ from .build import build from .multibuild import Multibuild -__version__ = '1.2.1_a1' +__version__ = '1.2.2' diff --git a/snakehouse/multibuild.py b/snakehouse/multibuild.py index c1605d1..f17b06a 100644 --- a/snakehouse/multibuild.py +++ b/snakehouse/multibuild.py @@ -1,7 +1,10 @@ import hashlib import os +import logging import collections import typing as tp +import warnings + import pkg_resources from satella.files import split from mako.template import Template @@ -13,6 +16,8 @@ GetDefinitionSection = collections.namedtuple('GetDefinitionSection', ( 'module_name', 'pyinit_name', 'coded_module_name' )) +logger = logging.getLogger(__name__) + def load_mako_lines(template_name: str) -> tp.List[str]: return pkg_resources.resource_string('snakehouse', os.path.join('templates', template_name)).decode('utf8') @@ -42,48 +47,66 @@ LINES_IN_HFILE = len(load_mako_lines('hfile.mako').split('\n')) class Multibuild: """ This specifies a single Cython extension, called {extension_name}.__bootstrap__ + + All kwargs will be sent straight to Cython's Extension + :param extension_name: the module name + :param files: list of pyx and c files """ - def __init__(self, extension_name: str, files: tp.Iterator[str]): - """ - :param extension_name: the module name - :param files: list of pyx and c files - """ + def __init__(self, extension_name: str, files: tp.Iterator[str], **kwargs): # sanitize path separators so that Linux-style paths are supported on Windows files = list(files) - files = [os.path.join(*split(file)) for file in files] - self.files = list([file for file in files if not file.endswith('__bootstrap__.pyx')]) - - self.pyx_files = [file for file in files if file.endswith('.pyx')] + self.kwargs = kwargs + if files: + files = [os.path.join(*split(file)) for file in files] + self.files = list([file for file in files if not file.endswith('__bootstrap__.pyx')]) + logger.warning(str(self.files)) + self.pyx_files = [file for file in self.files if file.endswith('.pyx')] + else: + self.pyx_files = [] - self.extension_name = extension_name # type: str - if len(self.files) == 1: - self.bootstrap_directory, _ = os.path.split(self.files[0]) # type: str + self.do_generate = True + if not self.pyx_files: + warnings.warn('No pyx files, probably installing from a source archive, skipping ' + 'generating files', RuntimeWarning) + self.do_generate = False else: - self.bootstrap_directory = os.path.commonpath(self.files) # type: str - self.modules = [] # type: tp.List[tp.Tuple[str, str, str]] - self.module_name_to_loader_function = {} - for filename in self.pyx_files: - with open(filename, 'rb') as f_in: - self.module_name_to_loader_function[filename] = hashlib.sha256(f_in.read()).hexdigest() + self.extension_name = extension_name # type: str + if len(self.files) == 1: + self.bootstrap_directory, _ = os.path.split(self.files[0]) # type: str + else: + self.bootstrap_directory = os.path.commonpath(self.files) # type: str + self.modules = [] # type: tp.List[tp.Tuple[str, str, str]] + self.module_name_to_loader_function = {} + for filename in self.pyx_files: + with open(filename, 'rb') as f_in: + self.module_name_to_loader_function[filename] = hashlib.sha256(f_in.read()).hexdigest() def generate_header_files(self): for filename in self.pyx_files: path, name, cmod_name_path, module_name, coded_module_name, complete_module_name = self.transform_module_name(filename) + logger.warning('Generating header file for %s' % (filename, )) if not name.endswith('.pyx'): continue h_file = filename.replace('.pyx', '.h') + if os.path.exists(h_file): with open(h_file, 'r') as f_in: data = f_in.readlines() linesep = 'cr' if '\r\n' in data[0] else 'lf' - - if not any('PyObject* PyInit_' in line for line in data): - data = [render_mako('hfile.mako', initpy_name=coded_module_name)+ \ - '\r\n' if linesep == 'cr' else '\n'] + data[LINES_IN_HFILE:] + rendered_mako = render_mako('hfile.mako', initpy_name=coded_module_name) + \ + '\r\n' if linesep == 'cr' else '\n' + assert len(rendered_mako) > 0 + + if any('#define SNAKEHOUSE_FILE' in line for line in data): + data = [rendered_mako, *data[LINES_IN_HFILE:]] + else: + data = [rendered_mako, *data] else: - data = render_mako('hfile.mako', initpy_name=coded_module_name) + rendered_mako = render_mako('hfile.mako', initpy_name=coded_module_name) + assert len(rendered_mako) > 0 + data = rendered_mako with open(h_file, 'w') as f_out: f_out.write(''.join(data)) @@ -113,6 +136,7 @@ class Multibuild: return path, name, cmod_name_path, module_name, coded_module_name, complete_module_name def do_after_cython(self): + self.generate_header_files() for filename in self.pyx_files: path, name, cmod_name_path, module_name, coded_module_name, complete_module_name = self.transform_module_name(filename) to_replace = '__Pyx_PyMODINIT_FUNC PyInit_%s' % (module_name, ) @@ -164,11 +188,12 @@ class Multibuild: f_out.write(data) def generate(self): - self.generate_header_files() - self.write_bootstrap_file() - self.alter_init() + if self.do_generate: + self.write_bootstrap_file() + self.alter_init() def for_cythonize(self, *args, **kwargs): + kwargs.update(self.kwargs) for_cythonize = [*self.files, os.path.join(self.bootstrap_directory, '__bootstrap__.pyx')] return Extension(self.extension_name+".__bootstrap__", for_cythonize, diff --git a/snakehouse/templates/bootstrap.mako b/snakehouse/templates/bootstrap.mako index acfacd7..9bebeb0 100644 --- a/snakehouse/templates/bootstrap.mako +++ b/snakehouse/templates/bootstrap.mako @@ -25,7 +25,6 @@ cdef object get_definition_by_name(str name): % endfor - cdef class CythonPackageLoader: cdef PyModuleDef* definition cdef object def_o diff --git a/snakehouse/templates/hfile.mako b/snakehouse/templates/hfile.mako index aee3bff..309812b 100644 --- a/snakehouse/templates/hfile.mako +++ b/snakehouse/templates/hfile.mako @@ -1,3 +1,3 @@ #include "Python.h" - +#define SNAKEHOUSE_FILE PyObject* PyInit_${initpy_name}(void); -- GitLab