diff --git a/CHANGELOG.md b/CHANGELOG.md index 3507f4a3d1faa7d8d9e4ee19ac24c00f58a1a424..bc5b23cfa69b2d600bdaf2c902aa2f331017290f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +# v1.2 + +* fixed issue #1 + # v1.1.2 * bugfix release: fixed behaviour if there was only diff --git a/example/example_module/test3/test2.pyx b/example/example_module/test3/test2.pyx new file mode 100644 index 0000000000000000000000000000000000000000..503bfe88f7a8caba07b15a05393a9e45ed05e3bb --- /dev/null +++ b/example/example_module/test3/test2.pyx @@ -0,0 +1,2 @@ +def times_seven(a): + return a*7 diff --git a/example/setup.py b/example/setup.py index e124631f79463f698214b3fd3cc99b0518f87cd6..bcec7bf58991c736715dda1b703eb26aa30bd804 100644 --- a/example/setup.py +++ b/example/setup.py @@ -10,6 +10,7 @@ cython_multibuilds = [ # the reverse not necessarily being true (issue #5) Multibuild('example_module', ['example_module/test.pyx', 'example_module/test2.pyx', 'example_module/test3/test3.pyx', + 'example_module/test3/test2.pyx', 'example_module/test_n.c']), Extension('example2.example', ['example2/example.pyx']), Multibuild('example3.example3.example3', ['example3/example3/example3/test.pyx']) diff --git a/example/tests/test_test.py b/example/tests/test_test.py index 0c0288cffa25e10e56953a78977124694a66100d..cb226877fed3ef07929d5dd2c9fee412e224e3fe 100644 --- a/example/tests/test_test.py +++ b/example/tests/test_test.py @@ -1,12 +1,16 @@ from example_module.test import times_two from example_module.test2 import times_three, times_five from example_module.test3.test3 import times_four +from example_module.test3.test2 import times_seven from example2.example import test from example3.example3.example3.test import test as test_three import unittest class TestExample(unittest.TestCase): + def test_seven(self): + self.assertEqual(times_seven(4), 4*7) + def test_three(self): self.assertEqual(test_three(2, 3), 5) diff --git a/snakehouse/__init__.py b/snakehouse/__init__.py index 8ee34e415eb1987cf0ee1b5930dac69015df8d49..23448b0ee1871ad685c87c51dea8113dd89ee664 100644 --- a/snakehouse/__init__.py +++ b/snakehouse/__init__.py @@ -1,4 +1,4 @@ from .build import build from .multibuild import Multibuild -__version__ = '1.1.2' +__version__ = '1.2' diff --git a/snakehouse/build.py b/snakehouse/build.py index ce92b3702adadc56439415bdf8cebd423f7ce7dd..1d4329ef237a668194a92e7809f9047ddb80a8f4 100644 --- a/snakehouse/build.py +++ b/snakehouse/build.py @@ -1,3 +1,4 @@ +import logging import typing as tp from Cython.Build import cythonize from setuptools import Extension @@ -5,16 +6,24 @@ from .multibuild import Multibuild MultiBuildType = tp.Union[Multibuild, Exception] +logger = logging.getLogger(__name__) + def build(extensions: tp.List[MultiBuildType], *args, **kwargs): returns = [] + multi_builds = [] for multi_build in extensions: if isinstance(multi_build, Extension): returns.append(multi_build) elif isinstance(multi_build, Multibuild): multi_build.generate() + multi_builds.append(multi_build) returns.append(multi_build.for_cythonize()) else: raise ValueError('Invalid value in list, expected either an instance of Multibuild ' 'or an Extension') - return cythonize(returns, *args, **kwargs) + values = cythonize(returns, *args, **kwargs) + for multi_build in multi_builds: + multi_build.do_after_cython() + logger.warning(multi_build.module_name_to_loader_function) + return values \ No newline at end of file diff --git a/snakehouse/multibuild.py b/snakehouse/multibuild.py index 1f75124fcdc8c7dca4831006b52814f8ff9949b5..6dcb7c550adb26773de2f44f89819937bab47a36 100644 --- a/snakehouse/multibuild.py +++ b/snakehouse/multibuild.py @@ -1,7 +1,9 @@ +import hashlib import os import collections import typing as tp import logging + import pkg_resources from satella.files import split from mako.template import Template @@ -10,18 +12,37 @@ from setuptools import Extension logger = logging.getLogger(__name__) -CdefSection = collections.namedtuple('CdefSection', ('h_file_name', 'module_name')) +CdefSection = collections.namedtuple('CdefSection', ('h_file_name', 'module_name', 'coded_module_name')) GetDefinitionSection = collections.namedtuple('GetDefinitionSection', ( - 'module_name', 'pyinit_name' + 'module_name', 'pyinit_name', 'coded_module_name' )) +def load_mako_lines(template_name: str) -> tp.List[str]: + return pkg_resources.resource_string('snakehouse', os.path.join('templates', template_name)).decode('utf8') + + +def cull_path(path): + if not path: + return path + if path[0] == os.path.sep: + if len(path) > 1: + if path[1] == os.path.sep: + return path[2:] + return path[1:] + else: + return path + + def render_mako(template_name: str, **kwargs) -> str: tpl = Template(pkg_resources.resource_string( 'snakehouse', os.path.join('templates', template_name)).decode('utf8')) return tpl.render(**kwargs) +LINES_IN_HFILE = len(load_mako_lines('hfile.mako').split('\n')) + + class Multibuild: """ This specifies a single Cython extension, called {extension_name}.__bootstrap__ @@ -32,12 +53,9 @@ class Multibuild: :param files: list of pyx and c files """ # sanitize path separators so that Linux-style paths are supported on Windows + files = list(files) files = [os.path.join(*split(file)) for file in files] self.files = list([file for file in files if not file.endswith('__bootstrap__.pyx')]) - file_name_set = set(os.path.split(file)[1] for file in self.files) - if len(self.files) != len(file_name_set): - raise ValueError('Two modules with the same name cannot appear together in a single ' - 'Multibuild') self.pyx_files = [file for file in files if file.endswith('.pyx')] @@ -46,55 +64,87 @@ class Multibuild: self.bootstrap_directory, _ = os.path.split(self.files[0]) # type: str else: self.bootstrap_directory = os.path.commonpath(self.files) # type: str - self.modules = set() # type: tp.Set[tp.Tuple[str, str]] + self.modules = [] # type: tp.List[tp.Tuple[str, str, str]] + self.module_name_to_loader_function = {} + for filename in self.pyx_files: + with open(filename, 'rb') as f_in: + self.module_name_to_loader_function[filename] = hashlib.sha256(f_in.read()).hexdigest() def generate_header_files(self): for filename in self.pyx_files: - path, name = os.path.split(filename) + path, name, cmod_name_path, module_name, coded_module_name, complete_module_name = self.transform_module_name(filename) if not name.endswith('.pyx'): continue - module_name = name.replace('.pyx', '') - h_name = name.replace('.pyx', '.h') + h_file = filename.replace('.pyx', '.h') + if os.path.exists(h_file): + with open(h_file, 'r') as f_in: + data = f_in.readlines() + + linesep = 'cr' if '\r\n' in data[0] else 'lf' + + if not any('PyObject* PyInit_' in line for line in data): + data = [render_mako('hfile.mako', initpy_name=coded_module_name)+ \ + '\r\n' if linesep == 'cr' else '\n'] + data[LINES_IN_HFILE:] + else: + data = render_mako('hfile.mako', initpy_name=coded_module_name) - if os.path.exists(h_name): - with open(os.path.join(path, h_name), 'r') as f_in: - data = f_in.read() + with open(h_file, 'w') as f_out: + f_out.write(''.join(data)) - if 'PyObject* PyInit_' not in data: - data = render_mako('hfile.mako', initpy_name=module_name) + data + def transform_module_name(self, filename): + path, name = os.path.split(filename) + module_name = name.replace('.pyx', '') + if path.startswith(self.bootstrap_directory): + cmod_name_path = path[len(self.bootstrap_directory):] + else: + cmod_name_path = path + path = cull_path(path) + cmod_name_path = cull_path(cmod_name_path) + + if path: + intro = '.'.join((e for e in cmod_name_path.split(os.path.sep) if e)) + if not intro: + complete_module_name = '%s.%s' % (self.extension_name, module_name) else: - data = render_mako('hfile.mako', initpy_name=module_name) + complete_module_name = '%s.%s.%s' % (self.extension_name, + intro, + module_name) + else: + complete_module_name = '%s.%s' % (self.extension_name, module_name) + + coded_module_name = self.module_name_to_loader_function[filename] + return path, name, cmod_name_path, module_name, coded_module_name, complete_module_name - with open(os.path.join(path, h_name), 'w') as f_out: + def do_after_cython(self): + for filename in self.pyx_files: + path, name, cmod_name_path, module_name, coded_module_name, complete_module_name = self.transform_module_name(filename) + to_replace = '__Pyx_PyMODINIT_FUNC PyInit_%s' % (module_name, ) + replace_with = '__Pyx_PyMODINIT_FUNC PyInit_%s' % (coded_module_name, ) + with open(filename.replace('.pyx', '.c'), 'r') as f_in: + data_in = f_in.read() + data = data_in.replace(to_replace, replace_with) + with open(filename.replace('.pyx', '.c'), 'w') as f_out: f_out.write(data) def generate_bootstrap(self) -> str: cdef_section = [] for filename in self.pyx_files: - path, name = os.path.split(filename) - if path.startswith(self.bootstrap_directory): - path = path[len(self.bootstrap_directory):] - module_name = name.replace('.pyx', '') - if path: - h_path_name = os.path.join(path[1:], name.replace('.pyx', '.h')).\ - replace('\\', '\\\\') - else: - h_path_name = name.replace('.pyx', '.h') - cdef_section.append(CdefSection(h_path_name, module_name)) + path, name, cmod_name_path, module_name, coded_module_name, complete_module_name = self.transform_module_name(filename) - if path: - complete_module_name = self.extension_name+'.'+'.'.join(path[1:].split( - os.path.sep))+'.'+module_name - else: - complete_module_name = self.extension_name + '.'+module_name + if os.path.exists(filename.replace('.pyx', '.c')): + os.unlink(filename.replace('.pyx', '.c')) - self.modules.add((complete_module_name, module_name, )) + h_path_name = os.path.join(cmod_name_path, name.replace('.pyx', '.h')).replace('\\', '\\\\') + + cdef_section.append(CdefSection(h_path_name, module_name, coded_module_name)) + + self.modules.append((complete_module_name, module_name, coded_module_name)) get_definition = [] - for mod_name, init_fun_name in self.modules: - get_definition.append(GetDefinitionSection(mod_name, init_fun_name)) + for mod_name, init_fun_name, coded_module_name in self.modules: + get_definition.append(GetDefinitionSection(mod_name, init_fun_name, coded_module_name)) return render_mako('bootstrap.mako', cdef_sections=cdef_section, get_definition_sections=get_definition, @@ -124,9 +174,9 @@ class Multibuild: def for_cythonize(self, *args, **kwargs): for_cythonize = [*self.files, os.path.join(self.bootstrap_directory, '__bootstrap__.pyx')] - logger.warning('For cythonize: %s', for_cythonize) - return Extension(self.extension_name+".__bootstrap__", for_cythonize, *args, **kwargs) + + diff --git a/snakehouse/templates/bootstrap.mako b/snakehouse/templates/bootstrap.mako index 508cb0f70e04988ebced27ff1facd0b58675bf0f..acfacd7602785f31953667b79fd10835a2219356 100644 --- a/snakehouse/templates/bootstrap.mako +++ b/snakehouse/templates/bootstrap.mako @@ -1,3 +1,5 @@ +import sys + cdef extern from "Python.h": ctypedef struct PyModuleDef: const char* m_name; @@ -8,21 +10,21 @@ cdef extern from "Python.h": % for cdef_section in cdef_sections: cdef extern from "${cdef_section.h_file_name}": - object PyInit_${cdef_section.module_name}() + object PyInit_${cdef_section.coded_module_name}() % endfor cdef object get_definition_by_name(str name): % for i, getdef_section in enumerate(get_definition_sections): % if i == 0: if name == "${getdef_section.module_name}": - return PyInit_${getdef_section.pyinit_name}() + return PyInit_${getdef_section.coded_module_name}() % else: elif name == "${getdef_section.module_name}": - return PyInit_${getdef_section.pyinit_name}() + return PyInit_${getdef_section.coded_module_name}() % endif % endfor -import sys + cdef class CythonPackageLoader: cdef PyModuleDef* definition