From 0c42bd64cee6945c4ee58ff479005883f99fb7ba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Piotr=20Ma=C5=9Blanka?= <piotr.maslanka@ericsson.com>
Date: Wed, 21 Aug 2024 13:26:53 +0200
Subject: [PATCH] v2.0
---
.codeclimate.yml | 19 -----------
.gitattributes | 3 --
.github/workflows/ci.yml | 25 ---------------
.gitignore | 3 +-
.gitlab-ci.yml | 4 +--
CHANGELOG.md | 1 +
README.md | 2 +-
build.sh | 10 ------
example/setup.py | 8 ++---
setup.cfg | 2 +-
snakehouse/__init__.py | 6 ----
snakehouse/bootstrap.py | 1 +
snakehouse/build.py | 2 +-
snakehouse/multibuild.py | 68 +++++++++++++++-------------------------
snakehouse/templating.py | 5 +--
15 files changed, 40 insertions(+), 119 deletions(-)
delete mode 100644 .codeclimate.yml
delete mode 100644 .github/workflows/ci.yml
delete mode 100644 build.sh
diff --git a/.codeclimate.yml b/.codeclimate.yml
deleted file mode 100644
index b177cae..0000000
--- a/.codeclimate.yml
+++ /dev/null
@@ -1,19 +0,0 @@
-engines:
- duplication:
- enabled: true
- config:
- languages:
- python:
- fixme:
- enabled: true
- markdownlint:
- enabled: true
- pep8:
- enabled: true
- radon:
- enabled: true
-exclude_paths:
-- example/**
-ratings:
- paths:
- - snakehouse/**
diff --git a/.gitattributes b/.gitattributes
index c096f8e..e69de29 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -1,3 +0,0 @@
-*.mako text eol=lf
-*.sh text eol=lf
-snakehouse/mako.py text eol=lf
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
deleted file mode 100644
index 38d3735..0000000
--- a/.github/workflows/ci.yml
+++ /dev/null
@@ -1,25 +0,0 @@
-name: CI
-run-name: ${{ github.actor }}
-on: [ push ]
-jobs:
- tests:
- runs-on: ubuntu-20.04
- strategy:
- matrix:
- python-version: [ "3.5", "3.6", "3.7", "3.8", "3.9", "3.10", "3.11", "3.12" ]
- steps:
- - uses: actions/checkout@main
- - uses: actions/setup-python@main
- with:
- python-version: ${{ matrix.python-version }}
- cache: pip
- - name: Install everything
- run: |
- pip install -U pip setuptools wheel disttools packaging pyproject.toml
- python setup.py install
- - name: Test
- run: |
- cd example
- python setup.py test
- env:
- DEBUG: "1"
diff --git a/.gitignore b/.gitignore
index 1973bb1..4760533 100644
--- a/.gitignore
+++ b/.gitignore
@@ -13,5 +13,4 @@ dist
__bootstrap__.*
venv/
.venv/
-
-
+*.so
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index b96e8f8..f9b5336 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -3,7 +3,7 @@ stages:
- build
pages:
- image: zoo.smok.co/build/build
+ image: zoo.smok.co/build/build:latest
stage: build
script:
- cd docs
@@ -22,7 +22,7 @@ pages:
script:
- python setup.py install
- cd example
- - python setup.py bdist_wheel
+ - python setup.py build_ext --inplace
test_python35:
extends: .test
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 095fab1..478afb6 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,7 @@
* should work on latest Pythons
* moved the development to SMOK's cloud
* got rid of Satella dependency
+* more tests
# v1.7
diff --git a/README.md b/README.md
index 3216a31..34b281e 100644
--- a/README.md
+++ b/README.md
@@ -10,7 +10,7 @@ snakehouse
[](http://snakehouse.readthedocs.io/en/latest/?badge=latest)
[](https://github.com/smok-serwis/snakehouse)
-Docs are [here](https://smokserwis.docs.smok.co/snakehouse.
+Docs are [here](https://smokserwis.docs.smok.co/snakehouse)..
**IMPORTANT!**
diff --git a/build.sh b/build.sh
deleted file mode 100644
index 1f9ee20..0000000
--- a/build.sh
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/bin/bash
-
-set -x
-set -e
-
-if [ $TRAVIS_BRANCH == "master" ]; then
- pip install wheel twine
- python setup.py bdist bdist_wheel
- twine upload -u $PYPI_USER -p $PYPI_PWD dist/*
-fi
diff --git a/example/setup.py b/example/setup.py
index 179384d..163ef8d 100644
--- a/example/setup.py
+++ b/example/setup.py
@@ -19,12 +19,12 @@ if 'DEBUG' in os.environ:
cython_multibuilds = [
# note that Windows-style pathes are supported on Linux build environment,
# the reverse not necessarily being true (issue #5)
- Multibuild('example_module', find_all('example_module'),
+ Multibuild('example_module', find_all('example_module', include_c_files=True),
define_macros=[("CYTHON_TRACE_NOGIL", "1")],
dont_snakehouse=dont_snakehouse),
- # Extension('example2.example', ['example2/example.pyx']),
- # Multibuild('example3.example3.example3', ['example3/example3/example3/test.pyx'],
- # dont_snakehouse=dont_snakehouse)
+ Extension('example2.example', ['example2/example.pyx']),
+ Multibuild('example3.example3.example3', ['example3/example3/example3/test.pyx'],
+ dont_snakehouse=dont_snakehouse)
]
# first argument is used directly by snakehouse, the rest and **kwargs are passed to
diff --git a/setup.cfg b/setup.cfg
index 910a412..36078fd 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -9,7 +9,7 @@ license_files = LICENSE
author = Piotr MaĹlanka
author_email = pmaslanka@smok.co
description = Utilities for packing multiple pyx files into a single Cython extension
-url = https://github.com/smok-serwis/snakehouse
+url = https://git.dms-serwis.com.pl/smokserwis/snakehouse
project_urls =
Code = https://git.dms-serwis.com.pl/smokserwis/snakehouse
Issue tracker = https://git.dms-serwis.com.pl/smokserwis/snakehouse/-/issues
diff --git a/snakehouse/__init__.py b/snakehouse/__init__.py
index f9b0063..33311e7 100644
--- a/snakehouse/__init__.py
+++ b/snakehouse/__init__.py
@@ -1,10 +1,4 @@
-import pkg_resources
from .build import build
from .multibuild import Multibuild, find_all
from .faster_builds import monkey_patch_parallel_compilation
from .requirements import read_requirements_txt, find_c, find_pyx_and_c, find_pyx
-
-try:
- __version__ = pkg_resources.require('snakehouse')[0].version
-except pkg_resources.DistributionNotFound:
- __version__ = '1.7'
diff --git a/snakehouse/bootstrap.py b/snakehouse/bootstrap.py
index fdac192..addea52 100644
--- a/snakehouse/bootstrap.py
+++ b/snakehouse/bootstrap.py
@@ -18,6 +18,7 @@ cdef extern from "{cdef_section.h_file_name}":
""", cdef_section=cdef_section)
output.append('cdef object get_definition_by_name(str name):')
+ output.append(' sys.stderr.write("Importing "+name+"\\n")')
for i, getdef_section in enumerate(get_definition_sections):
if i == 0:
output.append_many("""
diff --git a/snakehouse/build.py b/snakehouse/build.py
index 39c4862..5a20ed9 100644
--- a/snakehouse/build.py
+++ b/snakehouse/build.py
@@ -18,7 +18,7 @@ def build(extensions: tp.List[MultiBuildType], *args, nthreads=None,
if nthreads is None:
nthreads = multiprocessing.cpu_count()
if sys.platform == 'win32':
- print('Sorry, multiprocessing is not yet supported on Windows')
+ logger.error('Sorry, multiprocessing is not yet supported on Windows')
nthreads = 0
kwargs['nthreads'] = nthreads
returns = []
diff --git a/snakehouse/multibuild.py b/snakehouse/multibuild.py
index cdd8e83..6036d3a 100644
--- a/snakehouse/multibuild.py
+++ b/snakehouse/multibuild.py
@@ -81,20 +81,23 @@ class Multibuild:
dont_snakehouse: bool = False,
**kwargs):
# sanitize path separators so that Linux-style paths are supported on Windows
- logger.warning('Building extension %s with files %s', extension_name, files)
files = list(files)
self.dont_snakehouse = dont_snakehouse
self.kwargs = kwargs
self.clear_all_bootstraps()
if files:
files = [os.path.join(*split(file)) for file in files]
- self.files = list([file for file in files if not file.endswith('__bootstrap__.pyx')])
+ self.files = [file for file in files if not file.endswith('__bootstrap__.pyx') and not file.endswith('.pyc')]
self.pyx_files = [file for file in self.files if file.endswith('.pyx')]
self.c_files = [file for file in self.files if file.endswith('.c') or file.endswith('.cpp')]
else:
self.pyx_files = []
self.c_files = []
+ self.files = list(set(self.files))
+ self.pyx_files = list(set(self.pyx_files))
+ self.c_files = list(set(self.c_files))
+
self.do_generate = True
if not self.pyx_files:
warnings.warn('No pyx files, probably installing from a source archive, skipping '
@@ -138,6 +141,16 @@ class Multibuild:
except BreakException:
pass
+ def is_h_file_ok(self, path: str) -> bool:
+ if not os.path.exists(path):
+ return False
+
+ with open(path, 'r', encoding='utf-8') as f_in:
+ data = f_in.read()
+ if '#define SNAKEHOUSE_LEAVE_ME_ALONE' in data:
+ return True
+ return False
+
def generate_header_files(self):
for filename in self.pyx_files:
path, name, cmod_name_path, module_name, coded_module_name, complete_module_name = self.transform_module_name(filename)
@@ -146,23 +159,9 @@ class Multibuild:
h_file = filename.replace('.pyx', '.h')
- if os.path.exists(h_file):
- with open(h_file, 'r', encoding='utf-8') as f_in:
- data = f_in.readlines()
- data = ['#include "Python.h"\n']
- rendered_mako = HFILE_MAKO.format(initpy_name=coded_module_name)
-
- if any('#define SNAKEHOUSE_FILE' in line for line in data):
- data = [*data, rendered_mako]
- else:
- data = [*data, rendered_mako]
- else:
- rendered_mako = HFILE_MAKO.format(initpy_name=coded_module_name)
- assert len(rendered_mako) > 0
- data = '#include "Python.h"\n' + rendered_mako
-
- with open(h_file, 'w') as f_out:
- f_out.write(''.join(data))
+ data = HFILE_MAKO.format(initpy_name=coded_module_name)
+ with open(h_file, 'w', encoding='utf-8') as f_out:
+ f_out.write(data)
def transform_module_name(self, filename):
path, name = os.path.split(filename)
@@ -201,24 +200,6 @@ class Multibuild:
data = data_in.replace(to_replace, replace_with)
with open(filename.replace('.pyx', '.c'), 'w') as f_out:
f_out.write(data)
- for root, dirs, files in os.walk('.'):
- for file in files:
- c_name = os.path.join(root, file)
- if file == '__bootstrap__.c':
- with open(c_name, 'r', encoding='utf-8') as fin:
- data = fin.read()
- data = data.replace('__Pyx_PyMODINIT_FUNC PyInit___bootstrap__(void)',
- 'PyMODINIT_FUNC PyInit__'+coded_module_name+'(void)')
- data = data.replace('__Pyx_PyMODINIT_FUNC init__bootstrap__(void)',
- 'PyMODINIT_FUNC init__'+coded_module_name+'(void)')
- with open(c_name, 'w', encoding='utf-8') as fout:
- for filename in self.pyx_files:
- path, name, cmod_name_path, module_name, coded_module_name, complete_module_name = self.transform_module_name(
- filename)
- h_path_name = os.path.join(cmod_name_path, name.replace('.pyx', '.h')).replace('\\', '\\\\')
- fout.write('#include "%s"\n' % (h_path_name, ))
- fout.write(data)
- c_name = os.path.join(root, file)
def generate_bootstrap(self) -> str:
@@ -238,7 +219,6 @@ class Multibuild:
get_definition = []
for mod_name, init_fun_name, coded_module_name in self.modules:
get_definition.append(GetDefinitionSection(mod_name, init_fun_name, coded_module_name))
-
return get_file(cdef_sections=cdef_section, get_definition_sections=get_definition,
module_set=[x[0] for x in self.modules])
@@ -250,12 +230,12 @@ class Multibuild:
os.unlink(c_name)
def write_bootstrap_file(self):
- with open(os.path.join(self.bootstrap_directory, '__bootstrap__.pyx'), 'w') as f_out:
+ with open(os.path.join(self.bootstrap_directory, '__bootstrap__.pyx'), 'w', encoding='utf-8') as f_out:
f_out.write(self.generate_bootstrap())
def alter_init(self):
if os.path.exists(os.path.join(self.bootstrap_directory, '__init__.py')):
- with open(os.path.join(self.bootstrap_directory, '__init__.py'), 'r') as f_in:
+ with open(os.path.join(self.bootstrap_directory, '__init__.py'), 'r', encoding='utf-8') as f_in:
data = f_in.read()
else:
data = ''
@@ -263,7 +243,7 @@ class Multibuild:
if 'bootstrap_cython_submodules' not in data:
data = INITPY_MAKO.format(module_name=self.extension_name) + data
- with open(os.path.join(self.bootstrap_directory, '__init__.py'), 'w') as f_out:
+ with open(os.path.join(self.bootstrap_directory, '__init__.py'), 'w', encoding='utf-8') as f_out:
f_out.write(data)
def generate(self):
@@ -283,8 +263,10 @@ class Multibuild:
return extensions
else:
kwargs.update(self.kwargs)
- for_cythonize = list(set([*self.files, os.path.join(self.bootstrap_directory, '__bootstrap__.pyx')]))
+ self.files = [file for file in self.files if file.endswith('.pyx')]
+ self.files.extend(self.c_files)
+ self.files.append(os.path.join(self.bootstrap_directory, '__bootstrap__.pyx'))
return [Extension(self.extension_name+".__bootstrap__",
- for_cythonize,
+ self.files,
*args,
**kwargs)]
diff --git a/snakehouse/templating.py b/snakehouse/templating.py
index 17aaf4d..30ae97e 100644
--- a/snakehouse/templating.py
+++ b/snakehouse/templating.py
@@ -5,8 +5,9 @@ import typing as tp
FOR_MATCH = re.compile(r'% for (.*?) in (.*?)\:')
-HFILE_MAKO = """
-extern PyObject* PyInit_{initpy_name}();
+HFILE_MAKO = """#include "Python.h"
+
+PyObject* PyInit_{initpy_name}();
"""
--
GitLab