diff --git a/.gitignore b/.gitignore index 7ae2d7be0f5f57ee038efe7ed0678437676722a0..003f6b90047ec3c8b1fc8272d2e4a8a89e8a3b99 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ build */build */dist +docs/_build dist .eggs *.c diff --git a/CHANGELOG.md b/CHANGELOG.md index e69ac16977a0fd6f4ab63901b91715f46c258a28..fcc088f841194f090c9249d66c256939f2cbd591 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ -# v1.3.3 +# v1.4 -* _TBA_ +* added `find_pyx`, `find_c` and `find_pyx_and_c` +* added documentation # v1.3.2 diff --git a/README.md b/README.md index 32258f398feda1e5f76e81af094b36d6d53e1af8..cb8b2bf2ef31e4a26e0e83657b4036615dc7a38c 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ snakehouse [](https://pypi.python.org/pypi/snakehouse) []() []() +[](http://snakehouse.readthedocs.io/en/latest/?badge=latest) snakehouse is a tool to pack mutiple .pyx files into a single extension. @@ -26,70 +27,5 @@ a part of your pull request as well! Note what have you changed in [CHANGELOG.md](/CHANGELOG.md) as well! -Accelerating builds -------------------- - -distutils by default compiles using a single process. To enable faster, multiprocess compilations -just type: - -```python -from snakehouse import monkey_patch_parallel_compilation - -monkey_patch_parallel_compilation() -```` - -Before your `setup()` call. - Usage notes - MANDATORY READING ------------------------------- -Take a look at [example](example/) on how to multi-build your Cython extensions. - -Don't place modules compiled that way in root .py file's top level imports. -Wrap them in a layer of indirection instead! - -This applies to unit tests as well! - -When something goes wrong (eg. the application throws an unhandled exception) -the built module has a tendency to dump core. -Try to debug it first by passing `dont_snakehouse=True` to your -modules in the debug mode. - -Also note that if you are compiling in `dont_snakehouse` -mode then your modules should have at least one of the following: -* a normal Python `def` -* a normal Python class (not `cdef class`) -* a line of Python initialization, eg. - -```python -a = None -``` - -or -```python -import logging - -logger = logging.getLogger(__name__) -``` - -Otherwise `PyInit` won't be generated by Cython -and such module will be unimportable in Python. Normal import won't suffice. - -Further streamlining your builds --------------------------------- - -If you add a MANIFEST.in file with contents: - -``` -include requirements.txt -``` - -Then you can write the following in your setup.py: - -```python -from snakehouse import read_requirements_txt - -setup(install_requires=read_requirements_txt()) -``` - -This will read in your requirements.txt and extract packages from there. -Be sure to entertain it's [pydoc](snakehouse/requirements.py)! diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..d4bb2cbb9eddb1bb1b4f366623044af8e4830919 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/accelerating.rst b/docs/accelerating.rst new file mode 100644 index 0000000000000000000000000000000000000000..5c71b5142d656949629dc2534e32b4e6fa8ad01c --- /dev/null +++ b/docs/accelerating.rst @@ -0,0 +1,14 @@ +Accelerating builds +=================== + +distutils by default compiles using a single process. To enable faster, multiprocess compilations +just type: + +.. code-block:: python + + from snakehouse import monkey_patch_parallel_compilation + + monkey_patch_parallel_compilation() + +Before your :code:`setup()` call. + diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000000000000000000000000000000000000..79d8e00afaf8ac6289d8ac5b29b7f3e4066b4d26 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,54 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys +sys.path.insert(0, os.path.abspath('..')) + + +# -- Project information ----------------------------------------------------- + +project = 'snakehouse' +copyright = '2020-2021 SMOK sp. z o. o.' +author = 'Piotr Maślanka' + +# The full version, including alpha/beta/rc tags +release = '1.4' + + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = ['sphinx.ext.autodoc'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'alabaster' + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..1fbf1ffabb65fc21661ac33ffc194bff49fce87e --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,57 @@ +Welcome to snakehouse's documentation! +====================================== + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + usage + accelerating + + +Mandatory reading +================= + +Take a look at example_ on how to multi-build your Cython extensions. + +.. _example: https://github.com/smok-serwis/snakehouse/blob/develop/example/setup.py + +Don't place modules compiled that way in root .py file's top level imports. +Wrap them in a layer of indirection instead! + +This applies to unit tests as well! + +When something goes wrong (eg. the application throws an unhandled exception) +the built module has a tendency to dump core. +Try to debug it first by passing :code:`dont_snakehouse=True` to your +modules in the debug mode. + +Also note that if you are compiling in :code:`dont_snakehouse` +mode then your modules should have at least one of the following: + +* a normal Python :code:`def` +* a normal Python class (not :code:`cdef class`) +* a line of Python initialization, eg. + +.. code-block:: python + + a = None + +or + +.. code-block:: python + + import logging + + logger = logging.getLogger(__name__) + +Otherwise :code:`PyInit` won't be generated by Cython +and such module will be unimportable in Python. Normal import won't suffice. + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000000000000000000000000000000000000..922152e96a04a242e6fc40f124261d74890617d8 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/usage.rst b/docs/usage.rst new file mode 100644 index 0000000000000000000000000000000000000000..43bb111aade349dacad596dcc97ba2cc078e376b --- /dev/null +++ b/docs/usage.rst @@ -0,0 +1,58 @@ +Usage +===== + +To use snakehouse just use the following in your :code:`setup.py`: + +.. code-block:: python + + list_of_pyx_files = [ ... ] + + setup(name='example_module', + version='0.1', + packages=['example_module'], + ext_modules=Multibuild('example_module', list_of_pyx_files) + ) + +Full documentation of Multibuild is here + +.. autoclass:: snakehouse.Multibuild + :members: + +You should use :code:`dont_snakehouse` for debugging and unit tests, as +snakehouse has a sad tendency to dump core on unhandled exceptions. To prevent that +from happening remember to handle your exceptions and debug using this flag. + +Helper functions +---------------- + +Finding files +~~~~~~~~~~~~~ + +Instead of manually specifying list of pyx and c files to compile you can use the following +functions: + +.. autofunction:: snakehouse.find_pyx + +.. autofunction:: snakehouse.find_c + +.. autofunction:: snakehouse.find_pyx_and_c + +Specifying requirements +~~~~~~~~~~~~~~~~~~~~~~~ + +If you add a MANIFEST.in file with contents: + +.. code-block:: + + include requirements.txt + +Then you can write the following in your setup.py: + +.. code-block:: python + + from snakehouse import read_requirements_txt + + setup(install_requires=read_requirements_txt()) + +.. autofunction:: snakehouse.read_requirements_txt + diff --git a/snakehouse/__init__.py b/snakehouse/__init__.py index 6d8ac0da99496e3aac9883a99b55c1ca027b4d9f..582dff18abdfc58a64cb1d3bf6ab04bc31461785 100644 --- a/snakehouse/__init__.py +++ b/snakehouse/__init__.py @@ -1,6 +1,6 @@ from .build import build from .multibuild import Multibuild from .faster_builds import monkey_patch_parallel_compilation -from .requirements import read_requirements_txt +from .requirements import read_requirements_txt, find_c, find_pyx_and_c, find_pyx -__version__ = '1.3.3a1' +__version__ = '1.4a1' diff --git a/snakehouse/multibuild.py b/snakehouse/multibuild.py index 50544a7da7f739a67d28708ceef7c8bf9da9b812..46bff48aa68a8392cda668d6ba90bc4b5f86071d 100644 --- a/snakehouse/multibuild.py +++ b/snakehouse/multibuild.py @@ -49,11 +49,12 @@ class Multibuild: This specifies a single Cython extension, called {extension_name}.__bootstrap__ All kwargs will be sent straight to Cython's Extension + :param extension_name: the module name :param files: list of pyx and c files :param kwargs: extra arguments to be passed to Extension() object :param dont_snakehouse: snakehouse won't be enabled, each element will be built - as a separate extension. It is for these cases when you're testing and something segfaults. + as a separate extension. It is for these cases when you're testing and something segfaults. """ def __init__(self, extension_name: str, files: tp.Iterator[str], dont_snakehouse: bool = False, diff --git a/snakehouse/requirements.py b/snakehouse/requirements.py index e7bedce03ef354494200e9e7d784b7cbd2fc63bd..e4bea2af66f45c1236ee4f6b7c9c43478d04717e 100644 --- a/snakehouse/requirements.py +++ b/snakehouse/requirements.py @@ -1,4 +1,40 @@ -from satella.files import read_lines +import typing as tp +from satella.coding import for_argument +from satella.files import read_lines, find_files + + +@for_argument(returns=list) +def find_pyx(directory_path: str) -> tp.List[str]: + """ + Return all .pyx files found in given directory. + + :param directory_path: directory to look through + :return: .pyx files found + """ + return find_files(directory_path, r'(.*)\.pyx', scan_subdirectories=True) + + +@for_argument(returns=list) +def find_c(directory_path: str) -> tp.List[str]: + """ + Return all .c files found in given directory. + + :param directory_path: directory to look through + :return: .c files found + """ + return find_files(directory_path, r'(.*)\.c', scan_subdirectories=True) + + +def find_pyx_and_c(directory_path: str) -> tp.List[str]: + """ + Return a list of all .pyx and .c files found in given directory. + + :param directory_path: + :return: list of all .pyx and .c files found in given directory + """ + files = find_pyx(directory_path) + files.extend(find_c(directory_path)) + return files def read_requirements_txt(path: str = 'requirements.txt'):