From 8996969c9f00bb52adf3a101a8f3eb6a752ed8fd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Piotr=20Ma=C5=9Blanka?= <piotr.maslanka@henrietta.com.pl>
Date: Mon, 14 Dec 2020 18:57:27 +0100
Subject: [PATCH] add and test gzip support
---
.coveragerc | 1 -
requirements.txt | 2 +-
setup.py | 9 +++++----
tempsdb/series.pyx | 10 ++++++++--
tempsdb/varlen.pxd | 2 +-
tempsdb/varlen.pyx | 8 ++++++--
unittest.Dockerfile | 5 ++---
7 files changed, 23 insertions(+), 14 deletions(-)
diff --git a/.coveragerc b/.coveragerc
index 001df73..62eb0ed 100644
--- a/.coveragerc
+++ b/.coveragerc
@@ -8,7 +8,6 @@ omit=
tests/*
.eggs/*
setup.py
- tempsdb/__init__.py
[report]
include=
diff --git a/requirements.txt b/requirements.txt
index 2fe238c..39ce8d3 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,4 @@
satella
ujson
-snakehouse
+snakehouse>=1.2.3
six
diff --git a/setup.py b/setup.py
index 86f800f..69dafaa 100644
--- a/setup.py
+++ b/setup.py
@@ -2,12 +2,11 @@ import os
import typing as tp
from Cython.Build import cythonize
-from satella.distutils import monkey_patch_parallel_compilation
from satella.files import find_files
from distutils.core import setup
from setuptools import Extension
-from snakehouse import Multibuild, build
+from snakehouse import Multibuild, build, monkey_patch_parallel_compilation
def find_pyx(*path) -> tp.List[str]:
@@ -23,15 +22,17 @@ monkey_patch_parallel_compilation()
# Extension('tempsdb.iterators', ['tempsdb/iterators.pyx'])]
#
directives = {'language_level': '3'}
+m_kwargs = {}
if 'CI' in os.environ:
directives.update(profile=True, linetrace=True, embedsignature=True)
+ m_kwargs['define_macros'] = [("CYTHON_TRACE_NOGIL", "1")]
setup(name='tempsdb',
- version='0.5.0a9',
+ version='0.5a10',
packages=['tempsdb'],
install_requires=['satella>=2.14.21', 'ujson'],
- ext_modules=build([Multibuild('tempsdb', find_pyx('tempsdb')), ],
+ ext_modules=build([Multibuild('tempsdb', find_pyx('tempsdb'), **m_kwargs), ],
compiler_directives=directives),
# ext_modules=cythonize(extensions,
# gdb_debug=True,
diff --git a/tempsdb/series.pyx b/tempsdb/series.pyx
index baaa9ee..c1966ff 100644
--- a/tempsdb/series.pyx
+++ b/tempsdb/series.pyx
@@ -177,14 +177,19 @@ cdef class TimeSeries:
Acquires a reference to the chunk.
:param name: name of the chunk
+ :param is_direct: is this a direct chunk?
+ :param is_gzip: is this a gzipped chunk?
:return: chunk
:raises DoesNotExist: chunk not found
:raises InvalidState: resource closed
+ :raises ValueError: chunk was gzipped but not direct
"""
if self.closed:
raise InvalidState('Series is closed')
if name not in (v[0] for v in self.chunks):
- raise DoesNotExist('Invalid chunk!')
+ raise DoesNotExist('Invalid chunk')
+ if is_gzip and not is_direct:
+ raise ValueError('Chunk that is gzipped must be direct')
cdef Chunk chunk
with self.open_lock:
if name not in self.open_chunks:
@@ -530,4 +535,5 @@ cpdef TimeSeries create_series(str path, str name, unsigned int block_size,
if gzip_level:
meta['gzip_level'] = gzip_level
write_json_to_file(os.path.join(path, 'metadata.txt'), meta)
- return TimeSeries(path, name)
+ return TimeSeries(path, name,
+ use_descriptor_based_access=use_descriptor_based_access)
diff --git a/tempsdb/varlen.pxd b/tempsdb/varlen.pxd
index 173e811..e792533 100644
--- a/tempsdb/varlen.pxd
+++ b/tempsdb/varlen.pxd
@@ -1,4 +1,4 @@
-from .series cimport TimeSeries
+from .series cimport TimeSeries, create_series
cdef class VarlenSeries:
diff --git a/tempsdb/varlen.pyx b/tempsdb/varlen.pyx
index 4100c47..4e1b393 100644
--- a/tempsdb/varlen.pyx
+++ b/tempsdb/varlen.pyx
@@ -7,7 +7,7 @@ import warnings
from .chunks.base cimport Chunk
from .exceptions import Corruption, AlreadyExists, StillOpen
from .iterators cimport Iterator
-from .series cimport TimeSeries, create_series
+from tempsdb.series cimport TimeSeries, create_series
cdef class VarlenEntry:
@@ -552,6 +552,8 @@ cdef class VarlenSeries:
Updates :attr:`~tempsdb.varlen.VarlenSeries.current_maximum_length`.
"""
+ from tempsdb.series import create_series
+
cdef:
int new_name = len(self.series)
int new_len = self.get_length_for(new_name)
@@ -561,6 +563,8 @@ cdef class VarlenSeries:
new_len,
self.max_entries_per_chunk,
gzip_level=self.gzip_level)
+ if self.mpm is not None:
+ series.register_memory_pressure_manager(self.mpm)
self.series.append(series)
self.current_maximum_length += new_len
@@ -636,7 +640,7 @@ cpdef VarlenSeries create_varlen_series(str path, str name, int size_struct, lis
:raises AlreadyExists: directory exists at given path
:raises ValueError: invalid length profile or max_entries_per_chunk or size_struct
"""
- from .series import create_series
+ from tempsdb.series import create_series
if os.path.exists(path):
raise AlreadyExists('directory present at paht')
diff --git a/unittest.Dockerfile b/unittest.Dockerfile
index bccaee1..e3aa113 100644
--- a/unittest.Dockerfile
+++ b/unittest.Dockerfile
@@ -1,15 +1,14 @@
FROM python:3.8
-RUN pip install satella>=2.14.24 snakehouse nose2 wheel ujson coverage
+RUN pip install satella snakehouse>=1.2.3 nose2 wheel ujson coverage
ADD tempsdb /app/tempsdb
ADD setup.py /app/setup.py
ADD .coveragerc /app/.coveragerc
ADD setup.cfg /app/setup.cfg
WORKDIR /app
-
ENV CI=true
RUN python setup.py build_ext --inplace
ADD tests /app/tests
-CMD ["nose2", "-vv"]
+CMD ["coverage", "run", "-m", "nose2", "-vv"]
--
GitLab