diff --git a/.dockerignore b/.dockerignore
index eb2762e9d02a4f5f7e09170a16017dbfc72df62b..677f96716c93178cfa0a7ac6d5e734e0a6b5c50a 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -1,3 +1,6 @@
.git
.circleci
docs
+build
+dist
+*.egg-info
diff --git a/LICENSE b/LICENSE
index 90f6d377a5a402925429c4210e6e8aa5704fec93..1915d01c9ad047c2c1b8ac964b794e3130469016 100644
--- a/LICENSE
+++ b/LICENSE
@@ -1,6 +1,6 @@
MIT License
-Copyright (c) 2020 SMOK
+Copyright (c) 2020 SMOK sp. z o. o.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
diff --git a/docs/conf.py b/docs/conf.py
index 194df61ba14bfc31289cda0eaa16c94c31afbfa0..c9865de954540efdce3caa24e521fe9ebcc3ba02 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -27,8 +27,8 @@ author = 'Piotr MaĹlanka'
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
-extensions = [
-]
+extensions = ['sphinx.ext.autodoc']
+
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
@@ -49,4 +49,4 @@ html_theme = 'alabaster'
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
\ No newline at end of file
+html_static_path = ['_static']
diff --git a/setup.py b/setup.py
index 4bfc77d89d856a93a8887bd4e44ebaef2e33cc81..bb82f047c69c3c87905017374eeda21dd9849be9 100644
--- a/setup.py
+++ b/setup.py
@@ -1,7 +1,6 @@
import os
import typing as tp
from satella.files import find_files
-from setuptools import find_packages
from distutils.core import setup
from snakehouse import Multibuild, build
@@ -12,11 +11,13 @@ def find_pyx(*path) -> tp.List[str]:
setup(name='tempsdb',
version='0.1_a1',
- packages=find_packages(include=['tempsdb', 'tempsdb.*']),
+ packages=['tempsdb'],
install_requires=['satella', 'ujson'],
ext_modules=build([Multibuild('tempsdb', find_pyx('tempsdb')), ],
compiler_directives={
'language_level': '3',
}),
- python_requires='!=2.7.*,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,!=3.7.*'
+ python_requires='!=2.7.*,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,!=3.7.*',
+ test_suite="tests",
+ zip_safe=False
)
diff --git a/tempsdb/__init__.py b/tempsdb/__init__.py
index 93a0b2e8f779eb4b71867716e1321d66782a5d9b..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 100644
--- a/tempsdb/__init__.py
+++ b/tempsdb/__init__.py
@@ -1,2 +0,0 @@
-from tempsdb.__bootstrap__ import bootstrap_cython_submodules
-bootstrap_cython_submodules()
diff --git a/tempsdb/chunks.pyx b/tempsdb/chunks.pyx
index 524f5a6b00e0b3ca03eb55683f326ec8c9db4c74..a2389a5136a187628a8f4be0d8ea39a2e62ee596 100644
--- a/tempsdb/chunks.pyx
+++ b/tempsdb/chunks.pyx
@@ -10,7 +10,9 @@ STRUCT_Q = struct.Struct('>Q')
cdef class Chunk:
"""
- Represents a single chunk of time series
+ Represents a single chunk of time series.
+
+ This also implements an iterator interface. This will iterate with tp.Tuple[int, bytes].
:param path: path to the chunk file
:type path: str
@@ -23,19 +25,29 @@ cdef class Chunk:
"""
def __init__(self, path: str):
self.closed = False
+ cdef unsigned long long file_size = os.path.getsize(path)
self.path = path
cdef bytes b
- self.file = open(self.path, 'rb')
+ print('Before open')
+ self.file = open(self.path, 'rb+')
try:
- self.mmap = mmap.mmap(self.file.fileno(), 0)
- except OSError:
- raise Corruption('Empty chunk file!')
+ self.mmap = mmap.mmap(self.file.fileno(), file_size, access=mmap.ACCESS_READ)
+ except OSError as e:
+ raise Corruption(f'Empty chunk file!')
try:
- self.min_ts, self.max_ts, self.block_size = STRUCT_QQL.unpack(self.file.read(16))
+ self.min_ts, self.max_ts, self.block_size = STRUCT_QQL.unpack(self.mmap[:20])
except struct.error:
raise Corruption('Could not read the header of the chunk file %s' % (self.path, ))
- self.pointer = 8
- self.entries = (os.path.getsize(self.path)-20) / self.block_size
+ self.pointer = 20
+ self.entries = (file_size-self.pointer) // self.block_size
+
+ def __iter__(self):
+ cdef unsigned long i = 0
+ for i in range(self.entries):
+ yield self.get_piece_at(i)
+
+ def __len__(self):
+ return self.entries
cpdef void close(self):
"""
@@ -59,9 +71,8 @@ cdef class Chunk:
if index >= self.entries:
raise IndexError('Index too large')
cdef:
- unsigned long starting_index = 20 + index * self.block_size
- unsigned long stopping_index = starting_index + self.block_size
- bytes bytes_at = self.mmap[starting_index:stopping_index]
+ unsigned long starting_index = 20 + index * (self.block_size+8)
+ unsigned long stopping_index = starting_index + self.block_size+8
unsigned long long ts = STRUCT_Q.unpack(self.mmap[starting_index:starting_index+8])[0]
return ts, self.mmap[starting_index+8:stopping_index]
@@ -107,10 +118,10 @@ cpdef Chunk create_chunk(str path, list data):
file.write(b)
last_ts = ts
first_element = False
- file.close()
except ValueError:
file.close()
os.unlink(path)
raise
+ file.close()
return Chunk(path)
diff --git a/tempsdb/series.pyx b/tempsdb/series.pyx
index d87eba5bd2cf72517f9f2d5bce441f49efebb112..7d0c20209f5266d551934a34f55241192a9c7ede 100644
--- a/tempsdb/series.pyx
+++ b/tempsdb/series.pyx
@@ -1,10 +1,9 @@
import threading
import time
-
import ujson
from satella.files import read_in_file
-from .chunks cimport Chunk, create_chunk
+from .chunks cimport create_chunk
from .database cimport Database
from .exceptions import DoesNotExist, Corruption
import os
diff --git a/tests/test_db.py b/tests/test_db.py
index 56413578c53be2d16df3ea72c7729c2183273795..12f678cd50bb8da374779ce12e7cbbe532be970b 100644
--- a/tests/test_db.py
+++ b/tests/test_db.py
@@ -1,15 +1,14 @@
import unittest
+from tempsdb.chunks import create_chunk
class TestDB(unittest.TestCase):
def test_chunk(self):
- from tempsdb.chunks import create_chunk
-
data = [(0, b'ala '), (1, b'ma '), (4, b'kota')]
chunk = create_chunk('chunk.db', data)
self.assertEqual(chunk.min_ts, 0)
self.assertEqual(chunk.max_ts, 4)
- self.assertEqual(chunk.bs, 4)
+ self.assertEqual(chunk.block_size, 4)
self.assertEqual(chunk.get_piece_at(0), (0, b'ala '))
- self.assertEqual(chunk.get_piece_at(1), (1, b'ma '))
+ self.assertEqual(chunk.get_piece_at(1), (1, b'ma '))
self.assertEqual(chunk.get_piece_at(2), (4, b'kota'))
diff --git a/unittest.Dockerfile b/unittest.Dockerfile
index cb64eb186826be299fcfd363890535d90601d8c4..610937e4c063e23c06aef87c709bc26b76ca20dc 100644
--- a/unittest.Dockerfile
+++ b/unittest.Dockerfile
@@ -1,8 +1,9 @@
FROM python:3.8
-RUN pip install satella snakehouse nose2
+RUN pip install satella snakehouse nose2 wheel
ADD . /app
WORKDIR /app
+RUN python setup.py build_ext --inplace
-CMD ["python", "setup.py", "test"]
+CMD ["nose2", "-vv"]