Skip to content
Snippets Groups Projects
Commit 61824143 authored by Piotr Maślanka's avatar Piotr Maślanka
Browse files

fix docs at the very least

parent 1794489f
No related branches found
No related tags found
No related merge requests found
.git .git
.circleci .circleci
docs docs
build
dist
*.egg-info
MIT License MIT License
Copyright (c) 2020 SMOK Copyright (c) 2020 SMOK sp. z o. o.
Permission is hereby granted, free of charge, to any person obtaining a copy Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal of this software and associated documentation files (the "Software"), to deal
......
...@@ -27,8 +27,8 @@ author = 'Piotr Maślanka' ...@@ -27,8 +27,8 @@ author = 'Piotr Maślanka'
# Add any Sphinx extension module names here, as strings. They can be # Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones. # ones.
extensions = [ extensions = ['sphinx.ext.autodoc']
]
# Add any paths that contain templates here, relative to this directory. # Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates'] templates_path = ['_templates']
...@@ -49,4 +49,4 @@ html_theme = 'alabaster' ...@@ -49,4 +49,4 @@ html_theme = 'alabaster'
# Add any paths that contain custom static files (such as style sheets) here, # Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files, # relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css". # so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static'] html_static_path = ['_static']
\ No newline at end of file
import os import os
import typing as tp import typing as tp
from satella.files import find_files from satella.files import find_files
from setuptools import find_packages
from distutils.core import setup from distutils.core import setup
from snakehouse import Multibuild, build from snakehouse import Multibuild, build
...@@ -12,11 +11,13 @@ def find_pyx(*path) -> tp.List[str]: ...@@ -12,11 +11,13 @@ def find_pyx(*path) -> tp.List[str]:
setup(name='tempsdb', setup(name='tempsdb',
version='0.1_a1', version='0.1_a1',
packages=find_packages(include=['tempsdb', 'tempsdb.*']), packages=['tempsdb'],
install_requires=['satella', 'ujson'], install_requires=['satella', 'ujson'],
ext_modules=build([Multibuild('tempsdb', find_pyx('tempsdb')), ], ext_modules=build([Multibuild('tempsdb', find_pyx('tempsdb')), ],
compiler_directives={ compiler_directives={
'language_level': '3', 'language_level': '3',
}), }),
python_requires='!=2.7.*,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,!=3.7.*' python_requires='!=2.7.*,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,!=3.7.*',
test_suite="tests",
zip_safe=False
) )
from tempsdb.__bootstrap__ import bootstrap_cython_submodules
bootstrap_cython_submodules()
...@@ -10,7 +10,9 @@ STRUCT_Q = struct.Struct('>Q') ...@@ -10,7 +10,9 @@ STRUCT_Q = struct.Struct('>Q')
cdef class Chunk: cdef class Chunk:
""" """
Represents a single chunk of time series Represents a single chunk of time series.
This also implements an iterator interface. This will iterate with tp.Tuple[int, bytes].
:param path: path to the chunk file :param path: path to the chunk file
:type path: str :type path: str
...@@ -23,19 +25,29 @@ cdef class Chunk: ...@@ -23,19 +25,29 @@ cdef class Chunk:
""" """
def __init__(self, path: str): def __init__(self, path: str):
self.closed = False self.closed = False
cdef unsigned long long file_size = os.path.getsize(path)
self.path = path self.path = path
cdef bytes b cdef bytes b
self.file = open(self.path, 'rb') print('Before open')
self.file = open(self.path, 'rb+')
try: try:
self.mmap = mmap.mmap(self.file.fileno(), 0) self.mmap = mmap.mmap(self.file.fileno(), file_size, access=mmap.ACCESS_READ)
except OSError: except OSError as e:
raise Corruption('Empty chunk file!') raise Corruption(f'Empty chunk file!')
try: try:
self.min_ts, self.max_ts, self.block_size = STRUCT_QQL.unpack(self.file.read(16)) self.min_ts, self.max_ts, self.block_size = STRUCT_QQL.unpack(self.mmap[:20])
except struct.error: except struct.error:
raise Corruption('Could not read the header of the chunk file %s' % (self.path, )) raise Corruption('Could not read the header of the chunk file %s' % (self.path, ))
self.pointer = 8 self.pointer = 20
self.entries = (os.path.getsize(self.path)-20) / self.block_size self.entries = (file_size-self.pointer) // self.block_size
def __iter__(self):
cdef unsigned long i = 0
for i in range(self.entries):
yield self.get_piece_at(i)
def __len__(self):
return self.entries
cpdef void close(self): cpdef void close(self):
""" """
...@@ -59,9 +71,8 @@ cdef class Chunk: ...@@ -59,9 +71,8 @@ cdef class Chunk:
if index >= self.entries: if index >= self.entries:
raise IndexError('Index too large') raise IndexError('Index too large')
cdef: cdef:
unsigned long starting_index = 20 + index * self.block_size unsigned long starting_index = 20 + index * (self.block_size+8)
unsigned long stopping_index = starting_index + self.block_size unsigned long stopping_index = starting_index + self.block_size+8
bytes bytes_at = self.mmap[starting_index:stopping_index]
unsigned long long ts = STRUCT_Q.unpack(self.mmap[starting_index:starting_index+8])[0] unsigned long long ts = STRUCT_Q.unpack(self.mmap[starting_index:starting_index+8])[0]
return ts, self.mmap[starting_index+8:stopping_index] return ts, self.mmap[starting_index+8:stopping_index]
...@@ -107,10 +118,10 @@ cpdef Chunk create_chunk(str path, list data): ...@@ -107,10 +118,10 @@ cpdef Chunk create_chunk(str path, list data):
file.write(b) file.write(b)
last_ts = ts last_ts = ts
first_element = False first_element = False
file.close()
except ValueError: except ValueError:
file.close() file.close()
os.unlink(path) os.unlink(path)
raise raise
file.close()
return Chunk(path) return Chunk(path)
import threading import threading
import time import time
import ujson import ujson
from satella.files import read_in_file from satella.files import read_in_file
from .chunks cimport Chunk, create_chunk from .chunks cimport create_chunk
from .database cimport Database from .database cimport Database
from .exceptions import DoesNotExist, Corruption from .exceptions import DoesNotExist, Corruption
import os import os
......
import unittest import unittest
from tempsdb.chunks import create_chunk
class TestDB(unittest.TestCase): class TestDB(unittest.TestCase):
def test_chunk(self): def test_chunk(self):
from tempsdb.chunks import create_chunk
data = [(0, b'ala '), (1, b'ma '), (4, b'kota')] data = [(0, b'ala '), (1, b'ma '), (4, b'kota')]
chunk = create_chunk('chunk.db', data) chunk = create_chunk('chunk.db', data)
self.assertEqual(chunk.min_ts, 0) self.assertEqual(chunk.min_ts, 0)
self.assertEqual(chunk.max_ts, 4) self.assertEqual(chunk.max_ts, 4)
self.assertEqual(chunk.bs, 4) self.assertEqual(chunk.block_size, 4)
self.assertEqual(chunk.get_piece_at(0), (0, b'ala ')) self.assertEqual(chunk.get_piece_at(0), (0, b'ala '))
self.assertEqual(chunk.get_piece_at(1), (1, b'ma ')) self.assertEqual(chunk.get_piece_at(1), (1, b'ma '))
self.assertEqual(chunk.get_piece_at(2), (4, b'kota')) self.assertEqual(chunk.get_piece_at(2), (4, b'kota'))
FROM python:3.8 FROM python:3.8
RUN pip install satella snakehouse nose2 RUN pip install satella snakehouse nose2 wheel
ADD . /app ADD . /app
WORKDIR /app WORKDIR /app
RUN python setup.py build_ext --inplace
CMD ["python", "setup.py", "test"] CMD ["nose2", "-vv"]
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment