From af06953a8e1c4e9abe9dc069c6d7849c1ce86f14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Ma=C5=9Blanka?= <piotr.maslanka@henrietta.com.pl> Date: Wed, 23 Dec 2020 22:05:28 +0100 Subject: [PATCH] add metadata and DB upgrade --- README.md | 3 ++- setup.py | 5 +++-- tempsdb/database.pxd | 3 +++ tempsdb/database.pyx | 40 +++++++++++++++++++++++++++++++++++++--- tests/test_database.py | 8 ++++++++ unittest.Dockerfile | 2 +- 6 files changed, 54 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 14b7851..14b92b3 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,8 @@ Then copy your resulting wheel and install it via pip on the target system. ## v0.5.4 -* _TBA_ +* older TempsDB databases that do not support varlens will be updated upon opening +* added metadata support for databases ## v0.5.3 diff --git a/setup.py b/setup.py index fec8254..df8396f 100644 --- a/setup.py +++ b/setup.py @@ -36,12 +36,13 @@ if 'CI' in os.environ: # Extension('tempsdb.chunks.base', ['tempsdb/chunks/base.pyx']), # ] # ext_modules = cythonize(extensions, compiler_directives=directives) -ext_modules = build([Multibuild('tempsdb', find_pyx('tempsdb'), **ext_kwargs), ], +ext_modules = build([Multibuild('tempsdb', find_pyx('tempsdb'), + **ext_kwargs), ], compiler_directives=directives, **cythonize_kwargs) setup(name='tempsdb', - version='0.5.4a1', + version='0.5.4a3', packages=find_packages(include=['tempsdb', 'tempsdb.*']), install_requires=['satella>=2.14.24', 'ujson'], ext_modules=ext_modules, diff --git a/tempsdb/database.pxd b/tempsdb/database.pxd index ddbabf8..f835ad9 100644 --- a/tempsdb/database.pxd +++ b/tempsdb/database.pxd @@ -10,7 +10,10 @@ cdef class Database: object mpm dict open_series dict open_varlen_series + readonly dict metadata + cpdef int reload_metadata(self) except -1 + cpdef int set_metadata(self, dict meta) except -1 cpdef int close(self) except -1 cpdef TimeSeries get_series(self, str name, bint use_descriptor_based_access=*) diff --git a/tempsdb/database.pyx b/tempsdb/database.pyx index c74171d..22a6f70 100644 --- a/tempsdb/database.pyx +++ b/tempsdb/database.pyx @@ -4,6 +4,7 @@ import threading import warnings from satella.coding import DictDeleter +from satella.json import read_json_from_file, write_json_to_file from tempsdb.exceptions import DoesNotExist, AlreadyExists, StillOpen from .series cimport TimeSeries, create_series @@ -23,16 +24,49 @@ cdef class Database: :raises DoesNotExist: database does not exist, use `create_database` :ivar path: path to the directory with the database (str) + :ivar metadata: metadata of this DB """ def __init__(self, path: str): if not os.path.isdir(path): raise DoesNotExist('Database does not exist') + + if not os.path.isdir(os.path.join(path, 'varlen')): + os.mkdir(os.path.join(path, 'varlen')) + self.path = path self.closed = False self.open_series = {} self.open_varlen_series = {} self.lock = threading.RLock() self.mpm = None + self.metadata = {} + self.reload_metadata() + + cpdef int reload_metadata(self) except -1: + """ + Try to load the metadata again. + + This will change `metadata` attribute. + """ + self.metadata = {} + if os.path.isfile(os.path.join(self.path, 'metadata.txt')): + try: + self.metadata = read_json_from_file(os.path.join(self.path, 'metadata.txt')).get('metadata', {}) + except ValueError: + pass + return 0 + + cpdef int set_metadata(self, dict metadata) except -1: + """ + Set metadata for this series. + + This will change `metadata` attribute. + + :param metadata: new metadata to set + """ + write_json_to_file(os.path.join(self.path, 'metadata.txt'), {'metadata': metadata}) + self.metadata = metadata + return 0 cpdef list get_open_series(self): """ @@ -282,7 +316,7 @@ cdef class Database: """ Create a new series. - Note that series cannot be named "varlen" + Note that series cannot be named "varlen" or "metadata.txt" :param name: name of the series :param block_size: size of the data field @@ -298,8 +332,8 @@ cdef class Database: """ if block_size > page_size + 8: raise ValueError('Invalid block size, pick larger page') - if name == 'varlen': - raise ValueError('Series cannot be named varlen') + if name == 'varlen' or name == 'metadata.txt': + raise ValueError('Series cannot be named varlen or metadata.txt') if os.path.isdir(os.path.join(self.path, name)): raise AlreadyExists('Series already exists') cdef TimeSeries series diff --git a/tests/test_database.py b/tests/test_database.py index c46991e..4fd9574 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -9,6 +9,14 @@ class TestDatabase(unittest.TestCase): def setUpClass(cls) -> None: cls.db = create_database('my_db') + def test_metadata(self): + meta = self.db.metadata + self.assertFalse(self.db.metadata) + meta = {'hello': 'world'} + self.db.set_metadata(meta) + self.db.reload_metadata() + self.assertEqual(self.db.metadata, meta) + def test_add_series(self): ser = self.db.create_series('hello-world', 1, 10) ser.append(10, b'\x00') diff --git a/unittest.Dockerfile b/unittest.Dockerfile index df003ba..1877bb5 100644 --- a/unittest.Dockerfile +++ b/unittest.Dockerfile @@ -1,6 +1,6 @@ FROM python:3.8 -RUN pip install satella>=2.14.24 snakehouse nose2 wheel ujson coverage indexed_gzip +RUN pip install satella>=2.14.24 snakehouse>=1.3 nose2 wheel ujson coverage ADD tempsdb /app/tempsdb ADD setup.py /app/setup.py -- GitLab