From af06953a8e1c4e9abe9dc069c6d7849c1ce86f14 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Piotr=20Ma=C5=9Blanka?= <piotr.maslanka@henrietta.com.pl>
Date: Wed, 23 Dec 2020 22:05:28 +0100
Subject: [PATCH] add metadata and DB upgrade

---
 README.md              |  3 ++-
 setup.py               |  5 +++--
 tempsdb/database.pxd   |  3 +++
 tempsdb/database.pyx   | 40 +++++++++++++++++++++++++++++++++++++---
 tests/test_database.py |  8 ++++++++
 unittest.Dockerfile    |  2 +-
 6 files changed, 54 insertions(+), 7 deletions(-)

diff --git a/README.md b/README.md
index 14b7851..14b92b3 100644
--- a/README.md
+++ b/README.md
@@ -54,7 +54,8 @@ Then copy your resulting wheel and install it via pip on the target system.
 
 ## v0.5.4
 
-* _TBA_
+* older TempsDB databases that do not support varlens will be updated upon opening
+* added metadata support for databases
 
 ## v0.5.3
 
diff --git a/setup.py b/setup.py
index fec8254..df8396f 100644
--- a/setup.py
+++ b/setup.py
@@ -36,12 +36,13 @@ if 'CI' in os.environ:
     #     Extension('tempsdb.chunks.base', ['tempsdb/chunks/base.pyx']),
     # ]
     # ext_modules = cythonize(extensions, compiler_directives=directives)
-ext_modules = build([Multibuild('tempsdb', find_pyx('tempsdb'), **ext_kwargs), ],
+ext_modules = build([Multibuild('tempsdb', find_pyx('tempsdb'),
+                                **ext_kwargs), ],
                      compiler_directives=directives,
                      **cythonize_kwargs)
 
 setup(name='tempsdb',
-      version='0.5.4a1',
+      version='0.5.4a3',
       packages=find_packages(include=['tempsdb', 'tempsdb.*']),
       install_requires=['satella>=2.14.24', 'ujson'],
       ext_modules=ext_modules,
diff --git a/tempsdb/database.pxd b/tempsdb/database.pxd
index ddbabf8..f835ad9 100644
--- a/tempsdb/database.pxd
+++ b/tempsdb/database.pxd
@@ -10,7 +10,10 @@ cdef class Database:
         object mpm
         dict open_series
         dict open_varlen_series
+        readonly dict metadata
 
+    cpdef int reload_metadata(self) except -1
+    cpdef int set_metadata(self, dict meta) except -1
     cpdef int close(self) except -1
     cpdef TimeSeries get_series(self, str name,
                                 bint use_descriptor_based_access=*)
diff --git a/tempsdb/database.pyx b/tempsdb/database.pyx
index c74171d..22a6f70 100644
--- a/tempsdb/database.pyx
+++ b/tempsdb/database.pyx
@@ -4,6 +4,7 @@ import threading
 import warnings
 
 from satella.coding import DictDeleter
+from satella.json import read_json_from_file, write_json_to_file
 
 from tempsdb.exceptions import DoesNotExist, AlreadyExists, StillOpen
 from .series cimport TimeSeries, create_series
@@ -23,16 +24,49 @@ cdef class Database:
     :raises DoesNotExist: database does not exist, use `create_database`
 
     :ivar path: path to  the directory with the database (str)
+    :ivar metadata: metadata of this DB
     """
     def __init__(self, path: str):
         if not os.path.isdir(path):
             raise DoesNotExist('Database does not exist')
+
+        if not os.path.isdir(os.path.join(path, 'varlen')):
+            os.mkdir(os.path.join(path, 'varlen'))
+
         self.path = path
         self.closed = False
         self.open_series = {}
         self.open_varlen_series = {}
         self.lock = threading.RLock()
         self.mpm = None
+        self.metadata = {}
+        self.reload_metadata()
+
+    cpdef int reload_metadata(self) except -1:
+        """
+        Try to load the metadata again.
+        
+        This will change `metadata` attribute.
+        """
+        self.metadata = {}
+        if os.path.isfile(os.path.join(self.path, 'metadata.txt')):
+            try:
+                self.metadata = read_json_from_file(os.path.join(self.path, 'metadata.txt')).get('metadata', {})
+            except ValueError:
+                pass
+        return 0
+
+    cpdef int set_metadata(self, dict metadata) except -1:
+        """
+        Set metadata for this series.
+        
+        This will change `metadata` attribute.
+        
+        :param metadata: new metadata to set
+        """
+        write_json_to_file(os.path.join(self.path, 'metadata.txt'), {'metadata': metadata})
+        self.metadata = metadata
+        return 0
 
     cpdef list get_open_series(self):
         """
@@ -282,7 +316,7 @@ cdef class Database:
         """
         Create a new series.
         
-        Note that series cannot be named "varlen"
+        Note that series cannot be named "varlen" or "metadata.txt"
         
         :param name: name of the series
         :param block_size: size of the data field
@@ -298,8 +332,8 @@ cdef class Database:
         """
         if block_size > page_size + 8:
             raise ValueError('Invalid block size, pick larger page')
-        if name == 'varlen':
-            raise ValueError('Series cannot be named varlen')
+        if name == 'varlen' or name == 'metadata.txt':
+            raise ValueError('Series cannot be named varlen or metadata.txt')
         if os.path.isdir(os.path.join(self.path, name)):
             raise AlreadyExists('Series already exists')
         cdef TimeSeries series
diff --git a/tests/test_database.py b/tests/test_database.py
index c46991e..4fd9574 100644
--- a/tests/test_database.py
+++ b/tests/test_database.py
@@ -9,6 +9,14 @@ class TestDatabase(unittest.TestCase):
     def setUpClass(cls) -> None:
         cls.db = create_database('my_db')
 
+    def test_metadata(self):
+        meta = self.db.metadata
+        self.assertFalse(self.db.metadata)
+        meta = {'hello': 'world'}
+        self.db.set_metadata(meta)
+        self.db.reload_metadata()
+        self.assertEqual(self.db.metadata, meta)
+
     def test_add_series(self):
         ser = self.db.create_series('hello-world', 1, 10)
         ser.append(10, b'\x00')
diff --git a/unittest.Dockerfile b/unittest.Dockerfile
index df003ba..1877bb5 100644
--- a/unittest.Dockerfile
+++ b/unittest.Dockerfile
@@ -1,6 +1,6 @@
 FROM python:3.8
 
-RUN pip install satella>=2.14.24 snakehouse nose2 wheel ujson coverage indexed_gzip
+RUN pip install satella>=2.14.24 snakehouse>=1.3 nose2 wheel ujson coverage
 
 ADD tempsdb /app/tempsdb
 ADD setup.py /app/setup.py
-- 
GitLab