From 01539b3b7285c66d1798e838bbd3b40255487e33 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Piotr=20Ma=C5=9Blanka?= <piotr.maslanka@henrietta.com.pl>
Date: Sat, 28 Nov 2020 21:02:32 +0100
Subject: [PATCH] use mmap for writes too
---
docs/chunks.rst | 8 ++++++++
docs/index.rst | 3 ++-
docs/usage.rst | 1 -
tempsdb/chunks.pxd | 1 +
tempsdb/chunks.pyx | 20 ++++++++++++--------
5 files changed, 23 insertions(+), 10 deletions(-)
create mode 100644 docs/chunks.rst
diff --git a/docs/chunks.rst b/docs/chunks.rst
new file mode 100644
index 0000000..cdf78eb
--- /dev/null
+++ b/docs/chunks.rst
@@ -0,0 +1,8 @@
+Chunk
+=====
+
+For your convenience the class :class:`~tempsdb.chunks.Chunk` was also documented, but don't use
+it directly:
+
+.. autoclass:: tempsdb.chunks.Chunk
+ :members:
diff --git a/docs/index.rst b/docs/index.rst
index 0f61f69..d4621d4 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -12,8 +12,9 @@ Welcome to tempsdb's documentation!
usage
exceptions
+ chunks
-It tries to use mmap for reads where possible, and in general be as zero-copy as possible (ie. the
+It tries to use mmap for reads and writes, and in general is as zero-copy as possible (ie. the
only time data is unserialized is when a particular entry is read).
Stored time series with a 8-bit timestamp and a fixed length of data.
diff --git a/docs/usage.rst b/docs/usage.rst
index 3daca86..fdc9540 100644
--- a/docs/usage.rst
+++ b/docs/usage.rst
@@ -14,4 +14,3 @@ Then you can create and retrieve particular series:
.. autoclass:: tempsdb.series.TimeSeries
:members:
-
diff --git a/tempsdb/chunks.pxd b/tempsdb/chunks.pxd
index 94aac24..d5a1eaf 100644
--- a/tempsdb/chunks.pxd
+++ b/tempsdb/chunks.pxd
@@ -18,6 +18,7 @@ cdef class Chunk:
cpdef void close(self)
cpdef tuple get_piece_at(self, unsigned int index)
cpdef int put(self, unsigned long long timestamp, bytes data) except -1
+ cpdef int sync(self) except -1
cdef inline int length(self):
return self.entries
diff --git a/tempsdb/chunks.pyx b/tempsdb/chunks.pyx
index 275d11a..0514bdb 100644
--- a/tempsdb/chunks.pyx
+++ b/tempsdb/chunks.pyx
@@ -56,6 +56,13 @@ cdef class Chunk:
self.max_ts, = STRUCT_Q.unpack(self.mmap[-TIMESTAMP_SIZE-self.block_size:-self.block_size])
self.min_ts, = STRUCT_Q.unpack(self.mmap[HEADER_SIZE:HEADER_SIZE+TIMESTAMP_SIZE])
+ cpdef int sync(self) except -1:
+ """
+ Synchronize the mmap
+ """
+ self.mmap.flush()
+ return 0
+
cpdef int put(self, unsigned long long timestamp, bytes data) except -1:
"""
Append a record to this chunk
@@ -73,14 +80,11 @@ cdef class Chunk:
raise ValueError('data not equal in length to block size!')
if timestamp <= self.max_ts:
raise ValueError('invalid timestamp')
-
- cdef bytearray data_to_write = bytearray(TIMESTAMP_SIZE+self.block_size)
- data_to_write[0:TIMESTAMP_SIZE] = STRUCT_Q.pack(timestamp)
- data_to_write[TIMESTAMP_SIZE:] = data
+ cdef unsigned long long pointer_at_end = (self.entries+1)*(TIMESTAMP_SIZE+self.block_size) + HEADER_SIZE
with self.write_lock:
- self.file.seek(0, 2)
- self.file.write(data_to_write)
- self.mmap.resize((self.entries+1)*(8+self.block_size)+HEADER_SIZE)
+ self.mmap.resize(pointer_at_end)
+ self.mmap[pointer_at_end-self.block_size-TIMESTAMP_SIZE:pointer_at_end-self.block_size] = STRUCT_Q.pack(timestamp)
+ self.mmap[pointer_at_end-self.block_size:pointer_at_end] = data
self.entries += 1
self.max_ts = timestamp
return 0
@@ -104,7 +108,7 @@ cdef class Chunk:
yield self.get_piece_at(i)
def __iter__(self) -> tp.Iterator[tp.Tuple[int, bytes]]:
- cdef unsigned long i = 0
+ cdef int i
for i in range(self.entries):
yield self.get_piece_at(i)
--
GitLab