Skip to content
Snippets Groups Projects
Commit 01539b3b authored by Piotr Maślanka's avatar Piotr Maślanka
Browse files

use mmap for writes too

parent e3ad397d
No related branches found
No related tags found
No related merge requests found
Chunk
=====
For your convenience the class :class:`~tempsdb.chunks.Chunk` was also documented, but don't use
it directly:
.. autoclass:: tempsdb.chunks.Chunk
:members:
...@@ -12,8 +12,9 @@ Welcome to tempsdb's documentation! ...@@ -12,8 +12,9 @@ Welcome to tempsdb's documentation!
usage usage
exceptions exceptions
chunks
It tries to use mmap for reads where possible, and in general be as zero-copy as possible (ie. the It tries to use mmap for reads and writes, and in general is as zero-copy as possible (ie. the
only time data is unserialized is when a particular entry is read). only time data is unserialized is when a particular entry is read).
Stored time series with a 8-bit timestamp and a fixed length of data. Stored time series with a 8-bit timestamp and a fixed length of data.
......
...@@ -14,4 +14,3 @@ Then you can create and retrieve particular series: ...@@ -14,4 +14,3 @@ Then you can create and retrieve particular series:
.. autoclass:: tempsdb.series.TimeSeries .. autoclass:: tempsdb.series.TimeSeries
:members: :members:
...@@ -18,6 +18,7 @@ cdef class Chunk: ...@@ -18,6 +18,7 @@ cdef class Chunk:
cpdef void close(self) cpdef void close(self)
cpdef tuple get_piece_at(self, unsigned int index) cpdef tuple get_piece_at(self, unsigned int index)
cpdef int put(self, unsigned long long timestamp, bytes data) except -1 cpdef int put(self, unsigned long long timestamp, bytes data) except -1
cpdef int sync(self) except -1
cdef inline int length(self): cdef inline int length(self):
return self.entries return self.entries
......
...@@ -56,6 +56,13 @@ cdef class Chunk: ...@@ -56,6 +56,13 @@ cdef class Chunk:
self.max_ts, = STRUCT_Q.unpack(self.mmap[-TIMESTAMP_SIZE-self.block_size:-self.block_size]) self.max_ts, = STRUCT_Q.unpack(self.mmap[-TIMESTAMP_SIZE-self.block_size:-self.block_size])
self.min_ts, = STRUCT_Q.unpack(self.mmap[HEADER_SIZE:HEADER_SIZE+TIMESTAMP_SIZE]) self.min_ts, = STRUCT_Q.unpack(self.mmap[HEADER_SIZE:HEADER_SIZE+TIMESTAMP_SIZE])
cpdef int sync(self) except -1:
"""
Synchronize the mmap
"""
self.mmap.flush()
return 0
cpdef int put(self, unsigned long long timestamp, bytes data) except -1: cpdef int put(self, unsigned long long timestamp, bytes data) except -1:
""" """
Append a record to this chunk Append a record to this chunk
...@@ -73,14 +80,11 @@ cdef class Chunk: ...@@ -73,14 +80,11 @@ cdef class Chunk:
raise ValueError('data not equal in length to block size!') raise ValueError('data not equal in length to block size!')
if timestamp <= self.max_ts: if timestamp <= self.max_ts:
raise ValueError('invalid timestamp') raise ValueError('invalid timestamp')
cdef unsigned long long pointer_at_end = (self.entries+1)*(TIMESTAMP_SIZE+self.block_size) + HEADER_SIZE
cdef bytearray data_to_write = bytearray(TIMESTAMP_SIZE+self.block_size)
data_to_write[0:TIMESTAMP_SIZE] = STRUCT_Q.pack(timestamp)
data_to_write[TIMESTAMP_SIZE:] = data
with self.write_lock: with self.write_lock:
self.file.seek(0, 2) self.mmap.resize(pointer_at_end)
self.file.write(data_to_write) self.mmap[pointer_at_end-self.block_size-TIMESTAMP_SIZE:pointer_at_end-self.block_size] = STRUCT_Q.pack(timestamp)
self.mmap.resize((self.entries+1)*(8+self.block_size)+HEADER_SIZE) self.mmap[pointer_at_end-self.block_size:pointer_at_end] = data
self.entries += 1 self.entries += 1
self.max_ts = timestamp self.max_ts = timestamp
return 0 return 0
...@@ -104,7 +108,7 @@ cdef class Chunk: ...@@ -104,7 +108,7 @@ cdef class Chunk:
yield self.get_piece_at(i) yield self.get_piece_at(i)
def __iter__(self) -> tp.Iterator[tp.Tuple[int, bytes]]: def __iter__(self) -> tp.Iterator[tp.Tuple[int, bytes]]:
cdef unsigned long i = 0 cdef int i
for i in range(self.entries): for i in range(self.entries):
yield self.get_piece_at(i) yield self.get_piece_at(i)
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment