From 7b23788fea5cb7a0be67d55b1a824ea26a4c9688 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Piotr=20Ma=C5=9Blanka?= <piotr.maslanka@henrietta.com.pl>
Date: Mon, 30 Dec 2019 21:13:12 +0100
Subject: [PATCH] multiple changes
---
CHANGELOG.md | 5 +-
docs/coding/concurrent.rst | 7 +-
docs/coding/functions.rst | 1 +
docs/coding/structures.rst | 13 ++--
docs/configuration/index.rst | 1 +
docs/configuration/schema.rst | 5 +-
docs/configuration/sources.rst | 6 +-
docs/exception_handling.rst | 72 +++++++++++++++++++
docs/exception_handling/index.rst | 13 ----
docs/index.rst | 2 +-
docs/json.rst | 1 +
satella/__init__.py | 2 +-
satella/exception_handling/__init__.py | 2 +
satella/exception_handling/dump_to_file.py | 10 +--
.../exception_handling/exception_handlers.py | 39 +++++++---
satella/exception_handling/global_eh.py | 10 +--
satella/exception_handling/memerrhandler.py | 27 ++++---
17 files changed, 159 insertions(+), 57 deletions(-)
create mode 100644 docs/exception_handling.rst
delete mode 100644 docs/exception_handling/index.rst
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 67e001f4..a5f40b00 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,9 @@
# v2.2.2
-* _TBA_
+* more docs
+* fixed `MemoryErrorExceptionHandler` not being importable from `satella.exception_handling`
+* fixed the custom_hook for `MemoryErrorExceptionHandler`: it won't
+ kill everything if the custom_hook returns True
# v2.2.1
diff --git a/docs/coding/concurrent.rst b/docs/coding/concurrent.rst
index f0684364..d6d67905 100644
--- a/docs/coding/concurrent.rst
+++ b/docs/coding/concurrent.rst
@@ -1,20 +1,21 @@
+==========================
Concurrent data structures
==========================
CallableGroup
--------------
+=============
.. autoclass:: satella.coding.concurrent.CallableGroup
:members:
LockedDataset
--------------
+=============
.. autoclass:: satella.coding.concurrent.LockedDataset
:members:
TerminableThread
-----------------
+================
Note that _force=True_ is not available on PyPy. If an attempt to use it on PyPy is made,
``RuntimeError`` will be thrown.
diff --git a/docs/coding/functions.rst b/docs/coding/functions.rst
index 3c5c4bcb..1cd67d44 100644
--- a/docs/coding/functions.rst
+++ b/docs/coding/functions.rst
@@ -1,3 +1,4 @@
+=========
Functions
=========
diff --git a/docs/coding/structures.rst b/docs/coding/structures.rst
index 2deb7ccc..267f28ec 100644
--- a/docs/coding/structures.rst
+++ b/docs/coding/structures.rst
@@ -1,10 +1,11 @@
+==========
Structures
==========
The following is a guide to all the data structures that Satella defines.
Heap
-----
+====
This essentially allows you to have a heap object that will pretty much
behave like the `heapq <https://docs.python.org/2/library/heapq.html>` library.
@@ -13,7 +14,7 @@ behave like the `heapq <https://docs.python.org/2/library/heapq.html>` library.
:members:
TimeBasedHeap
----------
+=============
Time-based heap is a good structure if you have many callbacks set to fire at a particular
time in the future. It functions very like a normal Heap.
@@ -22,7 +23,7 @@ time in the future. It functions very like a normal Heap.
:members:
typednamedtuple
----------------
+===============
It's a named tuple, but it has typed fields. You will get a TypeError if you
try to assign something else there.
@@ -30,7 +31,7 @@ try to assign something else there.
.. autofunction:: satella.coding.structures.typednamedtuple
OmniHashableMixin
------------------
+=================
If you need quick __hash__ and __eq__ operators from listed fields of the class.
@@ -38,7 +39,7 @@ If you need quick __hash__ and __eq__ operators from listed fields of the class.
:members:
Singleton
----------
+=========
Makes the resulting object's ``__init__()`` be called at most once, then caches the object and returns the same
upon each instantiation.
@@ -46,7 +47,7 @@ upon each instantiation.
.. autofunction:: satella.coding.structures.Singleton
DictObject
-----------
+==========
DictObject is an object constructed out of a dict, that allows it's values to be obtained as getattr(), and not only
getitem().
diff --git a/docs/configuration/index.rst b/docs/configuration/index.rst
index 5b45df2d..c00a54bb 100644
--- a/docs/configuration/index.rst
+++ b/docs/configuration/index.rst
@@ -1,3 +1,4 @@
+=============
Configuration
=============
diff --git a/docs/configuration/schema.rst b/docs/configuration/schema.rst
index 201924bd..45d5a02e 100644
--- a/docs/configuration/schema.rst
+++ b/docs/configuration/schema.rst
@@ -1,5 +1,6 @@
-Schema validation
-=================
+===============================
+Configuration schema validation
+===============================
As noted in index_, your configuration is mostly supposed to be a dict. To validate your schema,
you should instantiate a Descriptor. Descriptor reflects how your config is nested.
diff --git a/docs/configuration/sources.rst b/docs/configuration/sources.rst
index 0a98dc66..ab3b653e 100644
--- a/docs/configuration/sources.rst
+++ b/docs/configuration/sources.rst
@@ -1,5 +1,6 @@
-Sources
-=======
+=====================
+Configuration sources
+=====================
At the core of your config files, there are Sources. A Source is a single source of configuration - it could be
an environment variable, or a particular file, or a directory full of these files.
@@ -34,6 +35,7 @@ Then there are abstract sources of configuration.
.. autoclass:: satella.configuration.sources.MergingSource
:members:
+In order to actually load the configuration, use the method ``provide()``.
JSON schema
diff --git a/docs/exception_handling.rst b/docs/exception_handling.rst
new file mode 100644
index 00000000..548c2b06
--- /dev/null
+++ b/docs/exception_handling.rst
@@ -0,0 +1,72 @@
+==================
+Exception handling
+==================
+
+Satella provides a rich functionality to register exception hooks.
+
+Writing your own exception handlers
+===================================
+
+To write your own exception handlers, subclass the following class:
+
+.. autoclass:: satella.exception_handling.BaseExceptionHandler
+ :members:
+
+And then instantiate it and call ``install()``.
+
+If you got a callable of signature [type, BaseException, types.TracebackType] (type, value, traceback) that
+returns True upon a request to swallow the exception, you can convert it to a Satella exception handler in two ways.
+
+First:
+
+::
+
+ a = FunctionExceptionHandler(exception_handle)
+ a.install()
+
+Or
+
+::
+
+ @exception_handler
+ def exception_handle(type, value, traceback) -> bool
+ ...
+
+ exception_handle().install()
+
+.. autofunction:: satella.exception_handling.exception_handler
+
+.. autoclass:: satella.exception_handling.FunctionExceptionHandler
+ :members:
+
+Pre-defined exception handlers
+==============================
+
+MemoryErrorExceptionHandler
+---------------------------
+
+.. autoclass:: satella.exception_handling.MemoryErrorExceptionHandler
+
+This exception hook kills the entire process if a `MemoryError` is spotted, under the rule that it's better to fail
+early than admit undefined behaviour.
+
+DumpToFileHandler
+-----------------
+
+.. autoclass:: satella.exception_handling.DumpToFileHandler
+
+A handler that will dump each stack frame of the exception, along with it's variables, to a file (or stdout/stderr).
+Two file handles/names are admitted:
+
+* **human_readables** - where human-readable form of the exception and it's values will be output
+* **trace_pickles** - where pickled `Traceback`s from the exception will be put.
+
+You can throw there either:
+
+* a **str** - file of given name will be created and data will be output to it
+* a file-like object supporting ``write()`` (and optionally ``flush()``) - data will be output to it
+* a None - output will be piped to /dev/null
+
+Note that the arguments are lists, so you can specify multiple target sources.
+
+
diff --git a/docs/exception_handling/index.rst b/docs/exception_handling/index.rst
deleted file mode 100644
index 0ea44485..00000000
--- a/docs/exception_handling/index.rst
+++ /dev/null
@@ -1,13 +0,0 @@
-Exception handling
-==================
-
-Satella provides a rich functionality to register exception hooks.
-
-Writing your own exception handlers
------------------------------------
-
-To write your own exception handlers, subclass the following class:
-
-.. autoclass:: satella.exception_handling.BaseExceptionHandler
- :members:
-
diff --git a/docs/index.rst b/docs/index.rst
index 30e19e9e..8a69a3f3 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -14,7 +14,7 @@ Welcome to satella's documentation!
coding/concurrent
instrumentation/traceback
instrumentation/metrics
- exception_handling/index
+ exception_handling
json
posix
recipes
diff --git a/docs/json.rst b/docs/json.rst
index 5cab7d12..f9844a89 100644
--- a/docs/json.rst
+++ b/docs/json.rst
@@ -1,3 +1,4 @@
+====
JSON
====
diff --git a/satella/__init__.py b/satella/__init__.py
index 17ff25bc..5ce5a63d 100644
--- a/satella/__init__.py
+++ b/satella/__init__.py
@@ -1,2 +1,2 @@
# coding=UTF-8
-__version__ = '2.2.2a1'
+__version__ = '2.2.2a2'
diff --git a/satella/exception_handling/__init__.py b/satella/exception_handling/__init__.py
index 7a4e20bb..39502e18 100644
--- a/satella/exception_handling/__init__.py
+++ b/satella/exception_handling/__init__.py
@@ -1,10 +1,12 @@
from .dump_to_file import *
from .exception_handlers import *
from .global_eh import *
+from .memerrhandler import MemoryErrorExceptionHandler
__all__ = [
'GlobalExcepthook',
'BaseExceptionHandler', 'exception_handler', 'FunctionExceptionHandler',
'NORMAL_PRIORITY', 'ALWAYS_LAST', 'ALWAYS_FIRST',
+ 'MemoryErrorExceptionHandler',
'DumpToFileHandler', 'AsStream'
]
diff --git a/satella/exception_handling/dump_to_file.py b/satella/exception_handling/dump_to_file.py
index 1ade0230..c32af116 100644
--- a/satella/exception_handling/dump_to_file.py
+++ b/satella/exception_handling/dump_to_file.py
@@ -12,12 +12,12 @@ from .exception_handlers import BaseExceptionHandler
logger = logging.getLogger(__name__)
__all__ = [
- 'DumpToFileHandler', 'AsStream'
+ 'DumpToFileHandler'
]
AsStreamTypeAccept = tp.Union[str, tp.IO, None]
AsStreamTypeAcceptHR = tp.Union[str, tp.TextIO]
-AsStreamTypeAcceptpIN = tp.Union[str, tp.BinaryIO]
+AsStreamTypeAcceptIN = tp.Union[str, tp.BinaryIO]
class AsStream:
@@ -80,11 +80,13 @@ class AsStream:
class DumpToFileHandler(BaseExceptionHandler):
"""
- Write the stack trace to a stream-file
+ Write the stack trace to a stream-file.
+
+ Note that your file-like objects you throw into that must support only .write() and optionally .flush()
"""
def __init__(self, human_readables: tp.Iterable[AsStreamTypeAcceptHR],
- trace_pickles: tp.Iterable[AsStreamTypeAcceptpIN] = None):
+ trace_pickles: tp.Iterable[AsStreamTypeAcceptIN] = None):
"""
Handler that dumps an exception to a file.
diff --git a/satella/exception_handling/exception_handlers.py b/satella/exception_handling/exception_handlers.py
index cecc9f68..e36c4420 100644
--- a/satella/exception_handling/exception_handlers.py
+++ b/satella/exception_handling/exception_handlers.py
@@ -1,5 +1,5 @@
-import types
import typing as tp
+import types
from abc import abstractmethod
__all__ = [
@@ -8,7 +8,8 @@ __all__ = [
'exception_handler',
'ALWAYS_FIRST',
'ALWAYS_LAST',
- 'NORMAL_PRIORITY'
+ 'NORMAL_PRIORITY',
+ 'ExceptionHandlerCallable'
]
ALWAYS_FIRST = -1000
@@ -16,28 +17,50 @@ NORMAL_PRIORITY = 0
ALWAYS_LAST = 1000
+ExceptionHandlerCallable = tp.Callable[[type, BaseException, types.TracebackType], bool]
+
+
class BaseExceptionHandler:
def __init__(self, priority=NORMAL_PRIORITY):
+ """
+ Instantiate an exception handler with provided priority.
+ Handlers with smaller priorities run sooner.
+
+ :param priority: Priority to use for this handler
+ """
self.priority = priority
- def install(self):
+ def install(self) -> 'BaseExceptionHandler':
+ """
+ Register this handler to run upon exceptions
+ """
from .global_eh import GlobalExcepthook
GlobalExcepthook().add_hook(self)
return self
def uninstall(self):
+ """
+ Unregister this handler to run on exceptions
+ """
from .global_eh import GlobalExcepthook
GlobalExcepthook().remove_hook(self)
@abstractmethod
- def handle_exception(self, type_: tp.Optional[type], value,
+ def handle_exception(self, type_: tp.Callable[[type, BaseException, types.TracebackType], None], value,
traceback: types.TracebackType) -> tp.Optional[bool]:
- """Return True to intercept the exception. It won't be propagated to other handlers."""
+ """
+ Return True to intercept the exception, so that it won't be propagated to other handlers.
+ """
pass
class FunctionExceptionHandler(BaseExceptionHandler):
- def __init__(self, fun: tp.Callable, priority: int = NORMAL_PRIORITY):
+ """
+ A exception handler to make callables of given signature into Satella's exception handlers.
+
+ Your exception handler must return a bool, whether to intercept the exception and not propagate it.
+ """
+ def __init__(self, fun: ExceptionHandlerCallable, priority: int = NORMAL_PRIORITY):
super(FunctionExceptionHandler, self).__init__(priority)
self.fun = fun
@@ -52,7 +75,7 @@ def exception_handler(priority: int = NORMAL_PRIORITY):
Convert a callable to an FunctionExceptionHandler. Usage
>>> @exception_handler(priority=-10)
- >>> def handle_exc(type, val, traceback):
+ >>> def handle_exc(type_, val, traceback):
>>> ...
:return: ExceptionHandler instance
@@ -61,7 +84,7 @@ def exception_handler(priority: int = NORMAL_PRIORITY):
if not isinstance(priority, int):
raise TypeError('Did you forget to use it as @exception_handler() ?')
- def outer(fun):
+ def outer(fun: ExceptionHandlerCallable) -> FunctionExceptionHandler:
return FunctionExceptionHandler(fun, priority=priority)
return outer
diff --git a/satella/exception_handling/global_eh.py b/satella/exception_handling/global_eh.py
index 015a213b..2d9b72a6 100644
--- a/satella/exception_handling/global_eh.py
+++ b/satella/exception_handling/global_eh.py
@@ -36,7 +36,7 @@ class GlobalExcepthook:
self.installed_hooks.remove(hook)
def add_hook(self, new_hook: tp.Union[
- tp.Callable, BaseExceptionHandler]) -> BaseExceptionHandler:
+ tp.Callable, BaseExceptionHandler]) -> BaseExceptionHandler:
"""
Register a hook to fire in case of an exception.
@@ -78,7 +78,7 @@ class GlobalExcepthook:
my_self.__except_handle(
*sys.exc_info()) # by now, it's our handler :D
except AttributeError:
- eh = None # Python interpreter is in an advanced state of shutdown, just let it go
+ pass # Python interpreter is in an advanced state of shutdown, just let it go
raise # re-raise if running on debug
@@ -88,13 +88,13 @@ class GlobalExcepthook:
threading.Thread.__init__ = init
- def __except_handle(self, type, value, traceback) -> None:
+ def __except_handle(self, type_, value, traceback) -> None:
hooks_to_run = self.installed_hooks + [self.old_excepthook]
- for hook in sorted(hooks_to_run, key=lambda hook: hook.priority):
+ for hook in sorted(hooks_to_run, key=lambda h: h.priority):
# noinspection PyBroadException
try:
- if hook.handle_exception(type, value, traceback):
+ if hook.handle_exception(type_, value, traceback):
break
except Exception as e:
tb = Traceback()
diff --git a/satella/exception_handling/memerrhandler.py b/satella/exception_handling/memerrhandler.py
index 03abea26..74f808aa 100644
--- a/satella/exception_handling/memerrhandler.py
+++ b/satella/exception_handling/memerrhandler.py
@@ -1,23 +1,27 @@
-import logging
import sys
import time
import typing as tp
from satella.posix import suicide
-from .exception_handlers import BaseExceptionHandler, ALWAYS_FIRST
-
-logger = logging.getLogger(__name__)
+from .exception_handlers import BaseExceptionHandler, ALWAYS_FIRST, ExceptionHandlerCallable
class MemoryErrorExceptionHandler(BaseExceptionHandler):
- def __init__(self, custom_hook: tp.Callable = lambda type, value, traceback: None,
+ """
+ A handler that terminates the entire process (or process group) is a MemoryError is seen.
+
+ `custom_hook` is an exception callable to implement you own behavior. If it returns True,
+ then MemoryErrorExceptionHandler won't kill anyone.
+ """
+ def __init__(self, custom_hook: ExceptionHandlerCallable = lambda type_, value, traceback: False,
kill_pg: bool = False):
"""
- :param kill_pg: kill entire process group, if applicable
+ :param kill_pg: whether to kill entire process group, if applicable
"""
super(MemoryErrorExceptionHandler, self).__init__()
self.priority = ALWAYS_FIRST # always run first!
- self._free_on_memoryerror = {'a': bytearray(1024 * 2)}
+ # so that we have some spare space in case a MemoryError is thrown
+ self._free_on_memory_error = {'a': bytearray(1024 * 2)}
self.custom_hook = custom_hook
self.kill_pg = kill_pg
self.installed = False
@@ -29,15 +33,16 @@ class MemoryErrorExceptionHandler(BaseExceptionHandler):
from .global_eh import GlobalExcepthook
GlobalExcepthook().add_hook(self)
- def handle_exception(self, type, value, traceback) -> tp.Optional[bool]:
- if not issubclass(type, MemoryError):
+ def handle_exception(self, type_, value, traceback) -> tp.Optional[bool]:
+ if not issubclass(type_, MemoryError):
return
- del self._free_on_memoryerror['a']
+ del self._free_on_memory_error['a']
# noinspection PyBroadException
try:
- self.custom_hook(type, value, traceback)
+ if self.custom_hook(type_, value, traceback):
+ return
except Exception as e:
pass
--
GitLab