From bff0fb99d041905dfa901b3458b4405abb640e5b Mon Sep 17 00:00:00 2001
From: Piotr Maslanka <piotr.maslanka@henrietta.com.pl>
Date: Wed, 17 May 2017 20:14:25 +0200
Subject: [PATCH] try
---
.coveragerc | 2 +-
.dockerignore | 2 +
.gitlab-ci.yml | 28 +++----------
CHANGELOG.md | 3 --
CONTRIBUTING.md | 1 -
Dockerfile | 11 +++++
LICENSE | 2 +-
README.md | 44 +++++---------------
Vagrantfile | 10 -----
circle/__init__.py | 15 +++++++
circle/metrics/__init__.py | 23 +++++++++++
circle/metrics/alarms.py | 43 +++++++++++++++++++
circle/metrics/devices_online.py | 42 +++++++++++++++++++
circle/metrics/sms_sent.py | 71 ++++++++++++++++++++++++++++++++
circle/metrics/utils.py | 24 +++++++++++
circle/run.py | 29 +++++++++++++
circle/serve.py | 63 ++++++++++++++++++++++++++++
docs/README.md | 5 ---
setup.py | 18 +++-----
your_project_source/__init__.py | 1 -
20 files changed, 347 insertions(+), 90 deletions(-)
create mode 100644 .dockerignore
delete mode 100644 CHANGELOG.md
delete mode 100644 CONTRIBUTING.md
create mode 100644 Dockerfile
delete mode 100644 Vagrantfile
create mode 100644 circle/__init__.py
create mode 100644 circle/metrics/__init__.py
create mode 100644 circle/metrics/alarms.py
create mode 100644 circle/metrics/devices_online.py
create mode 100644 circle/metrics/sms_sent.py
create mode 100644 circle/metrics/utils.py
create mode 100644 circle/run.py
create mode 100644 circle/serve.py
delete mode 100644 docs/README.md
delete mode 100644 your_project_source/__init__.py
diff --git a/.coveragerc b/.coveragerc
index 3e2716d..7819b9d 100644
--- a/.coveragerc
+++ b/.coveragerc
@@ -1,7 +1,7 @@
[run]
branch=1
include=
- your_project_source/*
+ circle/*
omit=
tests/*
diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 0000000..e9dcab0
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,2 @@
+.git
+tests
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index bfc356c..8941cfc 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,26 +1,8 @@
+
build:
stage: build
+ variables:
+ DOCKER_HOST: "192.168.224.200:2375"
script:
- - python setup.py sdist bdist bdist_wheel
- artifacts:
- paths:
- - dist/
- only:
- - staging
- - master
-unit_tests:
- stage: test
- coverage: /^TOTAL\s+\d+\s+\d+\s+\d+\s+\d+\s+(\d+)%$/
- script:
- - python setup.py nosetests
-
-# vagrant_backed_tests
-# stage: test
-# coverage: /^TOTAL\s+\d+\s+\d+\s+\d+\s+\d+\s+(\d+)%$/
-# script:
-# - vagrant ssh -c 'sudo shutdown -P +10'
-# - vagrant ssh -c 'cd /vagrant; python setup.py nosetests'
-# before_script:
-# - vagrant up
-# after_script:
-# - vagrant destroy -f
\ No newline at end of file
+ - docker build -t "zoo.smok.co:5000/smok4/circle:$CI_COMMIT_REF_NAME" .
+ - docker push "zoo.smok.co:5000/smok4/circle:$CI_COMMIT_REF_NAME"
diff --git a/CHANGELOG.md b/CHANGELOG.md
deleted file mode 100644
index 5ff8628..0000000
--- a/CHANGELOG.md
+++ /dev/null
@@ -1,3 +0,0 @@
-# v1.0
-
-Nothing there
\ No newline at end of file
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
deleted file mode 100644
index f7e7a75..0000000
--- a/CONTRIBUTING.md
+++ /dev/null
@@ -1 +0,0 @@
-Ask your maintainer. Keep _master_ working.
\ No newline at end of file
diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 0000000..cad3506
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,11 @@
+FROM zoo.smok.co:5000/smok4/core:master
+
+RUN mkdir /tmp/install
+ADD . /tmp/install
+WORKDIR /tmp/install
+RUN python setup.py install
+RUN rm -rf /tmp/install
+
+ENV SMOK4_SERVICE=circle
+CMD /usr/bin/python -m circle.run
+
diff --git a/LICENSE b/LICENSE
index 0cb5855..6464ea0 100644
--- a/LICENSE
+++ b/LICENSE
@@ -1 +1 @@
-Copyright (c) X-Y Z. All rights reserved.
+Copyright DMS Serwis (c) 2017. All rights reserved.
diff --git a/README.md b/README.md
index 4e7295c..05ad777 100644
--- a/README.md
+++ b/README.md
@@ -1,34 +1,12 @@
-your_project
+circle
============
-[](http://git.dms-serwis.com.pl/henrietta/py-scaffold/commits/master)
-[](http://git.dms-serwis.com.pl/henrietta/py-scaffold/commits/master)
-
-This is a generic scaffold for Python projects that use:
-* _GitLab CI_ for builds/tests/deploys
-* _coverage.py_ for test coverage
-* _nose_ for unit tests
-* _Vagrant_ for environment
-* _setuptools_ for packaging
-* _git_ for version control
-
-# How to use
-
-1. `git clone http://git.dms-serwis.com.pl/henrietta/py-scaffold.git`
-2. delete `.git` directory
-3. Rename [your_project_source](your_project_source) to match your project
-4. adjust [setup.py](setup.py), and optionally [setup.cfg](setup.cfg)
-5. adjust [.coveragerc](.coveragerc)
-6. adjust [license](LICENSE)
-7. adjust or delete [contribution guide](CONTRIBUTING.md)
-8. adjust or delete [change log](CHANGELOG.md)
-9. adjust [Vagrantfile](Vagrantfile), or remove it
-10. adjust [MANIFEST.in](MANIFEST.in) if you have data files
-11. adjust [docs](docs/)
-12. adjust [gitlab-ci.yml](.gitlab-ci.yml), especially if your tests require a
- Vagrant environment
-13. Set up a repository on GitLab
-14. Ensure you have _master_ permission on this repository
-15. `git init`
-16. Add suitable remotes, commit the data, push to repo
-17. Done!
-
+[](http://git.dms-serwis.com.pl/smok4/circle/commits/master)
+[](http://git.dms-serwis.com.pl/smok4/circle/commits/master)
+
+
+ Zdefiniuj Ĺrodowiskowe:
+
+* `SMOK4_CIRCLE_BINDADDR` - interfejs do nasĹuchu HTTP
+* `SMOK4_CIRCLE_BINDPORT` - port do nasĹuchu HTTP
+
+`CMD` jako `/usr/bin/python -m circle.run`
diff --git a/Vagrantfile b/Vagrantfile
deleted file mode 100644
index 206aa0c..0000000
--- a/Vagrantfile
+++ /dev/null
@@ -1,10 +0,0 @@
-
-Vagrant.configure("2") do |config|
-
- config.vm.box = "debian/contrib-jessie64"
-
- config.vm.provision "shell", inline: <<-SHELL
- apt-get update
-
- SHELL
-end
diff --git a/circle/__init__.py b/circle/__init__.py
new file mode 100644
index 0000000..8dde00a
--- /dev/null
+++ b/circle/__init__.py
@@ -0,0 +1,15 @@
+# coding=UTF-8
+"""
+Prometheus metrics gatherer
+
+Supported envs:
+ SMOK4_CIRCLE_BINDPORT (default 7998)
+ SMOK4_CIRCLE_BINDADDR (default 0.0.0.0)
+"""
+from __future__ import print_function, absolute_import, division
+import six
+import logging
+
+logger = logging.getLogger(__name__)
+
+
diff --git a/circle/metrics/__init__.py b/circle/metrics/__init__.py
new file mode 100644
index 0000000..c6d8818
--- /dev/null
+++ b/circle/metrics/__init__.py
@@ -0,0 +1,23 @@
+# coding=UTF-8
+"""
+This will return a list of Metric objects, each of which can return a line to write out.
+"""
+from __future__ import print_function, absolute_import, division
+import six
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+def setup_metrics():
+ """
+ Return a list of callable/0 that return generators with lines.
+
+ Called once per start
+ """
+ from circle.metrics.devices_online import device_status_metric
+ from circle.metrics.sms_sent import oob_sent
+ from circle.metrics.alarms import get_alarms
+
+ return [oob_sent, device_status_metric, get_alarms]
+
diff --git a/circle/metrics/alarms.py b/circle/metrics/alarms.py
new file mode 100644
index 0000000..60dcab2
--- /dev/null
+++ b/circle/metrics/alarms.py
@@ -0,0 +1,43 @@
+# coding=UTF-8
+"""
+It sounds like a melody
+"""
+from __future__ import print_function, absolute_import, division
+import six
+from sai.basic import SQL
+from collections import defaultdict
+import time
+from circle.metrics.utils import CounterHelper
+
+CNTRS = {
+ 'started': CounterHelper(),
+ 'ended': CounterHelper()
+}
+
+
+def get_alarms():
+ with SQL.cursor('catalog', 'smok4_sars') as cur:
+
+ cur.execute('SELECT start_on, token FROM event WHERE start_on > %s', (int(time.time() - 24*60*60),))
+ started = cur.fetchall()
+
+ cur.execute('SELECT end_on, token FROM event WHERE end_on > %s', (int(time.time() - 24*60*60),))
+ stopped = cur.fetchall()
+
+ by_tokens = defaultdict(lambda: defaultdict(lambda: 0)) # by token, action
+
+ for action, lst in zip(['started', 'ended'], [started, stopped]):
+ for ts, token in lst:
+ by_tokens[token][action] += 1
+ CNTRS[action].feed(ts)
+
+ st_v = len(started[0])
+
+ for action, cntr in CNTRS.iteritems():
+ yield 'smok4_alarms_total{action="%s"} %s' % (action, cntr.value())
+
+ for token, stat in by_tokens.iteritems():
+ for action, v in stat.iteritems():
+ yield 'smok4_alarms_rollup{rollup="24h", token="%s", action="%s"} %s' % (token, action, v)
+
+
diff --git a/circle/metrics/devices_online.py b/circle/metrics/devices_online.py
new file mode 100644
index 0000000..f2d7b1e
--- /dev/null
+++ b/circle/metrics/devices_online.py
@@ -0,0 +1,42 @@
+# coding=UTF-8
+from __future__ import print_function, absolute_import, division
+import six
+from collections import defaultdict
+import logging
+from sai.basic import Cassandra, Configuration2
+
+
+def device_status_metric():
+ with Cassandra.cursor() as cur:
+ p = cur.execute('SELECT node, online FROM s4obj.devices')
+
+ status = {
+ 0: defaultdict(lambda: 0), # offline
+ 1: defaultdict(lambda: 0) # online
+ }
+ for node, online in p:
+ status[1 if online else 0][node] += 1
+
+ for status, stat in status.iteritems():
+ for node, count in stat.iteritems():
+ yield u'smok4_device_status{node="%s", online="%s"} %s' % (node, status, count)
+
+ for node in Configuration2.get_device_nodes():
+
+ dev_status = defaultdict(lambda: defaultdict(lambda: 0))
+ han_status = defaultdict(lambda: defaultdict(lambda: 0))
+
+ with Cassandra.cursor() as cur:
+ p = cur.execute('SELECT online, handler_online, device_id, master_controller FROM sfcore.devices_on_nodes WHERE node=%s', (node, ))
+
+ for online, han_online, device_id, master_controller in p:
+
+ is_mc = int(device_id == master_controller)
+
+ dev_status[1 if online else 0][is_mc] += 1
+ han_status[1 if han_online else 0][is_mc] += 1
+
+ for st in (0, 1):
+ for is_mc in (0, 1):
+ yield u'smok4_thrall_online_status{node="%s", online="%s", master_controller="%s"} %s' % (node, st, is_mc, dev_status[st][is_mc])
+ yield u'smok4_thrall_handler_status{node="%s", online="%s", master_controller="%s"} %s' % (node, st, is_mc, han_status[st][is_mc])
diff --git a/circle/metrics/sms_sent.py b/circle/metrics/sms_sent.py
new file mode 100644
index 0000000..c88340a
--- /dev/null
+++ b/circle/metrics/sms_sent.py
@@ -0,0 +1,71 @@
+# coding=UTF-8
+from __future__ import print_function, absolute_import, division
+import six
+from collections import defaultdict
+import logging
+from sai.basic import Cassandra
+import time
+from .utils import CounterHelper
+import datetime
+
+"""
+ Dimensions are
+ - rollup
+ - msg_class
+"""
+
+ROLLUPS = { # name => seconds
+ '5m': 5*60,
+ '15m': 15*60,
+ '60m': 60*60,
+ '3h': 60*60*3,
+ '24h': 60*60*24
+}
+
+TOTALS = defaultdict(CounterHelper)
+
+def oob_sent():
+ p = datetime.datetime.utcfromtimestamp(time.time())
+ now_day = datetime.date(year=p.year, month=p.month, day=p.day)
+
+ # We will execute 2 queries and check that
+
+ # you need to be bigger than this to qualify (milliseconds)
+ cutoff_point_for = dict(
+ (rollname, (time.time() - ROLLUPS[rollname])*1000) for rollname in ROLLUPS.keys()
+ )
+ metrics = dict((rollname, defaultdict(lambda: 0)) for rollname in ROLLUPS.keys())
+
+
+ supl_sms_by_owner_24 = defaultdict(lambda: 0) # supplemental metric - rollup 24h, by owner.
+
+ with Cassandra.cursor() as cur:
+ p = cur.execute('SELECT sent_on, msg_class, owner FROM s4obj.oob_messages_sent WHERE day IN (%s, %s)',
+ (now_day, now_day-datetime.timedelta(1)))
+
+ msg_classes_found = set()
+
+ for sent_on, msg_class, owner in p:
+ TOTALS[msg_class].feed(sent_on)
+ msg_classes_found.add(msg_class)
+
+ for rollname, cutoff in cutoff_point_for.iteritems():
+ if sent_on < cutoff:
+ continue
+
+ metrics[rollname][msg_class] += 1
+
+ if rollname == '24h' and msg_class == 'sms':
+ supl_sms_by_owner_24[owner] += 1
+
+ for rollup, stat in metrics.iteritems():
+
+ for msg_class_name in msg_classes_found:
+ yield u'smok4_oob_messages{rollup="%s", msg_class="%s"} %s' % (rollup, msg_class_name,
+ metrics[rollup][msg_class_name])
+
+ for owner, val in supl_sms_by_owner_24.iteritems():
+ yield u'smok4_sms_by_owner_24h{owner="%s"} %s' % (owner, val)
+
+ for msg_class in msg_classes_found:
+ yield u'smok4_total{msg_class="%s"} %s' % (msg_class, TOTALS[msg_class].value())
diff --git a/circle/metrics/utils.py b/circle/metrics/utils.py
new file mode 100644
index 0000000..87979cd
--- /dev/null
+++ b/circle/metrics/utils.py
@@ -0,0 +1,24 @@
+# coding=UTF-8
+from __future__ import print_function, absolute_import, division
+import six
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class CounterHelper(object):
+ """
+ States that up to timestamp X, Y objects have been received
+ """
+
+ def __init__(self):
+ self.objects = 0
+ self.timestamp = 0
+
+ def feed(self, ts):
+ if ts > self.timestamp:
+ self.objects += 1
+ self.timestamp = ts
+
+ def value(self):
+ return self.objects
diff --git a/circle/run.py b/circle/run.py
new file mode 100644
index 0000000..b6c6d74
--- /dev/null
+++ b/circle/run.py
@@ -0,0 +1,29 @@
+# coding=UTF-8
+from __future__ import print_function, absolute_import, division
+import signal, os, socket
+from sai.moves.satella1 import hang_until_sig
+from sai.basic import SAIContext
+from circle.serve import start_serving_thread
+
+
+if __name__ == '__main__':
+ with SAIContext('circle', intercept_exceptions=False):
+
+ s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+
+ bindat = (
+ os.environ.get('SMOK4_CIRCLE_BINDADDR', '0.0.0.0'),
+ int(os.environ.get('SMOK4_CIRCLE_BINDPORT', '7998'))
+ )
+
+ s.bind(bindat)
+ s.listen(0)
+ s.shutdown(socket.SHUT_RDWR)
+ s.close()
+
+
+ start_serving_thread(bindat)
+
+ hang_until_sig(extra_signals=[signal.SIGQUIT, signal.SIGCONT])
+
diff --git a/circle/serve.py b/circle/serve.py
new file mode 100644
index 0000000..4441254
--- /dev/null
+++ b/circle/serve.py
@@ -0,0 +1,63 @@
+# coding=UTF-8
+"""
+Serves HTTP requests
+"""
+from __future__ import print_function, absolute_import, division
+import six
+import logging
+import threading
+import SimpleHTTPServer
+import SocketServer
+from .metrics import setup_metrics
+
+logger = logging.getLogger(__name__)
+
+
+class CircleRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
+ """
+ WARNING!
+ SimpleHTTPServer.SimpleHTTPRequestHandler is an
+
+ OLD
+
+ STYLE
+
+ OBJECT
+
+
+ exercise due care!
+ """
+ def do_GET(self):
+ if 'favicon.ico' in self.path:
+ self.send_error(404)
+ return
+
+ self.send_response(200)
+ self.send_header('Content-Type', 'text/plain')
+ self.send_header('Content-Encoding', 'utf8')
+ self.end_headers()
+
+ for metric in self.server.metrics:
+
+ lines_for_this_metric = list(metric())
+ response = (u'\n'.join(lines_for_this_metric)).encode('utf8')
+ self.wfile.write(response)
+ print(response)
+ self.wfile.write(b'\n')
+
+
+def start_serving_thread(bindifc):
+ """
+ Start a daemonic serving thread and return.
+ :param bindifc: tuple of (addr, port) to listen on
+ """
+
+ class _InnerServe(threading.Thread):
+ def run(self):
+ httpd = SocketServer.TCPServer(bindifc, CircleRequestHandler)
+ httpd.metrics = setup_metrics()
+ httpd.serve_forever()
+
+ t = _InnerServe()
+ t.daemon = True
+ t.start()
diff --git a/docs/README.md b/docs/README.md
deleted file mode 100644
index 5468404..0000000
--- a/docs/README.md
+++ /dev/null
@@ -1,5 +0,0 @@
-# your-project docs
-
-This is index file for documentation, if any is needed,
-or any should remain here and not somewhere else (eg.
-Confluence).
diff --git a/setup.py b/setup.py
index ccdaf52..b6c3abc 100644
--- a/setup.py
+++ b/setup.py
@@ -3,22 +3,16 @@
#todo adjust this
from setuptools import setup
setup(
- name="your-project",
- version="0.0rc0",
- author=u'Banana',
- author_email='banana@example.com',
- description=u'banana banana banana',
- url='http://example.com',
+ name="smok4-circle",
+ version="1.0rc0",
+ author=u'DMS s.c.',
+ description=u'SMOK4 metric exporter for Prometheus',
packages=[
- 'your_project_source',
+ 'circle',
+ 'circle.metrics'
],
data_files=[
],
- classifiers=[
- # See https://pypi.python.org/pypi?%3Aaction=list_classifiers for list of classifiers
- 'Development Status :: 1 - Planning',
- 'Programming Language :: Python'
- ],
tests_require=['nose', 'mock', 'coverage'],
test_suite='nose.collector'
)
diff --git a/your_project_source/__init__.py b/your_project_source/__init__.py
deleted file mode 100644
index 9599562..0000000
--- a/your_project_source/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-# coding=UTF-8
--
GitLab