From b893a532f214a7d537507f5a7f670cdf554b6810 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Piotr=20Ma=C5=9Blanka?= <piotr.maslanka@henrietta.com.pl>
Date: Thu, 26 Aug 2021 16:32:56 +0200
Subject: [PATCH] add migrations and remaining two apps
---
Dockerfile | 4 ++-
README.md | 24 +++++++++++++++--
agent/__init__.py | 0
agent/admin.py | 3 +++
agent/apps.py | 6 +++++
agent/middleware.py | 18 +++++++++++++
agent/migrations/0001_initial.py | 25 +++++++++++++++++
agent/migrations/__init__.py | 0
agent/models.py | 7 +++++
agent/tests.py | 3 +++
agent/views.py | 3 +++
counting/__init__.py | 0
counting/admin.py | 3 +++
counting/apps.py | 6 +++++
counting/migrations/0001_initial.py | 23 ++++++++++++++++
counting/migrations/__init__.py | 0
counting/models.py | 7 +++++
counting/tests.py | 3 +++
counting/views.py | 3 +++
netguru/settings.py | 23 +++++++++++++---
shares/migrations/0001_initial.py | 4 +--
shares/models.py | 42 ++++++++++++++++++++++++++++-
shares/views.py | 17 +++++++-----
23 files changed, 207 insertions(+), 17 deletions(-)
create mode 100644 agent/__init__.py
create mode 100644 agent/admin.py
create mode 100644 agent/apps.py
create mode 100644 agent/middleware.py
create mode 100644 agent/migrations/0001_initial.py
create mode 100644 agent/migrations/__init__.py
create mode 100644 agent/models.py
create mode 100644 agent/tests.py
create mode 100644 agent/views.py
create mode 100644 counting/__init__.py
create mode 100644 counting/admin.py
create mode 100644 counting/apps.py
create mode 100644 counting/migrations/0001_initial.py
create mode 100644 counting/migrations/__init__.py
create mode 100644 counting/models.py
create mode 100644 counting/tests.py
create mode 100644 counting/views.py
diff --git a/Dockerfile b/Dockerfile
index 0037322..9084a98 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -9,11 +9,13 @@ ADD netguru /app/netguru
ADD manage.py /app/manage.py
ADD shares /app/shares
ADD templates /app/templates
+ADD counting /app/counting
+ADD agent /app/agent
ADD test.sh /app/test.sh
RUN python manage.py collectstatic && \
chmod ugo+x /app/test.sh
VOLUME /data
-CMD ["gunicorn", "-w", "4", "-b", "0.0.0.0:80", "netguru.wsgi:application"]
+CMD ["gunicorn", "-w", "1", "--threads", "4", "-b", "0.0.0.0:80", "netguru.wsgi:application"]
diff --git a/README.md b/README.md
index 98e37f9..f36d79c 100644
--- a/README.md
+++ b/README.md
@@ -15,6 +15,15 @@ To run locally just run:
docker-compose up -d run_local
```
+This will expose both ports 80 (for TCP traffic)
+and ports 81 (for metrics).
+
+I expect you to terminate SSL at a reverse proxy.
+Please watch out for generated links, as they will
+invariable link to SSL, because I've got no easy way
+to detect if I've been called with SSL already, or is
+it just a reverse proxy handing over the requests.
+
# Production
## Links
@@ -47,6 +56,15 @@ provide it, a reasonably default value is supplied,
but since it has already been used it isn't safe anymore.
**CHANGE IT!**
+You also need to provide a list of hosts from which
+this Django instance can be accessed. You do that
+by defining an environment variable called
+ALLOWED_HOSTS and giving a list of hosts separated
+by a comma. If you don't define it, a default
+value of `*` will be used
+
+**WARNING!!** This is not secure, don't do it!
+
### Volumes
The application needs a volume marked at
@@ -61,7 +79,7 @@ You will just have to forgive me that I failed
to hook up any loggers for it. Any Python-based
logger will do, I sincerely recommends
[seq-log](https://github.com/tintoy/seqlog)
-- full disclosure: I'm a contributor there
+- full disclosure: I'm a [contributor](https://github.com/tintoy/seqlog/blob/master/AUTHORS.rst) there
#### Metrics
@@ -82,4 +100,6 @@ I failed to deploy that on your test environment since I'm
pretty much strapped for time.
BTW: If you're not already doing
-[tracing](https://opentracing.io/), you should totally consider it.
+[tracing](https://opentracing.io/),
+you should totally consider it.
+
diff --git a/agent/__init__.py b/agent/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/agent/admin.py b/agent/admin.py
new file mode 100644
index 0000000..8c38f3f
--- /dev/null
+++ b/agent/admin.py
@@ -0,0 +1,3 @@
+from django.contrib import admin
+
+# Register your models here.
diff --git a/agent/apps.py b/agent/apps.py
new file mode 100644
index 0000000..f1442df
--- /dev/null
+++ b/agent/apps.py
@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class AgentConfig(AppConfig):
+ default_auto_field = 'django.db.models.BigAutoField'
+ name = 'agent'
diff --git a/agent/middleware.py b/agent/middleware.py
new file mode 100644
index 0000000..b5efb0e
--- /dev/null
+++ b/agent/middleware.py
@@ -0,0 +1,18 @@
+from agent.models import UserAgentStorage
+
+
+class UANotingMiddleware:
+ def __init__(self, get_response):
+ self.get_response = get_response
+
+ def __call__(self, request):
+ response = self.get_response(request)
+ if request.user.is_authenticated:
+ try:
+ ua = UserAgentStorage.objects.get(user=request.user)
+ ua.ua = request.headers.get('User-Agent')
+ except UserAgentStorage.DoesNotExist:
+ ua = UserAgentStorage(user=request.user, ua=request.headers.get('User-Agent'))
+ ua.save()
+
+ return response
diff --git a/agent/migrations/0001_initial.py b/agent/migrations/0001_initial.py
new file mode 100644
index 0000000..fd5b309
--- /dev/null
+++ b/agent/migrations/0001_initial.py
@@ -0,0 +1,25 @@
+# Generated by Django 3.2.6 on 2021-08-26 14:26
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+ initial = True
+
+ dependencies = [
+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='UserAgentStorage',
+ fields=[
+ ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('ua', models.TextField(blank=True, null=True, verbose_name='User agent')),
+ ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL, verbose_name='User')),
+ ],
+ ),
+ ]
diff --git a/agent/migrations/__init__.py b/agent/migrations/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/agent/models.py b/agent/models.py
new file mode 100644
index 0000000..2b80242
--- /dev/null
+++ b/agent/models.py
@@ -0,0 +1,7 @@
+from django.db import models
+
+
+class UserAgentStorage(models.Model):
+ user = models.ForeignKey('auth.User', verbose_name='User', db_index=True,
+ on_delete=models.CASCADE)
+ ua = models.TextField(verbose_name='User agent', null=True, blank=True)
diff --git a/agent/tests.py b/agent/tests.py
new file mode 100644
index 0000000..7ce503c
--- /dev/null
+++ b/agent/tests.py
@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.
diff --git a/agent/views.py b/agent/views.py
new file mode 100644
index 0000000..91ea44a
--- /dev/null
+++ b/agent/views.py
@@ -0,0 +1,3 @@
+from django.shortcuts import render
+
+# Create your views here.
diff --git a/counting/__init__.py b/counting/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/counting/admin.py b/counting/admin.py
new file mode 100644
index 0000000..8c38f3f
--- /dev/null
+++ b/counting/admin.py
@@ -0,0 +1,3 @@
+from django.contrib import admin
+
+# Register your models here.
diff --git a/counting/apps.py b/counting/apps.py
new file mode 100644
index 0000000..7548273
--- /dev/null
+++ b/counting/apps.py
@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class CountingConfig(AppConfig):
+ default_auto_field = 'django.db.models.BigAutoField'
+ name = 'counting'
diff --git a/counting/migrations/0001_initial.py b/counting/migrations/0001_initial.py
new file mode 100644
index 0000000..8fddbec
--- /dev/null
+++ b/counting/migrations/0001_initial.py
@@ -0,0 +1,23 @@
+# Generated by Django 3.2.6 on 2021-08-26 14:26
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ initial = True
+
+ dependencies = [
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='StoryOfADay',
+ fields=[
+ ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('day', models.DateField(verbose_name='Date')),
+ ('links_visited', models.IntegerField(verbose_name='Links visited')),
+ ('files_visited', models.IntegerField(verbose_name='Files visited')),
+ ],
+ ),
+ ]
diff --git a/counting/migrations/__init__.py b/counting/migrations/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/counting/models.py b/counting/models.py
new file mode 100644
index 0000000..2a8760c
--- /dev/null
+++ b/counting/models.py
@@ -0,0 +1,7 @@
+from django.db import models
+
+
+class StoryOfADay(models.Model):
+ day = models.DateField(verbose_name='Date')
+ links_visited = models.IntegerField(verbose_name='Links visited')
+ files_visited = models.IntegerField(verbose_name='Files visited')
diff --git a/counting/tests.py b/counting/tests.py
new file mode 100644
index 0000000..7ce503c
--- /dev/null
+++ b/counting/tests.py
@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.
diff --git a/counting/views.py b/counting/views.py
new file mode 100644
index 0000000..91ea44a
--- /dev/null
+++ b/counting/views.py
@@ -0,0 +1,3 @@
+from django.shortcuts import render
+
+# Create your views here.
diff --git a/netguru/settings.py b/netguru/settings.py
index 5f7a836..ec3f823 100644
--- a/netguru/settings.py
+++ b/netguru/settings.py
@@ -1,4 +1,5 @@
import os
+import logging
from satella.instrumentation.metrics import getMetric
from satella.instrumentation.metrics.exporters import PrometheusHTTPExporterThread
@@ -12,8 +13,7 @@ SECRET_KEY = os.environ.get('SECRET_KEY', 'r6(!w-%glre916esjft0^lds4u)=fym=v*26$
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = 'DEBUG' in os.environ
-
-ALLOWED_HOSTS = ['*']
+ALLOWED_HOSTS = os.environ.get('ALLOWED_HOSTS', '*').split(',')
# Application definition
@@ -25,6 +25,8 @@ INSTALLED_APPS = [
'django.contrib.messages',
'django.contrib.staticfiles',
'shares.apps.SharesConfig',
+ 'agent.apps.AgentConfig',
+ 'counting.apps.CountingConfig',
]
MIDDLEWARE = [
@@ -37,6 +39,7 @@ MIDDLEWARE = [
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
+ 'agent.middleware.UANotingMiddleware',
]
ROOT_URLCONF = 'netguru.urls'
@@ -115,6 +118,7 @@ STATIC_ROOT = os.path.join(BASE_DIR, 'media')
FILE_SAVE_LOCATION = '/data'
# Configure tracing
+# =================
OPENTRACING_TRACE_ALL = True
# Callable that returns an `opentracing.Tracer` implementation.
@@ -125,15 +129,26 @@ OPENTRACING_TRACER_PARAMETERS = {
}
# Set up metrics
+# ==============
DJANGO_SATELLA_METRICS = {
'summary_metric': getMetric('netguru.time.summary', 'summary'),
'histogram_metric': getMetric('netguru.time.histogram', 'histogram'),
'status_codes_metric': getMetric('netguru.status_codes', 'counter')
}
-
-
+# Set up metric exporter thread
+# -----------------------------
phet = PrometheusHTTPExporterThread('0.0.0.0', 81, {'service_name': 'netguru'})
phet.start()
+# Configure logging
+# =================
+
+logger = logging.getLogger(__name__)
+if DEBUG:
+ logging.basicConfig(level=logging.DEBUG)
+ logger.warning('WARNING! You are running in debug mode. Don\'t do it on production!')
+else:
+ logging.basicConfig(level=logging.INFO)
+
diff --git a/shares/migrations/0001_initial.py b/shares/migrations/0001_initial.py
index 0c460cb..0242f07 100644
--- a/shares/migrations/0001_initial.py
+++ b/shares/migrations/0001_initial.py
@@ -1,4 +1,4 @@
-# Generated by Django 3.2.6 on 2021-08-25 16:46
+# Generated by Django 3.2.6 on 2021-08-26 14:26
import datetime
from django.conf import settings
@@ -23,7 +23,7 @@ class Migration(migrations.Migration):
('created_on', models.DateTimeField(db_index=True, default=datetime.datetime.now, verbose_name='Created on')),
('pwd_hash', models.CharField(max_length=64, verbose_name='Hashed password salt')),
('resource', models.TextField(verbose_name='Resource')),
- ('views_count', models.IntegerField(default=0, verbose_name='Amount of displays')),
+ ('used', models.BooleanField(default=False, verbose_name='Has been used?')),
('creator', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL, verbose_name='Creator')),
],
),
diff --git a/shares/models.py b/shares/models.py
index bfadb68..0c6e9f5 100644
--- a/shares/models.py
+++ b/shares/models.py
@@ -1,8 +1,12 @@
+import os
from datetime import datetime
from django.db import models
# Create your models here.
+from satella.coding import silence_excs
+
+from netguru.settings import FILE_SAVE_LOCATION
SHARE_URL = 0
SHARE_FILE = 1
@@ -14,6 +18,11 @@ SHARE_TYPES = [
class Share(models.Model):
+ """
+ A share is a single shared resource.
+
+ It is created on being added and removed upon the reaper task collecting it.
+ """
creator = models.ForeignKey('auth.User', verbose_name='Creator', on_delete=models.CASCADE)
share_type = models.IntegerField(choices=SHARE_TYPES, verbose_name='Share type')
created_on = models.DateTimeField(default=datetime.now, verbose_name='Created on',
@@ -21,4 +30,35 @@ class Share(models.Model):
pwd_hash = models.CharField(max_length=64, verbose_name='Hashed password salt')
# this is either an URL or a f'{UUID}.real_file_name' in /data
resource = models.TextField(verbose_name='Resource')
- views_count = models.IntegerField(verbose_name='Amount of displays', default=0)
+ used = models.BooleanField(verbose_name='Has been used?', default=False)
+
+ @property
+ def file_name(self) -> str:
+ """
+ Return real file name (as submitted by the user).
+ Must be a SHARE_FILE
+ """
+ assert self.share_type == SHARE_FILE, 'Not a file!'
+ return self.resource.split('.', 1)[1]
+
+ @property
+ def uuid_name(self) -> str:
+ """Return file name. Must be a SHARE_FILE"""
+ assert self.share_type == SHARE_FILE, 'Not a file!'
+ return self.resource.split('.', 1)[0]
+
+ @property
+ def path(self) -> str:
+ """Return path to file"""
+ return os.path.join(FILE_SAVE_LOCATION, self.uuid_name)
+
+ def delete(self, *args, **kwargs):
+ if self.share_type == SHARE_FILE:
+ path = os.path.join(FILE_SAVE_LOCATION, self.uuid_name)
+ # Since we might have already deleted it, but our transaction was rolled
+ # back because of reasons.
+ # This is both EAFP (more Pythonic) and shorter 2 lines than a try ... catch
+ # besides it's a great showcase of my Satella
+ with silence_excs(FileNotFoundError):
+ os.unlink(path)
+ super().delete(*args, **kwargs)
diff --git a/shares/views.py b/shares/views.py
index a82e16f..3ed8c74 100644
--- a/shares/views.py
+++ b/shares/views.py
@@ -1,3 +1,4 @@
+import datetime
import mimetypes
import typing as tp
import os
@@ -109,6 +110,10 @@ class PasswordForm(forms.Form):
def view_share(request, share_id: str):
share = get_object_or_404(Share, id=share_id)
+
+ if share.created_on < datetime.datetime.now() - datetime.timedelta(days=1):
+ raise Http404()
+
if request.method == 'POST':
form = PasswordForm(request.POST)
if form.is_valid():
@@ -120,9 +125,7 @@ def view_share(request, share_id: str):
links_visited.runtime(+1)
return redirect(share.resource)
else:
- file_uuid, file_name = share.resource.split('.', 1)
- file_loc = os.path.join(FILE_SAVE_LOCATION, file_uuid)
- if not os.path.exists(file_loc):
+ if not os.path.exists(share.path):
raise Http404()
def file_iterator(path: str) -> tp.Iterator[bytes]:
@@ -133,16 +136,16 @@ def view_share(request, share_id: str):
p = f.read(1024)
# Django will have the UTF-8 mumbo jumbo for us
- cd = f'attachment; filename="{file_name}"'
- mimetype = mimetypes.guess_type(file_name)[0] or 'application/octet-stream'
+ cd = f'attachment; filename="{share.file_name}"'
+ mimetype = mimetypes.guess_type(share.file_name)[0] or 'application/octet-stream'
files_visited.runtime(+1)
- return StreamingHttpResponse(file_iterator(file_loc),
+ return StreamingHttpResponse(file_iterator(share.path),
content_type=mimetype,
headers={
'Content-Disposition': cd
})
finally:
- share.views_count += 1
+ share.used = True
share.save()
else:
form = PasswordForm()
--
GitLab