From 04906f78b6cc815a926b49c78baf7f3ccda14a70 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Piotr=20Ma=C5=9Blanka?= <piotr.maslanka@henrietta.com.pl>
Date: Wed, 25 Aug 2021 21:40:41 +0200
Subject: [PATCH] added submitting links
---
.gitattributes | 1 +
.gitignore | 1 +
.gitlab-ci.yml | 8 +--
Dockerfile | 12 ++---
docker-compose.yml | 15 +++---
netguru/settings.py | 20 ++++---
netguru/urls.py | 3 +-
requirements.txt | 3 +-
shares/cron.py | 14 ++++-
shares/migrations/0001_initial.py | 30 +++++++++++
shares/models.py | 3 +-
shares/templates/share/add.html | 11 ++--
shares/templates/share/view.html | 13 +++++
shares/tests.py | 57 +++++++++++++++++++-
shares/views.py | 86 ++++++++++++++++++++++++++++---
templates/base.html | 8 ++-
test.sh | 6 +++
tests/__init__.py | 0
18 files changed, 241 insertions(+), 50 deletions(-)
create mode 100644 .gitattributes
create mode 100644 shares/migrations/0001_initial.py
create mode 100644 shares/templates/share/view.html
create mode 100644 test.sh
delete mode 100644 tests/__init__.py
diff --git a/.gitattributes b/.gitattributes
new file mode 100644
index 0000000..dfdb8b7
--- /dev/null
+++ b/.gitattributes
@@ -0,0 +1 @@
+*.sh text eol=lf
diff --git a/.gitignore b/.gitignore
index 3bd56f3..3a92663 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
.idea/
build
dist
+db/
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 1ffbe61..e817dac 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -7,11 +7,13 @@ stages:
- deploy
unittest:
- image: python:3.9
before_script:
- - pip install -r requirements.txt
+ - echo "Not logging in"
script:
- - python manage.py test
+ - docker-compose up --build unittest
+ after_script:
+ - docker-compose down
+
build_nginx:
script:
diff --git a/Dockerfile b/Dockerfile
index 955ccdc..0037322 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -9,15 +9,11 @@ ADD netguru /app/netguru
ADD manage.py /app/manage.py
ADD shares /app/shares
ADD templates /app/templates
-
-RUN python manage.py collectstatic
+ADD test.sh /app/test.sh
+RUN python manage.py collectstatic && \
+ chmod ugo+x /app/test.sh
VOLUME /data
-CMD ["gunicorn", "-w", "4", "-b", "0.0.0.0:80", "netguru.wsgi:application"]]
-
-FROM runtime AS unittest
-
-ADD tests /app/tests
+CMD ["gunicorn", "-w", "4", "-b", "0.0.0.0:80", "netguru.wsgi:application"]
-CMD ["python", "manage.py", "test"]
diff --git a/docker-compose.yml b/docker-compose.yml
index 0cc7e20..269d93a 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -6,18 +6,17 @@ services:
- POSTGRES_DB=postgres
- POSTGRES_USER=postgres
- POSTGRES_PASSWORD=postgres
+ volumes:
+ - ./db:/var/lib/postgresql
unittest:
- build:
- context: .
- dockerfile: Dockerfile
- target: unittest
+ command: "./test.sh"
+ build: .
+ environment:
+ - CI=1
depends_on:
- postgres
run_local:
- build:
- context: .
- dockerfile: Dockerfile
- target: runtime
+ build: .
depends_on:
- postgres
ports:
diff --git a/netguru/settings.py b/netguru/settings.py
index 95de413..8cd17bc 100644
--- a/netguru/settings.py
+++ b/netguru/settings.py
@@ -21,19 +21,16 @@ INSTALLED_APPS = [
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
+ 'django.contrib.messages',
'django.contrib.staticfiles',
'shares.apps.SharesConfig',
- 'django_cron',
-]
-
-CRON_CLASSES = [
- "shares.cron.ClearExpiredLinks"
]
MIDDLEWARE = [
'django.middleware.security.SecurityMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.common.CommonMiddleware',
+ 'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
@@ -51,6 +48,7 @@ TEMPLATES = [
'django.template.context_processors.debug',
'django.template.context_processors.request',
'django.contrib.auth.context_processors.auth',
+ 'django.contrib.messages.context_processors.messages',
'netguru.context.add_user',
],
},
@@ -62,8 +60,8 @@ WSGI_APPLICATION = 'netguru.wsgi.application'
# Database
# https://docs.djangoproject.com/en/3.0/ref/settings/#databases
-DATABASES = {
- 'default': {
+if True:
+ db = {
'ENGINE': 'django.db.backends.postgresql',
'NAME': os.environ.get('DB_NAME', 'postgres'),
'USER': os.environ.get('DB_USER', 'postgres'),
@@ -72,6 +70,12 @@ DATABASES = {
'PORT': int(os.environ.get('DB_PORT', '5432')),
'ATOMIC_REQUESTS': True # transactions aren't just a good idea
}
+else:
+ db = {'ENGINE': 'django.db.backends.sqlite3',
+ 'NAME': 'test-db.sqlite'}
+
+DATABASES = {
+ 'default': db
}
# Password validation
@@ -110,3 +114,5 @@ USE_TZ = True
STATIC_URL = '/static/'
STATIC_ROOT = os.path.join(BASE_DIR, 'media')
+
+FILE_SAVE_LOCATION = '/data'
diff --git a/netguru/urls.py b/netguru/urls.py
index 3bbb55b..764e36c 100644
--- a/netguru/urls.py
+++ b/netguru/urls.py
@@ -1,11 +1,12 @@
from django.conf.urls.static import static
from django.contrib import admin
-from django.urls import path, include
+from django.urls import path, include, re_path
from shares import views
urlpatterns = [
path('accounts/', include('django.contrib.auth.urls')),
path('accounts/profile', views.add_share),
+ re_path(r'shares/(?P<share_id>[0-9]+)$',views.view_share),
path('admin/', admin.site.urls),
] + static('/static/', document_root='/app/media/')
diff --git a/requirements.txt b/requirements.txt
index 039b9ce..7f6d31d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,5 @@
django
-psycopg2-binary
+psycopg2
gunicorn
django_cron
+satella
diff --git a/shares/cron.py b/shares/cron.py
index 23e35fc..34e80cb 100644
--- a/shares/cron.py
+++ b/shares/cron.py
@@ -1,7 +1,10 @@
import logging
+import os
from datetime import datetime, timedelta
from django_cron import CronJobBase, Schedule
-from .models import Share
+
+from netguru.settings import FILE_SAVE_LOCATION
+from .models import Share, SHARE_FILE
logger = logging.getLogger(__name__)
@@ -13,5 +16,12 @@ class ClearExpiredLinks(CronJobBase):
code = 'clear-expired-links' # a unique code
def do(self):
- Share.objects.filter(created_on__lt=datetime.now() - timedelta.days(1)).delete()
+ shares = Share.objects.filter(created_on__lt=datetime.now() - timedelta.days(1))
+ for share in shares:
+ if share.share_type == SHARE_FILE:
+ path = os.path.join(FILE_SAVE_LOCATION, share.resource.split('.', 1)[0])
+ if os.path.exists(path):
+ # we might have deleted in earlier and this call might have aborted with exception
+ # the transaction won't proceed so we might have already deleted this file
+ os.unlink(path)
logger.info('Deleted expired shares')
diff --git a/shares/migrations/0001_initial.py b/shares/migrations/0001_initial.py
new file mode 100644
index 0000000..0c460cb
--- /dev/null
+++ b/shares/migrations/0001_initial.py
@@ -0,0 +1,30 @@
+# Generated by Django 3.2.6 on 2021-08-25 16:46
+
+import datetime
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+ initial = True
+
+ dependencies = [
+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='Share',
+ fields=[
+ ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('share_type', models.IntegerField(choices=[(0, 'URL'), (1, 'File')], verbose_name='Share type')),
+ ('created_on', models.DateTimeField(db_index=True, default=datetime.datetime.now, verbose_name='Created on')),
+ ('pwd_hash', models.CharField(max_length=64, verbose_name='Hashed password salt')),
+ ('resource', models.TextField(verbose_name='Resource')),
+ ('views_count', models.IntegerField(default=0, verbose_name='Amount of displays')),
+ ('creator', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL, verbose_name='Creator')),
+ ],
+ ),
+ ]
diff --git a/shares/models.py b/shares/models.py
index df95b4b..bfadb68 100644
--- a/shares/models.py
+++ b/shares/models.py
@@ -19,5 +19,6 @@ class Share(models.Model):
created_on = models.DateTimeField(default=datetime.now, verbose_name='Created on',
db_index=True)
pwd_hash = models.CharField(max_length=64, verbose_name='Hashed password salt')
- # this is either an URL or a file name in /data
+ # this is either an URL or a f'{UUID}.real_file_name' in /data
resource = models.TextField(verbose_name='Resource')
+ views_count = models.IntegerField(verbose_name='Amount of displays', default=0)
diff --git a/shares/templates/share/add.html b/shares/templates/share/add.html
index 7e6de44..ada7993 100644
--- a/shares/templates/share/add.html
+++ b/shares/templates/share/add.html
@@ -7,12 +7,13 @@
{% block body %}
{% if added %}
<div><em>Resource successfully added:</em><br/>
- Your password is <em>{{ added.password }}</em><br/>
- Your link is: <span><a href="{{ added.url }}" id="url">{{ added.url }}</a></span>
- <button onclick="copyTextToClipboard('{{ added.url }}')">Copy to clipboard</button><br/>
- </div></dv>
+ Your password is <em id="link_password">{{ added.password }}</em>
+ <button onclick="copyTextToClipboard('{{ added.password }}')">Copy the link to clipboard</button><br>
+ Your link is: <span><a href="{{ added.url }}" id="link_url">{{ added.url }}</a></span>
+ <button onclick="copyTextToClipboard('{{ added.url }}')">Copy the password to clipboard</button><br>
+ </div>
{% endif %}
- <form class="form-group text-left" role="form" action="/accounts/profile" method="POST">
+ <form class="form-group text-left" role="form" action="/accounts/profile" method="POST" enctype="multipart/form-data">
{% csrf_token %}
{{ form.as_p }}
<p>
diff --git a/shares/templates/share/view.html b/shares/templates/share/view.html
new file mode 100644
index 0000000..b8b3b3f
--- /dev/null
+++ b/shares/templates/share/view.html
@@ -0,0 +1,13 @@
+{% extends "base.html" %}{% load static %}
+{% load static %}
+{% block title %}View a resource{% endblock %}
+{% block head %}
+{% endblock %}
+{% block body %}
+ <form class="form-group text-left" role="form" action="/shares/{{ share_id }}" method="POST">
+ {% csrf_token %}
+ {{ form.as_p }}
+ <p>
+ <input type="submit" value="Submit">
+ </form>
+{% endblock %}
diff --git a/shares/tests.py b/shares/tests.py
index 7ce503c..1ef9aa9 100644
--- a/shares/tests.py
+++ b/shares/tests.py
@@ -1,3 +1,58 @@
-from django.test import TestCase
+from django.contrib.auth.hashers import make_password
+from django.contrib.auth.models import User
+from django.test import TestCase, Client
+from django.test.html import parse_html
+
+
+def find_element(document, field, value):
+ if isinstance(document, str):
+ return
+ attrs = dict(document.attributes)
+ if attrs.get(field) == value:
+ document.attributes = attrs
+ return document
+ for child in document.children:
+ a = find_element(child, field, value)
+ if a is not None:
+ return a
+
# Create your tests here.
+class TestShares(TestCase):
+
+ def setUp(self) -> None:
+ super().setUp()
+ user = User.objects.create(username='testuser', email='admin@admin.com',
+ password='12345')
+ self.client = Client() # May be you have missed this line
+ self.client.force_login(user)
+
+ def test_add_url(self):
+ """
+ Test that an URL can be added and that it can be visited
+ """
+ response = self.client.get('http://127.0.0.1/accounts/profile')
+
+ self.assertEqual(response.status_code, 200)
+ html = parse_html(response.content.decode('utf-8'))
+ csrf_token = find_element(html, 'name', 'csrfmiddlewaretoken').attributes['value']
+
+ response = self.client.post('http://127.0.0.1/accounts/profile', data={
+ 'url': 'https://example.com',
+ 'csrfmiddlewaretoken': csrf_token
+ })
+ self.assertEqual(response.status_code, 200)
+
+ html = parse_html(response.content.decode('utf-8'))
+ link = find_element(html, 'id', 'link_url').children[0].replace('https', 'http')
+ password = find_element(html, 'id', 'link_password').children[0]
+
+ response = self.client.get(link)
+ self.assertEqual(response.status_code, 200)
+ html = parse_html(response.content.decode('utf-8'))
+ csrf_token = find_element(html, 'name', 'csrfmiddlewaretoken').attributes['value']
+ response = self.client.post(link, data={
+ 'password': password,
+ 'csrfmiddlewaretoken': csrf_token
+ })
+ self.assertEqual(response.status_code, 302)
diff --git a/shares/views.py b/shares/views.py
index e52ebac..5e55355 100644
--- a/shares/views.py
+++ b/shares/views.py
@@ -1,34 +1,49 @@
+import mimetypes
+import typing as tp
+import os
import random
import string
-
+from email.header import Header
+import uuid
from django import forms
+from django.core.validators import URLValidator
from django.forms import fields
from django.core.exceptions import ValidationError
-from django.shortcuts import render
+from django.http import Http404, StreamingHttpResponse
+from django.shortcuts import render, get_object_or_404, redirect
from django.contrib.auth.decorators import login_required
from django.utils.safestring import mark_safe
from shares.models import Share, SHARE_URL, SHARE_FILE
import hashlib
-from netguru.settings import SECRET_KEY
+from netguru.settings import SECRET_KEY, FILE_SAVE_LOCATION
def hash_password(password: str) -> str:
return hashlib.sha256(password.encode('utf-8') + SECRET_KEY.encode('utf-8')).hexdigest()
+def generate_correct_uuid() -> str:
+ f = uuid.uuid4().hex
+ # I know that hash collisions practically don't happen, but since it's so cheap for us
+ # to go that extra mile
+ while os.path.exists(FILE_SAVE_LOCATION, f):
+ f = uuid.uuid4().hex
+ return f
+
# Cut off O, I and S so that the user doesn't confuse them with 0, 1 and 5 respectively
-CHARACTERS_TO_USE = string.ascii_uppercase.replace('O', '').replace('I', '').replace('S',
- '') + string.digits
+CHARACTERS_TO_USE = string.ascii_uppercase.replace('O', '') \
+ .replace('I', '').replace('S', '') + string.digits
def random_password() -> str:
- """Generate a eight character reasonably safe passwordpy"""
+ """Generate a eight character reasonably safe password"""
return ''.join(random.choice(CHARACTERS_TO_USE) for i in range(6))
class AddShareURLForm(forms.Form):
- url = fields.CharField(label='URL', required=False)
+ url = fields.CharField(label='URL', required=False,
+ validators=[URLValidator()])
file = fields.FileField(label='File', required=False)
def clean(self):
@@ -57,8 +72,63 @@ def add_share(request):
share.save()
data_added['url'] = mark_safe(f'https://{request.get_host()}/shares/{share.id}')
else:
- pass
+ file_name = generate_correct_uuid()
+ file = request.FILES['file']
+ resource_name = f'{file_name}.{file.name}'
+ with open(os.path.join(FILE_SAVE_LOCATION, file_name), 'wb') as f_out:
+ for chunk in file.chunks():
+ f_out.write(chunk)
+ share = Share(creator=request.user,
+ resource=resource_name,
+ pwd_hash=pwd_hash,
+ share_type=SHARE_FILE)
+ share.save()
+ data_added['url'] = mark_safe(f'https://{request.get_host()}/shares/{share.id}')
else:
form = AddShareURLForm()
return render(request, 'share/add.html', {'form': form, 'added': data_added})
+
+
+class PasswordForm(forms.Form):
+ password = fields.CharField(label='Password', widget=forms.PasswordInput)
+
+
+def view_share(request, share_id: str):
+ share = get_object_or_404(Share, id=share_id)
+ if request.method == 'POST':
+ form = PasswordForm(request.POST)
+ if form.is_valid():
+ pwd = form.cleaned_data['password']
+ if hash_password(pwd) == share.pwd_hash:
+ try:
+ if share.share_type == SHARE_URL:
+ return redirect(share.resource)
+ else:
+ file_uuid, file_name = share.resource.split('.', 1)
+ file_loc = os.path.join(FILE_SAVE_LOCATION, file_uuid)
+ if not os.path.exists(file_loc):
+ raise Http404()
+ else:
+ def file_iterator(path: str) -> tp.Iterator[bytes]:
+ with open(path, 'rb') as f:
+ p = f.read(1024)
+ while p:
+ yield p
+ p = f.read(1024)
+
+ # Django will have the UTF-8 mumbo jumbo for us
+ cd = f'attachment; filename="{file_name}"'
+ mimetype = mimetypes.guess_type(file_name)[0] or 'application/octet-stream'
+ return StreamingHttpResponse(file_iterator(file_loc),
+ content_type=mimetype,
+ headers={
+ 'Content-Disposition': cd
+ })
+ finally:
+ share.views_count += 1
+ share.save()
+ else:
+ form = PasswordForm()
+
+ return render(request, 'share/view.html', {'form': form, 'share_id': share_id})
diff --git a/templates/base.html b/templates/base.html
index 95c0cb2..e2c255c 100644
--- a/templates/base.html
+++ b/templates/base.html
@@ -13,11 +13,9 @@
crossorigin="anonymous"></script>
<link href="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/css/bootstrap.min.css"
rel="stylesheet">
- <link href="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/js/bootstrap.min.js"
- rel="stylesheet">
- <link href="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/js/bootstrap.bundle.min.js"
- rel="stylesheet">
- <link href="https://use.fontawesome.com/releases/v5.3.1/css/all.css" rel="stylesheet"/>
+ <script src="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/js/bootstrap.min.js"></script>
+ <script src="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/js/bootstrap.bundle.min.js"></script>
+ <link href="https://use.fontawesome.com/releases/v5.3.1/css/all.css" rel="stylesheet">
{% block head %}{% endblock %}
</head>
<body>
diff --git a/test.sh b/test.sh
new file mode 100644
index 0000000..a84325e
--- /dev/null
+++ b/test.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+set -e
+
+sleep 3 # wait for postgres to start up
+python manage.py makemigrations shares
+python manage.py test --no-input
diff --git a/tests/__init__.py b/tests/__init__.py
deleted file mode 100644
index e69de29..0000000
--
GitLab