From 8b97379e4bcb67c5132552757f4bca9ee87968fb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Piotr=20Ma=C5=9Blanka?= <piotr.maslanka@henrietta.com.pl>
Date: Fri, 27 Aug 2021 10:21:00 +0200
Subject: [PATCH] fix everything, prepare to deployment
---
.gitlab-ci.yml | 8 +-------
NOTES.md | 5 ++---
agent/models.py | 2 +-
counting/cron.py | 20 +++++++++----------
deployment/nginx/Dockerfile | 3 ---
deployment/nginx/conf.d/.gitkeep | 0
netguru/settings.py | 1 -
.../migrations/0002_alter_share_resource.py | 19 ++++++++++++++++++
shares/tests.py | 3 ++-
test.sh | 1 -
10 files changed, 34 insertions(+), 28 deletions(-)
delete mode 100644 deployment/nginx/Dockerfile
delete mode 100644 deployment/nginx/conf.d/.gitkeep
create mode 100644 shares/migrations/0002_alter_share_resource.py
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 8582b7e..35c0bc2 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -15,16 +15,10 @@ unittest:
- docker-compose down
-build_nginx:
- script:
- - DOCKERIT_NO_BRANCH=1 docker-it nginx zoo.smok.co/henrietta/netguru/nginx deployment/nginx
- only:
- - master
-
build:
stage: build
script:
- - DOCKERIT_NO_BRANCH=1 docker-it netguru zoo.smok.co/henrietta/netguru/backend . --target runtime
+ - DOCKERIT_NO_BRANCH=1 docker-it netguru zoo.smok.co/henrietta/netguru .
deploy:
stage: deploy
diff --git a/NOTES.md b/NOTES.md
index c83a172..a45206c 100644
--- a/NOTES.md
+++ b/NOTES.md
@@ -21,7 +21,6 @@ I know that I'm ignoring specification, and you are free to call me out
on that - but as I got some experience with frontend,
I'd rather do **the right thing**.
-
## Documentation
I couldn't get the DRF documentation to cooperate with me (most
@@ -54,7 +53,7 @@ I realize that I would be best practice to deduplicate some code contained
within tests, but since I'm running about 1,5 of full-time equvialents you just have to forgive me
for sparing the effort to do so. Thanks "from the mountain!"
-# The [reaper job](counting/cron.py)
+# The [reaper job](counting/cron.py#L27)
I came up with the reaper job trying to think up a reasonable solution
that wouldn't load the processing server too much. It processes way-past-due
@@ -62,7 +61,7 @@ links into a nice compact representation and stores them in the database.
Two last days (since the job fires only every 24 hours) are computed on-the-fly,
and cached as necessary, for up to 5 minutes.
-# Nginx serving static content
+# Nginx not serving static content
Since it's not ideal for Django to be serving large static files,
I tried to optimize it as much as possible by using a streaming iterator.
diff --git a/agent/models.py b/agent/models.py
index ab2d677..6998978 100644
--- a/agent/models.py
+++ b/agent/models.py
@@ -4,7 +4,7 @@ from django.db import models
class UserAgentStorage(models.Model):
user = models.ForeignKey('auth.User', verbose_name='User', db_index=True,
on_delete=models.CASCADE)
- ua = models.TextField(verbose_name='User agent', null=True, blank=True)
+ ua = models.TextField(verbose_name='User agent')
def __str__(self):
return f'{self.user} - {self.ua}'
diff --git a/counting/cron.py b/counting/cron.py
index a457c13..0e97eb3 100644
--- a/counting/cron.py
+++ b/counting/cron.py
@@ -27,10 +27,8 @@ class DayType(enum.IntEnum):
class ReaperJob(CronJobBase):
"""
Reaper's job is to collect dead links within the database and compile a history out of them.
- It will delete only links from previous days, only if they have already expired.
-
- Let's talk a moment about it's logic - days can be divided into one of 3 categories
- displayed above in DayType
+ It will delete only links from previous days, and only for days that have all of their
+ links expired.
"""
RUN_EVERY_MINS = 24 * 60 # once each day
@@ -46,6 +44,7 @@ class ReaperJob(CronJobBase):
StoryOfADay.objects.get(date=date)
return DayType.DAY_WITH_HISTORY
except StoryOfADay.DoesNotExist:
+ # this is going to do the right thing even if the iterator's empty
if all(item.expired for item in Share.objects.get_for_day(date)):
return DayType.DAY_UNPROCESSED_BUT_PROCESSABLE
else:
@@ -65,11 +64,10 @@ class ReaperJob(CronJobBase):
files += 1
else:
links += 1
- sod = StoryOfADay(day=cur_day, links=links,
- files=files)
- logger.info('Historic info for %s compiled, %s files visited, %s links visited',
- cur_day, files, links)
- entries_compiled.runtime(+1)
- sod.save()
- share.delete()
+ sod = StoryOfADay(day=cur_day, links=links, files=files)
+ logger.info('Historic info for %s compiled, %s files visited, %s links visited',
+ cur_day, files, links)
+ entries_compiled.runtime(+1)
+ sod.save()
+ share.delete()
cur_day = cur_day + datetime.timedelta(days=1)
diff --git a/deployment/nginx/Dockerfile b/deployment/nginx/Dockerfile
deleted file mode 100644
index f44279c..0000000
--- a/deployment/nginx/Dockerfile
+++ /dev/null
@@ -1,3 +0,0 @@
-FROM nginx
-
-ADD conf.d/ /etc/nginx/conf.d/
diff --git a/deployment/nginx/conf.d/.gitkeep b/deployment/nginx/conf.d/.gitkeep
deleted file mode 100644
index e69de29..0000000
diff --git a/netguru/settings.py b/netguru/settings.py
index 801da6b..861a3c8 100644
--- a/netguru/settings.py
+++ b/netguru/settings.py
@@ -152,7 +152,6 @@ REST_FRAMEWORK = {
'UNICODE_JSON': True,
}
-
# Configure tracing
# =================
OPENTRACING_TRACE_ALL = True
diff --git a/shares/migrations/0002_alter_share_resource.py b/shares/migrations/0002_alter_share_resource.py
new file mode 100644
index 0000000..584cf96
--- /dev/null
+++ b/shares/migrations/0002_alter_share_resource.py
@@ -0,0 +1,19 @@
+# Generated by Django 3.2.6 on 2021-08-26 23:49
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ('shares', '0001_initial'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='share',
+ name='resource',
+ field=models.TextField(
+ help_text='URL if this is an URL. Otherwise it will be name of the file as it is on filesystem, then a dot,and then the file name the user submitted it as',
+ verbose_name='Resource'),
+ ),
+ ]
diff --git a/shares/tests.py b/shares/tests.py
index 154d1b6..a8d7895 100644
--- a/shares/tests.py
+++ b/shares/tests.py
@@ -75,7 +75,8 @@ class TestShares(TestCase):
share = Share.objects.get(id=response.json()['url'].rsplit('/', 1)[-1])
share.created_on = datetime.datetime.now() - datetime.timedelta(days=2)
share.save()
- response = self.api_client.post(f'http://127.0.0.1/api/get/{share.id}', {'password': response.json()['password']},
+ response = self.api_client.post(f'http://127.0.0.1/api/get/{share.id}',
+ {'password': response.json()['password']},
format='json')
self.assertEqual(response.status_code, 404)
diff --git a/test.sh b/test.sh
index 507ad9a..30ffcf9 100644
--- a/test.sh
+++ b/test.sh
@@ -2,5 +2,4 @@
set -e
sleep 3 # wait for postgres to start up
-python manage.py makemigrations shares
python manage.py test --no-input
--
GitLab