From e4b650900390e051fa9057a2035b10cf04643def Mon Sep 17 00:00:00 2001 From: Dmitriy Lyalyuev Date: Sat, 5 Nov 2022 23:46:14 +0300 Subject: [PATCH] feat: get url content in background --- docker-compose.yaml | 16 +++++++++++++- later42/__init__.py | 3 +++ later42/celery.py | 22 +++++++++++++++++++ later42/migrations/0004_articles.py | 22 +++++++++++++++++++ .../0005_rename_articles_article.py | 17 ++++++++++++++ later42/models/article.py | 9 ++++++++ later42/settings.py | 7 ++++++ later42/tasks.py | 13 +++++++++++ later42/templates/archive.html | 2 +- later42/templates/reader.html | 6 ++--- later42/views/api.py | 2 ++ later42/views/reader.py | 11 ++++++++-- requirements.txt | 1 + 13 files changed, 124 insertions(+), 7 deletions(-) create mode 100644 later42/celery.py create mode 100644 later42/migrations/0004_articles.py create mode 100644 later42/migrations/0005_rename_articles_article.py create mode 100644 later42/models/article.py create mode 100644 later42/tasks.py diff --git a/docker-compose.yaml b/docker-compose.yaml index 0515c54..1e1f713 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -46,6 +46,7 @@ services: DB_PASS: later42 DOMAIN: later42.com READABILITY_HOST: http://ureadability:8080/ + REDIS_URL: redis://redis:6379 logging: driver: json-file options: @@ -108,5 +109,18 @@ services: - MONGO_INITDB_DATABASE=admin - MONGO_INITDB_ROOT_USERNAME=root - MONGO_INITDB_ROOT_PASSWORD=aldbhvaygvavASDVSDFVQFQgfwvsav + ports: + - "27017:27017" volumes: - - /opt/docker/mongodb:/data/db \ No newline at end of file + - /opt/docker/mongodb:/data/db + + redis: + image: "redis:alpine" + hostname: redis + container_name: redis + ports: + - "6379:6379" + volumes: + - /opt/docker/redis-data:/var/lib/redis + environment: + - REDIS_REPLICATION_MODE=master diff --git a/later42/__init__.py b/later42/__init__.py index e69de29..fb989c4 100644 --- a/later42/__init__.py +++ b/later42/__init__.py @@ -0,0 +1,3 @@ +from .celery import app as celery_app + +__all__ = ('celery_app',) diff --git a/later42/celery.py b/later42/celery.py new file mode 100644 index 0000000..3605c12 --- /dev/null +++ b/later42/celery.py @@ -0,0 +1,22 @@ +import os + +from celery import Celery + +# Set the default Django settings module for the 'celery' program. +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'later42.settings') + +app = Celery('later42') + +# Using a string here means the worker doesn't have to serialize +# the configuration object to child processes. +# - namespace='CELERY' means all celery-related configuration keys +# should have a `CELERY_` prefix. +app.config_from_object('django.conf:settings', namespace='CELERY') + +# Load task modules from all registered Django apps. +app.autodiscover_tasks() + + +@app.task(bind=True) +def debug_task(self): + print(f'Request: {self.request!r}') diff --git a/later42/migrations/0004_articles.py b/later42/migrations/0004_articles.py new file mode 100644 index 0000000..b65e97b --- /dev/null +++ b/later42/migrations/0004_articles.py @@ -0,0 +1,22 @@ +# Generated by Django 4.1.2 on 2022-11-05 17:10 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('later42', '0003_url_content'), + ] + + operations = [ + migrations.CreateModel( + name='Articles', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False)), + ('content', models.TextField(blank=True, null=True)), + ('url', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='later42.url')), + ], + ), + ] diff --git a/later42/migrations/0005_rename_articles_article.py b/later42/migrations/0005_rename_articles_article.py new file mode 100644 index 0000000..6bc258b --- /dev/null +++ b/later42/migrations/0005_rename_articles_article.py @@ -0,0 +1,17 @@ +# Generated by Django 4.1.2 on 2022-11-05 17:11 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('later42', '0004_articles'), + ] + + operations = [ + migrations.RenameModel( + old_name='Articles', + new_name='Article', + ), + ] diff --git a/later42/models/article.py b/later42/models/article.py new file mode 100644 index 0000000..97606d0 --- /dev/null +++ b/later42/models/article.py @@ -0,0 +1,9 @@ +from django.contrib.auth.models import User +from django.db import models +from later42.models.urls import URL + + +class Article(models.Model): + id = models.AutoField(auto_created=True, primary_key=True) + url = models.ForeignKey(URL, on_delete=models.CASCADE) + content = models.TextField(blank=True, null=True) diff --git a/later42/settings.py b/later42/settings.py index 1067fa8..a4881ca 100644 --- a/later42/settings.py +++ b/later42/settings.py @@ -158,6 +158,13 @@ REST_FRAMEWORK = { ), } +# Celery Configuration Options +CELERY_TIMEZONE = "UTC" +CELERY_TASK_TRACK_STARTED = True +CELERY_TASK_TIME_LIMIT = 30 * 60 +CELERY_BROKER_URL = os.getenv('REDIS_URL', 'redis://localhost:6379') +CELERY_RESULT_BACKEND = os.getenv('REDIS_URL', 'redis://localhost:6379') + URLS_PER_PAGE = 20 READABILITY_HOST = os.getenv('READABILITY_HOST', None) diff --git a/later42/tasks.py b/later42/tasks.py new file mode 100644 index 0000000..b764e25 --- /dev/null +++ b/later42/tasks.py @@ -0,0 +1,13 @@ +from celery import shared_task +from later42.models.urls import URL +from later42.models.article import Article +from later42.libs.content import get_content + + +@shared_task() +def get_url_content_task(id): + url = URL.objects.get(id=id) + article = Article.objects.create(url=url) + content = get_content(url.url)['rich_content'] + article.content = content + article.save() diff --git a/later42/templates/archive.html b/later42/templates/archive.html index eff861b..4384d2f 100644 --- a/later42/templates/archive.html +++ b/later42/templates/archive.html @@ -8,7 +8,7 @@
- +

{{ url.title }}
diff --git a/later42/templates/reader.html b/later42/templates/reader.html index 55f7d3d..bc8a7d9 100644 --- a/later42/templates/reader.html +++ b/later42/templates/reader.html @@ -12,7 +12,7 @@  Отметить прочитанным
- Открыть оригинал статьи +  Открыть оригинал статьи
@@ -24,12 +24,12 @@  Отметить прочитанным
- Открыть оригинал статьи +  Открыть оригинал статьи
{% else %}

Что-то пошло не так.

- Открыть оригинал ссылки. +  Открыть оригинал ссылки. {% endif %} {% endblock %} \ No newline at end of file diff --git a/later42/views/api.py b/later42/views/api.py index de78b86..95f2686 100644 --- a/later42/views/api.py +++ b/later42/views/api.py @@ -2,6 +2,7 @@ from rest_framework.response import Response from rest_framework.views import APIView from later42.libs.content import get_content from later42.models.urls import URL as URLModel +from later42.tasks import get_url_content_task from django.conf import settings @@ -26,6 +27,7 @@ class URL(APIView): url = URLModel(url=url, user=request.user, title=title, content=content) url.save() + get_url_content_task.delay(url.id) return Response({'status': 'success'}) else: return Response({'status': 'error'}) diff --git a/later42/views/reader.py b/later42/views/reader.py index 6c1cb07..eb99194 100644 --- a/later42/views/reader.py +++ b/later42/views/reader.py @@ -4,14 +4,21 @@ from django.shortcuts import render, redirect from django.core.paginator import Paginator from django.conf import settings from later42.libs.content import get_content, sanitize_img_size +from later42.models.article import Article from later42.models.urls import URL @login_required def get(request, url_id=None): url = URL.objects.get( - user=request.user, archived=False, id=url_id) - content = get_content(url.url) + user=request.user, id=url_id) + + content = {} + + article = Article.objects.get(url=url) + content['rich_content'] = article.content + content['title'] = url.title + content['url'] = url.url content['rich_content'] = sanitize_img_size(content['rich_content']) context = {'url': url, 'content': content} return render(request, 'reader.html', context) diff --git a/requirements.txt b/requirements.txt index be382a6..30efd41 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,3 +7,4 @@ djangorestframework==3.14.0 whitenoise==6.2.0 psycopg2-binary==2.9.4 six==1.16.0 +celery[redis]==5.2.7