feat: get url content in background

docker-compose.yaml

@@ -46,6 +46,7 @@ services:
       DB_PASS: later42
       DOMAIN: later42.com
       READABILITY_HOST: http://ureadability:8080/
+      REDIS_URL: redis://redis:6379
     logging:
       driver: json-file
       options:
@@ -108,5 +109,18 @@ services:
       - MONGO_INITDB_DATABASE=admin
       - MONGO_INITDB_ROOT_USERNAME=root
       - MONGO_INITDB_ROOT_PASSWORD=aldbhvaygvavASDVSDFVQFQgfwvsav
+    ports:
+      - "27017:27017"
     volumes:
       - /opt/docker/mongodb:/data/db
+
+  redis:
+    image: "redis:alpine"
+    hostname: redis
+    container_name: redis
+    ports:
+      - "6379:6379"
+    volumes:
+      - /opt/docker/redis-data:/var/lib/redis
+    environment:
+      - REDIS_REPLICATION_MODE=master

later42/__init__.py

@@ -0,0 +1,3 @@
+from .celery import app as celery_app
+
+__all__ = ('celery_app',)

later42/celery.py

@@ -0,0 +1,22 @@
+import os
+
+from celery import Celery
+
+# Set the default Django settings module for the 'celery' program.
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'later42.settings')
+
+app = Celery('later42')
+
+# Using a string here means the worker doesn't have to serialize
+# the configuration object to child processes.
+# - namespace='CELERY' means all celery-related configuration keys
+#   should have a `CELERY_` prefix.
+app.config_from_object('django.conf:settings', namespace='CELERY')
+
+# Load task modules from all registered Django apps.
+app.autodiscover_tasks()
+
+
+@app.task(bind=True)
+def debug_task(self):
+    print(f'Request: {self.request!r}')

later42/migrations/0004_articles.py

@@ -0,0 +1,22 @@
+# Generated by Django 4.1.2 on 2022-11-05 17:10
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('later42', '0003_url_content'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Articles',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False)),
+                ('content', models.TextField(blank=True, null=True)),
+                ('url', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='later42.url')),
+            ],
+        ),
+    ]

later42/migrations/0005_rename_articles_article.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.1.2 on 2022-11-05 17:11
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('later42', '0004_articles'),
+    ]
+
+    operations = [
+        migrations.RenameModel(
+            old_name='Articles',
+            new_name='Article',
+        ),
+    ]

later42/models/article.py

@@ -0,0 +1,9 @@
+from django.contrib.auth.models import User
+from django.db import models
+from later42.models.urls import URL
+
+
+class Article(models.Model):
+    id = models.AutoField(auto_created=True, primary_key=True)
+    url = models.ForeignKey(URL, on_delete=models.CASCADE)
+    content = models.TextField(blank=True, null=True)

later42/settings.py

@@ -158,6 +158,13 @@ REST_FRAMEWORK = {
     ),
 }
 
+# Celery Configuration Options
+CELERY_TIMEZONE = "UTC"
+CELERY_TASK_TRACK_STARTED = True
+CELERY_TASK_TIME_LIMIT = 30 * 60
+CELERY_BROKER_URL = os.getenv('REDIS_URL', 'redis://localhost:6379')
+CELERY_RESULT_BACKEND = os.getenv('REDIS_URL', 'redis://localhost:6379')
+
 URLS_PER_PAGE = 20
 READABILITY_HOST = os.getenv('READABILITY_HOST', None)
 

later42/tasks.py

@@ -0,0 +1,13 @@
+from celery import shared_task
+from later42.models.urls import URL
+from later42.models.article import Article
+from later42.libs.content import get_content
+
+
+@shared_task()
+def get_url_content_task(id):
+    url = URL.objects.get(id=id)
+    article = Article.objects.create(url=url)
+    content = get_content(url.url)['rich_content']
+    article.content = content
+    article.save()

later42/templates/archive.html

@@ -8,7 +8,7 @@
 <div class="post-preview">
     <div class="d-flex justify-content-between">
         <div class="p-2">
-            <a href="{{ url.url }}">
+            <a href="{% url 'reader' url.id %}">
                 <h2 class="post-title">{{ url.title }}</h2>
             </a>
         </div>

later42/templates/reader.html

@@ -12,7 +12,7 @@
                     <a href="{% url 'archive_url' url.id %}"><span class="fa-regular fa-square-check"></span>&nbsp;Отметить прочитанным</a>
                 </div>
                 <div class="p-2">
-                    <a href="{{ url.url }}">Открыть оригинал статьи</a>
+                    <a href="{{ url.url }}"><span class="fa-solid fa-arrow-up-right-from-square"></span>&nbsp;Открыть оригинал статьи</a>
                 </div>
             </div>
         </div>
@@ -24,12 +24,12 @@
             <a href="{% url 'archive_url' url.id %}"><span class="fa-regular fa-square-check"></span>&nbsp;Отметить прочитанным</a>
         </div>
         <div class="p-2">
-            <a href="{{ url.url }}">Открыть оригинал статьи</a>
+            <a href="{{ url.url }}"><span class="fa-solid fa-arrow-up-right-from-square"></span>&nbsp;Открыть оригинал статьи</a>
         </div>
     </div>
 {% else %}
     <h2 class="post-title">Что-то пошло не так.</h2>
-    Открыть <a href="{{ url.url }}">оригинал ссылки</a>.
+    <span class="fa-solid fa-arrow-up-right-from-square"></span>&nbsp;Открыть <a href="{{ url.url }}">оригинал ссылки</a>.
 {% endif %}
 </div>
 {% endblock %}

later42/views/api.py

@@ -2,6 +2,7 @@ from rest_framework.response import Response
 from rest_framework.views import APIView
 from later42.libs.content import get_content
 from later42.models.urls import URL as URLModel
+from later42.tasks import get_url_content_task
 from django.conf import settings
 
 
@@ -26,6 +27,7 @@ class URL(APIView):
             url = URLModel(url=url, user=request.user,
                            title=title, content=content)
             url.save()
+            get_url_content_task.delay(url.id)
             return Response({'status': 'success'})
         else:
             return Response({'status': 'error'})

later42/views/reader.py

@@ -4,14 +4,21 @@ from django.shortcuts import render, redirect
 from django.core.paginator import Paginator
 from django.conf import settings
 from later42.libs.content import get_content, sanitize_img_size
+from later42.models.article import Article
 from later42.models.urls import URL
 
 
 @login_required
 def get(request, url_id=None):
     url = URL.objects.get(
-        user=request.user, archived=False, id=url_id)
+        user=request.user, id=url_id)
-    content = get_content(url.url)
+
+    content = {}
+
+    article = Article.objects.get(url=url)
+    content['rich_content'] = article.content
+    content['title'] = url.title
+    content['url'] = url.url
     content['rich_content'] = sanitize_img_size(content['rich_content'])
     context = {'url': url, 'content': content}
     return render(request, 'reader.html', context)

requirements.txt

@@ -7,3 +7,4 @@ djangorestframework==3.14.0
 whitenoise==6.2.0
 psycopg2-binary==2.9.4
 six==1.16.0
+celery[redis]==5.2.7