✨ Added basic upload endpoint

Pipfile

@@ -37,6 +37,8 @@ scheduler = "*"
 webauthn = "*"
 pyotp = "*"
 minio = "*"
+xxhash = "*"
+arq = "*"
 
 [dev-packages]
 coverage = "*"

classquiz/__init__.py

@@ -1,7 +1,6 @@
 #  This Source Code Form is subject to the terms of the Mozilla Public
 #  License, v. 2.0. If a copy of the MPL was not distributed with this
 #  file, You can obtain one at https://mozilla.org/MPL/2.0/.
-import asyncio
 
 import sentry_sdk
 from fastapi import FastAPI, Request
@@ -11,7 +10,6 @@ from starlette.middleware.sessions import SessionMiddleware
 
 from classquiz.config import settings
 from classquiz.db import database
-from datetime import timedelta
 
 from classquiz.oauth import rememberme_middleware
 from classquiz.routers import (
@@ -36,8 +34,7 @@ from classquiz.routers import (
     quiztivity,
 )
 from classquiz.socket_server import sio
-from classquiz.helpers import meilisearch_init, telemetry_ping, bg_tasks
+from classquiz.helpers import meilisearch_init, telemetry_ping
-from scheduler.asyncio import Scheduler
 
 settings = settings()
 if settings.sentry_dsn:
@@ -58,13 +55,6 @@ async def sentry_exception(request: Request, call_next):
         raise e
 
 
-async def background_tasks():
-    schedule = Scheduler()
-    schedule.cyclic(timedelta(hours=6), bg_tasks.clean_editor_images_up)
-    while True:
-        await asyncio.sleep(1)
-
-
 @app.on_event("startup")
 async def startup() -> None:
     database_ = app.state.database
@@ -72,7 +62,6 @@ async def startup() -> None:
         await database_.connect()
     await meilisearch_init()
     await telemetry_ping()
-    asyncio.create_task(background_tasks())
 
 
 @app.on_event("shutdown")

classquiz/config.py

@@ -9,6 +9,8 @@ import redis as redis_base_lib
 from pydantic import BaseSettings, RedisDsn, PostgresDsn, BaseModel
 import meilisearch as MeiliSearch
 from typing import Optional
+from arq import create_pool
+from arq.connections import RedisSettings, ArqRedis
 
 from classquiz.storage import Storage
 
@@ -70,12 +72,22 @@ class Settings(BaseSettings):
         env_nested_delimiter = "__"
 
 
+async def initialize_arq():
+    global arq
+    arq = await create_pool(RedisSettings.from_dsn(settings.redis))
+
+
 @lru_cache()
 def settings() -> Settings:
     return Settings()
 
 
-redis: redis_base_lib.client.Redis = redis_lib.Redis().from_url(settings().redis)
+# asyncio.run(initialize_arq())
+
+pool = redis_lib.ConnectionPool().from_url(settings().redis)
+
+redis: redis_base_lib.client.Redis = redis_lib.Redis(connection_pool=pool)
+arq: ArqRedis = ArqRedis(pool_or_conn=pool)
 storage: Storage = Storage(
     backend=settings().storage_backend,
     deta_key=settings().deta_project_key,

classquiz/db/models.py

@@ -378,3 +378,23 @@ class PublicQuizTivityShare(BaseModel):
             quiztivity=OnlyId(id=data.quiztivity.id),
             user=OnlyId(id=data.user.id),
         )
+
+
+class StorageItem(ormar.Model):
+    id: uuid.UUID = ormar.UUID(primary_key=True)
+    uploaded_at: datetime = ormar.DateTime(nullable=False, default=datetime.now())
+    mime_type: str = ormar.Text(nullable=False)
+    hash: bytes | None = ormar.LargeBinary(nullable=True, min_length=16, max_length=16)
+    user: User | None = ormar.ForeignKey(User)
+    size: int = ormar.BigInteger(nullable=False)
+    storage_path: str | None = ormar.Text(nullable=True)
+    deleted_at: datetime | None = ormar.DateTime(nullable=True, default=None)
+    quiztivities: list[QuizTivity] | None = ormar.ManyToMany(QuizTivity)
+    quizzes: list[Quiz] | None = ormar.ManyToMany(Quiz)
+    alt_text: str | None = ormar.Text(default=None, nullable=True)
+    filename: str | None = ormar.Text(default=None, nullable=True)
+
+    class Meta:
+        tablename = "storage_items"
+        metadata = metadata
+        database = database

classquiz/routers/storage.py

@@ -1,14 +1,17 @@
 #  This Source Code Form is subject to the terms of the Mozilla Public
 #  License, v. 2.0. If a copy of the MPL was not distributed with this
 #  file, You can obtain one at https://mozilla.org/MPL/2.0/.
-
 import re
+from datetime import datetime
 
-from fastapi import APIRouter, HTTPException
+from fastapi import APIRouter, HTTPException, UploadFile, File, Depends
 from fastapi.responses import StreamingResponse, RedirectResponse
 
-from classquiz.config import settings, storage
+from classquiz.auth import get_current_user
+from classquiz.config import settings, storage, arq
+from classquiz.db.models import User, StorageItem
 from classquiz.storage.errors import DownloadingFailedError
+from uuid import uuid4
 
 settings = settings()
 
@@ -39,3 +42,32 @@ async def download_file(file_name: str):
         media_type="image/*",
         headers={"Cache-Control": "public, immutable, max-age=31536000"},
     )
+
+
+@router.post("/")
+async def upload_file(file: UploadFile = File(), user: User = Depends(get_current_user)):
+    file_id = uuid4()
+
+    size = 0
+    # if file.file.name is None:
+    #     size = len(await file.read())
+    #     print("MemorySize", size)
+    # else:
+    #     f = file.file
+    #     a = file.file.fileno()
+    #     os.path.getsize(file.file.name)
+    #     print("DiskSize", size, "name:", file.file.name, "a:", file.file.tell(), "size")
+    file_obj = StorageItem(
+        id=file_id,
+        uploaded_at=datetime.now(),
+        mime_type=file.content_type,
+        hash=None,
+        user=user,
+        size=size,
+        deleted_at=None,
+        alt_text=None,
+    )
+    file_data = await file.read()
+    await storage.upload(file_name=file_id.hex, file_data=file_data)
+    await file_obj.save()
+    await arq.enqueue_job("calculate_hash", file_id.hex)

classquiz/worker/__init__.py

@@ -0,0 +1,29 @@
+#  This Source Code Form is subject to the terms of the Mozilla Public
+#  License, v. 2.0. If a copy of the MPL was not distributed with this
+#  file, You can obtain one at https://mozilla.org/MPL/2.0/.
+from arq import cron
+from arq.connections import RedisSettings
+
+from classquiz import settings
+from classquiz.db import database
+from classquiz.worker.storage import clean_editor_images_up, calculate_hash
+
+
+async def startup(ctx):
+    ctx["db"] = database
+    if not ctx["db"].is_connected:
+        await ctx["db"].connect()
+
+
+async def shutdown(ctx):
+    if ctx["db"].is_connected:
+        await ctx["db"].disconnect()
+
+
+class WorkerSettings:
+    # functions = [add_track]
+    functions = [calculate_hash]
+    cron_jobs = [cron(clean_editor_images_up, hour={0, 6, 12, 18}, minute=0)]
+    on_startup = startup
+    on_shutdown = shutdown
+    redis_settings = RedisSettings.from_dsn(settings.redis)

classquiz/worker/storage.py

@@ -1,11 +1,18 @@
 #  This Source Code Form is subject to the terms of the Mozilla Public
 #  License, v. 2.0. If a copy of the MPL was not distributed with this
 #  file, You can obtain one at https://mozilla.org/MPL/2.0/.
+import uuid
+
+import xxhash
+
 from classquiz.config import redis, storage
+from tempfile import SpooledTemporaryFile
+
+from classquiz.db.models import StorageItem
 from classquiz.storage.errors import DeletionFailedError
 
 
-async def clean_editor_images_up():
+async def clean_editor_images_up(ctx):
     print("Cleaning images up")
     edit_sessions = await redis.smembers("edit_sessions")
     for session_id in edit_sessions:
@@ -19,3 +26,22 @@ async def clean_editor_images_up():
                     print("Deletion Error", images)
             await redis.srem("edit_sessions", session_id)
             await redis.delete(f"edit_session:{session_id}:images")
+
+
+async def calculate_hash(ctx, file_id_as_str: str):
+    print("Calculating hash")
+    file_id = uuid.UUID(file_id_as_str)
+    file_data = await StorageItem.objects.get(id=file_id)
+    file_path = file_id.hex
+    if file_data.storage_path is not None:
+        file_path = file_data.storage_path
+    file = SpooledTemporaryFile()
+    file.write((await storage.download(file_path)).getbuffer().tobytes())
+    hash_obj = xxhash.xxh3_128()
+    # assert hash_obj.block_size == 64
+    while chunk := file.read(6400):
+        hash_obj.update(chunk)
+    file_data.hash = hash_obj.digest()
+    print("Got hash!")
+    await file_data.update()
+    file.close()

migrations/versions/44255816ff7b_added_storageitem.py

@@ -0,0 +1,83 @@
+"""Added StorageItem
+
+Revision ID: 44255816ff7b
+Revises: 8ac2bed1718e
+Create Date: 2023-05-25 12:29:44.913484
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import ormar
+
+
+# revision identifiers, used by Alembic.
+revision = "44255816ff7b"
+down_revision = "8ac2bed1718e"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "storage_items",
+        sa.Column("id", ormar.fields.sqlalchemy_uuid.CHAR(32), nullable=False),
+        sa.Column("uploaded_at", sa.DateTime(), nullable=False),
+        sa.Column("mime_type", sa.Text(), nullable=False),
+        sa.Column("hash", sa.LargeBinary(length=16), nullable=True),
+        sa.Column("user", ormar.fields.sqlalchemy_uuid.CHAR(32), nullable=True),
+        sa.Column("size", sa.BigInteger(), nullable=False),
+        sa.Column("storage_path", sa.Text(), nullable=True),
+        sa.Column("deleted_at", sa.DateTime(), nullable=True),
+        sa.Column("alt_text", sa.Text(), nullable=True),
+        sa.Column("filename", sa.Text(), nullable=True),
+        sa.ForeignKeyConstraint(["user"], ["users.id"], name="fk_storage_items_users_id_user"),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_table(
+        "storageitems_quizs",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("quiz", ormar.fields.sqlalchemy_uuid.CHAR(32), nullable=True),
+        sa.Column("storageitem", ormar.fields.sqlalchemy_uuid.CHAR(32), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["quiz"], ["quiz.id"], name="fk_storageitems_quizs_quiz_quiz_id", onupdate="CASCADE", ondelete="CASCADE"
+        ),
+        sa.ForeignKeyConstraint(
+            ["storageitem"],
+            ["storage_items.id"],
+            name="fk_storageitems_quizs_storage_items_storageitem_id",
+            onupdate="CASCADE",
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_table(
+        "storageitems_quiztivitys",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("quiztivity", ormar.fields.sqlalchemy_uuid.CHAR(32), nullable=True),
+        sa.Column("storageitem", ormar.fields.sqlalchemy_uuid.CHAR(32), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["quiztivity"],
+            ["quiztivitys.id"],
+            name="fk_storageitems_quiztivitys_quiztivitys_quiztivity_id",
+            onupdate="CASCADE",
+            ondelete="CASCADE",
+        ),
+        sa.ForeignKeyConstraint(
+            ["storageitem"],
+            ["storage_items.id"],
+            name="fk_storageitems_quiztivitys_storage_items_storageitem_id",
+            onupdate="CASCADE",
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table("storageitems_quiztivitys")
+    op.drop_table("storageitems_quizs")
+    op.drop_table("storage_items")
+    # ### end Alembic commands ###