✨ Added streaming up- and downloads, fixed file sizes and added a (not working) dashboard

classquiz/config.py

@@ -90,8 +90,6 @@ redis: redis_base_lib.client.Redis = redis_lib.Redis(connection_pool=pool)
 arq: ArqRedis = ArqRedis(pool_or_conn=pool)
 storage: Storage = Storage(
     backend=settings().storage_backend,
-    deta_key=settings().deta_project_key,
-    deta_id=settings().deta_project_id,
     storage_path=settings().storage_path,
     access_key=settings().s3_access_key,
     secret_key=settings().s3_secret_key,

classquiz/db/models.py

@@ -393,6 +393,9 @@ class StorageItem(ormar.Model):
     quizzes: list[Quiz] | None = ormar.ManyToMany(Quiz)
     alt_text: str | None = ormar.Text(default=None, nullable=True)
     filename: str | None = ormar.Text(default=None, nullable=True)
+    thumbhash: str | None = ormar.Text(default=None, nullable=True)
+    server: str | None = ormar.Text(default=None, nullable=True)
+    imported: bool = ormar.Boolean(default=False, nullable=True)
 
     class Meta:
         tablename = "storage_items"
@@ -409,6 +412,9 @@ class PublicStorageItem(BaseModel):
     deleted_at: datetime | None
     alt_text: str | None
     filename: str | None
+    thumbhash: str | None
+    server: str | None
+    imported: bool
 
     @classmethod
     def from_db_model(cls, data: StorageItem):
@@ -424,6 +430,41 @@ class PublicStorageItem(BaseModel):
             deleted_at=data.deleted_at,
             alt_text=data.alt_text,
             filename=data.filename,
+            thumbhash=data.thumbhash,
+            server=data.server,
+            imported=data.imported,
+        )
+
+
+class PrivateStorageItem(PublicStorageItem):
+    quizzes: list[OnlyId]
+    quiztivities: list[OnlyId]
+
+    @classmethod
+    def from_db_model(cls, data: StorageItem):
+        hash_data = None
+        if data.hash is not None:
+            hash_data = data.hash.hex()
+        quiztivities = []
+        quizzes = []
+        for quiz in data.quizzes:
+            quizzes.append(OnlyId(id=quiz.id))
+        for quiztivity in data.quizzes:
+            quiztivities.append(OnlyId(id=quiztivity.id))
+        return cls(
+            id=data.id,
+            uploaded_at=data.uploaded_at,
+            mime_type=data.mime_type,
+            hash=hash_data,
+            size=data.size,
+            deleted_at=data.deleted_at,
+            alt_text=data.alt_text,
+            filename=data.filename,
+            quiztivities=quiztivities,
+            quizzes=quizzes,
+            thumbhash=data.thumbhash,
+            server=data.server,
+            imported=data.imported,
         )
 
 

classquiz/kahoot_importer/import_quiz.py

@@ -3,6 +3,7 @@
 #  file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
 import html
+import io
 import json
 import uuid
 from datetime import datetime
@@ -10,8 +11,8 @@ from datetime import datetime
 import bleach
 from aiohttp import ClientSession
 
-from classquiz.config import settings, storage, meilisearch, ALLOWED_TAGS_FOR_QUIZ
-from classquiz.db.models import Quiz, ABCDQuizAnswer, QuizQuestion, User
+from classquiz.config import settings, storage, meilisearch, ALLOWED_TAGS_FOR_QUIZ, arq
+from classquiz.db.models import Quiz, ABCDQuizAnswer, QuizQuestion, User, StorageItem
 from classquiz.kahoot_importer.get import get as get_quiz
 from classquiz.helpers import get_meili_data
 
@@ -26,6 +27,25 @@ async def _download_image(url: str) -> bytes:
 DEFAULT_COLORS = ["#D6EDC9", "#B07156", "#7F7057", "#4E6E58"]
 
 
+async def handle_image_upload(url: str, user: User) -> str:
+    image_bytes = await _download_image(url)
+    file_obj = StorageItem(
+        id=uuid.uuid4(),
+        uploaded_at=datetime.now(),
+        mime_type="application/octet-stream",
+        hash=None,
+        user=user,
+        size=0,
+        deleted_at=None,
+        alt_text=None,
+        imported=True,
+    )
+    await file_obj.save()
+    await storage.upload(file_name=file_obj.id.hex, file_data=io.BytesIO(image_bytes))
+    await arq.enqueue_job("calculate_hash", file_obj.id.hex)
+    return file_obj.id.hex
+
+
 async def import_quiz(quiz_id: str, user: User) -> Quiz | str:
     """
     Imports a quiz from Kahoot.
@@ -46,10 +66,7 @@ async def import_quiz(quiz_id: str, user: User) -> Quiz | str:
         answers: list[ABCDQuizAnswer] = []
         image = None
         if q.image is not None and q.image != "":
-            image_bytes = await _download_image(q.image)
-            image_name = f"{quiz_id}--{uuid.uuid4()}"
-            await storage.upload(file_name=image_name, file_data=image_bytes)
-            image = image_name
+            image = await handle_image_upload(q.image, user)
         for i, a in enumerate(q.choices):
             answers.append(
                 (
@@ -71,10 +88,7 @@ async def import_quiz(quiz_id: str, user: User) -> Quiz | str:
         )
     cover = None
     if quiz.kahoot.cover != "" and quiz.kahoot.cover is not None:
-        image_bytes = await _download_image(quiz.kahoot.cover)
-        image_name = f"{quiz_id}--{uuid.uuid4()}"
-        await storage.upload(file_name=image_name, file_data=image_bytes)
-        cover = image_name
+        cover = await handle_image_upload(quiz.kahoot.cover, user)
     quiz_data = Quiz(
         id=quiz_id,
         public=True,

classquiz/routers/storage.py

@@ -1,14 +1,14 @@
 #  This Source Code Form is subject to the terms of the Mozilla Public
 #  License, v. 2.0. If a copy of the MPL was not distributed with this
 #  file, You can obtain one at https://mozilla.org/MPL/2.0/.
-from datetime import datetime
+from datetime import datetime, timedelta
 
 from fastapi import APIRouter, HTTPException, UploadFile, File, Depends
 from fastapi.responses import StreamingResponse, RedirectResponse
 
 from classquiz.auth import get_current_user
 from classquiz.config import settings, storage, arq
-from classquiz.db.models import User, StorageItem, PublicStorageItem, UpdateStorageItem
+from classquiz.db.models import User, StorageItem, PublicStorageItem, UpdateStorageItem, PrivateStorageItem
 from classquiz.helpers import check_image_string
 from classquiz.storage.errors import DownloadingFailedError
 from uuid import uuid4, UUID
@@ -112,3 +112,25 @@ async def update_image_data(
     file_data.alt_text = data.alt_text
     await file_data.update()
     return PublicStorageItem.from_db_model(file_data)
+
+
+@router.get("/list")
+async def list_images(
+    since: datetime | None = None, user: User = Depends(get_current_user)
+) -> list[PrivateStorageItem]:
+    if since is None:
+        since = datetime.now() - timedelta(days=2)
+    storage_items = (
+        await StorageItem.objects.filter(user=user)
+        .filter(StorageItem.uploaded_at > since)
+        .order_by(StorageItem.uploaded_at.desc())
+        .select_related([StorageItem.quizzes, StorageItem.quiztivities])
+        .all()
+    )
+    if len(storage_items) == 0:
+        raise HTTPException(status_code=404, detail="No items found")
+    return_items: list[PrivateStorageItem] = []
+    for item in storage_items:
+        # print(item.quizzes)
+        return_items.append(PrivateStorageItem.from_db_model(item))
+    return return_items

classquiz/storage/__init__.py

@@ -2,20 +2,17 @@
 #  License, v. 2.0. If a copy of the MPL was not distributed with this
 #  file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
-from io import BytesIO
-from typing import Optional
+from typing import Optional, BinaryIO
 
-from .deta_storage import DetaStorage
 from .local_storage import LocalStorage
 from .s3_storage import S3Storage
+from typing import Generator
 
 
 class Storage:
     def __init__(
         self,
         backend: str,
-        deta_key: Optional[str],
-        deta_id: Optional[str],
         storage_path: Optional[str],
         access_key: str | None = None,
         secret_key: str | None = None,
@@ -23,65 +20,45 @@ class Storage:
         base_url: str | None = None,
     ):
         self.backend = backend
-        self.deta_key: str | None = deta_key
-        self.deta_id: str | None = deta_id
-        # self.deta_base_url = f"https://drive.deta.sh/v1/{deta_id}/classquiz1"
         self.access_key = access_key
         self.secret_key = secret_key
         self.bucket_name = bucket_name
         self.base_url = base_url
-        self.deta_instance = None
-        if backend == "deta":
-            if deta_key is None or deta_id is None:
-                raise ValueError("deta_key and deta_id must be provided")
-            if self.base_url is None:
-                self.base_url = f"https://drive.deta.sh/v1/{deta_id}/classquiz1"
-            self.deta_instance = DetaStorage(
-                deta_base_url=self.base_url,
-                deta_key=self.deta_key,
-                deta_id=self.deta_id,
-            )
+        self.instance: LocalStorage | S3Storage | None = None
 
-        elif backend == "local":
+        if backend == "local":
             if storage_path is None:
                 raise ValueError("storage_path must be provided")
             else:
-                self.local_instance = LocalStorage(base_path=storage_path)
+                self.instance = LocalStorage(base_path=storage_path)
 
         elif backend == "s3":
             if access_key is None or secret_key is None or bucket_name is None or base_url is None:
                 raise ValueError("Not all parameters given")
-            self.s3_instance = S3Storage(
+            self.instance = S3Storage(
                 base_url=base_url, access_key=access_key, secret_key=secret_key, bucket_name=bucket_name
             )
 
         else:
             raise NotImplementedError(f"Backend {backend} not implemented")
 
-    async def download(self, file_name: str) -> BytesIO | None:
-        if self.backend == "deta":
-            return await self.deta_instance.download(file_name)
-        elif self.backend == "local":
-            return await self.local_instance.get_file(file_name)
-        elif self.backend == "s3":
-            return await self.s3_instance.download(file_name)
+    async def download(self, file_name: str) -> Generator | None:
+        """
+        No support for s3 since it doesn't make sense relaying the traffic through this backend
+        :param file_name:
+        :return:
+        """
+        yield self.instance.download(file_name)
 
-    async def upload(self, file_name: str, file_data: bytes) -> None:
-        if self.backend == "deta":
-            return await self.deta_instance.upload(file=file_data, file_name=file_name)
-        elif self.backend == "local":
-            return await self.local_instance.write_file(file_name=file_name, data=file_data)
-        elif self.backend == "s3":
-            return await self.s3_instance.upload(file=file_data, file_name=file_name)
+    async def upload(self, file_name: str, file_data: BinaryIO) -> None:
+        return await self.instance.upload(file=file_data, file_name=file_name)
 
     async def delete(self, file_names: [str]) -> None:
-        if self.backend == "deta":
-            return await self.deta_instance.delete(file_names=file_names)
-        elif self.backend == "local":
-            return await self.local_instance.delete_file(file_names=file_names)
-        elif self.backend == "s3":
-            return await self.s3_instance.delete(file_names=file_names)
+        return await self.instance.delete(file_names=file_names)
 
     async def get_url(self, file_name: str, expiry: int) -> str:
         if self.backend == "s3":
-            return self.s3_instance.get_url(file_name=file_name, expire=expiry)
+            return self.instance.get_url(file_name=file_name, expire=expiry)
+
+    async def get_file_size(self, file_name: str) -> int | None:
+        return self.instance.size(file_name)

classquiz/storage/deta_storage.py

@@ -1,58 +0,0 @@
-#  This Source Code Form is subject to the terms of the Mozilla Public
-#  License, v. 2.0. If a copy of the MPL was not distributed with this
-#  file, You can obtain one at https://mozilla.org/MPL/2.0/.
-
-from io import BytesIO
-from classquiz.storage.errors import DeletionFailedError, SavingFailedError, DownloadingFailedError
-
-from aiohttp import ClientSession
-
-
-class DetaStorage:
-    def __init__(self, deta_base_url: str, deta_id: str, deta_key: str):
-        self.deta_url = deta_base_url
-        self.deta_id = deta_id
-        self.deta_key = deta_key
-        self.headers = {
-            "X-Api-Key": self.deta_key,
-        }
-
-    async def download(self, file_name: str) -> BytesIO | None:
-        """
-
-        :param file_name: The name of the file to be downloaded
-        :return: Either bytes if successful download or None if failed
-        """
-        async with ClientSession(headers=self.headers) as session, session.get(
-            f"{self.deta_url}/files/download?name={file_name}"
-        ) as response:
-            if response.status == 200:
-                return BytesIO(await response.read())
-            elif response.status == 404:
-                return None
-            else:
-                raise DownloadingFailedError
-
-    async def upload(self, file: bytes, file_name: str) -> None:
-        """
-        :param file: The file in bytes
-        :param file_name: The name of the file
-        :return:
-        """
-        async with ClientSession(headers=self.headers) as session, session.post(
-            f"{self.deta_url}/files?name={file_name}", data=file
-        ) as response:
-            if response.status == 201:
-                return None
-            else:
-                print(response.status, await response.json())
-                raise SavingFailedError
-
-    async def delete(self, file_names: [str]) -> None:
-        async with ClientSession(headers=self.headers) as session, session.delete(
-            f"{self.deta_url}/files", json={"names": file_names}
-        ) as response:
-            if response.status == 200:
-                return None
-            else:
-                raise DeletionFailedError

classquiz/storage/local_storage.py

@@ -2,32 +2,45 @@
 #  License, v. 2.0. If a copy of the MPL was not distributed with this
 #  file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
-import io
 import os
+from typing import BinaryIO, Generator
 
 import aiofiles
 import aiofiles.os
 
+_DEFAULT_CHUNK_SIZE = 32768  # bytes; arbitrary
+
+
+async def aioshutil_copyfileobj(async_fsrc, async_fdst, *, chunksize: int = _DEFAULT_CHUNK_SIZE) -> None:
+    while (chunk := await async_fsrc.read(chunksize)) != b"":
+        await async_fdst.write(chunk)
+
 
 class LocalStorage:
     def __init__(self, base_path: str):
         self.base_path = base_path
 
-    async def get_file(self, file_name: str) -> io.BytesIO | None:
+    async def download(self, file_name: str) -> Generator | None:
         try:
             async with aiofiles.open(file=os.path.join(self.base_path, file_name), mode="rb") as f:
-                return io.BytesIO(await f.read())
+                yield f.read()
         except FileNotFoundError:
-            return None
+            yield None
 
-    async def write_file(self, file_name: str, data: bytes) -> None:
+    async def upload(self, file_name: str, data: BinaryIO) -> None:
         async with aiofiles.open(file=os.path.join(self.base_path, file_name), mode="wb") as f:
-            await f.write(data)
+            await aioshutil_copyfileobj(data, f)
 
-    async def delete_file(self, file_names: [str]) -> None:
+    async def delete(self, file_names: [str]) -> None:
         for i in file_names:
             try:
                 await aiofiles.os.remove(os.path.join(self.base_path, i))
             except FileNotFoundError:
                 pass
         return None
+
+    def size(self, file_name: str) -> int | None:
+        try:
+            return os.stat(os.path.join(self.base_path, file_name))
+        except FileNotFoundError:
+            return None

classquiz/storage/s3_storage.py

@@ -8,8 +8,7 @@
 import hashlib
 import hmac
 from datetime import datetime, timedelta
-from io import BytesIO
-from typing import Tuple
+from typing import Tuple, BinaryIO, Generator
 
 from aiohttp import ClientSession
 import minio
@@ -113,17 +112,9 @@ class S3Storage:
 
         return headers, request_url
 
-    async def download(self, file_name: str):
-        headers, url = self._generate_aws_signature_v4(method="GET", path=f"/{file_name}")
-        async with ClientSession() as session, session.get(url, headers=headers) as resp:
-            if resp.status == 200:
-                return BytesIO(await resp.read())
-            elif resp.status == 404:
-                return None
-            else:
-                raise DownloadingFailedError
-
-    async def upload(self, file: bytes, file_name: str, content_type: str | None = "application/octet-stream") -> None:
+    async def upload(
+        self, file: BinaryIO, file_name: str, content_type: str | None = "application/octet-stream"
+    ) -> None:
         headers, url = self._generate_aws_signature_v4(method="PUT", path=f"/{file_name}")
         async with ClientSession() as session, session.put(url, headers=headers, data=file) as resp:
             if resp.status == 200:
@@ -145,3 +136,25 @@ class S3Storage:
         return self.client.presigned_get_object(
             object_name=file_name, bucket_name=self.bucket_name, expires=timedelta(seconds=expire)
         )
+
+    def size(self, file_name: str) -> int | None:
+        res = self.client.stat_object(bucket_name=self.bucket_name, object_name=file_name)
+        if res is None:
+            return None
+        return res.size
+
+    async def download(self, file_name: str) -> Generator:
+        headers, url = self._generate_aws_signature_v4(method="GET", path=f"/{file_name}")
+
+        async with ClientSession() as session, session.get(url, headers=headers) as resp:
+            if resp.status == 200:
+                async for i in resp.content.iter_chunked(1024):
+                    yield i
+            elif resp.status == 404:
+                yield None
+            else:
+                raise DownloadingFailedError
+        # client = httpx.AsyncClient()
+        # async with client.stream("GET", url, headers=headers) as resp:
+        #     if resp.status == 200:
+        #         yield resp.aiter_bytes()

classquiz/worker/storage.py

@@ -13,6 +13,7 @@ from tempfile import SpooledTemporaryFile
 from classquiz.db.models import StorageItem, Quiz
 from classquiz.helpers import extract_image_ids_from_quiz
 from classquiz.storage.errors import DeletionFailedError
+from thumbhash import image_to_thumbhash
 
 
 async def clean_editor_images_up(ctx):
@@ -32,19 +33,27 @@ async def clean_editor_images_up(ctx):
 
 
 async def calculate_hash(ctx, file_id_as_str: str):
-    print("Calculating hash")
     file_id = uuid.UUID(file_id_as_str)
-    file_data = await StorageItem.objects.get(id=file_id)
+    file_data: StorageItem = await StorageItem.objects.get(id=file_id)
     file_path = file_id.hex
     if file_data.storage_path is not None:
         file_path = file_data.storage_path
     file = SpooledTemporaryFile()
-    file_bytes = await storage.download(file_path)
+    file_data.size = await storage.get_file_size(file_name=file_path)
+    if file_data.size is None:
+        file_data.size = 0
+    file_bytes = storage.download(file_path)
     if file_bytes is None:
         print("Retry raised!")
         raise Retry(defer=ctx["job_try"] * 10)
-    file.write(file_bytes.getbuffer().tobytes())
+    async for chunk in file_bytes:
+        async for c in chunk:
+            file.write(c)
+
+    if 0 < file_data.size < 20_970_000:  # greater than 0 but smaller than 20mbytes
+        file_data.thumbhash = image_to_thumbhash(file)
     hash_obj = xxhash.xxh3_128()
+
     # assert hash_obj.block_size == 64
     while chunk := file.read(6400):
         hash_obj.update(chunk)