✨ Added streaming up- and downloads, fixed file sizes and added a (not working) dashboard

Pipfile

@@ -39,6 +39,7 @@ pyotp = "*"
 minio = "*"
 xxhash = "*"
 arq = "*"
+thumbhash-python = "==1.0.0"
 
 [dev-packages]
 coverage = "*"

Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "5d6d03750063f2d3ea5912afb98742ee45a0c51a02dbf78059793c523b9d4c7c"
+            "sha256": "dc7977c3d6c0c72f41144ddb7bc0821c1bd34c856cbf2a93077c809e6b7a9048"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -135,11 +135,11 @@
         },
         "anyio": {
             "hashes": [
-                "sha256:25ea0d673ae30af41a0c442f81cf3b38c7e79fdc7b60335a4c14e05eb0947421",
+                "sha256:275d9973793619a5374e1c89a4f4ad3f4b0a5510a2b5b939444bee8f4c4d37ce",
-                "sha256:fbbe32bd270d2a2ef3ed1c5d45041250284e31fc0a4df4a5a6071842051a51e3"
+                "sha256:eddca883c4175f14df8aedce21054bfca3adb70ffe76a9f607aef9d7fa2ea7f0"
             ],
-            "markers": "python_full_version >= '3.6.2'",
+            "markers": "python_version >= '3.7'",
-            "version": "==3.6.2"
+            "version": "==3.7.0"
         },
         "argon2-cffi": {
             "hashes": [
@@ -491,6 +491,20 @@
             "markers": "python_version >= '3.7'",
             "version": "==8.1.3"
         },
+        "colorama": {
+            "hashes": [
+                "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44",
+                "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"
+            ],
+            "version": "==0.4.6"
+        },
+        "commonmark": {
+            "hashes": [
+                "sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60",
+                "sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9"
+            ],
+            "version": "==0.9.1"
+        },
         "cryptography": {
             "hashes": [
                 "sha256:05dc219433b14046c476f6f09d7636b92a1c3e5808b9a6536adf4932b3b2c440",
@@ -548,6 +562,14 @@
             "index": "pypi",
             "version": "==2.0.0.post2"
         },
+        "exceptiongroup": {
+            "hashes": [
+                "sha256:232c37c63e4f682982c8b6459f33a8981039e5fb8756b2074364e5055c498c9e",
+                "sha256:d484c3090ba2889ae2928419117447a14daf3c1231d5e30d0aae34f354f01785"
+            ],
+            "markers": "python_version < '3.11'",
+            "version": "==1.1.1"
+        },
         "fastapi": {
             "hashes": [
                 "sha256:4d9d3e8c71c73f11874bcf5e33626258d143252e329a01002f767306c64fb982",
@@ -1289,6 +1311,14 @@
             "index": "pypi",
             "version": "==1.10.4"
         },
+        "pygments": {
+            "hashes": [
+                "sha256:8ace4d3c1dd481894b2005f560ead0f9f19ee64fe983366be1a21e171d12775c",
+                "sha256:db2db3deb4b4179f399a09054b023b6a586b76499d36965813c71aa8ed7b5fd1"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==2.15.1"
+        },
         "pyopenssl": {
             "hashes": [
                 "sha256:841498b9bec61623b1b6c47ebbc02367c07d60e0e195f19790817f10cc8db0b7",
@@ -1420,6 +1450,13 @@
             "markers": "python_version >= '3.7'",
             "version": "==2.31.0"
         },
+        "rich": {
+            "hashes": [
+                "sha256:a4eb26484f2c82589bd9a17c73d32a010b1e29d89f1604cd9bf3a2097b81bb5e",
+                "sha256:ba3a3775974105c221d31141f2c116f4fd65c5ceb0698657a11e9f295ec93fd0"
+            ],
+            "version": "==12.6.0"
+        },
         "rsa": {
             "hashes": [
                 "sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7",
@@ -1452,6 +1489,13 @@
             "markers": "python_version >= '3.7'",
             "version": "==67.8.0"
         },
+        "shellingham": {
+            "hashes": [
+                "sha256:368bf8c00754fd4f55afb7bbb86e272df77e4dc76ac29dbcbb81a59e9fc15744",
+                "sha256:823bc5fb5c34d60f285b624e7264f4dda254bc803a3774a147bf99c0e3004a28"
+            ],
+            "version": "==1.5.0.post1"
+        },
         "six": {
             "hashes": [
                 "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
@@ -1523,6 +1567,14 @@
             "markers": "python_version >= '3.7'",
             "version": "==0.27.0"
         },
+        "thumbhash-python": {
+            "hashes": [
+                "sha256:17ad1f8cb8fb4dd3c0738cc2c60e8c5ef18c345290b7ff5f22fa16973c7b356c",
+                "sha256:755379a45c994cc0bcb06a7cac41880fb7966d65373560b1631e96ad2a59c6a6"
+            ],
+            "index": "pypi",
+            "version": "==1.0.0"
+        },
         "typeguard": {
             "hashes": [
                 "sha256:194fb3dbcb06ea9caf7088f3befee014de57961689f9c859ac5239b1ef61d987",
@@ -1531,13 +1583,24 @@
             "markers": "python_full_version >= '3.7.4'",
             "version": "==4.0.0"
         },
+        "typer": {
+            "extras": [
+                "all"
+            ],
+            "hashes": [
+                "sha256:b5e704f4e48ec263de1c0b3a2387cd405a13767d2f907f44c1a08cbad96f606d",
+                "sha256:ff797846578a9f2a201b53442aedeb543319466870fbe1c701eab66dd7681165"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==0.7.0"
+        },
         "typing-extensions": {
             "hashes": [
-                "sha256:558bc0c4145f01e6405f4a5fdbd82050bd221b119f4bf72a961a1cfd471349d6",
+                "sha256:06006244c70ac8ee83fa8282cb188f697b8db25bc8b4df07be1873c43897060c",
-                "sha256:6bac751f4789b135c43228e72de18637e9a6c29d12777023a703fd1a6858469f"
+                "sha256:3a8b36f13dd5fdc5d1b16fe317f5668545de77fa0b8e02006381fd49d731ab98"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==4.6.1"
+            "version": "==4.6.2"
         },
         "urllib3": {
             "hashes": [

classquiz/config.py

@@ -90,8 +90,6 @@ redis: redis_base_lib.client.Redis = redis_lib.Redis(connection_pool=pool)
 arq: ArqRedis = ArqRedis(pool_or_conn=pool)
 storage: Storage = Storage(
     backend=settings().storage_backend,
-    deta_key=settings().deta_project_key,
-    deta_id=settings().deta_project_id,
     storage_path=settings().storage_path,
     access_key=settings().s3_access_key,
     secret_key=settings().s3_secret_key,

classquiz/db/models.py

@@ -393,6 +393,9 @@ class StorageItem(ormar.Model):
     quizzes: list[Quiz] | None = ormar.ManyToMany(Quiz)
     alt_text: str | None = ormar.Text(default=None, nullable=True)
     filename: str | None = ormar.Text(default=None, nullable=True)
+    thumbhash: str | None = ormar.Text(default=None, nullable=True)
+    server: str | None = ormar.Text(default=None, nullable=True)
+    imported: bool = ormar.Boolean(default=False, nullable=True)
 
     class Meta:
         tablename = "storage_items"
@@ -409,6 +412,9 @@ class PublicStorageItem(BaseModel):
     deleted_at: datetime | None
     alt_text: str | None
     filename: str | None
+    thumbhash: str | None
+    server: str | None
+    imported: bool
 
     @classmethod
     def from_db_model(cls, data: StorageItem):
@@ -424,6 +430,41 @@ class PublicStorageItem(BaseModel):
             deleted_at=data.deleted_at,
             alt_text=data.alt_text,
             filename=data.filename,
+            thumbhash=data.thumbhash,
+            server=data.server,
+            imported=data.imported,
+        )
+
+
+class PrivateStorageItem(PublicStorageItem):
+    quizzes: list[OnlyId]
+    quiztivities: list[OnlyId]
+
+    @classmethod
+    def from_db_model(cls, data: StorageItem):
+        hash_data = None
+        if data.hash is not None:
+            hash_data = data.hash.hex()
+        quiztivities = []
+        quizzes = []
+        for quiz in data.quizzes:
+            quizzes.append(OnlyId(id=quiz.id))
+        for quiztivity in data.quizzes:
+            quiztivities.append(OnlyId(id=quiztivity.id))
+        return cls(
+            id=data.id,
+            uploaded_at=data.uploaded_at,
+            mime_type=data.mime_type,
+            hash=hash_data,
+            size=data.size,
+            deleted_at=data.deleted_at,
+            alt_text=data.alt_text,
+            filename=data.filename,
+            quiztivities=quiztivities,
+            quizzes=quizzes,
+            thumbhash=data.thumbhash,
+            server=data.server,
+            imported=data.imported,
         )
 
 

classquiz/kahoot_importer/import_quiz.py

@@ -3,6 +3,7 @@
 #  file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
 import html
+import io
 import json
 import uuid
 from datetime import datetime
@@ -10,8 +11,8 @@ from datetime import datetime
 import bleach
 from aiohttp import ClientSession
 
-from classquiz.config import settings, storage, meilisearch, ALLOWED_TAGS_FOR_QUIZ
+from classquiz.config import settings, storage, meilisearch, ALLOWED_TAGS_FOR_QUIZ, arq
-from classquiz.db.models import Quiz, ABCDQuizAnswer, QuizQuestion, User
+from classquiz.db.models import Quiz, ABCDQuizAnswer, QuizQuestion, User, StorageItem
 from classquiz.kahoot_importer.get import get as get_quiz
 from classquiz.helpers import get_meili_data
 
@@ -26,6 +27,25 @@ async def _download_image(url: str) -> bytes:
 DEFAULT_COLORS = ["#D6EDC9", "#B07156", "#7F7057", "#4E6E58"]
 
 
+async def handle_image_upload(url: str, user: User) -> str:
+    image_bytes = await _download_image(url)
+    file_obj = StorageItem(
+        id=uuid.uuid4(),
+        uploaded_at=datetime.now(),
+        mime_type="application/octet-stream",
+        hash=None,
+        user=user,
+        size=0,
+        deleted_at=None,
+        alt_text=None,
+        imported=True,
+    )
+    await file_obj.save()
+    await storage.upload(file_name=file_obj.id.hex, file_data=io.BytesIO(image_bytes))
+    await arq.enqueue_job("calculate_hash", file_obj.id.hex)
+    return file_obj.id.hex
+
+
 async def import_quiz(quiz_id: str, user: User) -> Quiz | str:
     """
     Imports a quiz from Kahoot.
@@ -46,10 +66,7 @@ async def import_quiz(quiz_id: str, user: User) -> Quiz | str:
         answers: list[ABCDQuizAnswer] = []
         image = None
         if q.image is not None and q.image != "":
-            image_bytes = await _download_image(q.image)
+            image = await handle_image_upload(q.image, user)
-            image_name = f"{quiz_id}--{uuid.uuid4()}"
-            await storage.upload(file_name=image_name, file_data=image_bytes)
-            image = image_name
         for i, a in enumerate(q.choices):
             answers.append(
                 (
@@ -71,10 +88,7 @@ async def import_quiz(quiz_id: str, user: User) -> Quiz | str:
         )
     cover = None
     if quiz.kahoot.cover != "" and quiz.kahoot.cover is not None:
-        image_bytes = await _download_image(quiz.kahoot.cover)
+        cover = await handle_image_upload(quiz.kahoot.cover, user)
-        image_name = f"{quiz_id}--{uuid.uuid4()}"
-        await storage.upload(file_name=image_name, file_data=image_bytes)
-        cover = image_name
     quiz_data = Quiz(
         id=quiz_id,
         public=True,

classquiz/routers/storage.py

@@ -1,14 +1,14 @@
 #  This Source Code Form is subject to the terms of the Mozilla Public
 #  License, v. 2.0. If a copy of the MPL was not distributed with this
 #  file, You can obtain one at https://mozilla.org/MPL/2.0/.
-from datetime import datetime
+from datetime import datetime, timedelta
 
 from fastapi import APIRouter, HTTPException, UploadFile, File, Depends
 from fastapi.responses import StreamingResponse, RedirectResponse
 
 from classquiz.auth import get_current_user
 from classquiz.config import settings, storage, arq
-from classquiz.db.models import User, StorageItem, PublicStorageItem, UpdateStorageItem
+from classquiz.db.models import User, StorageItem, PublicStorageItem, UpdateStorageItem, PrivateStorageItem
 from classquiz.helpers import check_image_string
 from classquiz.storage.errors import DownloadingFailedError
 from uuid import uuid4, UUID
@@ -112,3 +112,25 @@ async def update_image_data(
     file_data.alt_text = data.alt_text
     await file_data.update()
     return PublicStorageItem.from_db_model(file_data)
+
+
+@router.get("/list")
+async def list_images(
+    since: datetime | None = None, user: User = Depends(get_current_user)
+) -> list[PrivateStorageItem]:
+    if since is None:
+        since = datetime.now() - timedelta(days=2)
+    storage_items = (
+        await StorageItem.objects.filter(user=user)
+        .filter(StorageItem.uploaded_at > since)
+        .order_by(StorageItem.uploaded_at.desc())
+        .select_related([StorageItem.quizzes, StorageItem.quiztivities])
+        .all()
+    )
+    if len(storage_items) == 0:
+        raise HTTPException(status_code=404, detail="No items found")
+    return_items: list[PrivateStorageItem] = []
+    for item in storage_items:
+        # print(item.quizzes)
+        return_items.append(PrivateStorageItem.from_db_model(item))
+    return return_items

classquiz/storage/__init__.py

@@ -2,20 +2,17 @@
 #  License, v. 2.0. If a copy of the MPL was not distributed with this
 #  file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
-from io import BytesIO
+from typing import Optional, BinaryIO
-from typing import Optional
 
-from .deta_storage import DetaStorage
 from .local_storage import LocalStorage
 from .s3_storage import S3Storage
+from typing import Generator
 
 
 class Storage:
     def __init__(
         self,
         backend: str,
-        deta_key: Optional[str],
-        deta_id: Optional[str],
         storage_path: Optional[str],
         access_key: str | None = None,
         secret_key: str | None = None,
@@ -23,65 +20,45 @@ class Storage:
         base_url: str | None = None,
     ):
         self.backend = backend
-        self.deta_key: str | None = deta_key
-        self.deta_id: str | None = deta_id
-        # self.deta_base_url = f"https://drive.deta.sh/v1/{deta_id}/classquiz1"
         self.access_key = access_key
         self.secret_key = secret_key
         self.bucket_name = bucket_name
         self.base_url = base_url
-        self.deta_instance = None
+        self.instance: LocalStorage | S3Storage | None = None
-        if backend == "deta":
-            if deta_key is None or deta_id is None:
-                raise ValueError("deta_key and deta_id must be provided")
-            if self.base_url is None:
-                self.base_url = f"https://drive.deta.sh/v1/{deta_id}/classquiz1"
-            self.deta_instance = DetaStorage(
-                deta_base_url=self.base_url,
-                deta_key=self.deta_key,
-                deta_id=self.deta_id,
-            )
 
-        elif backend == "local":
+        if backend == "local":
             if storage_path is None:
                 raise ValueError("storage_path must be provided")
             else:
-                self.local_instance = LocalStorage(base_path=storage_path)
+                self.instance = LocalStorage(base_path=storage_path)
 
         elif backend == "s3":
             if access_key is None or secret_key is None or bucket_name is None or base_url is None:
                 raise ValueError("Not all parameters given")
-            self.s3_instance = S3Storage(
+            self.instance = S3Storage(
                 base_url=base_url, access_key=access_key, secret_key=secret_key, bucket_name=bucket_name
             )
 
         else:
             raise NotImplementedError(f"Backend {backend} not implemented")
 
-    async def download(self, file_name: str) -> BytesIO | None:
+    async def download(self, file_name: str) -> Generator | None:
-        if self.backend == "deta":
+        """
-            return await self.deta_instance.download(file_name)
+        No support for s3 since it doesn't make sense relaying the traffic through this backend
-        elif self.backend == "local":
+        :param file_name:
-            return await self.local_instance.get_file(file_name)
+        :return:
-        elif self.backend == "s3":
+        """
-            return await self.s3_instance.download(file_name)
+        yield self.instance.download(file_name)
 
-    async def upload(self, file_name: str, file_data: bytes) -> None:
+    async def upload(self, file_name: str, file_data: BinaryIO) -> None:
-        if self.backend == "deta":
+        return await self.instance.upload(file=file_data, file_name=file_name)
-            return await self.deta_instance.upload(file=file_data, file_name=file_name)
-        elif self.backend == "local":
-            return await self.local_instance.write_file(file_name=file_name, data=file_data)
-        elif self.backend == "s3":
-            return await self.s3_instance.upload(file=file_data, file_name=file_name)
 
     async def delete(self, file_names: [str]) -> None:
-        if self.backend == "deta":
+        return await self.instance.delete(file_names=file_names)
-            return await self.deta_instance.delete(file_names=file_names)
-        elif self.backend == "local":
-            return await self.local_instance.delete_file(file_names=file_names)
-        elif self.backend == "s3":
-            return await self.s3_instance.delete(file_names=file_names)
 
     async def get_url(self, file_name: str, expiry: int) -> str:
         if self.backend == "s3":
-            return self.s3_instance.get_url(file_name=file_name, expire=expiry)
+            return self.instance.get_url(file_name=file_name, expire=expiry)
+
+    async def get_file_size(self, file_name: str) -> int | None:
+        return self.instance.size(file_name)

classquiz/storage/deta_storage.py

@@ -1,58 +0,0 @@
-#  This Source Code Form is subject to the terms of the Mozilla Public
-#  License, v. 2.0. If a copy of the MPL was not distributed with this
-#  file, You can obtain one at https://mozilla.org/MPL/2.0/.
-
-from io import BytesIO
-from classquiz.storage.errors import DeletionFailedError, SavingFailedError, DownloadingFailedError
-
-from aiohttp import ClientSession
-
-
-class DetaStorage:
-    def __init__(self, deta_base_url: str, deta_id: str, deta_key: str):
-        self.deta_url = deta_base_url
-        self.deta_id = deta_id
-        self.deta_key = deta_key
-        self.headers = {
-            "X-Api-Key": self.deta_key,
-        }
-
-    async def download(self, file_name: str) -> BytesIO | None:
-        """
-
-        :param file_name: The name of the file to be downloaded
-        :return: Either bytes if successful download or None if failed
-        """
-        async with ClientSession(headers=self.headers) as session, session.get(
-            f"{self.deta_url}/files/download?name={file_name}"
-        ) as response:
-            if response.status == 200:
-                return BytesIO(await response.read())
-            elif response.status == 404:
-                return None
-            else:
-                raise DownloadingFailedError
-
-    async def upload(self, file: bytes, file_name: str) -> None:
-        """
-        :param file: The file in bytes
-        :param file_name: The name of the file
-        :return:
-        """
-        async with ClientSession(headers=self.headers) as session, session.post(
-            f"{self.deta_url}/files?name={file_name}", data=file
-        ) as response:
-            if response.status == 201:
-                return None
-            else:
-                print(response.status, await response.json())
-                raise SavingFailedError
-
-    async def delete(self, file_names: [str]) -> None:
-        async with ClientSession(headers=self.headers) as session, session.delete(
-            f"{self.deta_url}/files", json={"names": file_names}
-        ) as response:
-            if response.status == 200:
-                return None
-            else:
-                raise DeletionFailedError

classquiz/storage/local_storage.py

@@ -2,32 +2,45 @@
 #  License, v. 2.0. If a copy of the MPL was not distributed with this
 #  file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
-import io
 import os
+from typing import BinaryIO, Generator
 
 import aiofiles
 import aiofiles.os
 
+_DEFAULT_CHUNK_SIZE = 32768  # bytes; arbitrary
+
+
+async def aioshutil_copyfileobj(async_fsrc, async_fdst, *, chunksize: int = _DEFAULT_CHUNK_SIZE) -> None:
+    while (chunk := await async_fsrc.read(chunksize)) != b"":
+        await async_fdst.write(chunk)
+
 
 class LocalStorage:
     def __init__(self, base_path: str):
         self.base_path = base_path
 
-    async def get_file(self, file_name: str) -> io.BytesIO | None:
+    async def download(self, file_name: str) -> Generator | None:
         try:
             async with aiofiles.open(file=os.path.join(self.base_path, file_name), mode="rb") as f:
-                return io.BytesIO(await f.read())
+                yield f.read()
         except FileNotFoundError:
-            return None
+            yield None
 
-    async def write_file(self, file_name: str, data: bytes) -> None:
+    async def upload(self, file_name: str, data: BinaryIO) -> None:
         async with aiofiles.open(file=os.path.join(self.base_path, file_name), mode="wb") as f:
-            await f.write(data)
+            await aioshutil_copyfileobj(data, f)
 
-    async def delete_file(self, file_names: [str]) -> None:
+    async def delete(self, file_names: [str]) -> None:
         for i in file_names:
             try:
                 await aiofiles.os.remove(os.path.join(self.base_path, i))
             except FileNotFoundError:
                 pass
         return None
+
+    def size(self, file_name: str) -> int | None:
+        try:
+            return os.stat(os.path.join(self.base_path, file_name))
+        except FileNotFoundError:
+            return None

classquiz/storage/s3_storage.py

@@ -8,8 +8,7 @@
 import hashlib
 import hmac
 from datetime import datetime, timedelta
-from io import BytesIO
+from typing import Tuple, BinaryIO, Generator
-from typing import Tuple
 
 from aiohttp import ClientSession
 import minio
@@ -113,17 +112,9 @@ class S3Storage:
 
         return headers, request_url
 
-    async def download(self, file_name: str):
+    async def upload(
-        headers, url = self._generate_aws_signature_v4(method="GET", path=f"/{file_name}")
+        self, file: BinaryIO, file_name: str, content_type: str | None = "application/octet-stream"
-        async with ClientSession() as session, session.get(url, headers=headers) as resp:
+    ) -> None:
-            if resp.status == 200:
-                return BytesIO(await resp.read())
-            elif resp.status == 404:
-                return None
-            else:
-                raise DownloadingFailedError
-
-    async def upload(self, file: bytes, file_name: str, content_type: str | None = "application/octet-stream") -> None:
         headers, url = self._generate_aws_signature_v4(method="PUT", path=f"/{file_name}")
         async with ClientSession() as session, session.put(url, headers=headers, data=file) as resp:
             if resp.status == 200:
@@ -145,3 +136,25 @@ class S3Storage:
         return self.client.presigned_get_object(
             object_name=file_name, bucket_name=self.bucket_name, expires=timedelta(seconds=expire)
         )
+
+    def size(self, file_name: str) -> int | None:
+        res = self.client.stat_object(bucket_name=self.bucket_name, object_name=file_name)
+        if res is None:
+            return None
+        return res.size
+
+    async def download(self, file_name: str) -> Generator:
+        headers, url = self._generate_aws_signature_v4(method="GET", path=f"/{file_name}")
+
+        async with ClientSession() as session, session.get(url, headers=headers) as resp:
+            if resp.status == 200:
+                async for i in resp.content.iter_chunked(1024):
+                    yield i
+            elif resp.status == 404:
+                yield None
+            else:
+                raise DownloadingFailedError
+        # client = httpx.AsyncClient()
+        # async with client.stream("GET", url, headers=headers) as resp:
+        #     if resp.status == 200:
+        #         yield resp.aiter_bytes()

classquiz/worker/storage.py

@@ -13,6 +13,7 @@ from tempfile import SpooledTemporaryFile
 from classquiz.db.models import StorageItem, Quiz
 from classquiz.helpers import extract_image_ids_from_quiz
 from classquiz.storage.errors import DeletionFailedError
+from thumbhash import image_to_thumbhash
 
 
 async def clean_editor_images_up(ctx):
@@ -32,19 +33,27 @@ async def clean_editor_images_up(ctx):
 
 
 async def calculate_hash(ctx, file_id_as_str: str):
-    print("Calculating hash")
     file_id = uuid.UUID(file_id_as_str)
-    file_data = await StorageItem.objects.get(id=file_id)
+    file_data: StorageItem = await StorageItem.objects.get(id=file_id)
     file_path = file_id.hex
     if file_data.storage_path is not None:
         file_path = file_data.storage_path
     file = SpooledTemporaryFile()
-    file_bytes = await storage.download(file_path)
+    file_data.size = await storage.get_file_size(file_name=file_path)
+    if file_data.size is None:
+        file_data.size = 0
+    file_bytes = storage.download(file_path)
     if file_bytes is None:
         print("Retry raised!")
         raise Retry(defer=ctx["job_try"] * 10)
-    file.write(file_bytes.getbuffer().tobytes())
+    async for chunk in file_bytes:
+        async for c in chunk:
+            file.write(c)
+
+    if 0 < file_data.size < 20_970_000:  # greater than 0 but smaller than 20mbytes
+        file_data.thumbhash = image_to_thumbhash(file)
     hash_obj = xxhash.xxh3_128()
+
     # assert hash_obj.block_size == 64
     while chunk := file.read(6400):
         hash_obj.update(chunk)

frontend/package.json

@@ -42,6 +42,7 @@
 		"@types/ua-parser-js": "^0.7.36",
 		"@typescript-eslint/eslint-plugin": "^5.59.0",
 		"@typescript-eslint/parser": "^5.59.0",
+		"@unlazy/svelte": "^0.8.9",
 		"@uppy/compressor": "^1.0.2",
 		"@uppy/core": "^3.2.0",
 		"@uppy/dashboard": "^3.4.0",
@@ -89,6 +90,7 @@
 		"svelte-tippy": "^1.3.2",
 		"swiper": "^8.4.7",
 		"tailwindcss": "^3.3.1",
+		"thumbhash": "^0.1.1",
 		"tippy.js": "^6.3.7",
 		"tslib": "^2.5.0",
 		"typescript": "~5.0.4",

frontend/pnpm-lock.yaml

@@ -29,6 +29,7 @@ specifiers:
     '@types/ua-parser-js': ^0.7.36
     '@typescript-eslint/eslint-plugin': ^5.59.0
     '@typescript-eslint/parser': ^5.59.0
+    '@unlazy/svelte': ^0.8.9
     '@uppy/compressor': ^1.0.2
     '@uppy/core': ^3.2.0
     '@uppy/dashboard': ^3.4.0
@@ -77,6 +78,7 @@ specifiers:
     svelte-tippy: ^1.3.2
     swiper: ^8.4.7
     tailwindcss: ^3.3.1
+    thumbhash: ^0.1.1
     tippy.js: ^6.3.7
     tslib: ^2.5.0
     typescript: ~5.0.4
@@ -117,6 +119,7 @@ devDependencies:
     '@types/ua-parser-js': 0.7.36
     '@typescript-eslint/eslint-plugin': 5.59.0_wsbkzerfphvgr4lle7vmeh64yi
     '@typescript-eslint/parser': 5.59.0_voubu7prgxjfsfbgx5d4sqnwiy
+    '@unlazy/svelte': 0.8.9_2a4gi7ewzykzc7boj45sshrguu
     '@uppy/compressor': 1.0.2_@uppy+core@3.2.0
     '@uppy/core': 3.2.0
     '@uppy/dashboard': 3.4.0_@uppy+core@3.2.0
@@ -164,6 +167,7 @@ devDependencies:
     svelte-tippy: 1.3.2
     swiper: 8.4.7
     tailwindcss: 3.3.1_postcss@8.4.22
+    thumbhash: 0.1.1
     tippy.js: 6.3.7
     tslib: 2.5.0
     typescript: 5.0.4
@@ -1705,6 +1709,31 @@ packages:
             eslint-visitor-keys: 3.4.0
         dev: true
 
+    /@unlazy/core/0.8.9:
+        resolution:
+            {
+                integrity: sha512-DQ4WB/cuEWTknU/59uRwpSipvMAJzBDmRyaHDUc1RcXi0Z7/Vcl0EE7BpROxEynqd1EI+2oMWQaDLyXffUdUiA==
+            }
+        dependencies:
+            fast-blurhash: 1.1.2
+            thumbhash: 0.1.1
+        dev: true
+
+    /@unlazy/svelte/0.8.9_2a4gi7ewzykzc7boj45sshrguu:
+        resolution:
+            {
+                integrity: sha512-vgOYuOlknE6l/QhzZzpncsU8dMaTgLKGd7Xn4qwqTWAa3TF+6NRL7QedPJb2XBY8Th7yRQ79ZqeXaSDxWtL9zA==
+            }
+        peerDependencies:
+            svelte: ^3.54.0
+        dependencies:
+            svelte: 3.58.0
+            unlazy: 0.8.9_thumbhash@0.1.1
+        transitivePeerDependencies:
+            - fast-blurhash
+            - thumbhash
+        dev: true
+
     /@uppy/companion-client/3.1.3:
         resolution:
             {
@@ -2982,6 +3011,13 @@ packages:
             }
         dev: true
 
+    /fast-blurhash/1.1.2:
+        resolution:
+            {
+                integrity: sha512-lJVOgYSlahqkRhrKumNx/SGB2F/qS0D1z7xjGYjb5EZJRtlzySGMniZjkQ9h9Rv8sPmM/V9orEgRiMwazDNH6A==
+            }
+        dev: true
+
     /fast-deep-equal/3.1.3:
         resolution:
             {
@@ -5466,6 +5502,13 @@ packages:
             any-promise: 1.3.0
         dev: true
 
+    /thumbhash/0.1.1:
+        resolution:
+            {
+                integrity: sha512-kH5pKeIIBPQXAOni2AiY/Cu/NKdkFREdpH+TLdM0g6WA7RriCv0kPLgP731ady67MhTAqrVG/4mnEeibVuCJcg==
+            }
+        dev: true
+
     /tiny-case/1.0.3:
         resolution:
             {
@@ -5628,6 +5671,19 @@ packages:
             '@types/unist': 2.0.6
         dev: true
 
+    /unlazy/0.8.9_thumbhash@0.1.1:
+        resolution:
+            {
+                integrity: sha512-lRCuXN20N1esqSQqtSVBLAw9GJz0lcBuOBs3UGGw7cFWHQlWJVZZ3OviwOl42f1CnVHjAON1rs2hIdJWgMAUyg==
+            }
+        peerDependencies:
+            fast-blurhash: ^1.1.2
+            thumbhash: ^0.1.1
+        dependencies:
+            '@unlazy/core': 0.8.9
+            thumbhash: 0.1.1
+        dev: true
+
     /update-browserslist-db/1.0.11_browserslist@4.21.5:
         resolution:
             {

frontend/src/lib/files/dashboard.svelte

@@ -0,0 +1,36 @@
+<!--
+  - This Source Code Form is subject to the terms of the Mozilla Public
+  - License, v. 2.0. If a copy of the MPL was not distributed with this
+  - file, You can obtain one at https://mozilla.org/MPL/2.0/.
+  -->
+
+<script lang="ts">
+	import Spinner from '$lib/Spinner.svelte';
+
+	export let files: {
+		id: string;
+		uploaded_at: string;
+		mime_type?: string;
+		hash?: string;
+		size: number;
+		deleted_at?: string;
+		alt_text?: string;
+		filename?: string;
+		thumbhash?: string;
+		server?: string;
+		quizzes: { id: string }[];
+		quiztivities: { id: string }[];
+	}[];
+
+	const get_files = async () => {
+		const res = await fetch('/api/v1/storage/list');
+		files = await res.json();
+	};
+	if (!files) {
+		get_files();
+	}
+</script>
+
+{#if files}{:else}
+	<Spinner />
+{/if}

frontend/src/routes/dashboard/files/+page.svelte

@@ -0,0 +1,34 @@
+<!--
+  - This Source Code Form is subject to the terms of the Mozilla Public
+  - License, v. 2.0. If a copy of the MPL was not distributed with this
+  - file, You can obtain one at https://mozilla.org/MPL/2.0/.
+  -->
+
+<script lang="ts">
+	import type { PageData } from './$types';
+	// import FileDahboard from "$lib/files/dashboard.svelte"
+	// import { thumbHashToDataURL } from 'thumbhash';
+
+	export let data: PageData;
+	const files = data.files;
+
+	/*	const base64ToBytes = (value: string) => {
+		const base64 = value.replace(/-/g, '+').replace(/_/g, '/');
+
+		const decodedData =
+			typeof Buffer !== 'undefined'
+				? Buffer.from(base64, 'base64')
+				: Uint8Array.from(atob(base64), (char) => char.charCodeAt(0));
+
+		return new Uint8Array(decodedData);
+	};*/
+	// let images_loaded = [];
+</script>
+
+<div class="w-full h-full">
+	<div class="grid grid-cols-2 w-full h-full gap-4">
+		{#each files as file, i}
+			<div class="w-full h-full flex" />
+		{/each}
+	</div>
+</div>

frontend/src/routes/dashboard/files/+page.ts

@@ -0,0 +1,13 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ */
+import type { PageLoad } from './$types';
+
+export const load = (async ({ fetch }) => {
+	const res = await fetch('/api/v1/storage/list');
+	if (res.ok) {
+		return { files: await res.json() };
+	}
+}) satisfies PageLoad;

migrations/versions/1310317108bd_added_thumbhash_and_server_storageitem.py

@@ -0,0 +1,33 @@
+"""Added thumbhash and server StorageItem
+
+Revision ID: 1310317108bd
+Revises: b74382b2c6ad
+Create Date: 2023-05-29 15:53:00.388788
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import ormar
+
+
+# revision identifiers, used by Alembic.
+revision = "1310317108bd"
+down_revision = "b74382b2c6ad"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column("storage_items", sa.Column("thumbhash", sa.Text(), nullable=True))
+    op.add_column("storage_items", sa.Column("server", sa.Text(), nullable=True))
+    op.add_column("storage_items", sa.Column("imported", sa.Boolean(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("storage_items", "imported")
+    op.drop_column("storage_items", "server")
+    op.drop_column("storage_items", "thumbhash")
+    # ### end Alembic commands ###