diff --git a/Pipfile b/Pipfile index c8dae70..d35597f 100644 --- a/Pipfile +++ b/Pipfile @@ -39,6 +39,7 @@ pyotp = "*" minio = "*" xxhash = "*" arq = "*" +thumbhash-python = "==1.0.0" [dev-packages] coverage = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 1b43a3e..cbdcf77 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "5d6d03750063f2d3ea5912afb98742ee45a0c51a02dbf78059793c523b9d4c7c" + "sha256": "dc7977c3d6c0c72f41144ddb7bc0821c1bd34c856cbf2a93077c809e6b7a9048" }, "pipfile-spec": 6, "requires": { @@ -135,11 +135,11 @@ }, "anyio": { "hashes": [ - "sha256:25ea0d673ae30af41a0c442f81cf3b38c7e79fdc7b60335a4c14e05eb0947421", - "sha256:fbbe32bd270d2a2ef3ed1c5d45041250284e31fc0a4df4a5a6071842051a51e3" + "sha256:275d9973793619a5374e1c89a4f4ad3f4b0a5510a2b5b939444bee8f4c4d37ce", + "sha256:eddca883c4175f14df8aedce21054bfca3adb70ffe76a9f607aef9d7fa2ea7f0" ], - "markers": "python_full_version >= '3.6.2'", - "version": "==3.6.2" + "markers": "python_version >= '3.7'", + "version": "==3.7.0" }, "argon2-cffi": { "hashes": [ @@ -491,6 +491,20 @@ "markers": "python_version >= '3.7'", "version": "==8.1.3" }, + "colorama": { + "hashes": [ + "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", + "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6" + ], + "version": "==0.4.6" + }, + "commonmark": { + "hashes": [ + "sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60", + "sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9" + ], + "version": "==0.9.1" + }, "cryptography": { "hashes": [ "sha256:05dc219433b14046c476f6f09d7636b92a1c3e5808b9a6536adf4932b3b2c440", @@ -548,6 +562,14 @@ "index": "pypi", "version": "==2.0.0.post2" }, + "exceptiongroup": { + "hashes": [ + "sha256:232c37c63e4f682982c8b6459f33a8981039e5fb8756b2074364e5055c498c9e", + "sha256:d484c3090ba2889ae2928419117447a14daf3c1231d5e30d0aae34f354f01785" + ], + "markers": "python_version < '3.11'", + "version": "==1.1.1" + }, "fastapi": { "hashes": [ "sha256:4d9d3e8c71c73f11874bcf5e33626258d143252e329a01002f767306c64fb982", @@ -1289,6 +1311,14 @@ "index": "pypi", "version": "==1.10.4" }, + "pygments": { + "hashes": [ + "sha256:8ace4d3c1dd481894b2005f560ead0f9f19ee64fe983366be1a21e171d12775c", + "sha256:db2db3deb4b4179f399a09054b023b6a586b76499d36965813c71aa8ed7b5fd1" + ], + "markers": "python_version >= '3.7'", + "version": "==2.15.1" + }, "pyopenssl": { "hashes": [ "sha256:841498b9bec61623b1b6c47ebbc02367c07d60e0e195f19790817f10cc8db0b7", @@ -1420,6 +1450,13 @@ "markers": "python_version >= '3.7'", "version": "==2.31.0" }, + "rich": { + "hashes": [ + "sha256:a4eb26484f2c82589bd9a17c73d32a010b1e29d89f1604cd9bf3a2097b81bb5e", + "sha256:ba3a3775974105c221d31141f2c116f4fd65c5ceb0698657a11e9f295ec93fd0" + ], + "version": "==12.6.0" + }, "rsa": { "hashes": [ "sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7", @@ -1452,6 +1489,13 @@ "markers": "python_version >= '3.7'", "version": "==67.8.0" }, + "shellingham": { + "hashes": [ + "sha256:368bf8c00754fd4f55afb7bbb86e272df77e4dc76ac29dbcbb81a59e9fc15744", + "sha256:823bc5fb5c34d60f285b624e7264f4dda254bc803a3774a147bf99c0e3004a28" + ], + "version": "==1.5.0.post1" + }, "six": { "hashes": [ "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", @@ -1523,6 +1567,14 @@ "markers": "python_version >= '3.7'", "version": "==0.27.0" }, + "thumbhash-python": { + "hashes": [ + "sha256:17ad1f8cb8fb4dd3c0738cc2c60e8c5ef18c345290b7ff5f22fa16973c7b356c", + "sha256:755379a45c994cc0bcb06a7cac41880fb7966d65373560b1631e96ad2a59c6a6" + ], + "index": "pypi", + "version": "==1.0.0" + }, "typeguard": { "hashes": [ "sha256:194fb3dbcb06ea9caf7088f3befee014de57961689f9c859ac5239b1ef61d987", @@ -1531,13 +1583,24 @@ "markers": "python_full_version >= '3.7.4'", "version": "==4.0.0" }, + "typer": { + "extras": [ + "all" + ], + "hashes": [ + "sha256:b5e704f4e48ec263de1c0b3a2387cd405a13767d2f907f44c1a08cbad96f606d", + "sha256:ff797846578a9f2a201b53442aedeb543319466870fbe1c701eab66dd7681165" + ], + "markers": "python_version >= '3.6'", + "version": "==0.7.0" + }, "typing-extensions": { "hashes": [ - "sha256:558bc0c4145f01e6405f4a5fdbd82050bd221b119f4bf72a961a1cfd471349d6", - "sha256:6bac751f4789b135c43228e72de18637e9a6c29d12777023a703fd1a6858469f" + "sha256:06006244c70ac8ee83fa8282cb188f697b8db25bc8b4df07be1873c43897060c", + "sha256:3a8b36f13dd5fdc5d1b16fe317f5668545de77fa0b8e02006381fd49d731ab98" ], "markers": "python_version >= '3.7'", - "version": "==4.6.1" + "version": "==4.6.2" }, "urllib3": { "hashes": [ diff --git a/classquiz/config.py b/classquiz/config.py index 1df4e37..b079f27 100644 --- a/classquiz/config.py +++ b/classquiz/config.py @@ -90,8 +90,6 @@ redis: redis_base_lib.client.Redis = redis_lib.Redis(connection_pool=pool) arq: ArqRedis = ArqRedis(pool_or_conn=pool) storage: Storage = Storage( backend=settings().storage_backend, - deta_key=settings().deta_project_key, - deta_id=settings().deta_project_id, storage_path=settings().storage_path, access_key=settings().s3_access_key, secret_key=settings().s3_secret_key, diff --git a/classquiz/db/models.py b/classquiz/db/models.py index dd6f63d..7f1d523 100644 --- a/classquiz/db/models.py +++ b/classquiz/db/models.py @@ -393,6 +393,9 @@ class StorageItem(ormar.Model): quizzes: list[Quiz] | None = ormar.ManyToMany(Quiz) alt_text: str | None = ormar.Text(default=None, nullable=True) filename: str | None = ormar.Text(default=None, nullable=True) + thumbhash: str | None = ormar.Text(default=None, nullable=True) + server: str | None = ormar.Text(default=None, nullable=True) + imported: bool = ormar.Boolean(default=False, nullable=True) class Meta: tablename = "storage_items" @@ -409,6 +412,9 @@ class PublicStorageItem(BaseModel): deleted_at: datetime | None alt_text: str | None filename: str | None + thumbhash: str | None + server: str | None + imported: bool @classmethod def from_db_model(cls, data: StorageItem): @@ -424,6 +430,41 @@ class PublicStorageItem(BaseModel): deleted_at=data.deleted_at, alt_text=data.alt_text, filename=data.filename, + thumbhash=data.thumbhash, + server=data.server, + imported=data.imported, + ) + + +class PrivateStorageItem(PublicStorageItem): + quizzes: list[OnlyId] + quiztivities: list[OnlyId] + + @classmethod + def from_db_model(cls, data: StorageItem): + hash_data = None + if data.hash is not None: + hash_data = data.hash.hex() + quiztivities = [] + quizzes = [] + for quiz in data.quizzes: + quizzes.append(OnlyId(id=quiz.id)) + for quiztivity in data.quizzes: + quiztivities.append(OnlyId(id=quiztivity.id)) + return cls( + id=data.id, + uploaded_at=data.uploaded_at, + mime_type=data.mime_type, + hash=hash_data, + size=data.size, + deleted_at=data.deleted_at, + alt_text=data.alt_text, + filename=data.filename, + quiztivities=quiztivities, + quizzes=quizzes, + thumbhash=data.thumbhash, + server=data.server, + imported=data.imported, ) diff --git a/classquiz/kahoot_importer/import_quiz.py b/classquiz/kahoot_importer/import_quiz.py index 87d7fb2..1dbff48 100644 --- a/classquiz/kahoot_importer/import_quiz.py +++ b/classquiz/kahoot_importer/import_quiz.py @@ -3,6 +3,7 @@ # file, You can obtain one at https://mozilla.org/MPL/2.0/. import html +import io import json import uuid from datetime import datetime @@ -10,8 +11,8 @@ from datetime import datetime import bleach from aiohttp import ClientSession -from classquiz.config import settings, storage, meilisearch, ALLOWED_TAGS_FOR_QUIZ -from classquiz.db.models import Quiz, ABCDQuizAnswer, QuizQuestion, User +from classquiz.config import settings, storage, meilisearch, ALLOWED_TAGS_FOR_QUIZ, arq +from classquiz.db.models import Quiz, ABCDQuizAnswer, QuizQuestion, User, StorageItem from classquiz.kahoot_importer.get import get as get_quiz from classquiz.helpers import get_meili_data @@ -26,6 +27,25 @@ async def _download_image(url: str) -> bytes: DEFAULT_COLORS = ["#D6EDC9", "#B07156", "#7F7057", "#4E6E58"] +async def handle_image_upload(url: str, user: User) -> str: + image_bytes = await _download_image(url) + file_obj = StorageItem( + id=uuid.uuid4(), + uploaded_at=datetime.now(), + mime_type="application/octet-stream", + hash=None, + user=user, + size=0, + deleted_at=None, + alt_text=None, + imported=True, + ) + await file_obj.save() + await storage.upload(file_name=file_obj.id.hex, file_data=io.BytesIO(image_bytes)) + await arq.enqueue_job("calculate_hash", file_obj.id.hex) + return file_obj.id.hex + + async def import_quiz(quiz_id: str, user: User) -> Quiz | str: """ Imports a quiz from Kahoot. @@ -46,10 +66,7 @@ async def import_quiz(quiz_id: str, user: User) -> Quiz | str: answers: list[ABCDQuizAnswer] = [] image = None if q.image is not None and q.image != "": - image_bytes = await _download_image(q.image) - image_name = f"{quiz_id}--{uuid.uuid4()}" - await storage.upload(file_name=image_name, file_data=image_bytes) - image = image_name + image = await handle_image_upload(q.image, user) for i, a in enumerate(q.choices): answers.append( ( @@ -71,10 +88,7 @@ async def import_quiz(quiz_id: str, user: User) -> Quiz | str: ) cover = None if quiz.kahoot.cover != "" and quiz.kahoot.cover is not None: - image_bytes = await _download_image(quiz.kahoot.cover) - image_name = f"{quiz_id}--{uuid.uuid4()}" - await storage.upload(file_name=image_name, file_data=image_bytes) - cover = image_name + cover = await handle_image_upload(quiz.kahoot.cover, user) quiz_data = Quiz( id=quiz_id, public=True, diff --git a/classquiz/routers/storage.py b/classquiz/routers/storage.py index 5e8bec1..3dcb617 100644 --- a/classquiz/routers/storage.py +++ b/classquiz/routers/storage.py @@ -1,14 +1,14 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. -from datetime import datetime +from datetime import datetime, timedelta from fastapi import APIRouter, HTTPException, UploadFile, File, Depends from fastapi.responses import StreamingResponse, RedirectResponse from classquiz.auth import get_current_user from classquiz.config import settings, storage, arq -from classquiz.db.models import User, StorageItem, PublicStorageItem, UpdateStorageItem +from classquiz.db.models import User, StorageItem, PublicStorageItem, UpdateStorageItem, PrivateStorageItem from classquiz.helpers import check_image_string from classquiz.storage.errors import DownloadingFailedError from uuid import uuid4, UUID @@ -112,3 +112,25 @@ async def update_image_data( file_data.alt_text = data.alt_text await file_data.update() return PublicStorageItem.from_db_model(file_data) + + +@router.get("/list") +async def list_images( + since: datetime | None = None, user: User = Depends(get_current_user) +) -> list[PrivateStorageItem]: + if since is None: + since = datetime.now() - timedelta(days=2) + storage_items = ( + await StorageItem.objects.filter(user=user) + .filter(StorageItem.uploaded_at > since) + .order_by(StorageItem.uploaded_at.desc()) + .select_related([StorageItem.quizzes, StorageItem.quiztivities]) + .all() + ) + if len(storage_items) == 0: + raise HTTPException(status_code=404, detail="No items found") + return_items: list[PrivateStorageItem] = [] + for item in storage_items: + # print(item.quizzes) + return_items.append(PrivateStorageItem.from_db_model(item)) + return return_items diff --git a/classquiz/storage/__init__.py b/classquiz/storage/__init__.py index 92e1c5a..fc56af4 100644 --- a/classquiz/storage/__init__.py +++ b/classquiz/storage/__init__.py @@ -2,20 +2,17 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. -from io import BytesIO -from typing import Optional +from typing import Optional, BinaryIO -from .deta_storage import DetaStorage from .local_storage import LocalStorage from .s3_storage import S3Storage +from typing import Generator class Storage: def __init__( self, backend: str, - deta_key: Optional[str], - deta_id: Optional[str], storage_path: Optional[str], access_key: str | None = None, secret_key: str | None = None, @@ -23,65 +20,45 @@ class Storage: base_url: str | None = None, ): self.backend = backend - self.deta_key: str | None = deta_key - self.deta_id: str | None = deta_id - # self.deta_base_url = f"https://drive.deta.sh/v1/{deta_id}/classquiz1" self.access_key = access_key self.secret_key = secret_key self.bucket_name = bucket_name self.base_url = base_url - self.deta_instance = None - if backend == "deta": - if deta_key is None or deta_id is None: - raise ValueError("deta_key and deta_id must be provided") - if self.base_url is None: - self.base_url = f"https://drive.deta.sh/v1/{deta_id}/classquiz1" - self.deta_instance = DetaStorage( - deta_base_url=self.base_url, - deta_key=self.deta_key, - deta_id=self.deta_id, - ) + self.instance: LocalStorage | S3Storage | None = None - elif backend == "local": + if backend == "local": if storage_path is None: raise ValueError("storage_path must be provided") else: - self.local_instance = LocalStorage(base_path=storage_path) + self.instance = LocalStorage(base_path=storage_path) elif backend == "s3": if access_key is None or secret_key is None or bucket_name is None or base_url is None: raise ValueError("Not all parameters given") - self.s3_instance = S3Storage( + self.instance = S3Storage( base_url=base_url, access_key=access_key, secret_key=secret_key, bucket_name=bucket_name ) else: raise NotImplementedError(f"Backend {backend} not implemented") - async def download(self, file_name: str) -> BytesIO | None: - if self.backend == "deta": - return await self.deta_instance.download(file_name) - elif self.backend == "local": - return await self.local_instance.get_file(file_name) - elif self.backend == "s3": - return await self.s3_instance.download(file_name) + async def download(self, file_name: str) -> Generator | None: + """ + No support for s3 since it doesn't make sense relaying the traffic through this backend + :param file_name: + :return: + """ + yield self.instance.download(file_name) - async def upload(self, file_name: str, file_data: bytes) -> None: - if self.backend == "deta": - return await self.deta_instance.upload(file=file_data, file_name=file_name) - elif self.backend == "local": - return await self.local_instance.write_file(file_name=file_name, data=file_data) - elif self.backend == "s3": - return await self.s3_instance.upload(file=file_data, file_name=file_name) + async def upload(self, file_name: str, file_data: BinaryIO) -> None: + return await self.instance.upload(file=file_data, file_name=file_name) async def delete(self, file_names: [str]) -> None: - if self.backend == "deta": - return await self.deta_instance.delete(file_names=file_names) - elif self.backend == "local": - return await self.local_instance.delete_file(file_names=file_names) - elif self.backend == "s3": - return await self.s3_instance.delete(file_names=file_names) + return await self.instance.delete(file_names=file_names) async def get_url(self, file_name: str, expiry: int) -> str: if self.backend == "s3": - return self.s3_instance.get_url(file_name=file_name, expire=expiry) + return self.instance.get_url(file_name=file_name, expire=expiry) + + async def get_file_size(self, file_name: str) -> int | None: + return self.instance.size(file_name) diff --git a/classquiz/storage/deta_storage.py b/classquiz/storage/deta_storage.py deleted file mode 100644 index 4620f16..0000000 --- a/classquiz/storage/deta_storage.py +++ /dev/null @@ -1,58 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. - -from io import BytesIO -from classquiz.storage.errors import DeletionFailedError, SavingFailedError, DownloadingFailedError - -from aiohttp import ClientSession - - -class DetaStorage: - def __init__(self, deta_base_url: str, deta_id: str, deta_key: str): - self.deta_url = deta_base_url - self.deta_id = deta_id - self.deta_key = deta_key - self.headers = { - "X-Api-Key": self.deta_key, - } - - async def download(self, file_name: str) -> BytesIO | None: - """ - - :param file_name: The name of the file to be downloaded - :return: Either bytes if successful download or None if failed - """ - async with ClientSession(headers=self.headers) as session, session.get( - f"{self.deta_url}/files/download?name={file_name}" - ) as response: - if response.status == 200: - return BytesIO(await response.read()) - elif response.status == 404: - return None - else: - raise DownloadingFailedError - - async def upload(self, file: bytes, file_name: str) -> None: - """ - :param file: The file in bytes - :param file_name: The name of the file - :return: - """ - async with ClientSession(headers=self.headers) as session, session.post( - f"{self.deta_url}/files?name={file_name}", data=file - ) as response: - if response.status == 201: - return None - else: - print(response.status, await response.json()) - raise SavingFailedError - - async def delete(self, file_names: [str]) -> None: - async with ClientSession(headers=self.headers) as session, session.delete( - f"{self.deta_url}/files", json={"names": file_names} - ) as response: - if response.status == 200: - return None - else: - raise DeletionFailedError diff --git a/classquiz/storage/local_storage.py b/classquiz/storage/local_storage.py index e75dbdd..894d47e 100644 --- a/classquiz/storage/local_storage.py +++ b/classquiz/storage/local_storage.py @@ -2,32 +2,45 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. -import io import os +from typing import BinaryIO, Generator import aiofiles import aiofiles.os +_DEFAULT_CHUNK_SIZE = 32768 # bytes; arbitrary + + +async def aioshutil_copyfileobj(async_fsrc, async_fdst, *, chunksize: int = _DEFAULT_CHUNK_SIZE) -> None: + while (chunk := await async_fsrc.read(chunksize)) != b"": + await async_fdst.write(chunk) + class LocalStorage: def __init__(self, base_path: str): self.base_path = base_path - async def get_file(self, file_name: str) -> io.BytesIO | None: + async def download(self, file_name: str) -> Generator | None: try: async with aiofiles.open(file=os.path.join(self.base_path, file_name), mode="rb") as f: - return io.BytesIO(await f.read()) + yield f.read() except FileNotFoundError: - return None + yield None - async def write_file(self, file_name: str, data: bytes) -> None: + async def upload(self, file_name: str, data: BinaryIO) -> None: async with aiofiles.open(file=os.path.join(self.base_path, file_name), mode="wb") as f: - await f.write(data) + await aioshutil_copyfileobj(data, f) - async def delete_file(self, file_names: [str]) -> None: + async def delete(self, file_names: [str]) -> None: for i in file_names: try: await aiofiles.os.remove(os.path.join(self.base_path, i)) except FileNotFoundError: pass return None + + def size(self, file_name: str) -> int | None: + try: + return os.stat(os.path.join(self.base_path, file_name)) + except FileNotFoundError: + return None diff --git a/classquiz/storage/s3_storage.py b/classquiz/storage/s3_storage.py index 4f499f0..55d22f1 100644 --- a/classquiz/storage/s3_storage.py +++ b/classquiz/storage/s3_storage.py @@ -8,8 +8,7 @@ import hashlib import hmac from datetime import datetime, timedelta -from io import BytesIO -from typing import Tuple +from typing import Tuple, BinaryIO, Generator from aiohttp import ClientSession import minio @@ -113,17 +112,9 @@ class S3Storage: return headers, request_url - async def download(self, file_name: str): - headers, url = self._generate_aws_signature_v4(method="GET", path=f"/{file_name}") - async with ClientSession() as session, session.get(url, headers=headers) as resp: - if resp.status == 200: - return BytesIO(await resp.read()) - elif resp.status == 404: - return None - else: - raise DownloadingFailedError - - async def upload(self, file: bytes, file_name: str, content_type: str | None = "application/octet-stream") -> None: + async def upload( + self, file: BinaryIO, file_name: str, content_type: str | None = "application/octet-stream" + ) -> None: headers, url = self._generate_aws_signature_v4(method="PUT", path=f"/{file_name}") async with ClientSession() as session, session.put(url, headers=headers, data=file) as resp: if resp.status == 200: @@ -145,3 +136,25 @@ class S3Storage: return self.client.presigned_get_object( object_name=file_name, bucket_name=self.bucket_name, expires=timedelta(seconds=expire) ) + + def size(self, file_name: str) -> int | None: + res = self.client.stat_object(bucket_name=self.bucket_name, object_name=file_name) + if res is None: + return None + return res.size + + async def download(self, file_name: str) -> Generator: + headers, url = self._generate_aws_signature_v4(method="GET", path=f"/{file_name}") + + async with ClientSession() as session, session.get(url, headers=headers) as resp: + if resp.status == 200: + async for i in resp.content.iter_chunked(1024): + yield i + elif resp.status == 404: + yield None + else: + raise DownloadingFailedError + # client = httpx.AsyncClient() + # async with client.stream("GET", url, headers=headers) as resp: + # if resp.status == 200: + # yield resp.aiter_bytes() diff --git a/classquiz/worker/storage.py b/classquiz/worker/storage.py index b6b76bb..2c2b003 100644 --- a/classquiz/worker/storage.py +++ b/classquiz/worker/storage.py @@ -13,6 +13,7 @@ from tempfile import SpooledTemporaryFile from classquiz.db.models import StorageItem, Quiz from classquiz.helpers import extract_image_ids_from_quiz from classquiz.storage.errors import DeletionFailedError +from thumbhash import image_to_thumbhash async def clean_editor_images_up(ctx): @@ -32,19 +33,27 @@ async def clean_editor_images_up(ctx): async def calculate_hash(ctx, file_id_as_str: str): - print("Calculating hash") file_id = uuid.UUID(file_id_as_str) - file_data = await StorageItem.objects.get(id=file_id) + file_data: StorageItem = await StorageItem.objects.get(id=file_id) file_path = file_id.hex if file_data.storage_path is not None: file_path = file_data.storage_path file = SpooledTemporaryFile() - file_bytes = await storage.download(file_path) + file_data.size = await storage.get_file_size(file_name=file_path) + if file_data.size is None: + file_data.size = 0 + file_bytes = storage.download(file_path) if file_bytes is None: print("Retry raised!") raise Retry(defer=ctx["job_try"] * 10) - file.write(file_bytes.getbuffer().tobytes()) + async for chunk in file_bytes: + async for c in chunk: + file.write(c) + + if 0 < file_data.size < 20_970_000: # greater than 0 but smaller than 20mbytes + file_data.thumbhash = image_to_thumbhash(file) hash_obj = xxhash.xxh3_128() + # assert hash_obj.block_size == 64 while chunk := file.read(6400): hash_obj.update(chunk) diff --git a/frontend/package.json b/frontend/package.json index cf535b5..6366615 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -42,6 +42,7 @@ "@types/ua-parser-js": "^0.7.36", "@typescript-eslint/eslint-plugin": "^5.59.0", "@typescript-eslint/parser": "^5.59.0", + "@unlazy/svelte": "^0.8.9", "@uppy/compressor": "^1.0.2", "@uppy/core": "^3.2.0", "@uppy/dashboard": "^3.4.0", @@ -89,6 +90,7 @@ "svelte-tippy": "^1.3.2", "swiper": "^8.4.7", "tailwindcss": "^3.3.1", + "thumbhash": "^0.1.1", "tippy.js": "^6.3.7", "tslib": "^2.5.0", "typescript": "~5.0.4", diff --git a/frontend/pnpm-lock.yaml b/frontend/pnpm-lock.yaml index 797eb29..a4c5017 100644 --- a/frontend/pnpm-lock.yaml +++ b/frontend/pnpm-lock.yaml @@ -29,6 +29,7 @@ specifiers: '@types/ua-parser-js': ^0.7.36 '@typescript-eslint/eslint-plugin': ^5.59.0 '@typescript-eslint/parser': ^5.59.0 + '@unlazy/svelte': ^0.8.9 '@uppy/compressor': ^1.0.2 '@uppy/core': ^3.2.0 '@uppy/dashboard': ^3.4.0 @@ -77,6 +78,7 @@ specifiers: svelte-tippy: ^1.3.2 swiper: ^8.4.7 tailwindcss: ^3.3.1 + thumbhash: ^0.1.1 tippy.js: ^6.3.7 tslib: ^2.5.0 typescript: ~5.0.4 @@ -117,6 +119,7 @@ devDependencies: '@types/ua-parser-js': 0.7.36 '@typescript-eslint/eslint-plugin': 5.59.0_wsbkzerfphvgr4lle7vmeh64yi '@typescript-eslint/parser': 5.59.0_voubu7prgxjfsfbgx5d4sqnwiy + '@unlazy/svelte': 0.8.9_2a4gi7ewzykzc7boj45sshrguu '@uppy/compressor': 1.0.2_@uppy+core@3.2.0 '@uppy/core': 3.2.0 '@uppy/dashboard': 3.4.0_@uppy+core@3.2.0 @@ -164,6 +167,7 @@ devDependencies: svelte-tippy: 1.3.2 swiper: 8.4.7 tailwindcss: 3.3.1_postcss@8.4.22 + thumbhash: 0.1.1 tippy.js: 6.3.7 tslib: 2.5.0 typescript: 5.0.4 @@ -1705,6 +1709,31 @@ packages: eslint-visitor-keys: 3.4.0 dev: true + /@unlazy/core/0.8.9: + resolution: + { + integrity: sha512-DQ4WB/cuEWTknU/59uRwpSipvMAJzBDmRyaHDUc1RcXi0Z7/Vcl0EE7BpROxEynqd1EI+2oMWQaDLyXffUdUiA== + } + dependencies: + fast-blurhash: 1.1.2 + thumbhash: 0.1.1 + dev: true + + /@unlazy/svelte/0.8.9_2a4gi7ewzykzc7boj45sshrguu: + resolution: + { + integrity: sha512-vgOYuOlknE6l/QhzZzpncsU8dMaTgLKGd7Xn4qwqTWAa3TF+6NRL7QedPJb2XBY8Th7yRQ79ZqeXaSDxWtL9zA== + } + peerDependencies: + svelte: ^3.54.0 + dependencies: + svelte: 3.58.0 + unlazy: 0.8.9_thumbhash@0.1.1 + transitivePeerDependencies: + - fast-blurhash + - thumbhash + dev: true + /@uppy/companion-client/3.1.3: resolution: { @@ -2982,6 +3011,13 @@ packages: } dev: true + /fast-blurhash/1.1.2: + resolution: + { + integrity: sha512-lJVOgYSlahqkRhrKumNx/SGB2F/qS0D1z7xjGYjb5EZJRtlzySGMniZjkQ9h9Rv8sPmM/V9orEgRiMwazDNH6A== + } + dev: true + /fast-deep-equal/3.1.3: resolution: { @@ -5466,6 +5502,13 @@ packages: any-promise: 1.3.0 dev: true + /thumbhash/0.1.1: + resolution: + { + integrity: sha512-kH5pKeIIBPQXAOni2AiY/Cu/NKdkFREdpH+TLdM0g6WA7RriCv0kPLgP731ady67MhTAqrVG/4mnEeibVuCJcg== + } + dev: true + /tiny-case/1.0.3: resolution: { @@ -5628,6 +5671,19 @@ packages: '@types/unist': 2.0.6 dev: true + /unlazy/0.8.9_thumbhash@0.1.1: + resolution: + { + integrity: sha512-lRCuXN20N1esqSQqtSVBLAw9GJz0lcBuOBs3UGGw7cFWHQlWJVZZ3OviwOl42f1CnVHjAON1rs2hIdJWgMAUyg== + } + peerDependencies: + fast-blurhash: ^1.1.2 + thumbhash: ^0.1.1 + dependencies: + '@unlazy/core': 0.8.9 + thumbhash: 0.1.1 + dev: true + /update-browserslist-db/1.0.11_browserslist@4.21.5: resolution: { diff --git a/frontend/src/lib/files/dashboard.svelte b/frontend/src/lib/files/dashboard.svelte new file mode 100644 index 0000000..b15940c --- /dev/null +++ b/frontend/src/lib/files/dashboard.svelte @@ -0,0 +1,36 @@ + + + + +{#if files}{:else} + +{/if} diff --git a/frontend/src/routes/dashboard/files/+page.svelte b/frontend/src/routes/dashboard/files/+page.svelte new file mode 100644 index 0000000..e449716 --- /dev/null +++ b/frontend/src/routes/dashboard/files/+page.svelte @@ -0,0 +1,34 @@ + + + + +
+
+ {#each files as file, i} +
+ {/each} +
+
diff --git a/frontend/src/routes/dashboard/files/+page.ts b/frontend/src/routes/dashboard/files/+page.ts new file mode 100644 index 0000000..92f4a01 --- /dev/null +++ b/frontend/src/routes/dashboard/files/+page.ts @@ -0,0 +1,13 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ +import type { PageLoad } from './$types'; + +export const load = (async ({ fetch }) => { + const res = await fetch('/api/v1/storage/list'); + if (res.ok) { + return { files: await res.json() }; + } +}) satisfies PageLoad; diff --git a/migrations/versions/1310317108bd_added_thumbhash_and_server_storageitem.py b/migrations/versions/1310317108bd_added_thumbhash_and_server_storageitem.py new file mode 100644 index 0000000..afe0eed --- /dev/null +++ b/migrations/versions/1310317108bd_added_thumbhash_and_server_storageitem.py @@ -0,0 +1,33 @@ +"""Added thumbhash and server StorageItem + +Revision ID: 1310317108bd +Revises: b74382b2c6ad +Create Date: 2023-05-29 15:53:00.388788 + +""" +from alembic import op +import sqlalchemy as sa +import ormar + + +# revision identifiers, used by Alembic. +revision = "1310317108bd" +down_revision = "b74382b2c6ad" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column("storage_items", sa.Column("thumbhash", sa.Text(), nullable=True)) + op.add_column("storage_items", sa.Column("server", sa.Text(), nullable=True)) + op.add_column("storage_items", sa.Column("imported", sa.Boolean(), nullable=True)) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("storage_items", "imported") + op.drop_column("storage_items", "server") + op.drop_column("storage_items", "thumbhash") + # ### end Alembic commands ###