From 0ec7c2b5197868276e5914770f9af7dc90eb4acd Mon Sep 17 00:00:00 2001 From: Mawoka Date: Fri, 1 Apr 2022 22:25:09 +0200 Subject: [PATCH] :sparkles: Added local-file-system storage --- Pipfile | 1 + Pipfile.lock | 16 ++++++++++---- classquiz/config.py | 7 ++++-- classquiz/kahoot_importer/import_quiz.py | 2 +- classquiz/routers/storage.py | 10 ++++++--- classquiz/storage/__init__.py | 11 ++++++++- classquiz/storage/local_storage.py | 17 ++++++++++++++ frontend/src/routes/docs/self-host.svelte | 27 +++++++++++++++++++---- 8 files changed, 76 insertions(+), 15 deletions(-) create mode 100644 classquiz/storage/local_storage.py diff --git a/Pipfile b/Pipfile index 62d10f2..a4ed59c 100644 --- a/Pipfile +++ b/Pipfile @@ -21,6 +21,7 @@ qrcode = "*" jinja2 = "*" argon2-cffi = "*" sentry-sdk = "*" +aiofiles = "*" [dev-packages] diff --git a/Pipfile.lock b/Pipfile.lock index 27dae22..d935524 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "0411374cb38f6ed3199ec87e5d1717b598f8af20c46cef66377af0c719c4b988" + "sha256": "99a926c60fbb456ac477587d91947b39dfce66d39d4a2b6a35d1c41c04b4e32f" }, "pipfile-spec": 6, "requires": { @@ -16,6 +16,14 @@ ] }, "default": { + "aiofiles": { + "hashes": [ + "sha256:7a973fc22b29e9962d0897805ace5856e6a566ab1f0c8e5c91ff6c866519c937", + "sha256:8334f23235248a3b2e83b2c3a78a22674f39969b96397126cc93664d9a901e59" + ], + "index": "pypi", + "version": "==0.8.0" + }, "aiohttp": { "hashes": [ "sha256:01d7bdb774a9acc838e6b8f1d114f45303841b89b95984cbb7d80ea41172a9e3", @@ -291,7 +299,7 @@ "sha256:2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597", "sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df" ], - "markers": "python_full_version >= '3.5.0'", + "markers": "python_version >= '3.5'", "version": "==2.0.12" }, "click": { @@ -536,7 +544,7 @@ "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff", "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d" ], - "markers": "python_full_version >= '3.5.0'", + "markers": "python_version >= '3.5'", "version": "==3.3" }, "jinja2": { @@ -957,7 +965,7 @@ "sha256:471b71698eac1c2112a40ce2752bb2f4a4814c22a54a3eed3676bc0f5ca9f663", "sha256:c4666eecec1d3f50960c6bdf61ab7bc350648da6c126e3cf6898d8cd4ddcd3de" ], - "markers": "python_full_version >= '3.5.0'", + "markers": "python_version >= '3.5'", "version": "==1.2.0" }, "sqlalchemy": { diff --git a/classquiz/config.py b/classquiz/config.py index 45aaae6..e025fdf 100644 --- a/classquiz/config.py +++ b/classquiz/config.py @@ -26,8 +26,11 @@ class Settings(BaseSettings): # storage_backend storage_backend: str | None = "deta" # if storage_backend == "deta": - deta_project_key: str - deta_project_id: str + deta_project_key: str | None + deta_project_id: str | None + + # if storage_backend == "local": + storage_path: str | None class Config: env_file = ".env" diff --git a/classquiz/kahoot_importer/import_quiz.py b/classquiz/kahoot_importer/import_quiz.py index be0c5fc..301b856 100644 --- a/classquiz/kahoot_importer/import_quiz.py +++ b/classquiz/kahoot_importer/import_quiz.py @@ -66,7 +66,7 @@ async def import_quiz(quiz_id: str, user: User) -> Quiz | str: quiz_questions: list[dict] = [] quiz_id = uuid.uuid4() storage = Storage(backend=settings.storage_backend, deta_key=settings.deta_project_key, - deta_id=settings.deta_project_id) + deta_id=settings.deta_project_id, storage_path=settings.storage_path) for q in quiz.kahoot.questions: answers: list[QuizAnswer] = [] diff --git a/classquiz/routers/storage.py b/classquiz/routers/storage.py index bab7939..a995d15 100644 --- a/classquiz/routers/storage.py +++ b/classquiz/routers/storage.py @@ -1,19 +1,23 @@ -from fastapi import APIRouter +from fastapi import APIRouter, HTTPException from fastapi.responses import StreamingResponse from classquiz.config import settings -import io +import re from classquiz.storage import Storage settings = settings() router = APIRouter() +file_regex = r"^[a-z0-9]{8}-[a-z0-9-]{27}--[a-z0-9-]{36}$" + @router.get('/download/{file_name}') async def download_file(file_name: str): storage = Storage(backend=settings.storage_backend, deta_key=settings.deta_project_key, - deta_id=settings.deta_project_id) + deta_id=settings.deta_project_id, storage_path=settings.storage_path) download = await storage.download(file_name) + if not re.match(file_regex, file_name): + raise HTTPException(status_code=400, detail="Invalid file name") def iter_file(): yield from download diff --git a/classquiz/storage/__init__.py b/classquiz/storage/__init__.py index 5e11cbd..11f54fd 100644 --- a/classquiz/storage/__init__.py +++ b/classquiz/storage/__init__.py @@ -1,17 +1,22 @@ from .deta_storage import DetaStorage +from .local_storage import LocalStorage from io import BytesIO class Storage: - def __init__(self, backend: str, deta_key: str | None, deta_id: str | None): + def __init__(self, backend: str, deta_key: str | None, deta_id: str | None, storage_path: str | None): self.backend = backend self.deta_key = deta_key self.deta_id = deta_id self.deta_base_url = f"https://drive.deta.sh/v1/{deta_id}/classquiz" self.deta_instance = DetaStorage(deta_base_url=self.deta_base_url, deta_key=self.deta_key, deta_id=self.deta_id) + self.local_instance = LocalStorage(base_path=storage_path) if backend == "deta": if deta_key is None or deta_id is None: raise ValueError("deta_key and deta_id must be provided") + if backend == "local": + if storage_path is None: + raise ValueError("storage_path must be provided") else: raise NotImplementedError(f"Backend {backend} not implemented") @@ -19,7 +24,11 @@ class Storage: if self.backend == "deta": return await self.deta_instance.download( file_name) + elif self.backend == "local": + return await self.local_instance.get_file(file_name) async def upload(self, file_name: str, file_data: bytes) -> None: if self.backend == "deta": return await self.deta_instance.upload(file=file_data, file_name=file_name) + elif self.backend == "local": + return await self.local_instance.write_file(file_name=file_name, data=file_data) diff --git a/classquiz/storage/local_storage.py b/classquiz/storage/local_storage.py new file mode 100644 index 0000000..f07598a --- /dev/null +++ b/classquiz/storage/local_storage.py @@ -0,0 +1,17 @@ +import io +import os +import aiofiles + + +class LocalStorage: + def __init__(self, base_path: str): + self.base_path = base_path + + async def get_file(self, file_name: str) -> io.BytesIO: + async with aiofiles.open(file=os.path.join(self.base_path, file_name), mode='rb') as f: + return io.BytesIO(await f.read()) + + async def write_file(self, file_name: str, data: bytes) -> None: + async with aiofiles.open(file=os.path.join(self.base_path, file_name), mode='wb') as f: + await f.write(data) + diff --git a/frontend/src/routes/docs/self-host.svelte b/frontend/src/routes/docs/self-host.svelte index 37a503a..243b29c 100644 --- a/frontend/src/routes/docs/self-host.svelte +++ b/frontend/src/routes/docs/self-host.svelte @@ -22,10 +22,6 @@

Since ClassQuiz is open-source, it can also be self-hosted.

-

Warning

-

This "warning" is just temporary, because you need a Deta account (which is free) - to store and serve the images getting imported with the kahoot-import function. I am planning to add more backends like s3 or the local file system. Untill then, Deta is needed.

-

Requirements