✨ Added s3 storage

Pipfile

@@ -36,6 +36,7 @@ cryptography = "*"
 scheduler = "*"
 webauthn = "*"
 pyotp = "*"
+minio = "*"
 
 [dev-packages]
 coverage = "*"

classquiz/config.py

@@ -58,6 +58,12 @@ class Settings(BaseSettings):
     # if storage_backend == "local":
     storage_path: str | None
 
+    # if storage_backend == "s3":
+    s3_access_key: str | None
+    s3_secret_key: str | None
+    s3_bucket_name: str = "classquiz"
+    s3_base_url: str | None
+
     class Config:
         env_file = ".env"
         env_file_encoding = "utf-8"
@@ -75,6 +81,10 @@ storage: Storage = Storage(
     deta_key=settings().deta_project_key,
     deta_id=settings().deta_project_id,
     storage_path=settings().storage_path,
+    access_key=settings().s3_access_key,
+    secret_key=settings().s3_secret_key,
+    bucket_name=settings().s3_bucket_name,
+    base_url=settings().s3_base_url,
 )
 
 meilisearch = MeiliSearch.Client(settings().meilisearch_url)

classquiz/routers/search.py

@@ -8,7 +8,7 @@ from fastapi import APIRouter
 from classquiz.config import settings, meilisearch
 from uuid import UUID
 from typing import Optional, List, Any
-from meilisearch.errors import MeiliSearchApiError
+from meilisearch.errors import MeilisearchApiError
 from classquiz.helpers import meilisearch_init
 
 settings = settings()
@@ -53,7 +53,7 @@ async def _perform_search(query: str, params: dict) -> dict[str, Any]:
     try:
         index = meilisearch.get_index(settings.meilisearch_index)
         return index.search(query, params)
-    except MeiliSearchApiError:
+    except MeilisearchApiError:
         await meilisearch_init()
         index = meilisearch.get_index(settings.meilisearch_index)
         return index.search(query, params)

classquiz/routers/storage.py

@@ -5,7 +5,7 @@
 import re
 
 from fastapi import APIRouter, HTTPException
-from fastapi.responses import StreamingResponse
+from fastapi.responses import StreamingResponse, RedirectResponse
 
 from classquiz.config import settings, storage
 from classquiz.storage.errors import DownloadingFailedError
@@ -21,6 +21,9 @@ file_regex = r"^[a-z0-9]{8}-[a-z0-9-]{27}--[a-z0-9-]{36}$"
 async def download_file(file_name: str):
     if not re.match(file_regex, file_name):
         raise HTTPException(status_code=400, detail="Invalid file name")
+    if storage.backend == "s3":
+        print("redir")
+        return RedirectResponse(url=await storage.get_url(file_name, 300))
     try:
         download = await storage.download(file_name)
     except DownloadingFailedError:

classquiz/storage/__init__.py

@@ -7,6 +7,7 @@ from typing import Optional
 
 from .deta_storage import DetaStorage
 from .local_storage import LocalStorage
+from .s3_storage import S3Storage
 
 
 class Storage:
@@ -16,20 +17,30 @@ class Storage:
         deta_key: Optional[str],
         deta_id: Optional[str],
         storage_path: Optional[str],
+        access_key: str | None = None,
+        secret_key: str | None = None,
+        bucket_name: str | None = None,
+        base_url: str | None = None,
     ):
         self.backend = backend
         self.deta_key: str | None = deta_key
         self.deta_id: str | None = deta_id
-        self.deta_base_url = f"https://drive.deta.sh/v1/{deta_id}/classquiz1"
+        # self.deta_base_url = f"https://drive.deta.sh/v1/{deta_id}/classquiz1"
+        self.access_key = access_key
+        self.secret_key = secret_key
+        self.bucket_name = bucket_name
+        self.base_url = base_url
+        self.deta_instance = None
         if backend == "deta":
             if deta_key is None or deta_id is None:
                 raise ValueError("deta_key and deta_id must be provided")
-            else:
-                self.deta_instance = DetaStorage(
-                    deta_base_url=self.deta_base_url,
-                    deta_key=self.deta_key,
-                    deta_id=self.deta_id,
-                )
+            if self.base_url is None:
+                self.base_url = f"https://drive.deta.sh/v1/{deta_id}/classquiz1"
+            self.deta_instance = DetaStorage(
+                deta_base_url=self.base_url,
+                deta_key=self.deta_key,
+                deta_id=self.deta_id,
+            )
 
         elif backend == "local":
             if storage_path is None:
@@ -37,6 +48,13 @@ class Storage:
             else:
                 self.local_instance = LocalStorage(base_path=storage_path)
 
+        elif backend == "s3":
+            if access_key is None or secret_key is None or bucket_name is None or base_url is None:
+                raise ValueError("Not all parameters given")
+            self.s3_instance = S3Storage(
+                base_url=base_url, access_key=access_key, secret_key=secret_key, bucket_name=bucket_name
+            )
+
         else:
             raise NotImplementedError(f"Backend {backend} not implemented")
 
@@ -45,15 +63,25 @@ class Storage:
             return await self.deta_instance.download(file_name)
         elif self.backend == "local":
             return await self.local_instance.get_file(file_name)
+        elif self.backend == "s3":
+            return await self.s3_instance.download(file_name)
 
     async def upload(self, file_name: str, file_data: bytes) -> None:
         if self.backend == "deta":
             return await self.deta_instance.upload(file=file_data, file_name=file_name)
         elif self.backend == "local":
             return await self.local_instance.write_file(file_name=file_name, data=file_data)
+        elif self.backend == "s3":
+            return await self.s3_instance.upload(file=file_data, file_name=file_name)
 
     async def delete(self, file_names: [str]) -> None:
         if self.backend == "deta":
             return await self.deta_instance.delete(file_names=file_names)
         elif self.backend == "local":
             return await self.local_instance.delete_file(file_names=file_names)
+        elif self.backend == "s3":
+            return await self.s3_instance.delete(file_names=file_names)
+
+    async def get_url(self, file_name: str, expiry: int) -> str:
+        if self.backend == "s3":
+            return self.s3_instance.get_url(file_name=file_name, expire=expiry)

classquiz/storage/deta_storage.py

@@ -21,7 +21,7 @@ class DetaStorage:
         """
 
         :param file_name: The name of the file to be downloaded
-        :return: Either bytes f successfull download or None if failed
+        :return: Either bytes if successful download or None if failed
         """
         async with ClientSession(headers=self.headers) as session, session.get(
             f"{self.deta_url}/files/download?name={file_name}"

classquiz/storage/s3_storage.py

@@ -0,0 +1,147 @@
+#  This Source Code Form is subject to the terms of the Mozilla Public
+#  License, v. 2.0. If a copy of the MPL was not distributed with this
+#  file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+#  This Source Code Form is subject to the terms of the Mozilla Public
+#  License, v. 2.0. If a copy of the MPL was not distributed with this
+#  file, You can obtain one at https://mozilla.org/MPL/2.0/.
+import hashlib
+import hmac
+from datetime import datetime, timedelta
+from io import BytesIO
+from typing import Tuple
+
+from aiohttp import ClientSession
+import minio
+from pydantic import BaseModel
+from classquiz.storage.errors import DeletionFailedError, SavingFailedError, DownloadingFailedError
+
+
+class S3Storage:
+    class _HeaderAndParams(BaseModel):
+        params: dict[str, str]
+        headers: dict[str, str]
+
+    def __init__(self, base_url: str, access_key: str, secret_key: str, bucket_name: str, region: str = "us-east-1"):
+        self.base_url = base_url
+        self.access_key = access_key
+        self.secret_key = secret_key
+        self.bucket_name = bucket_name
+        self.region = region
+        self.DATE_FORMAT = "%a, %d %b %Y %H:%M:%S GMT"
+        self.host = base_url.replace("http://", "").replace("https://", "")
+        self.client = minio.Minio(self.host, access_key=access_key, secret_key=secret_key)
+        if not self.client.bucket_exists(self.bucket_name):
+            self.client.make_bucket(self.bucket_name)
+
+    def _generate_aws_signature_v4(self, method: str, path: str, expiry: int = None) -> Tuple[dict, str]:
+        path = f"/{self.bucket_name}{path}"
+        service = "s3"
+
+        # Create a timestamp for the request
+        t = datetime.utcnow()
+        amz_date = t.strftime("%Y%m%dT%H%M%SZ")
+        datestamp = t.strftime("%Y%m%d")
+
+        # Create a canonical request
+        canonical_uri = path
+        canonical_querystring = ""
+        if expiry is not None:
+            canonical_querystring = f"Expires={expiry}"
+        canonical_headers = "host:" + self.host + "\n" + "x-amz-date:" + amz_date + "\n"
+        signed_headers = "host;x-amz-date"
+        payload_hash = hashlib.sha256("".encode("utf-8")).hexdigest()
+        canonical_request = (
+            method
+            + "\n"
+            + canonical_uri
+            + "\n"
+            + canonical_querystring
+            + "\n"
+            + canonical_headers
+            + "\n"
+            + signed_headers
+            + "\n"
+            + payload_hash
+        )
+
+        # Create a string to sign
+        algorithm = "AWS4-HMAC-SHA256"
+        credential_scope = datestamp + "/" + self.region + "/" + service + "/" + "aws4_request"
+        string_to_sign = (
+            algorithm
+            + "\n"
+            + amz_date
+            + "\n"
+            + credential_scope
+            + "\n"
+            + hashlib.sha256(canonical_request.encode("utf-8")).hexdigest()
+        )
+
+        # Create a signing key
+        k_date = hmac.new(
+            ("AWS4" + self.secret_key).encode("utf-8"), datestamp.encode("utf-8"), hashlib.sha256
+        ).digest()
+        k_region = hmac.new(k_date, self.region.encode("utf-8"), hashlib.sha256).digest()
+        k_service = hmac.new(k_region, service.encode("utf-8"), hashlib.sha256).digest()
+        k_signing = hmac.new(k_service, b"aws4_request", hashlib.sha256).digest()
+
+        # Calculate the signature
+        signature = hmac.new(k_signing, string_to_sign.encode("utf-8"), hashlib.sha256).hexdigest()
+
+        # Add the signature to the request as an Authorization header
+        authorization_header = (
+            algorithm
+            + " "
+            + "Credential="
+            + self.access_key
+            + "/"
+            + credential_scope
+            + ", "
+            + "SignedHeaders="
+            + signed_headers
+            + ", "
+            + "Signature="
+            + signature
+        )
+        # if expiry is not None:
+        #     authorization_header += f", Expires={expiry}"
+
+        # Send the request with the authorization header
+        headers = {"x-amz-date": amz_date, "Authorization": authorization_header}
+        request_url = self.base_url + path + "?" + canonical_querystring
+
+        return headers, request_url
+
+    async def download(self, file_name: str):
+        headers, url = self._generate_aws_signature_v4(method="GET", path=f"/{file_name}")
+        async with ClientSession() as session, session.get(url, headers=headers) as resp:
+            if resp.status == 200:
+                return BytesIO(await resp.read())
+            elif resp.status == 404:
+                return None
+            else:
+                raise DownloadingFailedError
+
+    async def upload(self, file: bytes, file_name: str, content_type: str | None = "application/octet-stream") -> None:
+        headers, url = self._generate_aws_signature_v4(method="PUT", path=f"/{file_name}")
+        async with ClientSession() as session, session.put(url, headers=headers, data=file) as resp:
+            if resp.status == 200:
+                return None
+            else:
+                print(await resp.text())
+                raise SavingFailedError
+
+    async def delete(self, file_names: list[str]) -> None:
+        for file in file_names:
+            headers, url = self._generate_aws_signature_v4(method="DELETE", path=f"/{file}")
+            async with ClientSession() as session, session.delete(url, headers=headers) as resp:
+                if resp.status == 204:
+                    return None
+                else:
+                    raise DeletionFailedError
+
+    def get_url(self, expire: int, file_name: str) -> str:
+        return self.client.presigned_get_object(
+            object_name=file_name, bucket_name=self.bucket_name, expires=timedelta(seconds=expire)
+        )

classquiz/tests/test_storage.py

@@ -54,3 +54,21 @@ async def test_deta():
 async def test_local():
     storage: Storage = Storage(backend="local", storage_path=settings.storage_path, deta_key=None, deta_id=None)
     await storage_tester(storage)
+
+
+@pytest.mark.asyncio
+async def test_minio():
+    storage: Storage = Storage(
+        backend="s3",
+        access_key="Q3AM3UQ867SPQQA43P2F",
+        secret_key="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG",
+        bucket_name="classquiz",
+        base_url="https://play.min.io",
+        deta_key=None,
+        deta_id=None,
+        storage_path=None,
+    )
+    await storage_tester(storage)
+    await storage.upload(file_name="test.txt", file_data=file_contents)
+    url = await storage.get_url(file_name="test.txt", expiry=20)
+    assert url is not None