From d43795b35f6431db528186225562a0f0a61d5674 Mon Sep 17 00:00:00 2001 From: Mawoka Date: Thu, 14 Apr 2022 12:43:28 +0200 Subject: [PATCH] :sparkles: Added search --- Caddyfile | 1 + Pipfile | 1 + Pipfile.lock | 54 ++++++++++++++-------- classquiz/__init__.py | 3 +- classquiz/config.py | 5 ++ classquiz/routers/quiz.py | 16 ++++++- classquiz/routers/search.py | 86 +++++++++++++++++++++++++++++++++++ classquiz/scripts/__init__.py | 0 import_to_meili.py | 26 +++++++++++ 9 files changed, 170 insertions(+), 22 deletions(-) create mode 100644 classquiz/routers/search.py create mode 100644 classquiz/scripts/__init__.py create mode 100644 import_to_meili.py diff --git a/Caddyfile b/Caddyfile index 5e4ff90..2a36f17 100644 --- a/Caddyfile +++ b/Caddyfile @@ -5,6 +5,7 @@ reverse_proxy /openapi.json localhost:8000 reverse_proxy /socket.io* localhost:8000 reverse_proxy /socket.io/* localhost:8000 + reverse_proxy /ms http://localhost:7700 } diff --git a/Pipfile b/Pipfile index dc9527d..7f6d76f 100644 --- a/Pipfile +++ b/Pipfile @@ -22,6 +22,7 @@ jinja2 = "*" argon2-cffi = "*" sentry-sdk = "*" aiofiles = "*" +meilisearch = "*" [dev-packages] coverage = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 1018ce8..cb60c85 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "075e07880b28878645b61e807c6091ae0ecdef50b3486e047aa853023fc2ae33" + "sha256": "7e2c8532f63f702ec73822fc0a4cab8fdf54de46dcd6be01aee4fbd0e5a8937e" }, "pipfile-spec": 6, "requires": { @@ -331,7 +331,7 @@ "sha256:0f7569a4a6ff151958b64304071d370daa3243d15941a7beedf0c9fe5105603e", "sha256:a851e51367fb93e9e1361732c1d60dab63eff98712e503ea7d92e6eccb109b4f" ], - "markers": "python_version >= '3.6' and python_version < '4'", + "markers": "python_version >= '3.6' and python_version < '4.0'", "version": "==2.2.1" }, "ecdsa": { @@ -609,6 +609,14 @@ "markers": "python_version >= '3.7'", "version": "==2.1.1" }, + "meilisearch": { + "hashes": [ + "sha256:437037dec8730d047e790e94d9af43b59b076fa7511593739aae857eafd83a1b", + "sha256:907986a37b8072b3e27b19b4ee2bb7a1a6f4c9fae961f240aaceadec3adbbab2" + ], + "index": "pypi", + "version": "==0.18.2" + }, "multidict": { "hashes": [ "sha256:0327292e745a880459ef71be14e709aaea2f783f3537588fb4ed09b6c01bca60", @@ -831,11 +839,11 @@ }, "pyparsing": { "hashes": [ - "sha256:18ee9022775d270c55187733956460083db60b37d0d0fb357445f3094eed3eea", - "sha256:a6c06a88f252e6c322f65faf8f418b16213b51bdfaece0524c1c1bc30c63c484" + "sha256:7bf433498c016c4314268d95df76c81b842a4cb2b276fa3312cfb1e1d85f6954", + "sha256:ef7b523f6356f763771559412c0d7134753f037822dad1b16945b7b846f7ad06" ], - "markers": "python_version >= '3.6'", - "version": "==3.0.7" + "markers": "python_full_version >= '3.6.8'", + "version": "==3.0.8" }, "python-dotenv": { "hashes": [ @@ -928,29 +936,37 @@ "index": "pypi", "version": "==4.2.2" }, + "requests": { + "hashes": [ + "sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61", + "sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", + "version": "==2.27.1" + }, "rsa": { "hashes": [ "sha256:5c6bd9dc7a543b7fe4304a631f8a8a3b674e2bbfc49c2ae96200cdbe55df6b17", "sha256:95c5d300c4e879ee69708c428ba566c59478fd653cc3a22243eeb8ed846950bb" ], - "markers": "python_version >= '3.6' and python_version < '4'", + "markers": "python_version >= '3.6' and python_version < '4.0'", "version": "==4.8" }, "sentry-sdk": { "hashes": [ - "sha256:32af1a57954576709242beb8c373b3dbde346ac6bd616921def29d68846fb8c3", - "sha256:38fd16a92b5ef94203db3ece10e03bdaa291481dd7e00e77a148aa0302267d47" + "sha256:02c0f79941964297341c14d53e4df09a81c949707bd91e2d5e66b565aad75c8b", + "sha256:afd9f92711556739172f3bc845a02fc921ca1560bee974ae65072ef6a30e3921" ], "index": "pypi", - "version": "==1.5.8" + "version": "==1.5.9" }, "setuptools": { "hashes": [ - "sha256:7999cbd87f1b6e1f33bf47efa368b224bed5e27b5ef2c4d46580186cbcb1a86a", - "sha256:a65e3802053e99fc64c6b3b29c11132943d5b8c8facbcc461157511546510967" + "sha256:26ead7d1f93efc0f8c804d9fafafbe4a44b179580a7105754b245155f9af05a8", + "sha256:47c7b0c0f8fc10eec4cf1e71c6fdadf8decaa74ffa087e68cd1c20db7ad6a592" ], "markers": "python_version >= '3.7'", - "version": "==62.0.0" + "version": "==62.1.0" }, "six": { "hashes": [ @@ -1031,7 +1047,7 @@ "sha256:44ece4d53fb1706f667c9bd1c648f5469a2ec925fcf3a776667042d645472c14", "sha256:aabaf16477806a5e1dd19aa41f8c2b7950dd3c746362d7e3223dbe6de6ac448e" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4.0'", "version": "==1.26.9" }, "uvicorn": { @@ -1454,11 +1470,11 @@ }, "pyparsing": { "hashes": [ - "sha256:18ee9022775d270c55187733956460083db60b37d0d0fb357445f3094eed3eea", - "sha256:a6c06a88f252e6c322f65faf8f418b16213b51bdfaece0524c1c1bc30c63c484" + "sha256:7bf433498c016c4314268d95df76c81b842a4cb2b276fa3312cfb1e1d85f6954", + "sha256:ef7b523f6356f763771559412c0d7134753f037822dad1b16945b7b846f7ad06" ], - "markers": "python_version >= '3.6'", - "version": "==3.0.7" + "markers": "python_full_version >= '3.6.8'", + "version": "==3.0.8" }, "pytest": { "hashes": [ @@ -1482,7 +1498,7 @@ "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc", "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f" ], - "markers": "python_version < '3.11'", + "markers": "python_version >= '3.7'", "version": "==2.0.1" } } diff --git a/classquiz/__init__.py b/classquiz/__init__.py index f91d596..8749237 100644 --- a/classquiz/__init__.py +++ b/classquiz/__init__.py @@ -5,7 +5,7 @@ from socketio import ASGIApp from classquiz.config import settings from classquiz.db import database -from classquiz.routers import users, quiz, utils, stats, storage +from classquiz.routers import users, quiz, utils, stats, storage, search from classquiz.socket_server import sio settings = settings() @@ -46,4 +46,5 @@ app.include_router(quiz.router, tags=["quiz"], prefix="/api/v1/quiz") app.include_router(utils.router, tags=["utils"], prefix="/api/v1/utils") app.include_router(stats.router, tags=["stats"], prefix="/api/v1/stats") app.include_router(storage.router, tags=["storage"], prefix="/api/v1/storage") +app.include_router(search.router, tags=["search"], prefix="/api/v1/search") app.mount("/", ASGIApp(sio)) diff --git a/classquiz/config.py b/classquiz/config.py index 0cde720..e93f769 100644 --- a/classquiz/config.py +++ b/classquiz/config.py @@ -2,6 +2,7 @@ from functools import lru_cache import redis.asyncio as redis_lib from pydantic import BaseSettings, RedisDsn, PostgresDsn +import meilisearch as MeiliSearch from classquiz.storage import Storage @@ -25,6 +26,8 @@ class Settings(BaseSettings): access_token_expire_minutes: int = 30 cache_expiry: int = 86400 sentry_dsn: str | None = "https://4981de1f72f24fd7b5e21b8913b93a02@o661934.ingest.sentry.io/6254641" + meilisearch_url: str + meilisearch_index: str = "classquiz" # storage_backend storage_backend: str | None = "deta" @@ -52,3 +55,5 @@ storage: Storage = Storage( deta_id=settings().deta_project_id, storage_path=settings().storage_path, ) + +meilisearch = MeiliSearch.Client(settings().meilisearch_url) diff --git a/classquiz/routers/quiz.py b/classquiz/routers/quiz.py index 4ba7a51..c4e8ed9 100644 --- a/classquiz/routers/quiz.py +++ b/classquiz/routers/quiz.py @@ -9,13 +9,12 @@ from fastapi.responses import JSONResponse from pydantic import ValidationError from classquiz.auth import get_current_user, get_current_user_optional -from classquiz.config import redis, settings, storage +from classquiz.config import redis, settings, storage, meilisearch from classquiz.db.models import Quiz, QuizInput, User, PlayGame from classquiz.kahoot_importer.import_quiz import import_quiz settings = settings() - router = APIRouter() @@ -31,6 +30,12 @@ async def create_quiz_lol(quiz_input: QuizInput, user: User = Depends(get_curren raise HTTPException(status_code=400, detail="image url is not valid") quiz = Quiz(**quiz_input.dict(), user_id=user.id, id=uuid.uuid4()) await redis.delete("global_quiz_count") + meilisearch.index(settings.meilisearch_index).add_documents([{ + "id": str(quiz.id), + "title": quiz.title, + "description": quiz.description, + "user": (await User.objects.filter(id=quiz.user_id).first()).username, + }]) return await quiz.save() @@ -117,6 +122,12 @@ async def update_quiz(quiz_id: str, quiz_input: QuizInput, user: User = Depends( quiz.description = quiz_input.description quiz.updated_at = datetime.now() quiz.questions = quiz_input.dict()["questions"] + meilisearch.index(settings.meilisearch_index).update_documents([{ + "id": str(quiz.id), + "title": quiz.title, + "description": quiz.description, + "user": (await User.objects.filter(id=quiz.user_id).first()).username, + }]) return await quiz.update() @@ -149,4 +160,5 @@ async def delete_quiz(quiz_id: str, user: User = Depends(get_current_user)): pass if len(pics_to_delete) != 0: await storage.delete(pics_to_delete) + meilisearch.index(settings.meilisearch_index).delete_document(str(quiz.id)) return await quiz.delete() diff --git a/classquiz/routers/search.py b/classquiz/routers/search.py new file mode 100644 index 0000000..0ed932f --- /dev/null +++ b/classquiz/routers/search.py @@ -0,0 +1,86 @@ +import json +import re +import uuid +from datetime import datetime +from random import randint + +import pydantic +from fastapi import APIRouter, Depends, HTTPException +from fastapi.responses import JSONResponse +from pydantic import ValidationError + +from classquiz.auth import get_current_user, get_current_user_optional +from classquiz.config import redis, settings, storage, meilisearch +from uuid import UUID +from typing import Optional, List +from classquiz.db.models import Quiz, QuizInput, User, PlayGame +from classquiz.kahoot_importer.import_quiz import import_quiz + +settings = settings() + +router = APIRouter() + + +class Hit(pydantic.BaseModel): + id: UUID + title: str + description: str + user: str + formatted: Optional['Hit'] = pydantic.Field(None, alias='_formatted') + + +class SearchResponse(pydantic.BaseModel): + hits: List[Hit] | list[None] + nbHits: int + exhaustiveNbHits: bool + query: str + limit: int + offset: int + processingTimeMs: int + + +class SearchData(pydantic.BaseModel): + q: str + offset: Optional[int] = 0 + limit: Optional[int] = 20 + filter: Optional[str] = None + facetsDistribution: Optional[list[str]] = None + attributesToRetrieve: Optional[list[str]] = ["*"] + attributesToCrop: Optional[list[str]]= None + cropLength: Optional[int] = 200 + attributesToHighlight: Optional[list[str]] = None + matches: Optional[bool] = False + sort: Optional[list[str]] = None + + +@router.post("/search", response_model=SearchResponse) +async def search(data: SearchData): + index = meilisearch.get_index(settings.meilisearch_index) + query = index.search(data.q, { + "offset": data.offset, + "limit": data.limit, + "filter": data.filter, + "cropLength": data.cropLength, + "matches": data.matches, + "facetsDistribution": data.facetsDistribution, + "attributesToRetrieve": data.attributesToRetrieve, + "attributesToCrop": data.attributesToCrop, + "sort": data.sort, + "attributesToHighlight": data.attributesToHighlight + }) + return query.serialize() + + +@router.get("/search", response_model=SearchResponse) +async def search_get(q: str, offset: int = 0, limit: int = 20, filter: str | None = None, + cropLength: int = 200, matches: bool = False, attributesToHighlight: Optional[str] = "*"): + index = meilisearch.get_index(settings.meilisearch_index) + query = index.search(q, { + "offset": offset, + "limit": limit, + "filter": filter, + "cropLength": cropLength, + "matches": matches, + "attributesToHighlight": [attributesToHighlight] + }) + return SearchResponse(**query) diff --git a/classquiz/scripts/__init__.py b/classquiz/scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/import_to_meili.py b/import_to_meili.py new file mode 100644 index 0000000..5e38254 --- /dev/null +++ b/import_to_meili.py @@ -0,0 +1,26 @@ +import meilisearch +from classquiz.db.models import Quiz, User +from pprint import pprint +from classquiz.config import settings +from asyncio import run + +settings = settings() + + +async def __main__(): + meili_data = [] + questions = await Quiz.objects.filter(public=True).all() + for question in questions: + meili_data.append({ + "id": str(question.id), + "title": question.title, + "description": question.description, + "user": (await User.objects.filter(id=question.user_id).first()).username, + }) + print(meili_data) + client = meilisearch.Client(settings.meilisearch_url) + client.index(settings.meilisearch_index).add_documents(meili_data) + + +if __name__ == "__main__": + run(__main__())