Feature 24/structrue folder changes (#25)

* Aligining the structure of the codebase to better reflect the layers

* Moved oauth to service folder as this module acts like a service for other layers

* Removed unwanted dependent packages and created a query service layer for query service functionalities

* Readding the requirments.txt file as pip-chill does not properly clean the requirements.txt

* Added req and added missing async to function

---------

Co-authored-by: tmeftah <46863341+tmeftah@users.noreply.github.com>
Co-authored-by: raikarn <nikhil.raikar@apl-landau.de>

backend/api/query.py

@@ -1,33 +1,18 @@
 from fastapi import APIRouter
-from fastapi.responses import StreamingResponse
-
-from backend.rag_llms_langchain import chain
-from backend.embeddings.ingest import get_vectorstore
-
-import json
-import uuid
-
+from fastapi import HTTPException
+from backend.service.query_service import query_service
 
 query_router = APIRouter()
 
 
 @query_router.get("/query")
 async def query(query: str):
-    # if current_user.role < 5:
-    #     raise HTTPException(status_code=403,
-    # detail="Only admin users can delete other users")
-    store = get_vectorstore()
-    docs = store.invoke(query)
-
-    print(20 * "*", "docs", 20 * "*", "\n", docs)
-
-    async def stream_generator():
-        # Use the LangChain model to generate text
-        print(20 * "*", "\n", query)
-        async for text in chain.astream({"input": query, "context": docs}):
-            yield json.dumps({"event_id": str(uuid.uuid4()), "data": text})
-
-        # TODO  here we have to add the metadata/source
-
-    return StreamingResponse(stream_generator(), 
-                            media_type="application/x-ndjson")
+    """
+    Handle query requests from user and
+    return appropriate response
+    """
+    try:
+        response = await query_service(query)
+        return response
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))

backend/api/token.py

@@ -1,6 +1,6 @@
 from fastapi import APIRouter, Depends, HTTPException
-from backend.pydantic_models import Token
-from backend.oauth import authenticate_user, create_access_token
+from backend.models.pydantic_models import Token
+from backend.service.oauth import authenticate_user, create_access_token
 from backend.config import ACCESS_TOKEN_EXPIRE_MINUTES
 from fastapi.security import OAuth2PasswordRequestForm
 from datetime import timedelta

backend/api/user.py

@@ -1,7 +1,7 @@
 from fastapi import APIRouter, Depends, HTTPException
-from backend.sqlalchemy_models import User
-from backend.sessions import session
-from backend.oauth import encrypt_password, get_current_user
+from backend.models.sqlalchemy_models import User
+from backend.db.sessions import session
+from backend.service.oauth import encrypt_password, get_current_user
 
 
 user_router = APIRouter()

backend/sessions.py → backend/db/sessions.py

@@ -2,7 +2,7 @@
 
 from sqlalchemy.orm import sessionmaker
 
-from backend.sqlalchemy_models import Base
+from backend.models.sqlalchemy_models import Base
 
 
 engine = create_engine("sqlite:///users.db")

backend/utils.py → backend/db/utils.py

@@ -1,6 +1,6 @@
-from backend.sqlalchemy_models import User
-from backend.sessions import session
-from backend.oauth import encrypt_password, get_user
+from backend.models.sqlalchemy_models import User
+from backend.db.sessions import session
+from backend.service.oauth import encrypt_password, get_user
 
 
 def populate_admin_user():

backend/main.py

@@ -5,7 +5,7 @@
 from backend.api.user import user_router
 from backend.api.token import token_router
 from backend.api.query import query_router
-from backend.utils import populate_admin_user
+from backend.db.utils import populate_admin_user
 
 
 app = FastAPI()

backend/requirements.txt

@@ -1,132 +1,10 @@
-aiohappyeyeballs==2.3.4
-aiohttp==3.10.0
-aiosignal==1.3.1
-annotated-types==0.7.0
-anyio==4.4.0
-asgiref==3.8.1
-async-timeout==4.0.3
-attrs==24.1.0
-backoff==2.2.1
-bcrypt==4.2.0
-build==1.2.1
-cachetools==5.4.0
-certifi==2024.7.4
-cffi==1.16.0
-cfgv==3.4.0
-charset-normalizer==3.3.2
-chroma-hnswlib==0.7.6
-chromadb==0.5.5
-click==8.1.7
-colorama==0.4.6
-coloredlogs==15.0.1
-cryptography==43.0.0
-dataclasses-json==0.6.7
-Deprecated==1.2.14
-distlib==0.3.8
-dnspython==2.6.1
-email_validator==2.2.0
-exceptiongroup==1.2.2
-fastapi==0.112.0
-fastapi-cli==0.0.5
-filelock==3.15.4
-flatbuffers==24.3.25
-frozenlist==1.4.1
-fsspec==2024.6.1
-google-auth==2.32.0
-googleapis-common-protos==1.63.2
-greenlet==3.0.3
-grpcio==1.65.4
-h11==0.14.0
-httpcore==1.0.5
-httptools==0.6.1
-httpx==0.27.0
-huggingface-hub==0.24.5
-humanfriendly==10.0
-identify==2.6.0
-idna==3.7
-importlib_metadata==8.0.0
-importlib_resources==6.4.0
-Jinja2==3.1.4
-jsonpatch==1.33
-jsonpointer==3.0.0
-jwt==1.3.1
-kubernetes==30.1.0
-langchain==0.2.12
-langchain-chroma==0.1.2
-langchain-community==0.2.11
-langchain-core==0.2.28
-langchain-ollama==0.1.1
-langchain-text-splitters==0.2.2
-langfuse==2.40.0
-langsmith==0.1.96
-markdown-it-py==3.0.0
-MarkupSafe==2.1.5
-marshmallow==3.21.3
-mdurl==0.1.2
-mmh3==4.1.0
-monotonic==1.6
-mpmath==1.3.0
-multidict==6.0.5
-mypy-extensions==1.0.0
-nodeenv==1.9.1
-numpy==1.26.4
-oauthlib==3.2.2
-ollama==0.3.1
-onnxruntime==1.18.1
-opentelemetry-api==1.26.0
-opentelemetry-exporter-otlp-proto-common==1.26.0
-opentelemetry-exporter-otlp-proto-grpc==1.26.0
-opentelemetry-instrumentation==0.47b0
-opentelemetry-instrumentation-asgi==0.47b0
-opentelemetry-instrumentation-fastapi==0.47b0
-opentelemetry-proto==1.26.0
-opentelemetry-sdk==1.26.0
-opentelemetry-semantic-conventions==0.47b0
-opentelemetry-util-http==0.47b0
-orjson==3.10.6
-overrides==7.7.0
-packaging==23.2
-platformdirs==4.2.2
-posthog==3.5.0
-protobuf==4.25.4
-pyasn1==0.6.0
-pyasn1_modules==0.4.0
-pycparser==2.22
-pydantic==2.8.2
-pydantic_core==2.20.1
-Pygments==2.18.0
-PyJWT==2.9.0
-pypdf==4.3.1
-PyPika==0.48.9
-pyproject_hooks==1.1.0
-pyreadline3==3.4.1
-python-dateutil==2.9.0.post0
-python-dotenv==1.0.1
-python-multipart==0.0.9
-PyYAML==6.0.1
-requests==2.32.3
-requests-oauthlib==2.0.0
-rich==13.7.1
-rsa==4.9
-shellingham==1.5.4
-six==1.16.0
-sniffio==1.3.1
-SQLAlchemy==2.0.31
-starlette==0.37.2
-sympy==1.13.1
-tenacity==8.5.0
-tokenizers==0.19.1
-tomli==2.0.1
-tqdm==4.66.5
-typer==0.12.3
-typing-inspect==0.9.0
-typing_extensions==4.12.2
-urllib3==2.2.2
-uvicorn==0.30.5
-virtualenv==20.26.3
-watchfiles==0.22.0
-websocket-client==1.8.0
-websockets==12.0
-wrapt==1.16.0
-yarl==1.9.4
-zipp==3.19.2
+langchain
+langchain_community
+langchain_core
+langchain_chroma
+langchain_ollama
+langfuse
+sqlalchemy
+fastapi[standard]
+pyjwt
+python-dotenv

backend/oauth.py → backend/service/oauth.py

@@ -4,9 +4,9 @@
 from fastapi import Depends, HTTPException
 
 from backend.config import SECRET_KEY, ALGORITHM
-from backend.sqlalchemy_models import User
-from backend.sessions import session
-from backend.pydantic_models import TokenData
+from backend.models.sqlalchemy_models import User
+from backend.db.sessions import session
+from backend.models.pydantic_models import TokenData
 
 from bcrypt import hashpw, gensalt, checkpw
 from datetime import datetime, timedelta

backend/service/query_service.py

@@ -0,0 +1,27 @@
+import json
+import uuid
+from fastapi.responses import StreamingResponse
+
+from backend.rag_llms_langchain import chain
+from backend.embeddings.ingest import get_vectorstore
+
+
+async def query_service(query: str):
+    """
+    Process a query and return a streaming response.
+    """
+
+    store = get_vectorstore()
+    docs = store.invoke(query)
+
+    print(20 * "*", "docs", 20 * "*", "\n", docs)
+
+    async def stream_generator():
+        print(20 * "*", "\n", query)
+        async for text in chain.astream({"input": query, "context": docs}):
+            yield json.dumps({"event_id": str(uuid.uuid4()), "data": text})
+
+        # TODO  here we have to add the metadata/source
+
+    return StreamingResponse(stream_generator(),
+                             media_type="application/x-ndjson")

backend/service/token_service.py

@@ -0,0 +1 @@
+pass

backend/service/user_service.py

@@ -0,0 +1 @@
+pass