Skip to content
This repository has been archived by the owner on Aug 27, 2024. It is now read-only.

Commit

Permalink
Refactored exception handling, and added top level integration tests (#…
Browse files Browse the repository at this point in the history
  • Loading branch information
maxyu1115 authored Sep 13, 2023
1 parent 8ccf77a commit 41d7d56
Show file tree
Hide file tree
Showing 9 changed files with 162 additions and 25 deletions.
5 changes: 5 additions & 0 deletions integration-tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,11 @@ def create_test_app():
app: Flask = create_test_app()


@pytest.fixture
def test_client():
yield app.test_client()


@pytest.fixture
def es_client():
with app.app_context():
Expand Down
24 changes: 24 additions & 0 deletions integration-tests/test_controlplane.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@


def test_create_user(test_client):
resp = test_client.post("/cp/create_user", json={"namespace_pathname": "create_user_1"})
assert resp.status_code == 200


def test_create_user_existing(test_client):
test_client.post("/cp/create_user", json={"namespace_pathname": "create_user_2"})
resp = test_client.post("/cp/create_user", json={"namespace_pathname": "create_user_2"})
assert resp.status_code == 400
assert resp.json["error_code"] == "namespace_exists"


def test_create_corpus(test_client):
resp = test_client.post("/cp/create_corpus", json={"corpus_pathname": "create_user_1:create_corpus_1"})
assert resp.status_code == 200


def test_create_corpus_existing(test_client):
test_client.post("/cp/create_corpus", json={"corpus_pathname": "create_user_2:create_corpus_2"})
resp = test_client.post("/cp/create_corpus", json={"corpus_pathname": "create_user_2:create_corpus_2"})
assert resp.status_code == 400
assert resp.json["error_code"] == "namespace_exists"
22 changes: 22 additions & 0 deletions integration-tests/test_dataplane.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
import time


def test_memorize_then_recall(test_client):
namespace_pathname = "memorize"
corpus_pathname = namespace_pathname + ":memorize_1"
resp1 = test_client.post("/cp/create_user", json={"namespace_pathname": namespace_pathname})
print(resp1)
resp2 = test_client.post("/cp/create_corpus", json={"corpus_pathname": corpus_pathname})
print(resp2)

resp3 = test_client.post(
"/dp/memorize", json={"corpus_pathname": corpus_pathname, "document": "What's MeMaS", "citation": {}})
assert resp3.status_code == 200
assert resp3.json["success"]

time.sleep(1)

resp4 = test_client.post("/dp/recall", json={"namespace_pathname": namespace_pathname, "clue": "What's MeMaS"})
assert resp4.status_code == 200
assert len(resp4.json) == 1
assert resp4.json[0]["document"] == "What's MeMaS"
8 changes: 8 additions & 0 deletions memas/app.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
import traceback
import yaml
from flask import Flask
from memas.context_manager import ContextManager
from memas.interface.exceptions import MemasException


def create_app(config_filename, *, first_init=False):
Expand All @@ -15,6 +17,12 @@ def create_app(config_filename, *, first_init=False):

app.ctx.init()

@app.errorhandler(MemasException)
def handle_memas_exception(e: MemasException):
app.logger.info(f"{e.__class__.__name__}: {e.return_obj()}")
# app.logger.info(traceback.format_exc())
return e.return_obj(), e.status_code.value

from memas.dataplane import dataplane
from memas.controlplane import controlplane
app.register_blueprint(dataplane)
Expand Down
8 changes: 4 additions & 4 deletions memas/context_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
from cassandra.cqlengine import connection as c_connection
from elasticsearch import Elasticsearch
from pymilvus import connections as milvus_connection
from memas.interface.exceptions import NotProperlyInitializedException
from memas.interface.exceptions import IllegalStateException
from memas.interface.storage_driver import CorpusDocumentMetadataStore, CorpusDocumentStore, CorpusVectorStore, MemasMetadataStore
from memas.storage_driver import corpus_doc_metadata, corpus_doc_store, corpus_vector_store, memas_metadata
from memas.corpus.corpus_provider import CorpusProvider
Expand Down Expand Up @@ -85,7 +85,7 @@ def setup_cassandra_keyspace(self):
'replication_factor': self.consts.cassandra_replication_factor
}
# NOTE: management.create_keyspace_simple doesn't work when we don't have a single keyspace...
create_keyspace_query = f"CREATE KEYSPACE IF NOT EXISTS {self.consts.cassandra_keyspace} WITH replication = {replication_options};"
create_keyspace_query = f"CREATE KEYSPACE {self.consts.cassandra_keyspace} WITH replication = {replication_options};"
session.execute(create_keyspace_query)
session.shutdown()

Expand All @@ -103,7 +103,7 @@ def init_clients(self) -> None:

def first_init_datastores(self) -> None:
if self.es is None:
raise NotProperlyInitializedException("Attempted to initialize data stores before connectors/clients")
raise IllegalStateException("Attempted to initialize data stores before connectors/clients")
self.corpus_doc = corpus_doc_store.ESDocumentStore(self.es)

self.memas_metadata.first_init()
Expand All @@ -113,7 +113,7 @@ def first_init_datastores(self) -> None:

def init_datastores(self) -> None:
if self.es is None:
raise NotProperlyInitializedException("Attempted to initialize data stores before connectors/clients")
raise IllegalStateException("Attempted to initialize data stores before connectors/clients")
self.corpus_doc = corpus_doc_store.ESDocumentStore(self.es)

self.memas_metadata.init()
Expand Down
61 changes: 52 additions & 9 deletions memas/interface/exceptions.py
Original file line number Diff line number Diff line change
@@ -1,25 +1,68 @@
from enum import Enum


class BadArgumentException(Exception):
class MemasInternalException(Exception):
def __init__(self, *args: object) -> None:
super().__init__(*args)


class IllegalStateException(MemasInternalException):
def __init__(self, msg: str) -> None:
super().__init__(msg)


class IllegalNameException(BadArgumentException):
def __init__(self, pathname: str) -> None:
super().__init__(f"\"{pathname}\" is not a valid pathname")
class StatusCode(Enum):
# The request could not be understood by the server due to incorrect syntax. The client SHOULD NOT repeat the request without modifications.
BAD_REQUEST = 400
# Indicates that the request requires user authentication information. The client MAY repeat the request with a suitable Authorization header field
UNAUTHORIZED = 401
# Unauthorized request. The client does not have access rights to the content. Unlike 401, the client’s identity is known to the server.
FORBIDDEN = 403
# The server can not find the requested resource.
NOT_FOUND = 404
# The request HTTP method is known by the server but has been disabled and cannot be used for that resource.
METHOD_NOT_ALLOWED = 405


class NamespaceExistsException(Exception):
def __init__(self, pathname: str) -> None:
super().__init__(f"\"{pathname}\" already exists")
# Each memas error needs to have an unique error code
class ErrorCode(Enum):
NamespaceExists = "namespace_exists"
NamespaceDoesNotExist = "namespace_does_not_exist"
NamespaceIllegalName = "namespace_illegal_name"


class NotProperlyInitializedException(Exception):
def __init__(self, msg: str) -> None:
class MemasException(Exception):
def __init__(self, error_code: ErrorCode, msg: str, additional_details: str = None) -> None:
super().__init__(msg)
self.status_code: StatusCode = StatusCode.BAD_REQUEST
self.error_code: ErrorCode = error_code
self.msg: str = msg
self.additional_details: str = additional_details

def return_obj(self):
resp = {"error_code": self.error_code.value, "msg": self.msg}
if self.additional_details:
resp["details"] = self.additional_details
return resp


class IllegalNameException(MemasException):
def __init__(self, pathname: str) -> None:
super().__init__(ErrorCode.NamespaceIllegalName, f"\"{pathname}\" is not a valid pathname")


class NamespaceExistsException(MemasException):
def __init__(self, pathname: str, additional_details: str = None) -> None:
super().__init__(ErrorCode.NamespaceExists, f"\"{pathname}\" already exists", additional_details)


class NamespaceDoesNotExistException(MemasException):
def __init__(self, pathname: str, additional_details: str = None) -> None:
super().__init__(ErrorCode.NamespaceDoesNotExist,
f"\"{pathname}\" does not exists, you need to create the resource first", additional_details)


# TODO: properly specify this exception type
class SentenceLengthOverflowException(Exception):
def __init__(self, sentence_len: int) -> None:
super().__init__("Sentence length is {len} which exceededs limit".format(len=sentence_len))
24 changes: 12 additions & 12 deletions memas/storage_driver/memas_metadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,10 @@
import uuid
from cassandra.cqlengine import columns, management
from cassandra.cqlengine.models import Model
from cassandra.cqlengine.query import BatchQuery, LWTException
from cassandra.cqlengine.query import BatchQuery, DoesNotExist, LWTException
from memas.interface import corpus
from memas.interface.corpus import CorpusType
from memas.interface.exceptions import BadArgumentException, IllegalNameException, NamespaceExistsException
from memas.interface.exceptions import IllegalNameException, NamespaceDoesNotExistException, NamespaceExistsException
from memas.interface.namespace import ROOT_ID, ROOT_NAME, NAMESPACE_SEPARATOR, CORPUS_SEPARATOR, is_pathname_format_valid, is_name_format_valid
from memas.interface.storage_driver import MemasMetadataStore

Expand Down Expand Up @@ -114,9 +114,14 @@ def first_init(self):
self.init()

def _get_id_by_name(self, fullname: str) -> uuid.UUID:
# the root user currently only exists logically
if fullname == ROOT_NAME:
return ROOT_ID
result = NamespaceNameToId.get(fullname=fullname)

try:
result = NamespaceNameToId.get(fullname=fullname)
except DoesNotExist as e:
raise NamespaceDoesNotExistException(fullname) from e
return result.id

def _get_ids_by_name(self, fullname: str) -> tuple[uuid.UUID, uuid.UUID]:
Expand All @@ -125,20 +130,15 @@ def _get_ids_by_name(self, fullname: str) -> tuple[uuid.UUID, uuid.UUID]:
else:
parent_pathname, child_name = split_namespace_pathname(fullname)

# if parent is root
if parent_pathname == ROOT_NAME:
child_id = NamespaceNameToId.get(fullname=fullname).id
return (ROOT_ID, child_id)

child_result = NamespaceNameToId.get(fullname=fullname)
parent_result = NamespaceNameToId.get(fullname=parent_pathname)
return (parent_result.id, child_result.id)
child_id = self._get_id_by_name(fullname=fullname)
parent_id = self._get_id_by_name(fullname=parent_pathname)
return (parent_id, child_id)

def create_namespace(self, namespace_pathname: str, *, parent_id: uuid.UUID = None) -> uuid.UUID:
_log.debug(f"Creating namespace for [namespace_pathname=\"{namespace_pathname}\"]")

if namespace_pathname == ROOT_NAME:
raise BadArgumentException("\"\" is reserved for the root namespace!")
raise NamespaceExistsException(namespace_pathname, "\"\" is reserved for the root namespace!")
if not is_pathname_format_valid(namespace_pathname):
raise IllegalNameException(namespace_pathname)

Expand Down
15 changes: 15 additions & 0 deletions tests/interface/test_exceptions.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
import pytest
from memas.interface.exceptions import ErrorCode, MemasException


def test_memas_exception_output_object():
e = MemasException(ErrorCode.NamespaceExists, "test message", additional_details="test details")

assert e.return_obj() == {"error_code": ErrorCode.NamespaceExists.value,
"msg": "test message", "details": "test details"}


def test_memas_exception_details():
e = MemasException(ErrorCode.NamespaceExists, "test message")

assert "details" not in e.return_obj()
20 changes: 20 additions & 0 deletions tests/storage_driver/test_memas_metadata.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,11 @@
import pytest
from unittest import mock
from cassandra.cqlengine.query import DoesNotExist

from memas.interface.exceptions import NamespaceDoesNotExistException
from memas.interface.namespace import ROOT_ID
from memas.storage_driver.memas_metadata import split_corpus_pathname, split_namespace_pathname
from memas.storage_driver.memas_metadata import MemasMetadataStoreImpl


def test_split_corpus_pathname():
Expand All @@ -8,3 +14,17 @@ def test_split_corpus_pathname():

def test_split_namespace_pathname():
assert split_namespace_pathname("namespace.user.bot") == ("namespace.user", "bot")


def test_get_id_by_name_root():
store = MemasMetadataStoreImpl()
assert ROOT_ID == store._get_id_by_name("")


@mock.patch('memas.storage_driver.memas_metadata.NamespaceNameToId')
def test_get_id_by_name_doesnt_exist(name_to_id):
name_to_id.get.side_effect = DoesNotExist()

store = MemasMetadataStoreImpl()
with pytest.raises(NamespaceDoesNotExistException):
store._get_id_by_name("XD")

0 comments on commit 41d7d56

Please sign in to comment.