handle comma in description

src/ficamp/classifier/preprocessing.py

@@ -21,11 +21,21 @@ def remove_pipes(s: str) -> str:
     return " ".join(s.split("|"))
 
 
+def remove_colon(s: str) -> str:
+    return " ".join(s.split(":"))
+
+
+def remove_comma(s: str) -> str:
+    return " ".join(s.split(","))
+
+
 def preprocess(s: str) -> str:
     "Clean up transaction description"
     steps = (
         lambda s: s.lower(),
         remove_pipes,
+        remove_colon,
+        remove_comma,
         remove_digits,
     )
     out = s

tests/test_preprocessing.py

@@ -2,6 +2,8 @@
 
 from ficamp.classifier.preprocessing import (
     preprocess,
+    remove_colon,
+    remove_comma,
     remove_digits,
     remove_pipes,
 )
@@ -35,6 +37,25 @@ def test_remove_pipes(inp, exp):
     assert remove_pipes(inp) == exp
 
 
+@pytest.mark.parametrize(
+    ("inp,exp"),
+    (
+        ("CSIDNL0213324324324", "CSIDNL0213324324324"),
+        ("CSID:NL0213324324324", "CSID NL0213324324324"),
+    ),
+)
+def test_remove_colon(inp, exp):
+    assert remove_colon(inp) == exp
+
+
+@pytest.mark.parametrize(
+    ("inp,exp"),
+    (("CSID,NL0213324324324", "CSID NL0213324324324"),),
+)
+def test_remove_comma(inp, exp):
+    assert remove_comma(inp) == exp
+
+
 @pytest.mark.parametrize(
     ("inp,exp"),
     (
@@ -47,6 +68,8 @@ def test_remove_pipes(inp, exp):
         ("SEPA 12312321 bic", "sepa bic"),
         ("SEPA 12312321 123BIC", "sepa"),
         ("SEPA 1231|AMSTERDAM 123BIC", "sepa amsterdam"),
+        ("CSID:NL0213324324324", "csid"),
+        ("CSID:NL0213324324324 HELLO,world1332", "csid hello"),
     ),
 )
 def test_preprocess(inp, exp):