feat: add list_latest_collections api

yiwen-ai · Dec 1, 2023 · 00ebe7f · 00ebe7f
1 parent a6b29b3
commit 00ebe7f
Show file tree

Hide file tree

Showing 6 changed files with 87 additions and 45 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "writing"
-version = "1.3.18"
+version = "1.4.0"
 edition = "2021"
 rust-version = "1.64"
 description = ""
@@ -29,7 +29,7 @@ bytes = "1"
 base64 = "0.21"
 ciborium = "0.2"
 ciborium-io = "0.2"
-isolang = { git = "https://github.com/yiwen-ai/isolang-rs.git", branch = "master", features = [
+isolang = { version = "2.4", features = [
   "english_names",
   "lowercase_names",
   "local_names",

diff --git a/src/api/collection.rs b/src/api/collection.rs
@@ -17,7 +17,7 @@ use scylla_orm::ColumnsMap;
 
 use super::{
     get_fields, message, token_from_xid, token_to_xid, AppState, GIDPagination, IDGIDPagination,
-    QueryGidCid, QueryGidId, QueryGidIdCid, QueryId, RFPInfo, SubscriptionInput,
+    Pagination, QueryGidCid, QueryGidId, QueryGidIdCid, QueryId, RFPInfo, SubscriptionInput,
     SubscriptionOutput, UpdateStatusInput, RFP,
 };
 
@@ -374,6 +374,37 @@ pub async fn list(
     }))
 }
 
+pub async fn list_latest(
+    State(app): State<Arc<AppState>>,
+    Extension(ctx): Extension<Arc<ReqContext>>,
+    to: PackObject<Pagination>,
+) -> Result<PackObject<SuccessResponse<Vec<CollectionOutput>>>, HTTPError> {
+    let (to, input) = to.unpack();
+    input.validate()?;
+    valid_user(ctx.user)?;
+
+    ctx.set_kvs(vec![("action", "list_latest_collections".into())])
+        .await;
+
+    let fields = input.fields.unwrap_or_default();
+    let (res, next_page_token) = db::Collection::list_latest(
+        &app.scylla,
+        fields,
+        token_to_xid(&input.page_token),
+        ctx.language,
+    )
+    .await?;
+
+    Ok(to.with(SuccessResponse {
+        total_size: None,
+        next_page_token: to.with_option(token_from_xid(next_page_token)),
+        result: res
+            .iter()
+            .map(|r| CollectionOutput::from(r.to_owned(), &to))
+            .collect(),
+    }))
+}
+
 #[derive(Debug, Deserialize, Validate)]
 pub struct UpdateCollectionInput {
     pub id: PackObject<xid::Id>,

diff --git a/src/db/model_collection.rs b/src/db/model_collection.rs
@@ -799,7 +799,7 @@ impl Collection {
 
         let mut res: Vec<Self> = Vec::new();
         let query = format!(
-                "SELECT {} FROM collection WHERE day=? AND status=2 LIMIT 1000 ALLOW FILTERING USING TIMEOUT 3s",
+                "SELECT {} FROM collection WHERE day=? AND status=2 LIMIT 100 ALLOW FILTERING USING TIMEOUT 3s",
                 fields.clone().join(",")
             );
 
@@ -809,7 +809,7 @@ impl Collection {
             (unix_ms() / (1000 * 3600 * 24)) as i32
         };
 
-        let min = (unix_ms() / (1000 * 3600 * 24)) as i32 - 30;
+        let min = (unix_ms() / (1000 * 3600 * 24)) as i32 - 90;
         while day > min {
             let params = (day,);
             let rows = db.execute_iter(query.as_str(), params).await?;
@@ -822,7 +822,7 @@ impl Collection {
                 res.push(doc);
             }
 
-            if (page_token.is_none() && res.len() >= 6) || (page_token.is_some() && res.len() >= 3)
+            if (page_token.is_none() && res.len() >= 2) || (page_token.is_some() && res.len() >= 1)
             {
                 break;
             }

diff --git a/src/db/model_publication.rs b/src/db/model_publication.rs
@@ -149,7 +149,7 @@ impl PublicationIndex {
             }
 
             // result should >= 6 for first page.
-            if (page_token.is_none() && res.len() >= 6) || (page_token.is_some() && res.len() >= 3)
+            if (page_token.is_none() && res.len() >= 2) || (page_token.is_some() && res.len() >= 1)
             {
                 let next_id = res.last().unwrap().cid;
                 res.sort_by(|a, b| b.cid.partial_cmp(&a.cid).unwrap());
@@ -178,7 +178,7 @@ impl PublicationIndex {
 
         let mut res: Vec<PublicationIndex> = Vec::new();
         let query = format!(
-            "SELECT {} FROM pub_index WHERE day=? AND gid=? LIMIT 1000 USING TIMEOUT 3s",
+            "SELECT {} FROM pub_index WHERE day=? AND gid=? LIMIT 100 USING TIMEOUT 3s",
             fields.clone().join(",")
         );
 
@@ -189,14 +189,15 @@ impl PublicationIndex {
         };
 
         let mut i = 0i8;
-        while day > 0 && i < 30 {
+        while day > 0 && i < 90 {
             for gid in gids.iter() {
                 if gid <= &MIN_ID {
                     continue;
                 }
 
                 let params = (day, gid.to_cql());
                 let rows = db.execute_iter(query.as_str(), params).await?;
+                let mut c = 0i8;
                 for row in rows {
                     let mut doc = PublicationIndex::default();
                     let mut cols = ColumnsMap::with_capacity(fields.len());
@@ -205,10 +206,12 @@ impl PublicationIndex {
                     doc._fields = fields.clone();
                     if res.is_empty() {
                         res.push(doc);
+                        c += 1;
                     } else {
                         let prev = res.last_mut().unwrap();
                         if prev.cid != doc.cid {
                             res.push(doc);
+                            c += 1;
                         } else if prev.language != doc.language {
                             match language {
                                 // prefer language match
@@ -219,11 +222,16 @@ impl PublicationIndex {
                             }
                         }
                     }
+
+                    // docs <= 5 for every group.
+                    if c >= 5 {
+                        break;
+                    }
                 }
             }
 
-            // result should >= 6 for first page.
-            if (page_token.is_none() && res.len() >= 6) || (page_token.is_some() && res.len() >= 3)
+            // result should >= 2 for first page.
+            if (page_token.is_none() && res.len() >= 2) || (page_token.is_some() && res.len() >= 1)
             {
                 let next_id = res.last().unwrap().cid;
                 res.sort_by(|a, b| b.cid.partial_cmp(&a.cid).unwrap());

diff --git a/src/router.rs b/src/router.rs
@@ -104,6 +104,7 @@ pub async fn new(cfg: conf::Conf) -> anyhow::Result<(Arc<api::AppState>, Router)
                     "/list_by_gids",
                     routing::post(api::publication::list_by_gids),
                 )
+                .route("/list_latest", routing::post(api::publication::list_latest))
                 .route(
                     "/update_status",
                     routing::patch(api::publication::update_status),
@@ -166,6 +167,7 @@ pub async fn new(cfg: conf::Conf) -> anyhow::Result<(Arc<api::AppState>, Router)
                         .delete(api::collection::remove_child),
                 )
                 .route("/list", routing::post(api::collection::list))
+                .route("/list_latest", routing::post(api::collection::list_latest))
                 .route(
                     "/list_children",
                     routing::post(api::collection::list_children),