improve /v1/publication/list_by_gids api.

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "writing"
-version = "0.10.3"
+version = "0.10.4"
 edition = "2021"
 rust-version = "1.64"
 description = ""

src/api/bookmark.rs

@@ -199,15 +199,14 @@ pub async fn list(
         token_to_xid(&input.page_token),
     )
     .await?;
-    let next_page_token = if res.len() >= page_size as usize {
-        to.with_option(token_from_xid(res.last().unwrap().id))
-    } else {
-        None
-    };
 
     Ok(to.with(SuccessResponse {
         total_size: None,
-        next_page_token,
+        next_page_token: to.with_option(token_from_xid(if res.len() >= page_size as usize {
+            Some(res.last().unwrap().id)
+        } else {
+            None
+        })),
         result: res
             .iter()
             .map(|r| BookmarkOutput::from(r.to_owned(), &to))

src/api/creation.rs

@@ -236,15 +236,14 @@ pub async fn list(
         input.status,
     )
     .await?;
-    let next_page_token = if res.len() >= page_size as usize {
-        to.with_option(token_from_xid(res.last().unwrap().id))
-    } else {
-        None
-    };
 
     Ok(to.with(SuccessResponse {
         total_size: None,
-        next_page_token,
+        next_page_token: to.with_option(token_from_xid(if res.len() >= page_size as usize {
+            Some(res.last().unwrap().id)
+        } else {
+            None
+        })),
         result: res
             .iter()
             .map(|r| CreationOutput::from(r.to_owned(), &to))

src/api/mod.rs

@@ -128,25 +128,12 @@ pub fn token_to_xid(page_token: &Option<PackObject<Vec<u8>>>) -> Option<xid::Id>
     }
 }
 
-pub fn token_from_xid(id: xid::Id) -> Option<Vec<u8>> {
-    cbor_to_vec(&PackObject::Cbor(id)).ok()
-}
-
-// pub fn token_to_publication(
-//     page_token: &Option<PackObject<Vec<u8>>>,
-// ) -> Option<(xid::Id, Language, i16)> {
-//     match page_token.as_ref().map(|v| v.unwrap_ref()) {
-//         Some(v) => cbor_from_slice::<(PackObject<xid::Id>, PackObject<Language>, i16)>(&v)
-//             .ok()
-//             .map(|v| (v.0.unwrap(), v.1.unwrap(), v.2)),
-//         _ => None,
-//     }
-// }
-
-// pub fn token_from_publication(v: (xid::Id, Language, i16)) -> Option<Vec<u8>> {
-//     let v = (PackObject::Cbor(v.0), PackObject::Cbor(v.1), v.2);
-//     cbor_to_vec(&v).ok()
-// }
+pub fn token_from_xid(id: Option<xid::Id>) -> Option<Vec<u8>> {
+    match id {
+        Some(id) => cbor_to_vec(&PackObject::Cbor(id)).ok(),
+        _ => None,
+    }
+}
 
 #[derive(Debug, Deserialize, Validate)]
 pub struct UpdateStatusInput {
@@ -194,7 +181,7 @@ mod tests {
     #[test]
     fn token_to_xid_works() {
         let input = xid::new();
-        let v = token_from_xid(input).unwrap();
+        let v = token_from_xid(Some(input)).unwrap();
         assert_eq!(hex_string(&v).len(), 26);
         let rt = token_to_xid(&Some(PackObject::Cbor(v)));
         assert_eq!(rt, Some(input));

src/api/publication.rs

@@ -297,16 +297,14 @@ pub async fn list(
         ctx.language,
     )
     .await?;
-    let next_page_token = if res.len() >= page_size as usize {
-        let v = res.last().unwrap();
-        to.with_option(token_from_xid(v.cid))
-    } else {
-        None
-    };
 
     Ok(to.with(SuccessResponse {
         total_size: None,
-        next_page_token,
+        next_page_token: to.with_option(token_from_xid(if res.len() >= page_size as usize {
+            Some(res.last().unwrap().cid)
+        } else {
+            None
+        })),
         result: res
             .iter()
             .map(|r| PublicationOutput::from(r.to_owned(), &to))

src/db/model_publication.rs

@@ -756,75 +756,69 @@ impl Publication {
         select_fields: Vec<String>,
         page_token: Option<xid::Id>,
         language: Option<Language>,
-    ) -> anyhow::Result<(Vec<Publication>, xid::Id)> {
+    ) -> anyhow::Result<(Vec<Publication>, Option<xid::Id>)> {
         let fields = Self::select_fields(select_fields, true)?;
 
-        let secs: u32 = 3600 * 48;
-        let query_size: i32 = match gids.len() {
-            v if v <= 5 => 5,
-            v if v <= 20 => 4,
-            v if v <= 50 => 3,
-            v if v <= 100 => 2,
-            _ => 1,
-        };
-
-        let mut res: Vec<Publication> = Vec::with_capacity(gids.len() * 2);
+        let secs: u32 = 3600 * 24;
+        let mut res: Vec<Publication> = Vec::new();
+        let query = format!(
+            "SELECT {} FROM publication WHERE gid=? AND status=? AND cid>=? AND cid<? BYPASS CACHE USING TIMEOUT 3s",
+            fields.clone().join(","));
 
-        let (start_id, end_id) = if let Some(cid) = page_token {
-            let raw = cid.as_bytes();
-            let unix_ts = u32::from_be_bytes([raw[0], raw[1], raw[2], raw[3]]);
-            let mut start_id = xid::Id::default();
-            start_id.0[0..=3].copy_from_slice(&(unix_ts - secs).to_be_bytes());
-            (start_id, cid)
+        let mut end_id = if let Some(cid) = page_token {
+            cid
         } else {
-            // from 1 days ago
             let unix_ts = (unix_ms() / 1000) as u32;
-            let mut start_id = xid::Id::default();
             let mut end_id = xid::Id::default();
-            start_id.0[0..=3].copy_from_slice(&(unix_ts - secs).to_be_bytes());
             end_id.0[0..=3].copy_from_slice(&unix_ts.to_be_bytes());
-            (start_id, end_id)
+            end_id
         };
 
-        for gid in gids {
-            let query = format!(
-                    "SELECT {} FROM publication WHERE gid=? AND status=? AND cid>=? AND cid<? BYPASS CACHE USING TIMEOUT 3s",
-                    fields.clone().join(","));
-            let params = (gid.to_cql(), 2i8, start_id.to_cql(), end_id.to_cql());
-            let rows = db.execute_iter(query, params).await?;
-            let mut local_size = 0i32;
-            for row in rows {
-                let mut doc = Publication::default();
-                let mut cols = ColumnsMap::with_capacity(fields.len());
-                cols.fill(row, &fields)?;
-                doc.fill(&cols);
-                doc._fields = fields.clone();
-                if res.is_empty() {
-                    res.push(doc);
-                    local_size += 1;
-                } else {
-                    let prev = res.last_mut().unwrap();
-                    if prev.cid != doc.cid {
+        let mut i = 0i8;
+        while i < 7 {
+            let raw = end_id.as_bytes();
+            let unix_ts = u32::from_be_bytes([raw[0], raw[1], raw[2], raw[3]]);
+            let mut start_id = xid::Id::default();
+            start_id.0[0..=3].copy_from_slice(&(unix_ts - secs).to_be_bytes());
+
+            for gid in gids.iter() {
+                let params = (gid.to_cql(), 2i8, start_id.to_cql(), end_id.to_cql());
+                let rows = db.execute_iter(query.as_str(), params).await?;
+                for row in rows {
+                    let mut doc = Publication::default();
+                    let mut cols = ColumnsMap::with_capacity(fields.len());
+                    cols.fill(row, &fields)?;
+                    doc.fill(&cols);
+                    doc._fields = fields.clone();
+                    if res.is_empty() {
                         res.push(doc);
-                        local_size += 1;
-                    } else if prev.language != doc.language {
-                        match language {
-                            // prefer language match
-                            Some(lang) if lang == doc.language => *prev = doc,
-                            // or original language
-                            None if doc.language == doc.from_language => *prev = doc,
-                            _ => {} // ignore
+                    } else {
+                        let prev = res.last_mut().unwrap();
+                        if prev.cid != doc.cid {
+                            res.push(doc);
+                        } else if prev.language != doc.language {
+                            match language {
+                                // prefer language match
+                                Some(lang) if lang == doc.language => *prev = doc,
+                                // or original language
+                                None if doc.language == doc.from_language => *prev = doc,
+                                _ => {} // ignore
+                            }
                         }
                     }
                 }
+            }
 
-                if local_size >= query_size {
-                    break;
-                }
+            if !res.is_empty() {
+                res.sort_by(|a, b| b.cid.partial_cmp(&a.cid).unwrap());
+                return Ok((res, Some(start_id)));
             }
+
+            i += 1;
+            end_id = start_id;
         }
 
-        Ok((res, start_id))
+        Ok((res, None))
     }
 
     pub async fn list_published_by_cid(