feat: duplicate published doc (#700)

* chore: test collab sync

* feat: folder view for user workspace

* feat: add private indicator

* chore: use collab cache instead

* chore: initial impl

* chore: cargo sqlx

* fix: write back to collab cache

* fix: assign page id

* fix: text map

* chore: connect api and refactor

* chore: switch to using mem storage

* fix: collab type

* feat: use group manager to manage sync

* feat: try add send command instead

* chore: add client api code

* feat: try use sink and stream from group collab

* chore: disable sync first

* fix: insert page before stopping group

* feat: add extra for view

* feat: add metadata to doc

* fix: icon

* fix: page circular dep

* fix: page circular dep

* fix: live update

* feat: database initial impl

* feat: workspace database

* chore: link database

* fix: database validation

* fix: workspace database oid

* fix: workspace database oid

* fix: specify view layout

* feat: add txn when inserting collab and rework database views

* fix: parent view id for new database

* fix: database view for folder

* fix: database ids

* fix: database row id

* fix: main database layout and write database before folder

* fix: metadata for non database main view

* chore: database in doc wip

* chore: filter visible views

* chore: use structure value instead of json

* chore: database in doc wip

* fix: remove child item push

* fix: main database view meta

* chore: rework inline database in doc

* fix: make duplicate more error forgiving

* chore: use get instead of get key value

* chore: merge with main

* feat: set max depth for folder view

* chore: update dependency counter

* chore: remove unneeded arg

* chore: simplify collab new from source

* chore: remove unneeded param

* chore: rename and doc

* feat: rework inline database and use duplicated ref

* chore: simplify params

* chore: fmt

* feat: deduplicate database wip

* fix: compatible with newer appflowy version

* fix: database views

* feat: database in doc

* chore: cargo clippy

* chore: update dockerfile rust toolchain

* fix: doc inline db parent id

* fix: database id

* fix: document data modification

* chore: minor refactor and todo

* chore: remove unneeded nesting

* fix: enforce ordering for views to add

* chore: add doc and test case for folder view

* fix: remove the need for cloning published collab

* fix: some review

* chore: move folder conversion logic

* fix: remove collab folder deps on shared entity

* chore: cargo fmt

* fix: compile

* chore: remove group manager dependency

* fix: dep count

* fix: add messages sent by server

* fix: dep count

* chore: add test for doc ref and inline doc database

* chore: cargo clippy

* chore: add more test scenarios

* chore: fix tests

* chore: get database id

* chore: update collab

* chore: add more assert and database row id checks

* fix: suggestions from review

* chore: sqlx

* fix: accumulate collab before insert

* chore: add tokio spawn blocking for encoding

* fix: reduce limit for publish collabs

---------

Co-authored-by: Bartosz Sypytkowski <[email protected]>
Co-authored-by: nathan <[email protected]>

Cargo.toml

@@ -139,6 +139,7 @@ shared-entity = { path = "libs/shared-entity", features = ["cloud"] }
 workspace-template = { workspace = true }
 collab-rt-entity.workspace = true
 collab-stream.workspace = true
+yrs.workspace = true
 
 tonic-build = "0.11.0"
 log = "0.4.20"
@@ -164,6 +165,7 @@ client-api = { path = "libs/client-api", features = [
 opener = "0.6.1"
 image = "0.23.14"
 collab-rt-entity.workspace = true
+hex = "0.4.3"
 
 [[bin]]
 name = "appflowy_cloud"

Dockerfile

@@ -1,5 +1,5 @@
 # Using cargo-chef to manage Rust build cache effectively
-FROM lukemathwalker/cargo-chef:latest-rust-1.77 as chef
+FROM lukemathwalker/cargo-chef:latest-rust-1.79 as chef
 
 WORKDIR /app
 RUN apt update && apt install lld clang -y

libs/client-api-test/Cargo.toml

@@ -26,6 +26,7 @@ image = "0.23.14"
 database-entity.workspace = true
 collab-entity.workspace = true
 shared-entity.workspace = true
+collab-database.workspace = true
 tracing-subscriber = { version = "0.3.18", features = ["registry", "env-filter", "ansi", "json"] }
 uuid = "1.6.1"
 lazy_static = "1.4.0"
@@ -43,4 +44,4 @@ web-sys = { version = "0.3", features = ["console"] }
 
 [features]
 collab-sync = ["client-api/collab-sync"]
-ai-test-enabled = []
+ai-test-enabled = []

libs/client-api-test/src/test_client.rs

@@ -1,7 +1,5 @@
 use std::borrow::BorrowMut;
 use std::collections::HashMap;
-use std::ops::Deref;
-use std::path::{Path, PathBuf};
 use std::sync::Arc;
 
 use anyhow::{anyhow, Error};
@@ -185,13 +183,38 @@ impl TestClient {
     Folder::from_collab_doc_state(
       uid,
       CollabOrigin::Empty,
-      DataSource::DocStateV1(data.encode_collab.doc_state.to_vec()),
+      data.encode_collab.into(),
       &workspace_id,
       vec![],
     )
     .unwrap()
   }
 
+  pub async fn get_workspace_database_collab(&mut self, workspace_id: &str) -> Collab {
+    let db_storage_id = self.open_workspace(workspace_id).await.database_storage_id;
+    let ws_db_doc_state = self
+      .get_collab(QueryCollabParams {
+        workspace_id: workspace_id.to_string(),
+        inner: QueryCollab {
+          object_id: db_storage_id.to_string(),
+          collab_type: CollabType::WorkspaceDatabase,
+        },
+      })
+      .await
+      .unwrap()
+      .encode_collab
+      .doc_state
+      .to_vec();
+    Collab::new_with_source(
+      CollabOrigin::Server,
+      &db_storage_id.to_string(),
+      DataSource::DocStateV1(ws_db_doc_state),
+      vec![],
+      false,
+    )
+    .unwrap()
+  }
+
   pub async fn get_user_awareness(&self) -> UserAwareness {
     let workspace_id = self.workspace_id().await;
     let profile = self.get_user_profile().await;
@@ -969,31 +992,3 @@ pub async fn get_collab_json_from_server(
   .unwrap()
   .to_json_value()
 }
-
-pub struct TestTempFile(PathBuf);
-
-impl TestTempFile {
-  fn cleanup(dir: &PathBuf) {
-    let _ = std::fs::remove_dir_all(dir);
-  }
-}
-
-impl AsRef<Path> for TestTempFile {
-  fn as_ref(&self) -> &Path {
-    &self.0
-  }
-}
-
-impl Deref for TestTempFile {
-  type Target = PathBuf;
-
-  fn deref(&self) -> &Self::Target {
-    &self.0
-  }
-}
-
-impl Drop for TestTempFile {
-  fn drop(&mut self) {
-    Self::cleanup(&self.0)
-  }
-}

libs/client-api/src/http.rs

@@ -1,5 +1,7 @@
 use crate::notify::{ClientToken, TokenStateReceiver};
 use app_error::AppError;
+use client_api_entity::workspace_dto::FolderView;
+use client_api_entity::workspace_dto::QueryWorkspaceFolder;
 use client_api_entity::workspace_dto::QueryWorkspaceParam;
 use client_api_entity::AuthProvider;
 use client_api_entity::CollabType;
@@ -657,6 +659,37 @@ impl Client {
       .into_data()
   }
 
+  /// List out the views in the workspace recursively.
+  /// The depth parameter specifies the depth of the folder view tree to return(default: 1).
+  /// e.g., depth=1 will return only up to `Shared` and `PrivateSpace`
+  /// depth=2 will return up to `mydoc1`, `mydoc2`, `mydoc3`, `mydoc4`
+  ///
+  /// . MyWorkspace
+  /// ├── Shared
+  /// │   ├── mydoc1
+  /// │   └── mydoc2
+  /// └── PrivateSpace
+  ///     ├── mydoc3
+  ///     └── mydoc4
+  #[instrument(level = "info", skip_all, err)]
+  pub async fn get_workspace_folder(
+    &self,
+    workspace_id: &str,
+    depth: Option<u32>,
+  ) -> Result<FolderView, AppResponseError> {
+    let url = format!("{}/api/workspace/{}/folder", self.base_url, workspace_id);
+    let resp = self
+      .http_client_with_auth(Method::GET, &url)
+      .await?
+      .query(&QueryWorkspaceFolder { depth })
+      .send()
+      .await?;
+    log_request_id(&resp);
+    AppResponse::<FolderView>::from_response(resp)
+      .await?
+      .into_data()
+  }
+
   #[instrument(level = "info", skip_all, err)]
   pub async fn open_workspace(&self, workspace_id: &str) -> Result<AFWorkspace, AppResponseError> {
     let url = format!("{}/api/workspace/{}/open", self.base_url, workspace_id);

libs/client-api/src/http_publish.rs

@@ -1,7 +1,8 @@
 use bytes::Bytes;
+use client_api_entity::{workspace_dto::PublishedDuplicate, PublishInfo, UpdatePublishNamespace};
 use client_api_entity::{
   CreateGlobalCommentParams, CreateReactionParams, DeleteGlobalCommentParams, DeleteReactionParams,
-  GetReactionQueryParams, GlobalComments, PublishInfo, Reactions, UpdatePublishNamespace,
+  GetReactionQueryParams, GlobalComments, Reactions,
 };
 use reqwest::Method;
 use shared_entity::response::{AppResponse, AppResponseError};
@@ -259,6 +260,24 @@ impl Client {
     Ok(bytes)
   }
 
+  pub async fn duplicate_published_to_workspace(
+    &self,
+    workspace_id: &str,
+    publish_duplicate: &PublishedDuplicate,
+  ) -> Result<(), AppResponseError> {
+    let url = format!(
+      "{}/api/workspace/{}/published-duplicate",
+      self.base_url, workspace_id
+    );
+    let resp = self
+      .http_client_with_auth(Method::POST, &url)
+      .await?
+      .json(publish_duplicate)
+      .send()
+      .await?;
+    AppResponse::<()>::from_response(resp).await?.into_error()
+  }
+
   pub async fn get_published_view_reactions(
     &self,
     view_id: &uuid::Uuid,

libs/database/Cargo.toml

@@ -8,6 +8,7 @@ edition = "2021"
 [dependencies]
 collab = { workspace = true }
 collab-entity = { workspace = true }
+collab-rt-entity = { workspace = true }
 validator = { version = "0.16", features = ["validator_derive", "derive"] }
 database-entity.workspace = true
 app-error = { workspace = true, features = ["sqlx_error", "validation_error"] }