Merge pull request #169 from Enraged-Dun-Cookie-Development-Team/fix-…

…修复最新饼id

修复最新饼id在七牛云缺失

Cargo.toml

@@ -151,6 +151,7 @@ persistence = { workspace = true, features = ["prelude", "migrate"] }
 [target.'cfg(not(target_env = "msvc"))'.dependencies]
 jemallocator = "0.5.4"
 
+
 [dependencies.tokio]
 workspace = true
 

logic/ceobe_cookie_logic/src/impletements/mod.rs

@@ -1,6 +1,7 @@
 pub mod basic_info;
 pub mod main_list;
 pub mod new_cookie;
+pub mod script;
 pub mod search;
 pub mod temp_list;
 pub mod terra_comic;

logic/ceobe_cookie_logic/src/impletements/script.rs

@@ -0,0 +1,34 @@
+use std::collections::HashMap;
+
+use ceobe_qiniu_upload::QiniuManager;
+use persistence::{operate::GetMutDatabaseConnect, redis::RedisConnect};
+use qiniu_service::QiniuService;
+use qq_channel_warning::QqChannelGrpcService;
+use redis::AsyncCommands;
+use redis_global::redis_key::cookie_list::CookieListKey;
+
+use super::CeobeCookieLogic;
+use crate::error::LogicResult;
+
+impl CeobeCookieLogic {
+    // 缓慢同步redis的combid数据到七牛云
+    pub async fn synchronous_qiniu_from_redis(
+        redis_client: &mut RedisConnect,
+        mut qq_channel: QqChannelGrpcService, qiniu: QiniuManager,
+    ) -> LogicResult<()> {
+        let redis = redis_client.mut_connect();
+        let comb_ids: HashMap<String, String> =
+            redis.hgetall(CookieListKey::NEWEST_COOKIES).await?;
+        for (comb_id, cookie_id) in comb_ids.into_iter() {
+            QiniuService::upload_newest_cookie_id_use_script(
+                qiniu.clone(),
+                cookie_id,
+                &mut qq_channel,
+                comb_id.to_owned(),
+            )
+            .await;
+        }
+
+        Ok(())
+    }
+}

service/qiniu_service/src/impletements/datasource_comb.rs

@@ -221,18 +221,78 @@ impl QiniuService {
         redis_client: RedisConnect, comb_ids: Vec<String>,
         datasource: Option<String>,
     ) {
-        let mut handles = Vec::<JoinHandle<()>>::new();
-        for comb_id in comb_ids {
-            handles.push(tokio::spawn(Self::update_datasource_comb(
-                qiniu.clone(),
-                qq_channel.clone(),
-                redis_client.clone(),
-                cookie_id,
-                update_cookie_id,
-                comb_id,
-                datasource.clone(),
-            )));
+        for comb_ids_array in comb_ids.chunks(200) {
+            let mut handles = Vec::<JoinHandle<()>>::new();
+            for comb_id in comb_ids_array {
+                handles.push(tokio::spawn(Self::update_datasource_comb(
+                    qiniu.clone(),
+                    qq_channel.clone(),
+                    redis_client.clone(),
+                    cookie_id,
+                    update_cookie_id,
+                    comb_id.to_owned(),
+                    datasource.clone(),
+                )));
+            }
+            futures::future::join_all(handles).await;
+        }
+    }
+
+    // 用于脚本的删除与上传最新饼id到七牛云
+    pub async fn upload_newest_cookie_id_use_script(
+        qiniu: QiniuManager, cookie_id: String,
+        qq_channel: &mut QqChannelGrpcService, comb_id: String,
+    ) {
+        // 先删除，后新增
+        let result = qiniu
+            .delete(DeleteObjectName {
+                file_name: comb_id.clone(),
+            })
+            .await
+            .err();
+        if let Some(err) = result {
+            let _ = qq_channel
+                .send_logger(
+                    LogRequest::builder()
+                        .level(LogType::Error)
+                        .manual()
+                        .info("删除七牛云数据源对应最新饼id文件失败".into())
+                        .extra(format!("报错：{err}\n组合id：{comb_id}"))
+                        .build(),
+                )
+                .await;
+        }
+
+        let source = CombIdToCookieId {
+            cookie_id: Some(&cookie_id),
+            update_cookie_id: None,
+        };
+        let payload = CombIdToCookieIdPlayLoad {
+            file_name: &comb_id,
+        };
+
+        // 上传数据源组合到对象储存[重试3次]
+        let mut result = Option::<ceobe_qiniu_upload::Error>::None;
+        for _ in 0..3 {
+            result = upload(&qiniu, &source, payload).await.err();
+            if result.is_none() {
+                break;
+            }
+        }
+        if let Some(err) = result {
+            let _ = qq_channel
+                .send_logger(
+                    LogRequest::builder()
+                        .level(LogType::Error)
+                        .manual()
+                        .info("上传七牛云数据源对应最新饼id文件失败".into())
+                        .extra(format!(
+                            "报错：{err}\n组合id：{comb_id}\n最新饼id：\
+                             {cookie_id:#?}\n",
+                        ))
+                        .build(),
+                )
+                .await;
         }
-        futures::future::join_all(handles).await;
     }
 }

src/router/back_end/ceobe_cookie/mod.rs

@@ -0,0 +1,22 @@
+use axum::Router;
+pub use newest::CeobeCookieNewestBackend;
+
+use self::newest::newest_router;
+use crate::{
+    middleware::authorize::AuthorizeLayer, new_auth_level,
+    utils::user_authorize::auth_level::prefabs::Chef,
+};
+
+mod newest;
+
+pub(super) fn ceobe_cookie_router() -> crate::router::ServerRoute {
+    Router::new()
+        .nest("/newest", newest_router())
+        .route_layer(AuthorizeLayer::<CeobeCookieAuth>::new())
+}
+
+new_auth_level! {
+    pub CeobeCookieAuth=>[
+        Chef
+    ]
+}

src/router/back_end/ceobe_cookie/newest.rs

@@ -0,0 +1,10 @@
+use axum::{routing::post, Router};
+
+pub struct CeobeCookieNewestBackend;
+
+pub(super) fn newest_router() -> crate::router::ServerRoute {
+    Router::new().route(
+        "/synchronousCombId",
+        post(CeobeCookieNewestBackend::synchronous_qiniu_from_redis),
+    )
+}

src/router/back_end/mod.rs

@@ -1,5 +1,6 @@
 use axum::Router;
 pub use bakery_mansion::BakeryMansionBackend;
+pub use ceobe_cookie::CeobeCookieNewestBackend;
 pub use ceobe_operation::{
     CeobeOpResource, CeobeOpVersion, CeobeOperationAnnouncement,
     CeobeOperationVideo,
@@ -8,7 +9,7 @@ pub use user_auth::UserAuthBackend;
 
 pub use self::fetcher::FetcherConfigControllers;
 use self::{
-    bakery_mansion::bakery_mansion_router,
+    bakery_mansion::bakery_mansion_router, ceobe_cookie::ceobe_cookie_router,
     ceobe_operation::ceobe_operation_router, fetcher::fetcher_config,
     user_auth::user_auth_router,
 };
@@ -17,12 +18,14 @@ mod bakery_mansion;
 pub mod ceobe_operation;
 pub mod fetcher;
 
+mod ceobe_cookie;
 mod user_auth;
 
 pub(super) fn back_end_router() -> crate::router::ServerRoute {
     Router::new()
         .nest("/fetcherConfig", fetcher_config())
         .nest("/user", user_auth_router())
         .nest("/mansion", bakery_mansion_router())
+        .nest("/cookie", ceobe_cookie_router())
         .merge(ceobe_operation_router())
 }

src/router/mod.rs

@@ -1,7 +1,7 @@
 use axum::{routing::get, Router};
 pub use back_end::{
-    BakeryMansionBackend, CeobeOpResource, CeobeOpVersion,
-    CeobeOperationAnnouncement, CeobeOperationVideo,
+    BakeryMansionBackend, CeobeCookieNewestBackend, CeobeOpResource,
+    CeobeOpVersion, CeobeOperationAnnouncement, CeobeOperationVideo,
     FetcherConfigControllers, UserAuthBackend,
 };
 pub use cdn::{CdnCookieMainListFrontend, CdnCookieTempFrontend};

src/serves/backend/ceobe_cookie/mod.rs

@@ -0,0 +1 @@
+pub mod newest;

src/serves/backend/ceobe_cookie/newest/controllers.rs

@@ -0,0 +1,28 @@
+use ceobe_cookie_logic::impletements::CeobeCookieLogic;
+use ceobe_qiniu_upload::QiniuManager;
+use persistence::redis::RedisConnect;
+use qq_channel_warning::QqChannelGrpcService;
+use resp_result::resp_try;
+use tracing::instrument;
+
+use super::error::CookieNewestRResult;
+use crate::router::CeobeCookieNewestBackend;
+
+impl CeobeCookieNewestBackend {
+    #[instrument(ret, skip(redis_client, qiniu))]
+    pub async fn synchronous_qiniu_from_redis(
+        mut redis_client: RedisConnect,
+        (qiniu, qq_channel): (QiniuManager, QqChannelGrpcService),
+    ) -> CookieNewestRResult<()> {
+        resp_try(async move {
+            CeobeCookieLogic::synchronous_qiniu_from_redis(
+                &mut redis_client,
+                qq_channel,
+                qiniu,
+            )
+            .await?;
+            Ok(())
+        })
+        .await
+    }
+}

src/serves/backend/ceobe_cookie/newest/error.rs

@@ -0,0 +1,12 @@
+use ceobe_cookie_logic::error::LogicError;
+use resp_result::RespResult;
+
+use crate::error_generate;
+
+error_generate! {
+    pub CookieNewestError
+
+    LogicError = LogicError
+}
+
+pub type CookieNewestRResult<T> = RespResult<T, CookieNewestError>;

src/serves/backend/ceobe_cookie/newest/mod.rs

@@ -0,0 +1,2 @@
+pub mod controllers;
+pub mod error;

src/serves/backend/mod.rs

@@ -1,4 +1,5 @@
 pub mod bakery_mansion;
+pub mod ceobe_cookie;
 pub mod ceobe_operation;
 mod fetcher;
 mod user_auth;