diff --git a/Cargo.toml b/Cargo.toml index 86e84cf10..e72be3f5d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -151,6 +151,7 @@ persistence = { workspace = true, features = ["prelude", "migrate"] } [target.'cfg(not(target_env = "msvc"))'.dependencies] jemallocator = "0.5.4" + [dependencies.tokio] workspace = true diff --git a/logic/ceobe_cookie_logic/src/impletements/mod.rs b/logic/ceobe_cookie_logic/src/impletements/mod.rs index 2aeb92683..6d7084e64 100644 --- a/logic/ceobe_cookie_logic/src/impletements/mod.rs +++ b/logic/ceobe_cookie_logic/src/impletements/mod.rs @@ -1,6 +1,7 @@ pub mod basic_info; pub mod main_list; pub mod new_cookie; +pub mod script; pub mod search; pub mod temp_list; pub mod terra_comic; diff --git a/logic/ceobe_cookie_logic/src/impletements/script.rs b/logic/ceobe_cookie_logic/src/impletements/script.rs new file mode 100644 index 000000000..d517eb3b9 --- /dev/null +++ b/logic/ceobe_cookie_logic/src/impletements/script.rs @@ -0,0 +1,34 @@ +use std::collections::HashMap; + +use ceobe_qiniu_upload::QiniuManager; +use persistence::{operate::GetMutDatabaseConnect, redis::RedisConnect}; +use qiniu_service::QiniuService; +use qq_channel_warning::QqChannelGrpcService; +use redis::AsyncCommands; +use redis_global::redis_key::cookie_list::CookieListKey; + +use super::CeobeCookieLogic; +use crate::error::LogicResult; + +impl CeobeCookieLogic { + // 缓慢同步redis的combid数据到七牛云 + pub async fn synchronous_qiniu_from_redis( + redis_client: &mut RedisConnect, + mut qq_channel: QqChannelGrpcService, qiniu: QiniuManager, + ) -> LogicResult<()> { + let redis = redis_client.mut_connect(); + let comb_ids: HashMap = + redis.hgetall(CookieListKey::NEWEST_COOKIES).await?; + for (comb_id, cookie_id) in comb_ids.into_iter() { + QiniuService::upload_newest_cookie_id_use_script( + qiniu.clone(), + cookie_id, + &mut qq_channel, + comb_id.to_owned(), + ) + .await; + } + + Ok(()) + } +} diff --git a/service/qiniu_service/src/impletements/datasource_comb.rs b/service/qiniu_service/src/impletements/datasource_comb.rs index 90abcee73..faf46fdc7 100644 --- a/service/qiniu_service/src/impletements/datasource_comb.rs +++ b/service/qiniu_service/src/impletements/datasource_comb.rs @@ -221,18 +221,78 @@ impl QiniuService { redis_client: RedisConnect, comb_ids: Vec, datasource: Option, ) { - let mut handles = Vec::>::new(); - for comb_id in comb_ids { - handles.push(tokio::spawn(Self::update_datasource_comb( - qiniu.clone(), - qq_channel.clone(), - redis_client.clone(), - cookie_id, - update_cookie_id, - comb_id, - datasource.clone(), - ))); + for comb_ids_array in comb_ids.chunks(200) { + let mut handles = Vec::>::new(); + for comb_id in comb_ids_array { + handles.push(tokio::spawn(Self::update_datasource_comb( + qiniu.clone(), + qq_channel.clone(), + redis_client.clone(), + cookie_id, + update_cookie_id, + comb_id.to_owned(), + datasource.clone(), + ))); + } + futures::future::join_all(handles).await; + } + } + + // 用于脚本的删除与上传最新饼id到七牛云 + pub async fn upload_newest_cookie_id_use_script( + qiniu: QiniuManager, cookie_id: String, + qq_channel: &mut QqChannelGrpcService, comb_id: String, + ) { + // 先删除，后新增 + let result = qiniu + .delete(DeleteObjectName { + file_name: comb_id.clone(), + }) + .await + .err(); + if let Some(err) = result { + let _ = qq_channel + .send_logger( + LogRequest::builder() + .level(LogType::Error) + .manual() + .info("删除七牛云数据源对应最新饼id文件失败".into()) + .extra(format!("报错：{err}\n组合id：{comb_id}")) + .build(), + ) + .await; + } + + let source = CombIdToCookieId { + cookie_id: Some(&cookie_id), + update_cookie_id: None, + }; + let payload = CombIdToCookieIdPlayLoad { + file_name: &comb_id, + }; + + // 上传数据源组合到对象储存[重试3次] + let mut result = Option::::None; + for _ in 0..3 { + result = upload(&qiniu, &source, payload).await.err(); + if result.is_none() { + break; + } + } + if let Some(err) = result { + let _ = qq_channel + .send_logger( + LogRequest::builder() + .level(LogType::Error) + .manual() + .info("上传七牛云数据源对应最新饼id文件失败".into()) + .extra(format!( + "报错：{err}\n组合id：{comb_id}\n最新饼id：\ + {cookie_id:#?}\n", + )) + .build(), + ) + .await; } - futures::future::join_all(handles).await; } } diff --git a/src/router/back_end/ceobe_cookie/mod.rs b/src/router/back_end/ceobe_cookie/mod.rs new file mode 100644 index 000000000..0790c1694 --- /dev/null +++ b/src/router/back_end/ceobe_cookie/mod.rs @@ -0,0 +1,22 @@ +use axum::Router; +pub use newest::CeobeCookieNewestBackend; + +use self::newest::newest_router; +use crate::{ + middleware::authorize::AuthorizeLayer, new_auth_level, + utils::user_authorize::auth_level::prefabs::Chef, +}; + +mod newest; + +pub(super) fn ceobe_cookie_router() -> crate::router::ServerRoute { + Router::new() + .nest("/newest", newest_router()) + .route_layer(AuthorizeLayer::::new()) +} + +new_auth_level! { + pub CeobeCookieAuth=>[ + Chef + ] +} diff --git a/src/router/back_end/ceobe_cookie/newest.rs b/src/router/back_end/ceobe_cookie/newest.rs new file mode 100644 index 000000000..07a3dda59 --- /dev/null +++ b/src/router/back_end/ceobe_cookie/newest.rs @@ -0,0 +1,10 @@ +use axum::{routing::post, Router}; + +pub struct CeobeCookieNewestBackend; + +pub(super) fn newest_router() -> crate::router::ServerRoute { + Router::new().route( + "/synchronousCombId", + post(CeobeCookieNewestBackend::synchronous_qiniu_from_redis), + ) +} diff --git a/src/router/back_end/mod.rs b/src/router/back_end/mod.rs index 56b36b4a1..0bcc7bff3 100644 --- a/src/router/back_end/mod.rs +++ b/src/router/back_end/mod.rs @@ -1,5 +1,6 @@ use axum::Router; pub use bakery_mansion::BakeryMansionBackend; +pub use ceobe_cookie::CeobeCookieNewestBackend; pub use ceobe_operation::{ CeobeOpResource, CeobeOpVersion, CeobeOperationAnnouncement, CeobeOperationVideo, @@ -8,7 +9,7 @@ pub use user_auth::UserAuthBackend; pub use self::fetcher::FetcherConfigControllers; use self::{ - bakery_mansion::bakery_mansion_router, + bakery_mansion::bakery_mansion_router, ceobe_cookie::ceobe_cookie_router, ceobe_operation::ceobe_operation_router, fetcher::fetcher_config, user_auth::user_auth_router, }; @@ -17,6 +18,7 @@ mod bakery_mansion; pub mod ceobe_operation; pub mod fetcher; +mod ceobe_cookie; mod user_auth; pub(super) fn back_end_router() -> crate::router::ServerRoute { @@ -24,5 +26,6 @@ pub(super) fn back_end_router() -> crate::router::ServerRoute { .nest("/fetcherConfig", fetcher_config()) .nest("/user", user_auth_router()) .nest("/mansion", bakery_mansion_router()) + .nest("/cookie", ceobe_cookie_router()) .merge(ceobe_operation_router()) } diff --git a/src/router/mod.rs b/src/router/mod.rs index cc598ffcb..e879e34d0 100644 --- a/src/router/mod.rs +++ b/src/router/mod.rs @@ -1,7 +1,7 @@ use axum::{routing::get, Router}; pub use back_end::{ - BakeryMansionBackend, CeobeOpResource, CeobeOpVersion, - CeobeOperationAnnouncement, CeobeOperationVideo, + BakeryMansionBackend, CeobeCookieNewestBackend, CeobeOpResource, + CeobeOpVersion, CeobeOperationAnnouncement, CeobeOperationVideo, FetcherConfigControllers, UserAuthBackend, }; pub use cdn::{CdnCookieMainListFrontend, CdnCookieTempFrontend}; diff --git a/src/serves/backend/ceobe_cookie/mod.rs b/src/serves/backend/ceobe_cookie/mod.rs new file mode 100644 index 000000000..35a6f65b4 --- /dev/null +++ b/src/serves/backend/ceobe_cookie/mod.rs @@ -0,0 +1 @@ +pub mod newest; diff --git a/src/serves/backend/ceobe_cookie/newest/controllers.rs b/src/serves/backend/ceobe_cookie/newest/controllers.rs new file mode 100644 index 000000000..90b8e1c59 --- /dev/null +++ b/src/serves/backend/ceobe_cookie/newest/controllers.rs @@ -0,0 +1,28 @@ +use ceobe_cookie_logic::impletements::CeobeCookieLogic; +use ceobe_qiniu_upload::QiniuManager; +use persistence::redis::RedisConnect; +use qq_channel_warning::QqChannelGrpcService; +use resp_result::resp_try; +use tracing::instrument; + +use super::error::CookieNewestRResult; +use crate::router::CeobeCookieNewestBackend; + +impl CeobeCookieNewestBackend { + #[instrument(ret, skip(redis_client, qiniu))] + pub async fn synchronous_qiniu_from_redis( + mut redis_client: RedisConnect, + (qiniu, qq_channel): (QiniuManager, QqChannelGrpcService), + ) -> CookieNewestRResult<()> { + resp_try(async move { + CeobeCookieLogic::synchronous_qiniu_from_redis( + &mut redis_client, + qq_channel, + qiniu, + ) + .await?; + Ok(()) + }) + .await + } +} diff --git a/src/serves/backend/ceobe_cookie/newest/error.rs b/src/serves/backend/ceobe_cookie/newest/error.rs new file mode 100644 index 000000000..3143eec41 --- /dev/null +++ b/src/serves/backend/ceobe_cookie/newest/error.rs @@ -0,0 +1,12 @@ +use ceobe_cookie_logic::error::LogicError; +use resp_result::RespResult; + +use crate::error_generate; + +error_generate! { + pub CookieNewestError + + LogicError = LogicError +} + +pub type CookieNewestRResult = RespResult; diff --git a/src/serves/backend/ceobe_cookie/newest/mod.rs b/src/serves/backend/ceobe_cookie/newest/mod.rs new file mode 100644 index 000000000..7ddc0393c --- /dev/null +++ b/src/serves/backend/ceobe_cookie/newest/mod.rs @@ -0,0 +1,2 @@ +pub mod controllers; +pub mod error; diff --git a/src/serves/backend/mod.rs b/src/serves/backend/mod.rs index e375e1de9..d464aab51 100644 --- a/src/serves/backend/mod.rs +++ b/src/serves/backend/mod.rs @@ -1,4 +1,5 @@ pub mod bakery_mansion; +pub mod ceobe_cookie; pub mod ceobe_operation; mod fetcher; mod user_auth;