From c466c1779312a41fd46457f68845a618841bff07 Mon Sep 17 00:00:00 2001 From: Tyrrrz <1935960+Tyrrrz@users.noreply.github.com> Date: Thu, 28 Sep 2023 16:16:20 +0300 Subject: [PATCH 1/4] Update NuGet packages --- .../DiscordChatExporter.Cli.Tests.csproj | 12 ++++++------ .../DiscordChatExporter.Core.csproj | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/DiscordChatExporter.Cli.Tests/DiscordChatExporter.Cli.Tests.csproj b/DiscordChatExporter.Cli.Tests/DiscordChatExporter.Cli.Tests.csproj index 95e7366304..f40d674387 100644 --- a/DiscordChatExporter.Cli.Tests/DiscordChatExporter.Cli.Tests.csproj +++ b/DiscordChatExporter.Cli.Tests/DiscordChatExporter.Cli.Tests.csproj @@ -12,18 +12,18 @@ + - - + + - + - - - + + diff --git a/DiscordChatExporter.Core/DiscordChatExporter.Core.csproj b/DiscordChatExporter.Core/DiscordChatExporter.Core.csproj index caba414f82..99fd64fd83 100644 --- a/DiscordChatExporter.Core/DiscordChatExporter.Core.csproj +++ b/DiscordChatExporter.Core/DiscordChatExporter.Core.csproj @@ -5,11 +5,11 @@ - - + + - + \ No newline at end of file From fbfff4e51fcc53f2dc70ecd06760eb82a5f28c46 Mon Sep 17 00:00:00 2001 From: Tyrrrz <1935960+Tyrrrz@users.noreply.github.com> Date: Thu, 28 Sep 2023 19:18:52 +0300 Subject: [PATCH 2/4] Make tests more resilient --- .../Specs/HtmlAttachmentSpecs.cs | 19 ++++++++++++++----- .../Specs/HtmlEmbedSpecs.cs | 4 ++-- .../Specs/HtmlStickerSpecs.cs | 7 +++++-- .../Specs/JsonAttachmentSpecs.cs | 13 +++++++++---- .../Specs/JsonStickerSpecs.cs | 4 ++-- 5 files changed, 32 insertions(+), 15 deletions(-) diff --git a/DiscordChatExporter.Cli.Tests/Specs/HtmlAttachmentSpecs.cs b/DiscordChatExporter.Cli.Tests/Specs/HtmlAttachmentSpecs.cs index 42f7acd18c..619c699682 100644 --- a/DiscordChatExporter.Cli.Tests/Specs/HtmlAttachmentSpecs.cs +++ b/DiscordChatExporter.Cli.Tests/Specs/HtmlAttachmentSpecs.cs @@ -1,4 +1,5 @@ -using System.Linq; +using System; +using System.Linq; using System.Threading.Tasks; using AngleSharp.Dom; using DiscordChatExporter.Cli.Tests.Infra; @@ -27,7 +28,11 @@ public async Task I_can_export_a_channel_that_contains_a_message_with_a_generic_ .Select(e => e.GetAttribute("href")) .Should() .Contain( - "https://cdn.discordapp.com/attachments/885587741654536192/885587844964417596/Test.txt" + u => + u.StartsWith( + "https://cdn.discordapp.com/attachments/885587741654536192/885587844964417596/Test.txt", + StringComparison.Ordinal + ) ); } @@ -48,7 +53,11 @@ public async Task I_can_export_a_channel_that_contains_a_message_with_an_image_a .Select(e => e.GetAttribute("src")) .Should() .Contain( - "https://cdn.discordapp.com/attachments/885587741654536192/885654862430359613/bird-thumbnail.png" + u => + u.StartsWith( + "https://cdn.discordapp.com/attachments/885587741654536192/885654862430359613/bird-thumbnail.png", + StringComparison.Ordinal + ) ); } @@ -69,7 +78,7 @@ public async Task I_can_export_a_channel_that_contains_a_message_with_a_video_at var videoUrl = message.QuerySelector("video source")?.GetAttribute("src"); videoUrl .Should() - .Be( + .StartWith( "https://cdn.discordapp.com/attachments/885587741654536192/885655761512968233/file_example_MP4_640_3MG.mp4" ); } @@ -91,7 +100,7 @@ public async Task I_can_export_a_channel_that_contains_a_message_with_an_audio_a var audioUrl = message.QuerySelector("audio source")?.GetAttribute("src"); audioUrl .Should() - .Be( + .StartWith( "https://cdn.discordapp.com/attachments/885587741654536192/885656175348187146/file_example_MP3_1MG.mp3" ); } diff --git a/DiscordChatExporter.Cli.Tests/Specs/HtmlEmbedSpecs.cs b/DiscordChatExporter.Cli.Tests/Specs/HtmlEmbedSpecs.cs index 4c155ff4ad..3cf6331f37 100644 --- a/DiscordChatExporter.Cli.Tests/Specs/HtmlEmbedSpecs.cs +++ b/DiscordChatExporter.Cli.Tests/Specs/HtmlEmbedSpecs.cs @@ -145,7 +145,7 @@ public async Task I_can_export_a_channel_that_contains_a_message_with_a_Spotify_ // Assert var iframeUrl = message.QuerySelector("iframe")?.GetAttribute("src"); - iframeUrl.Should().Be("https://open.spotify.com/embed/track/1LHZMWefF9502NPfArRfvP"); + iframeUrl.Should().StartWith("https://open.spotify.com/embed/track/1LHZMWefF9502NPfArRfvP"); } [Fact] @@ -161,7 +161,7 @@ public async Task I_can_export_a_channel_that_contains_a_message_with_a_YouTube_ // Assert var iframeUrl = message.QuerySelector("iframe")?.GetAttribute("src"); - iframeUrl.Should().Be("https://www.youtube.com/embed/qOWW4OlgbvE"); + iframeUrl.Should().StartWith("https://www.youtube.com/embed/qOWW4OlgbvE"); } [Fact] diff --git a/DiscordChatExporter.Cli.Tests/Specs/HtmlStickerSpecs.cs b/DiscordChatExporter.Cli.Tests/Specs/HtmlStickerSpecs.cs index 23b5b637a3..44d7932a8c 100644 --- a/DiscordChatExporter.Cli.Tests/Specs/HtmlStickerSpecs.cs +++ b/DiscordChatExporter.Cli.Tests/Specs/HtmlStickerSpecs.cs @@ -19,7 +19,7 @@ public async Task I_can_export_a_channel_that_contains_a_message_with_a_PNG_stic // Assert var stickerUrl = message.QuerySelector("[title='rock'] img")?.GetAttribute("src"); - stickerUrl.Should().Be("https://cdn.discordapp.com/stickers/904215665597120572.png"); + stickerUrl.Should().StartWith("https://cdn.discordapp.com/stickers/904215665597120572.png"); } [Fact] @@ -35,6 +35,9 @@ public async Task I_can_export_a_channel_that_contains_a_message_with_a_Lottie_s var stickerUrl = message .QuerySelector("[title='Yikes'] [data-source]") ?.GetAttribute("data-source"); - stickerUrl.Should().Be("https://cdn.discordapp.com/stickers/816087132447178774.json"); + + stickerUrl + .Should() + .StartWith("https://cdn.discordapp.com/stickers/816087132447178774.json"); } } diff --git a/DiscordChatExporter.Cli.Tests/Specs/JsonAttachmentSpecs.cs b/DiscordChatExporter.Cli.Tests/Specs/JsonAttachmentSpecs.cs index 26bc78d857..20539ebc1b 100644 --- a/DiscordChatExporter.Cli.Tests/Specs/JsonAttachmentSpecs.cs +++ b/DiscordChatExporter.Cli.Tests/Specs/JsonAttachmentSpecs.cs @@ -28,9 +28,10 @@ public async Task I_can_export_a_channel_that_contains_a_message_with_a_generic_ .GetProperty("url") .GetString() .Should() - .Be( + .StartWith( "https://cdn.discordapp.com/attachments/885587741654536192/885587844964417596/Test.txt" ); + attachments[0].GetProperty("fileName").GetString().Should().Be("Test.txt"); attachments[0].GetProperty("fileSizeBytes").GetInt64().Should().Be(11); } @@ -54,9 +55,10 @@ public async Task I_can_export_a_channel_that_contains_a_message_with_an_image_a .GetProperty("url") .GetString() .Should() - .Be( + .StartWith( "https://cdn.discordapp.com/attachments/885587741654536192/885654862430359613/bird-thumbnail.png" ); + attachments[0].GetProperty("fileName").GetString().Should().Be("bird-thumbnail.png"); attachments[0].GetProperty("fileSizeBytes").GetInt64().Should().Be(466335); } @@ -80,14 +82,16 @@ public async Task I_can_export_a_channel_that_contains_a_message_with_a_video_at .GetProperty("url") .GetString() .Should() - .Be( + .StartWith( "https://cdn.discordapp.com/attachments/885587741654536192/885655761512968233/file_example_MP4_640_3MG.mp4" ); + attachments[0] .GetProperty("fileName") .GetString() .Should() .Be("file_example_MP4_640_3MG.mp4"); + attachments[0].GetProperty("fileSizeBytes").GetInt64().Should().Be(3114374); } @@ -110,9 +114,10 @@ public async Task I_can_export_a_channel_that_contains_a_message_with_an_audio_a .GetProperty("url") .GetString() .Should() - .Be( + .StartWith( "https://cdn.discordapp.com/attachments/885587741654536192/885656175348187146/file_example_MP3_1MG.mp3" ); + attachments[0].GetProperty("fileName").GetString().Should().Be("file_example_MP3_1MG.mp3"); attachments[0].GetProperty("fileSizeBytes").GetInt64().Should().Be(1087849); } diff --git a/DiscordChatExporter.Cli.Tests/Specs/JsonStickerSpecs.cs b/DiscordChatExporter.Cli.Tests/Specs/JsonStickerSpecs.cs index f46cf847b1..d8d812021c 100644 --- a/DiscordChatExporter.Cli.Tests/Specs/JsonStickerSpecs.cs +++ b/DiscordChatExporter.Cli.Tests/Specs/JsonStickerSpecs.cs @@ -28,7 +28,7 @@ public async Task I_can_export_a_channel_that_contains_a_message_with_a_PNG_stic .GetProperty("sourceUrl") .GetString() .Should() - .Be("https://cdn.discordapp.com/stickers/904215665597120572.png"); + .StartWith("https://cdn.discordapp.com/stickers/904215665597120572.png"); } [Fact] @@ -50,6 +50,6 @@ public async Task I_can_export_a_channel_that_contains_a_message_with_a_Lottie_s .GetProperty("sourceUrl") .GetString() .Should() - .Be("https://cdn.discordapp.com/stickers/816087132447178774.json"); + .StartWith("https://cdn.discordapp.com/stickers/816087132447178774.json"); } } From a58509fda8ae97f638ea51653435f1588aea29e7 Mon Sep 17 00:00:00 2001 From: Tyrrrz <1935960+Tyrrrz@users.noreply.github.com> Date: Thu, 28 Sep 2023 19:30:12 +0300 Subject: [PATCH 3/4] Upgrade to Polly 8 usage --- .../Discord/DiscordClient.cs | 2 +- .../Exporting/ExportAssetDownloader.cs | 73 ++++++++++--------- DiscordChatExporter.Core/Utils/Http.cs | 59 +++++++++------ 3 files changed, 77 insertions(+), 57 deletions(-) diff --git a/DiscordChatExporter.Core/Discord/DiscordClient.cs b/DiscordChatExporter.Core/Discord/DiscordClient.cs index 9e94369506..169261e4a4 100644 --- a/DiscordChatExporter.Core/Discord/DiscordClient.cs +++ b/DiscordChatExporter.Core/Discord/DiscordClient.cs @@ -33,7 +33,7 @@ private async ValueTask GetResponseAsync( CancellationToken cancellationToken = default ) { - return await Http.ResponseResiliencePolicy.ExecuteAsync( + return await Http.ResponseResiliencePipeline.ExecuteAsync( async innerCancellationToken => { using var request = new HttpRequestMessage(HttpMethod.Get, new Uri(_baseUri, url)); diff --git a/DiscordChatExporter.Core/Exporting/ExportAssetDownloader.cs b/DiscordChatExporter.Core/Exporting/ExportAssetDownloader.cs index bc3173b9d3..2be9efd20f 100644 --- a/DiscordChatExporter.Core/Exporting/ExportAssetDownloader.cs +++ b/DiscordChatExporter.Core/Exporting/ExportAssetDownloader.cs @@ -53,44 +53,47 @@ public async ValueTask DownloadAsync( Directory.CreateDirectory(_workingDirPath); - await Http.ResiliencePolicy.ExecuteAsync(async () => - { - // Download the file - using var response = await Http.Client.GetAsync(url, cancellationToken); - await using (var output = File.Create(filePath)) - await response.Content.CopyToAsync(output, cancellationToken); - - // Try to set the file date according to the last-modified header - try + await Http.ResiliencePipeline.ExecuteAsync( + async innerCancellationToken => { - var lastModified = response.Content.Headers - .TryGetValue("Last-Modified") - ?.Pipe( - s => - DateTimeOffset.TryParse( - s, - CultureInfo.InvariantCulture, - DateTimeStyles.None, - out var instant - ) - ? instant - : (DateTimeOffset?)null - ); - - if (lastModified is not null) + // Download the file + using var response = await Http.Client.GetAsync(url, innerCancellationToken); + await using (var output = File.Create(filePath)) + await response.Content.CopyToAsync(output, innerCancellationToken); + + // Try to set the file date according to the last-modified header + try { - File.SetCreationTimeUtc(filePath, lastModified.Value.UtcDateTime); - File.SetLastWriteTimeUtc(filePath, lastModified.Value.UtcDateTime); - File.SetLastAccessTimeUtc(filePath, lastModified.Value.UtcDateTime); + var lastModified = response.Content.Headers + .TryGetValue("Last-Modified") + ?.Pipe( + s => + DateTimeOffset.TryParse( + s, + CultureInfo.InvariantCulture, + DateTimeStyles.None, + out var instant + ) + ? instant + : (DateTimeOffset?)null + ); + + if (lastModified is not null) + { + File.SetCreationTimeUtc(filePath, lastModified.Value.UtcDateTime); + File.SetLastWriteTimeUtc(filePath, lastModified.Value.UtcDateTime); + File.SetLastAccessTimeUtc(filePath, lastModified.Value.UtcDateTime); + } } - } - catch - { - // This can apparently fail for some reason. - // Updating the file date is not a critical task, so we'll just ignore exceptions thrown here. - // https://github.com/Tyrrrz/DiscordChatExporter/issues/585 - } - }); + catch + { + // This can apparently fail for some reason. + // Updating the file date is not a critical task, so we'll just ignore exceptions thrown here. + // https://github.com/Tyrrrz/DiscordChatExporter/issues/585 + } + }, + cancellationToken + ); return _previousPathsByUrl[url] = filePath; } diff --git a/DiscordChatExporter.Core/Utils/Http.cs b/DiscordChatExporter.Core/Utils/Http.cs index 041deae6f1..f08984f8eb 100644 --- a/DiscordChatExporter.Core/Utils/Http.cs +++ b/DiscordChatExporter.Core/Utils/Http.cs @@ -7,6 +7,7 @@ using System.Threading.Tasks; using DiscordChatExporter.Core.Utils.Extensions; using Polly; +using Polly.Retry; namespace DiscordChatExporter.Core.Utils; @@ -31,29 +32,45 @@ ex is TimeoutException or SocketException or AuthenticationException && IsRetryableStatusCode(hrex.StatusCode ?? HttpStatusCode.OK) ); - public static IAsyncPolicy ResiliencePolicy { get; } = - Policy - .Handle(IsRetryableException) - .WaitAndRetryAsync(4, i => TimeSpan.FromSeconds(Math.Pow(2, i) + 1)); + public static ResiliencePipeline ResiliencePipeline { get; } = + new ResiliencePipelineBuilder() + .AddRetry( + new RetryStrategyOptions + { + ShouldHandle = new PredicateBuilder().Handle(IsRetryableException), + MaxRetryAttempts = 4, + BackoffType = DelayBackoffType.Exponential, + Delay = TimeSpan.FromSeconds(1) + } + ) + .Build(); - public static IAsyncPolicy ResponseResiliencePolicy { get; } = - Policy - .Handle(IsRetryableException) - .OrResult(m => IsRetryableStatusCode(m.StatusCode)) - .WaitAndRetryAsync( - 8, - (i, result, _) => + public static ResiliencePipeline ResponseResiliencePipeline { get; } = + new ResiliencePipelineBuilder() + .AddRetry( + new RetryStrategyOptions { - // If rate-limited, use retry-after header as the guide. - // The response can be null here if an exception was thrown. - if (result.Result?.Headers.RetryAfter?.Delta is { } retryAfter) + ShouldHandle = new PredicateBuilder() + .Handle(IsRetryableException) + .HandleResult(m => IsRetryableStatusCode(m.StatusCode)), + MaxRetryAttempts = 8, + DelayGenerator = args => { - // Add some buffer just in case - return retryAfter + TimeSpan.FromSeconds(1); - } + // If rate-limited, use retry-after header as the guide. + // The response can be null here if an exception was thrown. + if (args.Outcome.Result?.Headers.RetryAfter?.Delta is { } retryAfter) + { + // Add some buffer just in case + return ValueTask.FromResult( + retryAfter + TimeSpan.FromSeconds(1) + ); + } - return TimeSpan.FromSeconds(Math.Pow(2, i) + 1); - }, - (_, _, _, _) => Task.CompletedTask - ); + return ValueTask.FromResult( + TimeSpan.FromSeconds(Math.Pow(2, args.AttemptNumber) + 1) + ); + } + } + ) + .Build(); } From 9180d51e5ee5eb24624ba193a4df884a8f9cb020 Mon Sep 17 00:00:00 2001 From: Tyrrrz <1935960+Tyrrrz@users.noreply.github.com> Date: Thu, 28 Sep 2023 19:30:21 +0300 Subject: [PATCH 4/4] Update version --- Changelog.md | 5 +++++ Directory.Build.props | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Changelog.md b/Changelog.md index 26f1b66d3b..2f260fac07 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,5 +1,10 @@ # Changelog +## v2.41.1 (28-Sep-2023) + +- CLI changes: + - Fixed an issue where the export failed to export channels with an empty name. + ## v2.41 (15-Sep-2023) - General changes: diff --git a/Directory.Build.props b/Directory.Build.props index 71398adc5d..07b42a6d7f 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -2,7 +2,7 @@ net7.0 - 2.41 + 2.41.1 Tyrrrz Copyright (c) Oleksii Holub preview