From c466c1779312a41fd46457f68845a618841bff07 Mon Sep 17 00:00:00 2001
From: Tyrrrz <1935960+Tyrrrz@users.noreply.github.com>
Date: Thu, 28 Sep 2023 16:16:20 +0300
Subject: [PATCH 1/4] Update NuGet packages
---
.../DiscordChatExporter.Cli.Tests.csproj | 12 ++++++------
.../DiscordChatExporter.Core.csproj | 6 +++---
2 files changed, 9 insertions(+), 9 deletions(-)
diff --git a/DiscordChatExporter.Cli.Tests/DiscordChatExporter.Cli.Tests.csproj b/DiscordChatExporter.Cli.Tests/DiscordChatExporter.Cli.Tests.csproj
index 95e7366304..f40d674387 100644
--- a/DiscordChatExporter.Cli.Tests/DiscordChatExporter.Cli.Tests.csproj
+++ b/DiscordChatExporter.Cli.Tests/DiscordChatExporter.Cli.Tests.csproj
@@ -12,18 +12,18 @@
+
-
-
+
+
-
+
-
-
-
+
+
diff --git a/DiscordChatExporter.Core/DiscordChatExporter.Core.csproj b/DiscordChatExporter.Core/DiscordChatExporter.Core.csproj
index caba414f82..99fd64fd83 100644
--- a/DiscordChatExporter.Core/DiscordChatExporter.Core.csproj
+++ b/DiscordChatExporter.Core/DiscordChatExporter.Core.csproj
@@ -5,11 +5,11 @@
-
-
+
+
-
+
\ No newline at end of file
From fbfff4e51fcc53f2dc70ecd06760eb82a5f28c46 Mon Sep 17 00:00:00 2001
From: Tyrrrz <1935960+Tyrrrz@users.noreply.github.com>
Date: Thu, 28 Sep 2023 19:18:52 +0300
Subject: [PATCH 2/4] Make tests more resilient
---
.../Specs/HtmlAttachmentSpecs.cs | 19 ++++++++++++++-----
.../Specs/HtmlEmbedSpecs.cs | 4 ++--
.../Specs/HtmlStickerSpecs.cs | 7 +++++--
.../Specs/JsonAttachmentSpecs.cs | 13 +++++++++----
.../Specs/JsonStickerSpecs.cs | 4 ++--
5 files changed, 32 insertions(+), 15 deletions(-)
diff --git a/DiscordChatExporter.Cli.Tests/Specs/HtmlAttachmentSpecs.cs b/DiscordChatExporter.Cli.Tests/Specs/HtmlAttachmentSpecs.cs
index 42f7acd18c..619c699682 100644
--- a/DiscordChatExporter.Cli.Tests/Specs/HtmlAttachmentSpecs.cs
+++ b/DiscordChatExporter.Cli.Tests/Specs/HtmlAttachmentSpecs.cs
@@ -1,4 +1,5 @@
-using System.Linq;
+using System;
+using System.Linq;
using System.Threading.Tasks;
using AngleSharp.Dom;
using DiscordChatExporter.Cli.Tests.Infra;
@@ -27,7 +28,11 @@ public async Task I_can_export_a_channel_that_contains_a_message_with_a_generic_
.Select(e => e.GetAttribute("href"))
.Should()
.Contain(
- "https://cdn.discordapp.com/attachments/885587741654536192/885587844964417596/Test.txt"
+ u =>
+ u.StartsWith(
+ "https://cdn.discordapp.com/attachments/885587741654536192/885587844964417596/Test.txt",
+ StringComparison.Ordinal
+ )
);
}
@@ -48,7 +53,11 @@ public async Task I_can_export_a_channel_that_contains_a_message_with_an_image_a
.Select(e => e.GetAttribute("src"))
.Should()
.Contain(
- "https://cdn.discordapp.com/attachments/885587741654536192/885654862430359613/bird-thumbnail.png"
+ u =>
+ u.StartsWith(
+ "https://cdn.discordapp.com/attachments/885587741654536192/885654862430359613/bird-thumbnail.png",
+ StringComparison.Ordinal
+ )
);
}
@@ -69,7 +78,7 @@ public async Task I_can_export_a_channel_that_contains_a_message_with_a_video_at
var videoUrl = message.QuerySelector("video source")?.GetAttribute("src");
videoUrl
.Should()
- .Be(
+ .StartWith(
"https://cdn.discordapp.com/attachments/885587741654536192/885655761512968233/file_example_MP4_640_3MG.mp4"
);
}
@@ -91,7 +100,7 @@ public async Task I_can_export_a_channel_that_contains_a_message_with_an_audio_a
var audioUrl = message.QuerySelector("audio source")?.GetAttribute("src");
audioUrl
.Should()
- .Be(
+ .StartWith(
"https://cdn.discordapp.com/attachments/885587741654536192/885656175348187146/file_example_MP3_1MG.mp3"
);
}
diff --git a/DiscordChatExporter.Cli.Tests/Specs/HtmlEmbedSpecs.cs b/DiscordChatExporter.Cli.Tests/Specs/HtmlEmbedSpecs.cs
index 4c155ff4ad..3cf6331f37 100644
--- a/DiscordChatExporter.Cli.Tests/Specs/HtmlEmbedSpecs.cs
+++ b/DiscordChatExporter.Cli.Tests/Specs/HtmlEmbedSpecs.cs
@@ -145,7 +145,7 @@ public async Task I_can_export_a_channel_that_contains_a_message_with_a_Spotify_
// Assert
var iframeUrl = message.QuerySelector("iframe")?.GetAttribute("src");
- iframeUrl.Should().Be("https://open.spotify.com/embed/track/1LHZMWefF9502NPfArRfvP");
+ iframeUrl.Should().StartWith("https://open.spotify.com/embed/track/1LHZMWefF9502NPfArRfvP");
}
[Fact]
@@ -161,7 +161,7 @@ public async Task I_can_export_a_channel_that_contains_a_message_with_a_YouTube_
// Assert
var iframeUrl = message.QuerySelector("iframe")?.GetAttribute("src");
- iframeUrl.Should().Be("https://www.youtube.com/embed/qOWW4OlgbvE");
+ iframeUrl.Should().StartWith("https://www.youtube.com/embed/qOWW4OlgbvE");
}
[Fact]
diff --git a/DiscordChatExporter.Cli.Tests/Specs/HtmlStickerSpecs.cs b/DiscordChatExporter.Cli.Tests/Specs/HtmlStickerSpecs.cs
index 23b5b637a3..44d7932a8c 100644
--- a/DiscordChatExporter.Cli.Tests/Specs/HtmlStickerSpecs.cs
+++ b/DiscordChatExporter.Cli.Tests/Specs/HtmlStickerSpecs.cs
@@ -19,7 +19,7 @@ public async Task I_can_export_a_channel_that_contains_a_message_with_a_PNG_stic
// Assert
var stickerUrl = message.QuerySelector("[title='rock'] img")?.GetAttribute("src");
- stickerUrl.Should().Be("https://cdn.discordapp.com/stickers/904215665597120572.png");
+ stickerUrl.Should().StartWith("https://cdn.discordapp.com/stickers/904215665597120572.png");
}
[Fact]
@@ -35,6 +35,9 @@ public async Task I_can_export_a_channel_that_contains_a_message_with_a_Lottie_s
var stickerUrl = message
.QuerySelector("[title='Yikes'] [data-source]")
?.GetAttribute("data-source");
- stickerUrl.Should().Be("https://cdn.discordapp.com/stickers/816087132447178774.json");
+
+ stickerUrl
+ .Should()
+ .StartWith("https://cdn.discordapp.com/stickers/816087132447178774.json");
}
}
diff --git a/DiscordChatExporter.Cli.Tests/Specs/JsonAttachmentSpecs.cs b/DiscordChatExporter.Cli.Tests/Specs/JsonAttachmentSpecs.cs
index 26bc78d857..20539ebc1b 100644
--- a/DiscordChatExporter.Cli.Tests/Specs/JsonAttachmentSpecs.cs
+++ b/DiscordChatExporter.Cli.Tests/Specs/JsonAttachmentSpecs.cs
@@ -28,9 +28,10 @@ public async Task I_can_export_a_channel_that_contains_a_message_with_a_generic_
.GetProperty("url")
.GetString()
.Should()
- .Be(
+ .StartWith(
"https://cdn.discordapp.com/attachments/885587741654536192/885587844964417596/Test.txt"
);
+
attachments[0].GetProperty("fileName").GetString().Should().Be("Test.txt");
attachments[0].GetProperty("fileSizeBytes").GetInt64().Should().Be(11);
}
@@ -54,9 +55,10 @@ public async Task I_can_export_a_channel_that_contains_a_message_with_an_image_a
.GetProperty("url")
.GetString()
.Should()
- .Be(
+ .StartWith(
"https://cdn.discordapp.com/attachments/885587741654536192/885654862430359613/bird-thumbnail.png"
);
+
attachments[0].GetProperty("fileName").GetString().Should().Be("bird-thumbnail.png");
attachments[0].GetProperty("fileSizeBytes").GetInt64().Should().Be(466335);
}
@@ -80,14 +82,16 @@ public async Task I_can_export_a_channel_that_contains_a_message_with_a_video_at
.GetProperty("url")
.GetString()
.Should()
- .Be(
+ .StartWith(
"https://cdn.discordapp.com/attachments/885587741654536192/885655761512968233/file_example_MP4_640_3MG.mp4"
);
+
attachments[0]
.GetProperty("fileName")
.GetString()
.Should()
.Be("file_example_MP4_640_3MG.mp4");
+
attachments[0].GetProperty("fileSizeBytes").GetInt64().Should().Be(3114374);
}
@@ -110,9 +114,10 @@ public async Task I_can_export_a_channel_that_contains_a_message_with_an_audio_a
.GetProperty("url")
.GetString()
.Should()
- .Be(
+ .StartWith(
"https://cdn.discordapp.com/attachments/885587741654536192/885656175348187146/file_example_MP3_1MG.mp3"
);
+
attachments[0].GetProperty("fileName").GetString().Should().Be("file_example_MP3_1MG.mp3");
attachments[0].GetProperty("fileSizeBytes").GetInt64().Should().Be(1087849);
}
diff --git a/DiscordChatExporter.Cli.Tests/Specs/JsonStickerSpecs.cs b/DiscordChatExporter.Cli.Tests/Specs/JsonStickerSpecs.cs
index f46cf847b1..d8d812021c 100644
--- a/DiscordChatExporter.Cli.Tests/Specs/JsonStickerSpecs.cs
+++ b/DiscordChatExporter.Cli.Tests/Specs/JsonStickerSpecs.cs
@@ -28,7 +28,7 @@ public async Task I_can_export_a_channel_that_contains_a_message_with_a_PNG_stic
.GetProperty("sourceUrl")
.GetString()
.Should()
- .Be("https://cdn.discordapp.com/stickers/904215665597120572.png");
+ .StartWith("https://cdn.discordapp.com/stickers/904215665597120572.png");
}
[Fact]
@@ -50,6 +50,6 @@ public async Task I_can_export_a_channel_that_contains_a_message_with_a_Lottie_s
.GetProperty("sourceUrl")
.GetString()
.Should()
- .Be("https://cdn.discordapp.com/stickers/816087132447178774.json");
+ .StartWith("https://cdn.discordapp.com/stickers/816087132447178774.json");
}
}
From a58509fda8ae97f638ea51653435f1588aea29e7 Mon Sep 17 00:00:00 2001
From: Tyrrrz <1935960+Tyrrrz@users.noreply.github.com>
Date: Thu, 28 Sep 2023 19:30:12 +0300
Subject: [PATCH 3/4] Upgrade to Polly 8 usage
---
.../Discord/DiscordClient.cs | 2 +-
.../Exporting/ExportAssetDownloader.cs | 73 ++++++++++---------
DiscordChatExporter.Core/Utils/Http.cs | 59 +++++++++------
3 files changed, 77 insertions(+), 57 deletions(-)
diff --git a/DiscordChatExporter.Core/Discord/DiscordClient.cs b/DiscordChatExporter.Core/Discord/DiscordClient.cs
index 9e94369506..169261e4a4 100644
--- a/DiscordChatExporter.Core/Discord/DiscordClient.cs
+++ b/DiscordChatExporter.Core/Discord/DiscordClient.cs
@@ -33,7 +33,7 @@ private async ValueTask GetResponseAsync(
CancellationToken cancellationToken = default
)
{
- return await Http.ResponseResiliencePolicy.ExecuteAsync(
+ return await Http.ResponseResiliencePipeline.ExecuteAsync(
async innerCancellationToken =>
{
using var request = new HttpRequestMessage(HttpMethod.Get, new Uri(_baseUri, url));
diff --git a/DiscordChatExporter.Core/Exporting/ExportAssetDownloader.cs b/DiscordChatExporter.Core/Exporting/ExportAssetDownloader.cs
index bc3173b9d3..2be9efd20f 100644
--- a/DiscordChatExporter.Core/Exporting/ExportAssetDownloader.cs
+++ b/DiscordChatExporter.Core/Exporting/ExportAssetDownloader.cs
@@ -53,44 +53,47 @@ public async ValueTask DownloadAsync(
Directory.CreateDirectory(_workingDirPath);
- await Http.ResiliencePolicy.ExecuteAsync(async () =>
- {
- // Download the file
- using var response = await Http.Client.GetAsync(url, cancellationToken);
- await using (var output = File.Create(filePath))
- await response.Content.CopyToAsync(output, cancellationToken);
-
- // Try to set the file date according to the last-modified header
- try
+ await Http.ResiliencePipeline.ExecuteAsync(
+ async innerCancellationToken =>
{
- var lastModified = response.Content.Headers
- .TryGetValue("Last-Modified")
- ?.Pipe(
- s =>
- DateTimeOffset.TryParse(
- s,
- CultureInfo.InvariantCulture,
- DateTimeStyles.None,
- out var instant
- )
- ? instant
- : (DateTimeOffset?)null
- );
-
- if (lastModified is not null)
+ // Download the file
+ using var response = await Http.Client.GetAsync(url, innerCancellationToken);
+ await using (var output = File.Create(filePath))
+ await response.Content.CopyToAsync(output, innerCancellationToken);
+
+ // Try to set the file date according to the last-modified header
+ try
{
- File.SetCreationTimeUtc(filePath, lastModified.Value.UtcDateTime);
- File.SetLastWriteTimeUtc(filePath, lastModified.Value.UtcDateTime);
- File.SetLastAccessTimeUtc(filePath, lastModified.Value.UtcDateTime);
+ var lastModified = response.Content.Headers
+ .TryGetValue("Last-Modified")
+ ?.Pipe(
+ s =>
+ DateTimeOffset.TryParse(
+ s,
+ CultureInfo.InvariantCulture,
+ DateTimeStyles.None,
+ out var instant
+ )
+ ? instant
+ : (DateTimeOffset?)null
+ );
+
+ if (lastModified is not null)
+ {
+ File.SetCreationTimeUtc(filePath, lastModified.Value.UtcDateTime);
+ File.SetLastWriteTimeUtc(filePath, lastModified.Value.UtcDateTime);
+ File.SetLastAccessTimeUtc(filePath, lastModified.Value.UtcDateTime);
+ }
}
- }
- catch
- {
- // This can apparently fail for some reason.
- // Updating the file date is not a critical task, so we'll just ignore exceptions thrown here.
- // https://github.com/Tyrrrz/DiscordChatExporter/issues/585
- }
- });
+ catch
+ {
+ // This can apparently fail for some reason.
+ // Updating the file date is not a critical task, so we'll just ignore exceptions thrown here.
+ // https://github.com/Tyrrrz/DiscordChatExporter/issues/585
+ }
+ },
+ cancellationToken
+ );
return _previousPathsByUrl[url] = filePath;
}
diff --git a/DiscordChatExporter.Core/Utils/Http.cs b/DiscordChatExporter.Core/Utils/Http.cs
index 041deae6f1..f08984f8eb 100644
--- a/DiscordChatExporter.Core/Utils/Http.cs
+++ b/DiscordChatExporter.Core/Utils/Http.cs
@@ -7,6 +7,7 @@
using System.Threading.Tasks;
using DiscordChatExporter.Core.Utils.Extensions;
using Polly;
+using Polly.Retry;
namespace DiscordChatExporter.Core.Utils;
@@ -31,29 +32,45 @@ ex is TimeoutException or SocketException or AuthenticationException
&& IsRetryableStatusCode(hrex.StatusCode ?? HttpStatusCode.OK)
);
- public static IAsyncPolicy ResiliencePolicy { get; } =
- Policy
- .Handle(IsRetryableException)
- .WaitAndRetryAsync(4, i => TimeSpan.FromSeconds(Math.Pow(2, i) + 1));
+ public static ResiliencePipeline ResiliencePipeline { get; } =
+ new ResiliencePipelineBuilder()
+ .AddRetry(
+ new RetryStrategyOptions
+ {
+ ShouldHandle = new PredicateBuilder().Handle(IsRetryableException),
+ MaxRetryAttempts = 4,
+ BackoffType = DelayBackoffType.Exponential,
+ Delay = TimeSpan.FromSeconds(1)
+ }
+ )
+ .Build();
- public static IAsyncPolicy ResponseResiliencePolicy { get; } =
- Policy
- .Handle(IsRetryableException)
- .OrResult(m => IsRetryableStatusCode(m.StatusCode))
- .WaitAndRetryAsync(
- 8,
- (i, result, _) =>
+ public static ResiliencePipeline ResponseResiliencePipeline { get; } =
+ new ResiliencePipelineBuilder()
+ .AddRetry(
+ new RetryStrategyOptions
{
- // If rate-limited, use retry-after header as the guide.
- // The response can be null here if an exception was thrown.
- if (result.Result?.Headers.RetryAfter?.Delta is { } retryAfter)
+ ShouldHandle = new PredicateBuilder()
+ .Handle(IsRetryableException)
+ .HandleResult(m => IsRetryableStatusCode(m.StatusCode)),
+ MaxRetryAttempts = 8,
+ DelayGenerator = args =>
{
- // Add some buffer just in case
- return retryAfter + TimeSpan.FromSeconds(1);
- }
+ // If rate-limited, use retry-after header as the guide.
+ // The response can be null here if an exception was thrown.
+ if (args.Outcome.Result?.Headers.RetryAfter?.Delta is { } retryAfter)
+ {
+ // Add some buffer just in case
+ return ValueTask.FromResult(
+ retryAfter + TimeSpan.FromSeconds(1)
+ );
+ }
- return TimeSpan.FromSeconds(Math.Pow(2, i) + 1);
- },
- (_, _, _, _) => Task.CompletedTask
- );
+ return ValueTask.FromResult(
+ TimeSpan.FromSeconds(Math.Pow(2, args.AttemptNumber) + 1)
+ );
+ }
+ }
+ )
+ .Build();
}
From 9180d51e5ee5eb24624ba193a4df884a8f9cb020 Mon Sep 17 00:00:00 2001
From: Tyrrrz <1935960+Tyrrrz@users.noreply.github.com>
Date: Thu, 28 Sep 2023 19:30:21 +0300
Subject: [PATCH 4/4] Update version
---
Changelog.md | 5 +++++
Directory.Build.props | 2 +-
2 files changed, 6 insertions(+), 1 deletion(-)
diff --git a/Changelog.md b/Changelog.md
index 26f1b66d3b..2f260fac07 100644
--- a/Changelog.md
+++ b/Changelog.md
@@ -1,5 +1,10 @@
# Changelog
+## v2.41.1 (28-Sep-2023)
+
+- CLI changes:
+ - Fixed an issue where the export failed to export channels with an empty name.
+
## v2.41 (15-Sep-2023)
- General changes:
diff --git a/Directory.Build.props b/Directory.Build.props
index 71398adc5d..07b42a6d7f 100644
--- a/Directory.Build.props
+++ b/Directory.Build.props
@@ -2,7 +2,7 @@
net7.0
- 2.41
+ 2.41.1
Tyrrrz
Copyright (c) Oleksii Holub
preview