Skip to content

Commit

Permalink
chore: Update Dotty to use Nuget.Protocol package, allow config to ig…
Browse files Browse the repository at this point in the history 
…nore patch or minor version bumps
  • Loading branch information
@tippmar-nr
tippmar-nr committed Jul 11, 2024
1 parent 82f601d commit 219e1e1
Show file tree
Hide file tree
Showing 4 changed files with 124 additions and 140 deletions.
33 changes: 1 addition & 32 deletions .github/workflows/nuget_slack_notifications.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,35 +71,4 @@ jobs:
NEW_RELIC_APP_NAME: Dotty
NEW_RELIC_HOST: staging-collector.newrelic.com
NEW_RELIC_LICENSE_KEY: ${{ secrets.STAGING_LICENSE_KEY }}
nugets:
"amazon.lambda.apigatewayevents
amazon.lambda.applicationloadbalancerevents
amazon.lambda.cloudwatchevents
amazon.lambda.dynamodbevents
amazon.lambda.kinesisevents
amazon.lambda.kinesisfirehoseevents
amazon.lambda.s3events
amazon.lambda.simpleemailevents
amazon.lambda.snsevents
amazon.lambda.sqsevents
elasticsearch.net
elastic.clients.elasticsearch
log4net
microsoft.extensions.logging
microsoft.data.sqlclient
microsoft.net.http
mongodb.driver
mysql.data
mysqlconnector
nest
nlog
rabbitmq.client
restsharp
serilog
serilog.extensions.logging
serilog.aspnetcore
serilog.sinks.file
serilog.sinks.console
stackexchange.redis
system.data.sqlclient"


194 changes: 87 additions & 107 deletions .github/workflows/scripts/nugetSlackNotifications/Program.cs
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,21 @@
using NewRelic.Api.Agent;
using Octokit;
//using Octokit;
using Serilog;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading.Tasks;

#pragma warning disable CS8618
using NuGet.Common;
using NuGet.Configuration;
using NuGet.Protocol;
using NuGet.Protocol.Core.Types;
using Octokit;
using Repository = NuGet.Protocol.Core.Types.Repository;

namespace nugetSlackNotifications
{
Expand All @@ -22,66 +27,102 @@ public class Program

private static readonly int _daysToSearch = int.TryParse(Environment.GetEnvironmentVariable("DOTTY_DAYS_TO_SEARCH"), out var days) ? days : 1; // How many days of package release history to scan for changes
private static readonly bool _testMode = bool.TryParse(Environment.GetEnvironmentVariable("DOTTY_TEST_MODE"), out var testMode) ? testMode : false;
private static readonly string? _webhook = Environment.GetEnvironmentVariable("DOTTY_WEBHOOK");
private static readonly string? _githubToken = Environment.GetEnvironmentVariable("DOTTY_TOKEN");
private static readonly string _webhook = Environment.GetEnvironmentVariable("DOTTY_WEBHOOK");
private static readonly string _githubToken = Environment.GetEnvironmentVariable("DOTTY_TOKEN");


static async Task Main(string[] args)
static async Task Main()
{
Log.Logger = new LoggerConfiguration().WriteTo.Console().CreateLogger();

foreach (string packageName in args)
// initialize nuget repo
var ps = new PackageSource("https://api.nuget.org/v3/index.json");
var sourceRepository = Repository.Factory.GetCoreV3(ps);
var metadataResource = await sourceRepository.GetResourceAsync<PackageMetadataResource>();
var sourceCacheContext = new SourceCacheContext();

if (!System.IO.File.Exists("packages.json"))
{
Log.Error("packages.json not found in the current directory. Exiting.");
return;
}

var packagesJson = await System.IO.File.ReadAllTextAsync("packages.json");
var packages = JsonSerializer.Deserialize<PackageInfo[]>(packagesJson);

foreach (var package in packages)
{
try
{
await CheckPackage(packageName);
await CheckPackage(package, metadataResource, sourceCacheContext);
}
catch (Exception ex)
{
Log.Error(ex, $"Caught exception while checking {packageName} for updates.");
await SendSlackNotification($"Dotty: caught exception while checking {packageName} for updates: {ex}");
Log.Error(ex, $"Caught exception while checking {package.PackageName} for updates.");
await SendSlackNotification($"Dotty: caught exception while checking {package.PackageName} for updates: {ex}");
}
}

await AlertOnNewVersions();
await CreateGithubIssuesForNewVersions();

}

[Transaction]
static async Task CheckPackage(string packageName)
static async Task CheckPackage(PackageInfo package, PackageMetadataResource metadataResource, SourceCacheContext sourceCacheContext)
{
var response = await _client.GetStringAsync($"https://api.nuget.org/v3/registration5-gz-semver2/{packageName}/index.json");
var packageName = package.PackageName;

var metaData = (await metadataResource.GetMetadataAsync(packageName, false, false, sourceCacheContext, NullLogger.Instance, System.Threading.CancellationToken.None)).OrderByDescending(p => p.Identity.Version).ToList();

SearchResult? searchResult = JsonSerializer.Deserialize<SearchResult>(response);
if (searchResult is null)
if (!metaData.Any())
{
Log.Warning($"CheckPackage: null search result for package {packageName}");
Log.Warning($"CheckPackage: No metadata found for package {packageName}");
return;
}

Item item = searchResult.items[^1]; // get the most recent group
// get the most recent version of the package
var latest = metaData.First();
packageName = latest.Identity.Id;

// need to make another request to get the page with individual release listings
Page? page = JsonSerializer.Deserialize<Page>(await _client.GetStringAsync(item.id));
if (page is null)
// get the second most recent version of the package (if there is one)
var previous = metaData.Skip(1).FirstOrDefault();

// see if it was published within the last _daysToSearch days
if (latest.Published > DateTime.Today.AddDays(-_daysToSearch))
{
Log.Warning($"CheckPackage: null page result for package {packageName}, item id {item.id}");
return;
}
if (previous != null && (package.IgnorePatch || package.IgnoreMinor))
{
var previousVersion = previous.Identity.Version;
var latestVersion = latest.Identity.Version;

// need to get the most recent and previous catalog entries to display previous and new version
Catalogentry? latestCatalogEntry = GetCatalogentry(packageName, page, 1);
Catalogentry? previousCatalogEntry = GetCatalogentry(packageName, page, 2);
if (package.IgnorePatch)
{
if (previousVersion.Major == latestVersion.Major && previousVersion.Minor == latestVersion.Minor)
{
Log.Information($"Package {packageName} ignores Patch version updates; the Minor version ({latestVersion.Major}.{latestVersion.Minor:2}) has not been updated in the past {_daysToSearch} days.");
return;
}
}

if (package.IgnoreMinor)
{

if (latestCatalogEntry?.published > DateTime.Now.AddDays(-_daysToSearch) && !await latestCatalogEntry.isPrerelease())
{
Log.Information($"Package {packageName} has been updated in the past {_daysToSearch} days.");
_newVersions.Add(new NugetVersionData(packageName, previousCatalogEntry?.version ?? "Unknown", latestCatalogEntry.version, $"https://www.nuget.org/packages/{packageName}/"));
if (previousVersion.Major == latestVersion.Major)
{
Log.Information($"Package {packageName} ignores Minor version updates; the Major version ({latestVersion.Major}) has not been updated in the past {_daysToSearch} days.");
return;
}
}
}

var previousVersionDescription = previous?.Identity.Version.ToNormalizedString() ?? "Unknown";
var latestVersionDescription = latest.Identity.Version.ToNormalizedString();
Log.Information($"Package {packageName} was updated from {previousVersionDescription} to {latestVersionDescription} on {latest.Published.Value.Date.ToShortDateString()}.");
_newVersions.Add(new NugetVersionData(packageName, previousVersionDescription, latestVersionDescription, latest.PackageDetailsUrl.ToString(), latest.Published.Value.Date));
}
else
{
Log.Information($"Package {packageName} has not been updated in the past {_daysToSearch} days.");
Log.Information($"Package {packageName} has NOT been updated in the past {_daysToSearch} days.");
}
}

Expand All @@ -91,7 +132,7 @@ static async Task AlertOnNewVersions()

if (_newVersions.Count > 0 && _webhook != null && !_testMode) // only message channel if there's package updates to report AND we have a webhook from the environment AND we're not in test mode
{
string msg = "Hi team! Dotty here :technologist::pager:\nThere's some new NuGet releases you should know about :arrow_heading_down::sparkles:";
var msg = "Hi team! Dotty here :technologist::pager:\nThere's some new NuGet releases you should know about :arrow_heading_down::sparkles:";
foreach (var versionData in _newVersions)
{
msg += $"\n\t:package: {char.ToUpper(versionData.PackageName[0]) + versionData.PackageName[1..]} {versionData.OldVersion} :point_right: <{versionData.Url}|{versionData.NewVersion}>";
Expand All @@ -117,8 +158,10 @@ static async Task CreateGithubIssuesForNewVersions()
ghClient.Credentials = tokenAuth;
foreach (var versionData in _newVersions)
{
var newIssue = new NewIssue($"Dotty: update tests for {versionData.PackageName} from {versionData.OldVersion} to {versionData.NewVersion}");
newIssue.Body = versionData.Url;
var newIssue = new NewIssue($"Dotty: update tests for {versionData.PackageName} from {versionData.OldVersion} to {versionData.NewVersion}")
{
Body = versionData.Url
};
newIssue.Labels.Add("testing");
newIssue.Labels.Add("Core Technologies");
var issue = await ghClient.Issue.Create("newrelic", "newrelic-dotnet-agent", newIssue);
Expand Down Expand Up @@ -160,29 +203,6 @@ static async Task SendSlackNotification(string msg)
Log.Error($"SendSlackNotification called but _webhook is null. msg={msg}");
}
}

[Trace]
static Catalogentry? GetCatalogentry(string packageName, Page page, int releaseIndex)
{
// release index = 1 for most recent release, 2 for the previous release, etc.
try
{
// alternative json structure (see mysql.data)
if (page.items[^1].catalogEntry is null)
{
return page.items[^1].items[^releaseIndex].catalogEntry; // latest release
}
else // standard structure
{
return page.items[^releaseIndex].catalogEntry;
}
}
catch (IndexOutOfRangeException)
{
Log.Warning($"GetCatalogEntry: array index issue for package {packageName} and releaseIndex {releaseIndex}");
return null;
}
}
}

public class NugetVersionData
Expand All @@ -191,65 +211,25 @@ public class NugetVersionData
public string OldVersion { get; set; }
public string NewVersion { get; set; }
public string Url { get; set; }
public DateTime PublishDate { get; set; }

public NugetVersionData(string packageName, string oldVersion, string newVersion, string url)
public NugetVersionData(string packageName, string oldVersion, string newVersion, string url, DateTime publishDate)
{
PackageName = packageName;
OldVersion = oldVersion;
NewVersion = newVersion;
Url = url;
PublishDate = publishDate;
}
}

public class SearchResult
{
public int count { get; set; }
public Item[] items { get; set; }
}

public class Item
{
[JsonPropertyName("@id")]
public string id { get; set; }
public DateTime commitTimeStamp { get; set; }
}

public class Page
{
public Release[] items { get; set; }
}

public class Release
public class PackageInfo
{
[JsonPropertyName("@id")]
public string id { get; set; }
public Catalogentry catalogEntry { get; set; }

// only packages with alternative json structure (i.e. mysql.data) will have this
public Release[] items { get; set; }
}

public class Version
{
public bool isPrerelease { get; set; }
}

public class Catalogentry
{
[JsonPropertyName("@id")]
public string id { get; set; }
public DateTime published { get; set; }
public string version { get; set; }

public async Task<bool> isPrerelease()
{
HttpClient client = new();
string response = await client.GetStringAsync(id);
Version? version = JsonSerializer.Deserialize<Version>(response);

return version is null ? false : version.isPrerelease;
}
[JsonPropertyName("packageName")]
public string PackageName { get; set; }
[JsonPropertyName("ignorePatch")]
public bool IgnorePatch { get; set; }
[JsonPropertyName("ignoreMinor")]
public bool IgnoreMinor { get; set; }
}
}

#pragma warning restore CS8618
5 changes: 4 additions & 1 deletion .github/workflows/scripts/nugetSlackNotifications/nugetSlackNotifications.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,20 +3,23 @@
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net8.0</TargetFramework>
<Nullable>enable</Nullable>
</PropertyGroup>

<ItemGroup>
<PackageReference Include="NewRelic.Agent.Api" Version="10.26.0" />
<PackageReference Include="Octokit" Version="13.0.1" />
<PackageReference Include="Serilog" Version="4.0.0" />
<PackageReference Include="Serilog.Sinks.Console" Version="6.0.0" />
<PackageReference Include="Nuget.Protocol" Version="6.10.1" />
</ItemGroup>

<ItemGroup>
<None Update="newrelic.config">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
<None Update="packages.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
</ItemGroup>

</Project>
32 changes: 32 additions & 0 deletions .github/workflows/scripts/nugetSlackNotifications/packages.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
[
{ "packageName": "amazon.lambda.apigatewayevents", "ignorePatch": false, "ignoreMinor": false },
{ "packageName": "amazon.lambda.applicationloadbalancerevents", "ignorePatch": false, "ignoreMinor": false },
{ "packageName": "amazon.lambda.cloudwatchevents", "ignorePatch": false, "ignoreMinor": false },
{ "packageName": "amazon.lambda.dynamodbevents", "ignorePatch": false, "ignoreMinor": false },
{ "packageName": "amazon.lambda.kinesisevents", "ignorePatch": false, "ignoreMinor": false },
{ "packageName": "amazon.lambda.kinesisfirehoseevents", "ignorePatch": false, "ignoreMinor": false },
{ "packageName": "amazon.lambda.s3events", "ignorePatch": false, "ignoreMinor": false },
{ "packageName": "amazon.lambda.simpleemailevents", "ignorePatch": false, "ignoreMinor": false },
{ "packageName": "amazon.lambda.snsevents", "ignorePatch": false, "ignoreMinor": false },
{ "packageName": "amazon.lambda.sqsevents", "ignorePatch": false, "ignoreMinor": false },
{ "packageName": "elasticsearch.net", "ignorePatch": false, "ignoreMinor": false },
{ "packageName": "elastic.clients.elasticsearch", "ignorePatch": true, "ignoreMinor": false },
{ "packageName": "log4net", "ignorePatch": false, "ignoreMinor": false },
{ "packageName": "microsoft.extensions.logging", "ignorePatch": false, "ignoreMinor": false },
{ "packageName": "microsoft.data.sqlclient", "ignorePatch": false, "ignoreMinor": false },
{ "packageName": "microsoft.net.http", "ignorePatch": false, "ignoreMinor": false },
{ "packageName": "mongodb.driver", "ignorePatch": false, "ignoreMinor": false },
{ "packageName": "mysql.data", "ignorePatch": false, "ignoreMinor": false },
{ "packageName": "mysqlconnector", "ignorePatch": false, "ignoreMinor": false },
{ "packageName": "nest", "ignorePatch": false, "ignoreMinor": false },
{ "packageName": "nlog", "ignorePatch": false, "ignoreMinor": false },
{ "packageName": "rabbitmq.client", "ignorePatch": false, "ignoreMinor": false },
{ "packageName": "restsharp", "ignorePatch": false, "ignoreMinor": false },
{ "packageName": "serilog", "ignorePatch": false, "ignoreMinor": false },
{ "packageName": "serilog.extensions.logging", "ignorePatch": false, "ignoreMinor": false },
{ "packageName": "serilog.aspnetcore", "ignorePatch": false, "ignoreMinor": false },
{ "packageName": "serilog.sinks.file", "ignorePatch": false, "ignoreMinor": false },
{ "packageName": "serilog.sinks.console", "ignorePatch": false, "ignoreMinor": false },
{ "packageName": "stackexchange.redis", "ignorePatch": false, "ignoreMinor": true },
{ "packageName": "system.data.sqlclient", "ignorePatch": false, "ignoreMinor": false}
]

0 comments on commit 219e1e1

Please sign in to comment.