Skip to content

Commit

Permalink
Fixes ENWS parser not working in full sets
Browse files Browse the repository at this point in the history 
Fixes regression caused by a fixed bug in ENWS; should now properly request for all pages in the website.
Updates DeckLog JP to 20221115.002.
  • Loading branch information
@ronelm2000
ronelm2000 committed Nov 16, 2022
1 parent 9eb3a15 commit 7ad999c
Show file tree
Hide file tree
Showing 4 changed files with 70 additions and 33 deletions.
12 changes: 7 additions & 5 deletions Montage.Weiss.Tools.Test/ENWS/ParserTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ namespace Montage.Weiss.Tools.Test.ENWS
public class ParserTests
{
[TestMethod("EN WS Parser Test")]
[Ignore("Currently not working due to a change in site. WIP")]
public async Task TestParser()
{
Serilog.Log.Logger = TestUtils.BootstrapLogging().CreateLogger();
Expand All @@ -29,11 +28,14 @@ public async Task TestParser()
Log.Information("Card: {@card}", card.Serial);

url = "https://en.ws-tcg.com/cardlist/list/?cardno=FS/S36-E018";
list = await new EnglishWSURLParser().Parse(url, progressReporter, CancellationToken.None).ToListAsync();
Log.Information("Cards Obtained: {length}", list.Count);
var dict = await new EnglishWSURLParser().Parse(url, progressReporter, CancellationToken.None)
.ToDictionaryAsync(c => c.Serial, c => c);
Log.Information("Cards Obtained: {length}", dict.Keys.Count);

foreach (var card in list)
Log.Information("Card: {@card}", card.Serial);
Action<string> serialAssertions = serial => Assert.IsTrue(dict.ContainsKey(serial), $"Could not find {serial} in output.");
serialAssertions("FS/S36-PE02");
serialAssertions("FS/S36-E012");
serialAssertions("FS/S36-PE01");
}
}
}
2 changes: 1 addition & 1 deletion MontageWeissTools/Entities/External/DeckLog/DeckLogSettings.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ namespace Montage.Weiss.Tools.Entities.External.DeckLog;
public class DeckLogSettings
{
public CardLanguage Language { get; set; } = CardLanguage.Japanese;
public string Version { get; set; } = "20221013.001";
public string Version { get; set; } = "20221115.002";
public string Authority { get; set; } = "https://decklog.bushiroad.com/";
public Regex DeckURLMatcher { get; set; } = new Regex(@"(.*):\/\/decklog\.bushiroad\.com\/view\/([^\?]*)(.*)");
public string VersionURL { get; set; } = "https://decklog.bushiroad.com/system/app/api/version/";
Expand Down
85 changes: 58 additions & 27 deletions MontageWeissTools/Impls/Parsers/Cards/EnglishWSURLParser.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,58 +137,89 @@ public async IAsyncEnumerable<WeissSchwarzCard> Parse(string urlOrLocalFile, IPr
await _WS_SEARCH_PAGE.WithClient(fc).WithHTMLHeaders().WithCookies(cs).GetAsync(cancellationToken); // To get some initial cookies.
var serial = uri.Query.Substring(_CARD_NO_QUERY.Length);
var serialID = WeissSchwarzCard.ParseSerial(serial);
var wsSearchPage = await _WS_SEARCH_PAGE_EXEC

Log.Debug("Cookie: {@c}", cs.Cookies.ToDictionary(k => k.GetKey(), k=> k.Value));

var wsSearchPage = await cs
.Request(_WS_SEARCH_PAGE_EXEC)
.WithClient(fc)
.WithCookies(cs)
.WithHTMLHeaders()
.WithHeader("Referer", _WS_SEARCH_PAGE)
.PostUrlEncodedAsync(new Dictionary<string,object>{
["cmd"] = "search",
["data[CardSearch][keyword]"] = serialID.ReleaseID,
["data[CardSearch][keyword_or]"] = "",
["data[CardSearch][keyword_not]"] = "",
["data[CardSearch][keyword_cardname]"] = "0",
["data[CardSearch][keyword_feature]"] = "0",
["data[CardSearch][keyword_text]"] = "0",
["data[CardSearch][keyword_cardnumber]"] = new[] { 0, 1 },
["data[CardSearch][expansion]"] = "",
["data[CardSearch][card_kind]"] = "",
["data[CardSearch][level_s]"] = "",
["data[CardSearch][level_e]"] = "",
["data[CardSearch][color]"] = "",
["data[CardSearch][soul_s]"] = "",
["data[CardSearch][soul_e]"] = "",
["data[CardSearch][cost_s]"] = "",
["data[CardSearch][cost_e]"] = "",
["data[CardSearch][trigger]"] = "",
["data[CardSearch][option_counter]"] = "0",
["data[CardSearch][option_clock]"] = "0",
["data[CardSearch][show_page_count]"] = "500",
["data[CardSearch][show_small]"] = "0",
["keyword"] = serialID.ReleaseID,
["keyword_or"] = "",
["keyword_not"] = "",
["keyword_cardname"] = new[] { "0", "1" },
["keyword_feature"] = new[] { "0", "1" },
["keyword_text"] = new[] { "0", "1" },
["keyword_cardnumber"] = new[] { "0", "1" },
["expansion"] = "",
["card_kind"] = "",
["level_s"] = "",
["level_e"] = "",
["color"] = "",
["soul_s"] = "",
["soul_e"] = "",
["cost_s"] = "",
["cost_e"] = "",
["trigger"] = "",
["option_counter"] = "0",
["option_clock"] = "0",
["show_page_count"] = "500",
["show_small"] = "0",
["button"] = "search"
})
.RecieveHTML();

var divsToProcess = wsSearchPage.QuerySelectorAll(_CARD_UNIT_SELECTOR);
var divsToProcess = wsSearchPage.QuerySelectorAll(_CARD_UNIT_SELECTOR).ToAsyncEnumerable();
var pageResultDiv = wsSearchPage.QuerySelector<IHtmlAnchorElement>(".pageLink :nth-last-child(2)")?.InnerHtml ?? null;
var resultCountString = wsSearchPage.QuerySelector("#exFilterForm ~ p").GetInnerText();

Func<int,ValueTask<IDocument>> followupDivs = async p => await cs
.Request(_WS_SEARCH_PAGE_EXEC)
.SetQueryParams(new
{
page = p
})
.WithClient(fc)
.WithHTMLHeaders()
.WithHeader("Referer", _WS_SEARCH_PAGE_EXEC)
.WithHeader("Cache-Control", "no-cache")
.WithHeader("Sec-Fetch-Site", "same-origin")
.WithHeader("Sec-Fetch-User", "?1")
.GetHTMLAsync(cancellationToken);

if (pageResultDiv is not null && int.TryParse(pageResultDiv, out int lastPage))
divsToProcess = Enumerable.Range(2, lastPage - 1)
.ToAsyncEnumerable()
.SelectAwait(followupDivs)
.Do(idoc => {
Log.Debug("Cookie: {@c}", cs.Cookies.ToDictionary(k => k.GetKey(), k => k.Value));
})
.SelectMany(html => html.QuerySelectorAll(_CARD_UNIT_SELECTOR).ToAsyncEnumerable())
.Concat(divsToProcess);


progressReport = progressReport with
{
ReportMessage = new MultiLanguageString { EN = $"(Possibly) Obtained [{divsToProcess.Length}] cards." },
ReportMessage = new MultiLanguageString { EN = $"Got {resultCountString}" },
Percentage = 10
};
progress.Report(progressReport);

foreach (var cardUnitTD in divsToProcess)
await foreach (var cardUnitTD in divsToProcess.WithCancellation(cancellationToken))
{
cancellationToken.ThrowIfCancellationRequested();
var result = await ParseCardAsync(cardUnitTD);
/*
progressReport = progressReport with
{
ReportMessage = new MultiLanguageString { EN = $"Parsed [{result.Serial}]." },
Percentage = 10 + (int)((progressReport.CardsParsed + 1f) * 90 / divsToProcess.Length),
CardsParsed = progressReport.CardsParsed + 1
};
progress.Report(progressReport);
*/
yield return result;
}

Expand Down
4 changes: 4 additions & 0 deletions MontageWeissTools/Properties/launchSettings.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,10 @@
"Export HHW via Blake": {
"commandName": "Project",
"commandLineArgs": "export https://www.encoredecks.com/deck/OkU82A_7E --exporter blake"
},
"Fetch BNJ": {
"commandName": "Project",
"commandLineArgs": "fetch BNJ"
}
}
}

0 comments on commit 7ad999c

Please sign in to comment.