More gracefully handle exceeding the API limit.

If another tool that uses the GGG API is used alongside Procurement, it
is very easy to exceed the API limit.  This change more gracefully
handles that case, but is very cautious about it.  If an overflow is
detected, completely fill up the window of active tasks, and wait for at
least as long as the window size.

POEApi.Infrastructure/Events/ThottledEventArgs.cs

@@ -5,9 +5,15 @@ namespace POEApi.Infrastructure.Events
     public class ThottledEventArgs : EventArgs
     {
         public TimeSpan WaitTime { get; private set; }
-        public ThottledEventArgs(TimeSpan waitTime)
+        /// <summary>
+        /// Whether the throttling event was expected.  If it was not expected, there might be other agents or
+        /// untracked actions using up resources towards the limit.
+        /// </summary>
+        public bool Expected { get; private set; }
+        public ThottledEventArgs(TimeSpan waitTime, bool expected = true)
         {
             WaitTime = waitTime;
+            Expected = expected;
         }
     }
 }

POEApi.Transport/HttpTransport.cs

@@ -205,11 +205,26 @@ protected HttpWebResponse BuildHttpRequestAndGetResponse(HttpMethod method, stri
         protected MemoryStream PerformHttpRequest(HttpMethod method, string url, bool? allowAutoRedirects = null,
             string requestData = null)
         {
-            using (var response = BuildHttpRequestAndGetResponse(method, url, allowAutoRedirects, requestData))
+            HttpWebResponse response = null;
+            // TODO: Don't retry an infinite number of times.
+            bool retry = true;
+            while (retry)
             {
-                MemoryStream responseStream = GetMemoryStreamFromResponse(response);
-                return responseStream;
+                try
+                {
+                    response = BuildHttpRequestAndGetResponse(method, url, allowAutoRedirects, requestData);
+                    retry = false;
+                }
+                catch (System.Net.WebException ex) when (
+                    !string.IsNullOrWhiteSpace(ex.Message) && ex.Message.Contains("(429) Too Many Requests."))
+                {
+                    Logger.Log("Exceeded API limit while performing HTTP request: " + ex.ToString());
+                    _taskThrottle.HandleUnexpectedOverload();
+                }
             }
+
+            MemoryStream responseStream = GetMemoryStreamFromResponse(response);
+            return responseStream;
         }
 
         // TODO(20180928): Remove the refresh parameter?

POEApi.Transport/TaskThrottle.cs

@@ -88,5 +88,22 @@ protected void RemvoeExpiredTasks()
                 CurrentTasks.Dequeue();
             }
         }
+
+        public void HandleUnexpectedOverload()
+        {
+            lock (_lockObject)
+            {
+                // We've unexpectedly gone over the number of allowed requests in a time period.  Fill up the set of
+                // current tasks as if we had started tasks.
+                while (CurrentTasks.Count < WindowLimit)
+                    CurrentTasks.Enqueue(DateTime.Now);
+
+                // Wait at least as long as the WindowSize before trying another task.
+                var timeUntilNextTask = CurrentTasks.Peek() - DateTime.Now;
+                TimeSpan waitTime = timeUntilNextTask > WindowSize ? timeUntilNextTask : WindowSize;
+                Throttled?.Invoke(this, new ThottledEventArgs(waitTime, false));  // Not an expected throttle event.
+                System.Threading.Thread.Sleep(waitTime);
+            }
+        }
     }
 }

Procurement/View/RefreshView.xaml.cs

@@ -77,9 +77,14 @@ private void model_StashLoading(POEApi.Model.POEModel sender, POEApi.Model.Event
 
         void model_Throttled(object sender, ThottledEventArgs e)
         {
-            if (e.WaitTime.TotalSeconds > 4)
+            if (!e.Expected)
+                update(string.Format("Exceeded GGG Server request limit; throttling activated.  Waiting {0} " +
+                    "seconds.  Ensure you do not have other instances of Procurement running or other apps using " +
+                    "the GGG API with your account.", Convert.ToInt32(e.WaitTime.TotalSeconds)),
+                    new POEEventArgs(POEEventState.BeforeEvent));
+            else if (e.WaitTime.TotalSeconds > 4)
                 update(string.Format("GGG Server request limit hit, throttling activated. Please wait {0} seconds",
-                    e.WaitTime.Seconds), new POEEventArgs(POEEventState.BeforeEvent));
+                    Convert.ToInt32(e.WaitTime.TotalSeconds)), new POEEventArgs(POEEventState.BeforeEvent));
         }
 
         private void update(string message, POEEventArgs e)

Procurement/ViewModel/LoginWindowViewModel.cs

@@ -337,8 +337,14 @@ void Model_Authenticating(POEModel sender, AuthenticateEventArgs e)
 
         void Model_Throttled(object sender, ThottledEventArgs e)
         {
-            if (e.WaitTime.TotalSeconds > 4)
-                Update(string.Format("GGG Server request limit hit, throttling activated. Please wait {0} seconds", e.WaitTime.Seconds), new POEEventArgs(POEEventState.BeforeEvent));
+            if (!e.Expected)
+                Update(string.Format("Exceeded GGG Server request limit; throttling activated.  Waiting {0} " +
+                    "seconds.  Ensure you do not have other instances of Procurement running or other apps using " +
+                    "the GGG API with your account.", Convert.ToInt32(e.WaitTime.TotalSeconds)),
+                    new POEEventArgs(POEEventState.BeforeEvent));
+            else if (e.WaitTime.TotalSeconds > 4)
+                Update(string.Format("GGG Server request limit hit, throttling activated. Please wait {0} seconds",
+                    Convert.ToInt32(e.WaitTime.TotalSeconds)), new POEEventArgs(POEEventState.BeforeEvent));
         }
 
         private void Update(string message, POEEventArgs e)