Overcome "Connection Refused" on some operations

.travis.yml

@@ -51,10 +51,10 @@ install:
   - pip install virtualenv
   - python -m virtualenv ~/.venv
   - source ~/.venv/bin/activate
+  - pip install -r requirements-test.txt
   - pip install coveralls
   - pip install .[credssp]
   - pip install .[kerberos]
-  - pip install -r requirements-test.txt
 script:
   - py.test -v --pep8 --cov=winrm --cov-report=term-missing winrm/tests/
 

requirements-test.txt

@@ -4,3 +4,4 @@ pytest<=3.2.5
 pytest-cov
 pytest-pep8
 mock
+pycparser<=2.18; python_version<"2.7"

winrm/protocol.py

@@ -26,6 +26,8 @@ class Protocol(object):
     DEFAULT_OPERATION_TIMEOUT_SEC = 20
     DEFAULT_MAX_ENV_SIZE = 153600
     DEFAULT_LOCALE = 'en-US'
+    DEFAULT_RECONNECTION_RETRIES = 0
+    DEFAULT_RECONNECTION_BACKOFF = 2.0
 
     def __init__(
             self, endpoint, transport='plaintext', username=None,
@@ -39,6 +41,8 @@ def __init__(
             message_encryption='auto',
             credssp_disable_tlsv1_2=False,
             send_cbt=True,
+            reconnection_retries = DEFAULT_RECONNECTION_RETRIES,
+            reconnection_backoff = DEFAULT_RECONNECTION_BACKOFF,
         ):
         """
         @param string endpoint: the WinRM webservice endpoint
@@ -57,6 +61,8 @@ def __init__(
         @param int operation_timeout_sec: maximum allowed time in seconds for any single wsman HTTP operation (default 20). Note that operation timeouts while receiving output (the only wsman operation that should take any significant time, and where these timeouts are expected) will be silently retried indefinitely. # NOQA
         @param string kerberos_hostname_override: the hostname to use for the kerberos exchange (defaults to the hostname in the endpoint URL)
         @param bool message_encryption_enabled: Will encrypt the WinRM messages if set to True and the transport auth supports message encryption (Default True).
+        @param int reconnection_retries: Number of retries on connection problems
+        @param float reconnection_backoff: Number of seconds to backoff in between reconnection attempts (first sleeps X, then sleeps 2*X, then sleeps 4*X, ...)
         """
 
         try:
@@ -88,7 +94,9 @@ def __init__(
             auth_method=transport,
             message_encryption=message_encryption,
             credssp_disable_tlsv1_2=credssp_disable_tlsv1_2,
-            send_cbt=send_cbt
+            send_cbt=send_cbt,
+            reconnection_retries=reconnection_retries,
+            reconnection_backoff=reconnection_backoff,
         )
 
         self.username = username
@@ -100,6 +108,8 @@ def __init__(
         self.kerberos_delegation = kerberos_delegation
         self.kerberos_hostname_override = kerberos_hostname_override
         self.credssp_disable_tlsv1_2 = credssp_disable_tlsv1_2
+        self.reconnection_retries = reconnection_retries
+        self.reconnection_backoff = reconnection_backoff
 
     def open_shell(self, i_stream='stdin', o_stream='stdout stderr',
                    working_directory=None, env_vars=None, noprofile=False,

winrm/transport.py

@@ -62,7 +62,10 @@ def __init__(
             auth_method='auto',
             message_encryption='auto',
             credssp_disable_tlsv1_2=False,
-            send_cbt=True):
+            send_cbt=True,
+            reconnection_retries=0,
+            reconnection_backoff=2.0,
+        ):
         self.endpoint = endpoint
         self.username = username
         self.password = password
@@ -78,6 +81,8 @@ def __init__(
         self.message_encryption = message_encryption
         self.credssp_disable_tlsv1_2 = credssp_disable_tlsv1_2
         self.send_cbt = send_cbt
+        self.reconnection_retries = reconnection_retries
+        self.reconnection_backoff = reconnection_backoff
 
         if self.server_cert_validation not in [None, 'validate', 'ignore']:
             raise WinRMError('invalid server_cert_validation mode: %s' % self.server_cert_validation)
@@ -113,7 +118,7 @@ def __init__(
                 from requests.packages.urllib3.exceptions import InsecureRequestWarning
                 warnings.simplefilter('ignore', category=InsecureRequestWarning)
             except: pass # oh well, we tried...
-            
+
             try:
                 from urllib3.exceptions import InsecureRequestWarning
                 warnings.simplefilter('ignore', category=InsecureRequestWarning)
@@ -152,6 +157,16 @@ def build_session(self):
         settings = session.merge_environment_settings(url=self.endpoint, proxies={}, stream=None,
                                                       verify=None, cert=None)
 
+        # Retry on connection errors, with a backoff factor
+        retries = requests.packages.urllib3.util.retry.Retry(total=self.reconnection_retries,
+                                                             connect=self.reconnection_retries,
+                                                             status=self.reconnection_retries,
+                                                             read=0,
+                                                             backoff_factor=self.reconnection_backoff,
+                                                             status_forcelist=(425, 429, 503))
+        session.mount('http://', requests.adapters.HTTPAdapter(max_retries=retries))
+        session.mount('https://', requests.adapters.HTTPAdapter(max_retries=retries))
+
         # get proxy settings from env
         # FUTURE: allow proxy to be passed in directly to supersede this value
         session.proxies = settings['proxies']
@@ -271,7 +286,6 @@ def _send_message_request(self, prepared_request, message):
             else:
                 response_text = ''
 
-
             raise WinRMTransportError('http', ex.response.status_code, response_text)
 
     def _get_message_response_text(self, response):