Python源码示例:aiohttp.ServerDisconnectedError()
示例1
def add_safe(self, name, url, author_id):
"""Try to add an emote. Returns a string that should be sent to the user."""
if not re.fullmatch(r'\w{2,32}', name, re.ASCII):
return _(
'{name} is not a valid emote name; use 2–32 English letters, numbers and underscores.'
).format(name=discord.utils.escape_mentions(name))
try:
emote = await self.add_from_url(name, url, author_id)
except discord.HTTPException as ex:
return (
_('An error occurred while creating the emote:\n')
+ utils.format_http_exception(ex))
except ValueError:
return _('Error: Invalid URL.')
except aiohttp.ServerDisconnectedError:
return _('Error: The connection was closed early by the remote host.')
except aiohttp.ClientResponseError as exc:
raise errors.HTTPException(exc.status)
else:
return _('Emote {emote} successfully created.').format(emote=emote)
示例2
def test_single_proxy(self, proxy):
"""
text one proxy, if valid, put them to usable_proxies.
"""
try:
async with aiohttp.ClientSession() as session:
try:
if isinstance(proxy, bytes):
proxy = proxy.decode('utf-8')
real_proxy = 'http://' + proxy
print('Testing', proxy)
async with session.get(self.test_api, proxy=real_proxy, timeout=get_proxy_timeout) as response:
if response.status == 200:
self._conn.put(proxy)
print('Valid proxy', proxy)
except (ProxyConnectionError, TimeoutError, ValueError):
print('Invalid proxy', proxy)
except (ServerDisconnectedError, ClientResponseError,ClientConnectorError) as s:
print(s)
pass
示例3
def fetch(self):
while True:
try:
hdrlen = constants.STREAM_HEADER_SIZE_BYTES
header = yield from self._response.content.readexactly(hdrlen)
_, length = struct.unpack(">BxxxL", header)
if not length:
continue
data = yield from self._response.content.readexactly(length)
except (
aiohttp.ClientConnectionError,
aiohttp.ServerDisconnectedError,
asyncio.IncompleteReadError,
):
break
return data
示例4
def _get(self, url, data=None, headers=None, method='GET'):
page = ''
try:
timeout = aiohttp.ClientTimeout(total=self._timeout)
async with self._sem_provider, self._session.request(
method, url, data=data, headers=headers, timeout=timeout
) as resp:
page = await resp.text()
if resp.status != 200:
log.debug(
'url: %s\nheaders: %s\ncookies: %s\npage:\n%s'
% (url, resp.headers, resp.cookies, page)
)
raise BadStatusError('Status: %s' % resp.status)
except (
UnicodeDecodeError,
BadStatusError,
asyncio.TimeoutError,
aiohttp.ClientOSError,
aiohttp.ClientResponseError,
aiohttp.ServerDisconnectedError,
) as e:
page = ''
log.debug('%s is failed. Error: %r;' % (url, e))
return page
示例5
def fetch(self, url):
"""Fetch request."""
error_msg = None
try:
async with aiohttp.ClientSession() as session:
body = await self.fetch_with_session(session, url)
except asyncio.TimeoutError:
error_msg = 'Request timed out'
raise ClashRoyaleAPIError(message=error_msg)
except aiohttp.ServerDisconnectedError as err:
error_msg = 'Server disconnected error: {}'.format(err)
raise ClashRoyaleAPIError(message=error_msg)
except (aiohttp.ClientError, ValueError) as err:
error_msg = 'Request connection error: {}'.format(err)
raise ClashRoyaleAPIError(message=error_msg)
except json.JSONDecodeError:
error_msg = "Non JSON returned"
raise ClashRoyaleAPIError(message=error_msg)
else:
return body
finally:
if error_msg is not None:
raise ClashRoyaleAPIError(message=error_msg)
示例6
def target_fetch(url, headers, timeout=15):
"""
:param url: target url
:return: text
"""
with async_timeout.timeout(timeout):
try:
async with aiohttp.ClientSession() as client:
async with client.get(url, headers=headers) as response:
assert response.status == 200
LOGGER.info('Task url: {}'.format(response.url))
try:
text = await response.text()
except:
try:
text = await response.read()
except aiohttp.ServerDisconnectedError as e:
LOGGER.exception(e)
text = None
return text
except Exception as e:
LOGGER.exception(str(e))
return None
示例7
def _arequest(self, url, **params):
method = params.get('method', 'GET')
json_data = params.get('json', {})
timeout = params.pop('timeout', None) or self.timeout
try:
async with self.session.request(
method, url, timeout=timeout, headers=self.headers, params=params, data=json_data
) as resp:
return self._raise_for_status(resp, await resp.text())
except asyncio.TimeoutError:
raise NotResponding
except aiohttp.ServerDisconnectedError:
raise NetworkError
示例8
def _arequest(self, url, **params):
timeout = params.pop('timeout', None) or self.timeout
try:
async with self.session.get(url, timeout=timeout, headers=self.headers, params=params) as resp:
return self._raise_for_status(resp, await resp.text())
except asyncio.TimeoutError:
raise NotResponding
except aiohttp.ServerDisconnectedError:
raise NetworkError
示例9
def fail_with_disconnected_error():
raise aiohttp.ServerDisconnectedError("Darn it, can't connect")
示例10
def handle_exception():
"""
Context manager translating network related exceptions
to custom :mod:`~galaxy.api.errors`.
"""
try:
yield
except asyncio.TimeoutError:
raise BackendTimeout()
except aiohttp.ServerDisconnectedError:
raise BackendNotAvailable()
except aiohttp.ClientConnectionError:
raise NetworkError()
except aiohttp.ContentTypeError:
raise UnknownBackendResponse()
except aiohttp.ClientResponseError as error:
if error.status == HTTPStatus.UNAUTHORIZED:
raise AuthenticationRequired()
if error.status == HTTPStatus.FORBIDDEN:
raise AccessDenied()
if error.status == HTTPStatus.SERVICE_UNAVAILABLE:
raise BackendNotAvailable()
if error.status == HTTPStatus.TOO_MANY_REQUESTS:
raise TooManyRequests()
if error.status >= 500:
raise BackendError()
if error.status >= 400:
logging.warning(
"Got status %d while performing %s request for %s",
error.status, error.request_info.method, str(error.request_info.url)
)
raise UnknownError()
except aiohttp.ClientError:
logging.exception("Caught exception while performing request")
raise UnknownError()
示例11
def __anext__(self):
while True:
try:
data = yield from self._response.content.readline()
if not data:
break
except (aiohttp.ClientConnectionError, aiohttp.ServerDisconnectedError):
break
return self._transform(json.loads(data.decode("utf8")))
raise StopAsyncIteration
示例12
def run(self, **params):
if self.response:
warnings.warn("already running", RuntimeWarning, stackelevel=2)
return
forced_params = {"follow": True}
default_params = {"stdout": True, "stderr": True}
params = ChainMap(forced_params, params, default_params)
try:
self.response = await self.docker._query(
"containers/{self.container._id}/logs".format(self=self), params=params
)
while True:
msg = await self.response.content.readline()
if not msg:
break
await self.channel.publish(msg)
except (aiohttp.ClientConnectionError, aiohttp.ServerDisconnectedError):
pass
finally:
# signal termination to subscribers
await self.channel.publish(None)
try:
await self.response.release()
except Exception:
pass
self.response = None
示例13
def test_body_match(aresponses):
aresponses.add("foo.com", "/", "get", aresponses.Response(text="hi"), body_pattern=re.compile(r".*?apple.*"))
url = "http://foo.com"
async with aiohttp.ClientSession() as session:
try:
async with session.get(url, data={"fruit": "pineapple"}) as response:
text = await response.text()
assert text == "hi"
except ServerDisconnectedError:
pass
aresponses.assert_plan_strictly_followed()
示例14
def test_failure_no_match(aresponses):
async with aiohttp.ClientSession() as session:
try:
async with session.get("http://foo.com") as response:
await response.text()
except ServerDisconnectedError:
pass
with pytest.raises(NoRouteFoundError):
aresponses.assert_all_requests_matched()
with pytest.raises(NoRouteFoundError):
aresponses.assert_plan_strictly_followed()
示例15
def fetch_multi(self, urls):
"""Perform parallel fetch"""
results = []
error_msg = None
try:
async with aiohttp.ClientSession() as session:
for url in urls:
await asyncio.sleep(0)
body = await self.fetch_with_session(session, url)
results.append(body)
except asyncio.TimeoutError:
error_msg = 'Request timed out'
raise ClashRoyaleAPIError(message=error_msg)
except aiohttp.ServerDisconnectedError as err:
error_msg = 'Server disconnected error: {}'.format(err)
raise ClashRoyaleAPIError(message=error_msg)
except (aiohttp.ClientError, ValueError) as err:
error_msg = 'Request connection error: {}'.format(err)
raise ClashRoyaleAPIError(message=error_msg)
except json.JSONDecodeError:
error_msg = "Non JSON returned"
raise ClashRoyaleAPIError(message=error_msg)
else:
return results
finally:
if error_msg is not None:
raise ClashRoyaleAPIError(message=error_msg)
示例16
def test_disconnect(self, mock_book, mock_connect):
mock_connect.return_value.aenter.receive_str = CoroutineMock()
mock_connect.return_value.aenter.send_json = CoroutineMock()
mock_book.return_value = {'bids': [], 'asks': [], 'sequence': 1}
messages_expected = [
json.dumps({
"type": "done",
"side": "sell",
"order_id": "4eef1226-4b38-422c-a5b1-56def7107f9a",
"reason": "canceled",
"product_id": "ETH-USD",
"price": "2601.76000000",
"remaining_size": "3.09000000",
"sequence": 2,
"time": "2017-06-25T11:23:14.775000Z"
}),
aiohttp.ServerDisconnectedError('error'),
json.dumps({
"type": "done",
"side": "sell",
"order_id": "4eef1226-4b38-422c-a5b1-56def7107f9a",
"reason": "canceled",
"product_id": "ETH-USD",
"price": "2601.76000000",
"remaining_size": "3.09000000",
"sequence": 2,
"time": "2017-06-25T11:23:14.775000Z"
})
]
mock_connect.return_value.aenter.receive_str.side_effect = \
messages_expected
async with gdax.orderbook.OrderBook() as orderbook:
message = await orderbook.handle_message()
assert message == json.loads(messages_expected[0])
message = await orderbook.handle_message()
assert message is None
message = await orderbook.handle_message()
assert message == json.loads(messages_expected[2])
示例17
def handle_exception():
"""
Context manager translating network related exceptions
to custom :mod:`~galaxy.api.errors`.
"""
try:
yield
except asyncio.TimeoutError:
raise BackendTimeout()
except aiohttp.ServerDisconnectedError:
raise BackendNotAvailable()
except aiohttp.ClientConnectionError:
raise NetworkError()
except aiohttp.ContentTypeError as error:
raise UnknownBackendResponse(error.message)
except aiohttp.ClientResponseError as error:
if error.status == HTTPStatus.UNAUTHORIZED:
raise AuthenticationRequired(error.message)
if error.status == HTTPStatus.FORBIDDEN:
raise AccessDenied(error.message)
if error.status == HTTPStatus.SERVICE_UNAVAILABLE:
raise BackendNotAvailable(error.message)
if error.status == HTTPStatus.TOO_MANY_REQUESTS:
raise TooManyRequests(error.message)
if error.status >= 500:
raise BackendError(error.message)
if error.status >= 400:
logger.warning(
"Got status %d while performing %s request for %s",
error.status, error.request_info.method, str(error.request_info.url)
)
raise UnknownError(error.message)
except aiohttp.ClientError as e:
logger.exception("Caught exception while performing request")
raise UnknownError(repr(e))
示例18
def request(self, method, endpoint, body, headers=None, query=None):
if headers is None:
headers = {}
if query is None:
query = {}
if body is not None:
if isinstance(body, str):
headers["Content-Type"] = "text/plain"
else:
body = json.dumps(body)
headers["Content-Type"] = "application/json"
if self.debug:
print(f"{method} {endpoint}: {body}")
data = None
async with aiohttp.ClientSession() as session:
for _ in range(MAX_TRIES):
try:
async with session.request(method, f"{self.base}{endpoint}", data=body, headers=headers,
params=query) as resp:
log.info(f"Dicecloud returned {resp.status} ({endpoint})")
if resp.status == 200:
data = await resp.json(encoding='utf-8')
break
elif resp.status == 429:
timeout = await resp.json(encoding='utf-8')
log.warning(f"Dicecloud ratelimit hit ({endpoint}) - resets in {timeout}ms")
await asyncio.sleep(timeout['timeToReset'] / 1000) # rate-limited, wait and try again
elif 400 <= resp.status < 600:
if resp.status == 403:
raise Forbidden(resp.reason)
elif resp.status == 404:
raise NotFound(resp.reason)
else:
raise HTTPException(resp.status, resp.reason)
else:
log.warning(f"Unknown response from Dicecloud: {resp.status}")
except aiohttp.ServerDisconnectedError:
raise HTTPException(None, "Server disconnected")
if not data: # we did 10 loops and always got either 200 or 429 but we have no data, so we must have 429ed
raise Timeout(f"Dicecloud failed to respond after {MAX_TRIES} tries. Please try again.")
return data
示例19
def _send(self, payload, processor):
"""Send a payload to be converted to JSON.
Handles temporary connection issues. Daemon reponse errors
are raise through DaemonError.
"""
def log_error(error):
nonlocal last_error_log, retry
now = time.time()
if now - last_error_log > 60:
last_error_log = now
self.logger.error(f'{error} Retrying occasionally...')
if retry == self.max_retry and self.failover():
retry = 0
on_good_message = None
last_error_log = 0
data = json.dumps(payload)
retry = self.init_retry
while True:
try:
result = await self._send_data(data)
result = processor(result)
if on_good_message:
self.logger.info(on_good_message)
return result
except asyncio.TimeoutError:
log_error('timeout error.')
except aiohttp.ServerDisconnectedError:
log_error('disconnected.')
on_good_message = 'connection restored'
except aiohttp.ClientConnectionError:
log_error('connection problem - is your daemon running?')
on_good_message = 'connection restored'
except aiohttp.ClientError as e:
log_error(f'daemon error: {e}')
on_good_message = 'running normally'
except WarmingUpError:
log_error('starting up checking blocks.')
on_good_message = 'running normally'
except WorkQueueFullError:
log_error('work queue full.')
on_good_message = 'running normally'
await asyncio.sleep(retry)
retry = max(min(self.max_retry, retry * 2), self.init_retry)
示例20
def read(self, url, *args, use_proxy=False, **kwargs) -> FeedResponse:
headers = content = None
try:
if use_proxy:
headers, content, url, status = await self._read_by_proxy(url, *args, **kwargs)
else:
headers, content, url, status = await self._read(url, *args, **kwargs)
except (socket.gaierror, aiodns.error.DNSError):
status = FeedResponseStatus.DNS_ERROR.value
except (socket.timeout, TimeoutError, aiohttp.ServerTimeoutError,
asyncio.TimeoutError, concurrent.futures.TimeoutError):
status = FeedResponseStatus.CONNECTION_TIMEOUT.value
except (ssl.SSLError, ssl.CertificateError,
aiohttp.ServerFingerprintMismatch,
aiohttp.ClientSSLError,
aiohttp.ClientConnectorSSLError,
aiohttp.ClientConnectorCertificateError):
status = FeedResponseStatus.SSL_ERROR.value
except (aiohttp.ClientProxyConnectionError,
aiohttp.ClientHttpProxyError):
status = FeedResponseStatus.PROXY_ERROR.value
except (ConnectionError,
aiohttp.ServerDisconnectedError,
aiohttp.ServerConnectionError,
aiohttp.ClientConnectionError,
aiohttp.ClientConnectorError):
status = FeedResponseStatus.CONNECTION_RESET.value
except (aiohttp.WSServerHandshakeError, aiohttp.ClientOSError):
status = FeedResponseStatus.CONNECTION_ERROR.value
except aiohttp.ClientPayloadError:
status = FeedResponseStatus.CHUNKED_ENCODING_ERROR.value
except UnicodeDecodeError:
status = FeedResponseStatus.CONTENT_DECODING_ERROR.value
except FeedReaderError as ex:
status = ex.status
LOG.warning(type(ex).__name__ + " url=%s %s", url, ex)
except (aiohttp.ClientResponseError, aiohttp.ContentTypeError) as ex:
status = ex.status
except (aiohttp.ClientError, aiohttp.InvalidURL):
status = FeedResponseStatus.UNKNOWN_ERROR.value
builder = FeedResponseBuilder(use_proxy=use_proxy)
builder.url(url)
builder.status(status)
builder.content(content)
builder.headers(headers)
return builder.build()
示例21
def _send(self, payload, processor):
"""Send a payload to be converted to JSON.
Handles temporary connection issues. Daemon response errors
are raise through DaemonError.
"""
def log_error(error):
nonlocal last_error_log, retry
now = time.time()
if now - last_error_log > 60:
last_error_log = now
self.logger.error(f'{error} Retrying occasionally...')
if retry == self.max_retry and self.failover():
retry = 0
on_good_message = None
last_error_log = 0
data = json.dumps(payload)
retry = self.init_retry
methods = tuple(
[payload['method']] if isinstance(payload, dict) else [request['method'] for request in payload]
)
while True:
try:
for method in methods:
self.lbrycrd_pending_count_metric.labels(method=method).inc()
result = await self._send_data(data)
result = processor(result)
if on_good_message:
self.logger.info(on_good_message)
return result
except asyncio.TimeoutError:
log_error('timeout error.')
except aiohttp.ServerDisconnectedError:
log_error('disconnected.')
on_good_message = 'connection restored'
except aiohttp.ClientConnectionError:
log_error('connection problem - is your daemon running?')
on_good_message = 'connection restored'
except aiohttp.ClientError as e:
log_error(f'daemon error: {e}')
on_good_message = 'running normally'
except WarmingUpError:
log_error('starting up checking blocks.')
on_good_message = 'running normally'
except WorkQueueFullError:
log_error('work queue full.')
on_good_message = 'running normally'
finally:
for method in methods:
self.lbrycrd_pending_count_metric.labels(method=method).dec()
await asyncio.sleep(retry)
retry = max(min(self.max_retry, retry * 2), self.init_retry)
示例22
def check(self, real_ext_ip):
# TODO: need refactoring
try:
self.ip = await self._resolver.resolve(self.host)
except ResolveError:
return
if self.scheme == 'SMTP':
self.is_working = True
self.available[self.scheme].append(self)
self.ev[self.scheme].set()
return
page = False
headers, rv = get_headers(rv=True)
connector = aiohttp.TCPConnector(
loop=self._loop, ssl=self.verify_ssl, force_close=True
)
try:
timeout = aiohttp.ClientTimeout(total=self.timeout)
async with aiohttp.ClientSession(
connector=connector, timeout=timeout, loop=self._loop
) as session, session.get(
url=self.url, headers=headers, allow_redirects=False
) as resp:
page = await resp.text()
except (
asyncio.TimeoutError,
aiohttp.ClientOSError,
aiohttp.ClientResponseError,
aiohttp.ServerDisconnectedError,
) as e:
log.debug('%s is failed. Error: %r;' % (self, e))
return
page = page.lower()
if resp.status == 200 and real_ext_ip in page and rv in page:
self.marks['via'] = page.count('via')
self.marks['proxy'] = page.count('proxy')
self.is_working = True
self.available[self.scheme].append(self)
self.ev[self.scheme].set()
log.debug('%s is verified' % self)
else:
log.debug(
(
'{j} is failed. HTTP status code: {code}; '
'Real IP on page: {ip}; Version: {word}; '
'Response: {page}'
).format(
j=self,
code=resp.status,
page=page,
ip=(real_ext_ip in page),
word=(rv in page),
)
)
示例23
def handle_message(self):
try:
message = await self._recv()
except aiohttp.ServerDisconnectedError as exc:
logging.error(
f'Error: Exception: f{exc}. Re-initializing websocket.')
await self.__aexit__(None, None, None)
await self.__aenter__()
return
msg_type = message['type']
if msg_type == 'error':
raise OrderBookError(f'Error: {message["message"]}')
if msg_type == 'subscriptions':
return # must filter out here because the subscriptions message does not have a product_id key
product_id = message['product_id']
assert self._sequences[product_id] is not None
sequence = message['sequence']
if sequence <= self._sequences[product_id]:
# ignore older messages (e.g. before order book initialization
# from getProductOrderBook)
return message
elif sequence > self._sequences[product_id] + 1:
logging.error(
'Error: messages missing ({} - {}). Re-initializing websocket.'
.format(sequence, self._sequences[product_id]))
await self.__aexit__(None, None, None)
await self.__aenter__()
return
if msg_type == 'open':
self.add(product_id, message)
elif msg_type == 'done' and 'price' in message:
self.remove(product_id, message)
elif msg_type == 'match':
self.match(product_id, message)
elif msg_type == 'change':
self.change(product_id, message)
elif msg_type == 'heartbeat':
pass
elif msg_type == 'received':
pass
elif msg_type == 'done':
pass
else:
raise OrderBookError(f'unknown message type {msg_type}')
self._sequences[product_id] = sequence
return message
示例24
def fetch(self, method, url, params=None, headers=None, data=None):
"""Make an HTTP request.
Automatically uses configured HTTP proxy, and adds Google authorization
header and cookies.
Failures will be retried MAX_RETRIES times before raising NetworkError.
Args:
method (str): Request method.
url (str): Request URL.
params (dict): (optional) Request query string parameters.
headers (dict): (optional) Request headers.
data: (str): (optional) Request body data.
Returns:
FetchResponse: Response data.
Raises:
NetworkError: If the request fails.
"""
logger.debug('Sending request %s %s:\n%r', method, url, data)
for retry_num in range(MAX_RETRIES):
try:
async with self.fetch_raw(method, url, params=params,
headers=headers, data=data) as res:
async with async_timeout.timeout(REQUEST_TIMEOUT):
body = await res.read()
logger.debug('Received response %d %s:\n%r',
res.status, res.reason, body)
except asyncio.TimeoutError:
error_msg = 'Request timed out'
except aiohttp.ServerDisconnectedError as err:
error_msg = 'Server disconnected error: {}'.format(err)
except (aiohttp.ClientError, ValueError) as err:
error_msg = 'Request connection error: {}'.format(err)
else:
break
logger.info('Request attempt %d failed: %s', retry_num, error_msg)
else:
logger.info('Request failed after %d attempts', MAX_RETRIES)
raise exceptions.NetworkError(error_msg)
if res.status != 200:
logger.info('Request returned unexpected status: %d %s',
res.status, res.reason)
raise exceptions.NetworkError(
'Request return unexpected status: {}: {}'
.format(res.status, res.reason)
)
return FetchResponse(res.status, body)
示例25
def _longpoll_request(self):
"""Open a long-polling request and receive arrays.
This method uses keep-alive to make re-opening the request faster, but
the remote server will set the "Connection: close" header once an hour.
Raises hangups.NetworkError or ChannelSessionError.
"""
params = {
'VER': 8, # channel protocol version
'gsessionid': self._gsessionid_param,
'RID': 'rpc', # request identifier
't': 1, # trial
'SID': self._sid_param, # session ID
'CI': 0, # 0 if streaming/chunked requests should be used
'ctype': 'hangouts', # client type
'TYPE': 'xmlhttp', # type of request
}
logger.info('Opening new long-polling request')
try:
async with self._session.fetch_raw('GET', CHANNEL_URL,
params=params) as res:
if res.status != 200:
if res.status == 400 and res.reason == 'Unknown SID':
raise ChannelSessionError('SID became invalid')
raise exceptions.NetworkError(
'Request return unexpected status: {}: {}'.format(
res.status, res.reason))
while True:
async with async_timeout.timeout(PUSH_TIMEOUT):
chunk = await res.content.read(MAX_READ_BYTES)
if not chunk:
break
await self._on_push_data(chunk)
except asyncio.TimeoutError:
raise exceptions.NetworkError('Request timed out')
except aiohttp.ServerDisconnectedError as err:
raise exceptions.NetworkError(
'Server disconnected error: %s' % err)
except aiohttp.ClientPayloadError:
raise ChannelSessionError('SID is about to expire')
except aiohttp.ClientError as err:
raise exceptions.NetworkError('Request connection error: %s' % err)