Python源码示例：sendfile.sendfile()
示例1
def _import_sendfile():
# By default attempt to use os.sendfile introduced in Python 3.3:
# http://bugs.python.org/issue10882
# ...otherwise fallback on using third-party pysendfile module:
# https://github.com/giampaolo/pysendfile/
if os.name == 'posix':
try:
return os.sendfile # py >= 3.3
except AttributeError:
try:
import sendfile as sf
# dirty hack to detect whether old 1.2.4 version is installed
if hasattr(sf, 'has_sf_hdtr'):
raise ImportError
return sf.sendfile
except ImportError:
pass
示例2
def _use_sendfile(self, producer):
if not self.cmd_channel.use_sendfile:
debug("starting transfer not using sendfile(2) as per server "
"config", self)
return False
if not isinstance(producer, FileProducer):
debug("starting transfer not using sendfile(2) (directory "
"listing)", self)
return False
else:
if not hasattr(self.file_obj, "fileno"):
debug("starting transfer not using sendfile(2) %r has no "
"fileno() method" % self.file_obj, self)
return False
if not producer.type == 'i':
debug("starting transfer not using sendfile(2) (text file "
"transfer)", self)
return False
debug("starting transfer using sendfile()", self)
return True
示例3
def _import_sendfile():
# By default attempt to use os.sendfile introduced in Python 3.3:
# http://bugs.python.org/issue10882
# ...otherwise fallback on using third-party pysendfile module:
# https://github.com/giampaolo/pysendfile/
if os.name == 'posix':
try:
return os.sendfile # py >= 3.3
except AttributeError:
try:
import sendfile as sf
# dirty hack to detect whether old 1.2.4 version is installed
if hasattr(sf, 'has_sf_hdtr'):
raise ImportError
return sf.sendfile
except ImportError:
pass
示例4
def _import_sendfile():
# By default attempt to use os.sendfile introduced in Python 3.3:
# http://bugs.python.org/issue10882
# ...otherwise fallback on using third-party pysendfile module:
# https://github.com/giampaolo/pysendfile/
if os.name == 'posix':
try:
return os.sendfile # py >= 3.3
except AttributeError:
try:
import sendfile as sf
# dirty hack to detect whether old 1.2.4 version is installed
if hasattr(sf, 'has_sf_hdtr'):
raise ImportError
return sf.sendfile
except ImportError:
pass
return None
示例5
def get_repr_info(self, as_str=False, extra_info={}):
info = OrderedDict()
info['id'] = id(self)
info['addr'] = "%s:%s" % (self.remote_ip, self.remote_port)
if _is_ssl_sock(self.socket):
info['ssl'] = True
if self.username:
info['user'] = self.username
# If threads are involved sometimes "self" may be None (?!?).
dc = getattr(self, 'data_channel', None)
if dc is not None:
if _is_ssl_sock(dc.socket):
info['ssl-data'] = True
if dc.file_obj:
if self.data_channel.receive:
info['sending-file'] = dc.file_obj
if dc.use_sendfile():
info['use-sendfile(2)'] = True
else:
info['receiving-file'] = dc.file_obj
info['bytes-trans'] = dc.get_transmitted_bytes()
info.update(extra_info)
if as_str:
return ', '.join(['%s=%r' % (k, v) for (k, v) in info.items()])
return info
示例6
def sendfile_wrapper(sock, file, offset, nbytes=BUFFER_LEN, header="",
trailer=""):
"""A higher level wrapper representing how an application is
supposed to use sendfile().
"""
while 1:
try:
if SUPPORT_HEADER_TRAILER:
sent = sendfile.sendfile(sock, file, offset, nbytes, header,
trailer)
else:
sent = sendfile.sendfile(sock, file, offset, nbytes)
except OSError:
err = sys.exc_info()[1]
if err.errno == errno.EAGAIN: # retry
continue
raise
else:
assert sent <= nbytes, (sent, nbytes)
return sent
示例7
def test_header(self):
total_sent = 0
header = b("x") * 512
sent = sendfile.sendfile(self.sockno, self.fileno, 0,
header=header)
total_sent += sent
offset = BUFFER_LEN
while 1:
sent = sendfile_wrapper(self.sockno, self.fileno, offset)
if sent == 0:
break
offset += sent
total_sent += sent
expected_data = header + DATA
self.assertEqual(total_sent, len(expected_data))
self.client.close()
self.server.wait()
data = self.server.handler_instance.get_data()
self.assertEqual(len(data), len(expected_data))
self.assertEqual(hash(data), hash(expected_data))
示例8
def _import_sendfile():
# By default attempt to use os.sendfile introduced in Python 3.3:
# http://bugs.python.org/issue10882
# ...otherwise fallback on using third-party pysendfile module:
# https://github.com/giampaolo/pysendfile/
if os.name == 'posix':
try:
return os.sendfile # py >= 3.3
except AttributeError:
try:
import sendfile as sf
# dirty hack to detect whether old 1.2.4 version is installed
if hasattr(sf, 'has_sf_hdtr'):
raise ImportError
return sf.sendfile
except ImportError:
pass
return None
示例9
def get_repr_info(self, as_str=False, extra_info={}):
info = OrderedDict()
info['id'] = id(self)
info['addr'] = "%s:%s" % (self.remote_ip, self.remote_port)
if _is_ssl_sock(self.socket):
info['ssl'] = True
if self.username:
info['user'] = self.username
# If threads are involved sometimes "self" may be None (?!?).
dc = getattr(self, 'data_channel', None)
if dc is not None:
if _is_ssl_sock(dc.socket):
info['ssl-data'] = True
if dc.file_obj:
if self.data_channel.receive:
info['sending-file'] = dc.file_obj
if dc.use_sendfile():
info['use-sendfile(2)'] = True
else:
info['receiving-file'] = dc.file_obj
info['bytes-trans'] = dc.get_transmitted_bytes()
info.update(extra_info)
if as_str:
return ', '.join(['%s=%r' % (k, v) for (k, v) in info.items()])
return info
示例10
def get_file(request, slug, attachment_id=None, filename=None):
attachment = get_object_or_404(Attachment, file__endswith='%s%s' % (posixpath.sep, filename), page__slug=slug)
as_attachment = ((not imghdr.what(attachment.file.path) and 'embed' not in request.GET)
or 'as_attachment' in request.GET)
# ref https://github.com/johnsensible/django-sendfile
return sendfile(request, attachment.file.path,
attachment=as_attachment, attachment_filename=text_type(attachment))
示例11
def initiate_sendfile(self):
"""A wrapper around sendfile."""
try:
sent = sendfile(self._fileno, self._filefd, self._offset,
self.ac_out_buffer_size)
except OSError as err:
if err.errno in _ERRNOS_RETRY or err.errno == errno.EBUSY:
return
elif err.errno in _ERRNOS_DISCONNECTED:
self.handle_close()
else:
if self.tot_bytes_sent == 0:
logger.warning(
"sendfile() failed; falling back on using plain send")
raise _GiveUpOnSendfile
else:
raise
else:
if sent == 0:
# this signals the channel that the transfer is completed
self.discard_buffers()
self.handle_close()
else:
self._offset += sent
self.tot_bytes_sent += sent
# --- utility methods
示例12
def _use_sendfile(self, producer):
if isinstance(self.socket, SSL.Connection):
debug(
"starting transfer not using sendfile(2) (SSL connection)",
self)
return False
else:
return super(TLS_DTPHandler, self)._use_sendfile(producer)
示例13
def initiate_sendfile(self):
"""A wrapper around sendfile."""
try:
sent = sendfile(self._fileno, self._filefd, self._offset,
self.ac_out_buffer_size)
except OSError as err:
if err.errno in _ERRNOS_RETRY or err.errno == errno.EBUSY:
return
elif err.errno in _ERRNOS_DISCONNECTED:
self.handle_close()
else:
if self.tot_bytes_sent == 0:
logger.warning(
"sendfile() failed; falling back on using plain send")
raise _GiveUpOnSendfile
else:
raise
else:
if sent == 0:
# this signals the channel that the transfer is completed
self.discard_buffers()
self.handle_close()
else:
self._offset += sent
self.tot_bytes_sent += sent
# --- utility methods
示例14
def _use_sendfile(self, producer):
if isinstance(self.socket, SSL.Connection):
debug(
"starting transfer not using sendfile(2) (SSL connection)",
self)
return False
else:
return super(TLS_DTPHandler, self)._use_sendfile(producer)
示例15
def export_user_data(request: Request):
"""
Export all user data in a zip blob and serve it through sendfile.
"""
user = request.user
# Cleanup old archives.
cache_expiration = datetime.fromtimestamp(datetime.now().timestamp() - USER_EXPORT_DATA_CACHE_PERIOD)
for archive in UserArchive.objects.filter(owner=user,
updated_on__lte=cache_expiration).iterator():
try:
archive.local_archive.delete()
except ValueError:
pass
finally:
archive.delete()
try:
archive = UserArchive.objects.get(owner=user)
return sendfile(request, archive.local_archive.path)
except (UserArchive.DoesNotExist, OSError, IOError, ValueError):
try:
builder = UserDataArchiveBuilder(user)
export(builder)
archive, _ = UserArchive.objects.get_or_create(owner=user)
builder.archive.close() # save the ZIP file.
archive.local_archive.save('data.zip',
File(open(builder.archive_filename, 'rb')))
builder.cleanup()
return sendfile(request, archive.local_archive.path)
except (OSError, IOError, django.db.Error) as e:
print(e)
return Response({}, status=status.HTTP_503_SERVICE_UNAVAILABLE)
示例16
def get(self, request, pk):
try:
odlc = find_odlc(request, int(pk))
except Odlc.DoesNotExist:
return HttpResponseNotFound('Odlc %s not found' % pk)
except ValueError as e:
return HttpResponseForbidden(str(e))
if not odlc.thumbnail or not odlc.thumbnail.name:
return HttpResponseNotFound('Odlc %s has no image' % pk)
# Tell sendfile to serve the thumbnail.
return sendfile(request, odlc.thumbnail.path)
示例17
def initiate_sendfile(self):
"""A wrapper around sendfile."""
try:
sent = sendfile(self._fileno, self._filefd, self._offset,
self.ac_out_buffer_size)
except OSError as err:
if err.errno in _ERRNOS_RETRY or err.errno == errno.EBUSY:
return
elif err.errno in _ERRNOS_DISCONNECTED:
self.handle_close()
else:
if self.tot_bytes_sent == 0:
logger.warning(
"sendfile() failed; falling back on using plain send")
raise _GiveUpOnSendfile
else:
raise
else:
if sent == 0:
# this signals the channel that the transfer is completed
self.discard_buffers()
self.handle_close()
else:
self._offset += sent
self.tot_bytes_sent += sent
# --- utility methods
示例18
def initiate_sendfile(self):
"""A wrapper around sendfile."""
try:
sent = sendfile(self._fileno, self._filefd, self._offset,
self.ac_out_buffer_size)
except OSError, err:
if err.errno in (errno.EAGAIN, errno.EWOULDBLOCK, errno.EBUSY):
return
elif err.errno in _DISCONNECTED:
self.handle_close()
else:
raise
示例19
def get_export_result(request, api_key, hash_id):
job = Job.objects.get(hashid=hash_id)
try:
user_id = ApiKey.objects.get(key=api_key).user_id
assert user_id == job.user.id
except:
return HttpResponseForbidden("This user did not request that file!")
if job.status != "Complete":
return HttpResponseNotFound("Export is not complete")
filepath = settings.SENDFILE_ROOT + job.data
return sendfile(request, filepath, attachment=True)
示例20
def get(self, request, *args, **kwargs):
attachment = get_r_object_or_404(self.request.user, Attachment,
issue__project__name_short=self.kwargs.get('project'),
issue__number=self.kwargs.get('sqn_i'),
seqnum=self.kwargs.get('sqn_a'))
# TODO BUG For some reason this is reported as a leak of resources: there is an unclosed file,
# which has been opened with mode='rb' and closefd=True
# So I assume the development-backend doesn't close the file properly
return sendfile(request, attachment.file.path, attachment=True)
示例21
def test_trailer(self):
with open(TESTFN2, 'wb') as f:
f.write(b("abcde"))
with open(TESTFN2, 'rb') as f:
sendfile.sendfile(self.sockno, f.fileno(), 0,
trailer=b("12345"))
time.sleep(.1)
self.client.close()
self.server.wait()
data = self.server.handler_instance.get_data()
self.assertEqual(data, b("abcde12345"))
示例22
def test_send_nbytes_0(self):
# On Mac OS X and FreeBSD a value of 0 for nbytes
# is supposed to send the whole file in one shot.
# OSX implementation appears to be just broken.
# On *BSD this works most of the times: sometimes
# EAGAIN is returned internally and here we get the
# number of bytes sent.
sent = sendfile.sendfile(self.sockno, self.fileno, 0, 0)
if sent == len(DATA):
self.client.close()
self.server.wait()
data = self.server.handler_instance.get_data()
self.assertEqual(len(data), len(DATA))
self.assertEqual(hash(data), hash(DATA))
示例23
def test_flags(self):
try:
sendfile.sendfile(self.sockno, self.fileno, 0, BUFFER_LEN,
flags=sendfile.SF_NODISKIO)
except OSError:
err = sys.exc_info()[1]
if err.errno not in (errno.EBUSY, errno.EAGAIN):
raise
# --- corner cases
示例24
def test_offset_overflow(self):
# specify an offset > file size
offset = len(DATA) + BUFFER_LEN
sent = sendfile.sendfile(self.sockno, self.fileno, offset, BUFFER_LEN)
self.assertEqual(sent, 0)
self.client.close()
self.server.wait()
data = self.server.handler_instance.get_data()
self.assertEqual(data, b(''))
示例25
def test_invalid_offset(self):
try:
sendfile.sendfile(self.sockno, self.fileno, -1, BUFFER_LEN)
except OSError:
err = sys.exc_info()[1]
self.assertEqual(err.errno, errno.EINVAL)
else:
self.fail("exception not raised")
示例26
def test_offset_none(self):
# on Linux offset == None sendfile() call is supposed
# to update the file offset
while 1:
sent = sendfile_wrapper(self.sockno, self.fileno, None)
if sent == 0:
break
self.client.close()
self.server.wait()
data = self.server.handler_instance.get_data()
self.assertEqual(len(data), len(DATA))
self.assertEqual(hash(data), hash(DATA))
示例27
def initiate_sendfile(self):
"""A wrapper around sendfile."""
try:
sent = sendfile(self._fileno, self._filefd, self._offset,
self.ac_out_buffer_size)
except OSError as err:
if err.errno in _ERRNOS_RETRY or err.errno == errno.EBUSY:
return
elif err.errno in _ERRNOS_DISCONNECTED:
self.handle_close()
else:
if self.tot_bytes_sent == 0:
logger.warning(
"sendfile() failed; falling back on using plain send")
raise _GiveUpOnSendfile
else:
raise
else:
if sent == 0:
# this signals the channel that the transfer is completed
self.discard_buffers()
self.handle_close()
else:
self._offset += sent
self.tot_bytes_sent += sent
# --- utility methods
示例28
def start_server(use_sendfile, keep_sending=False):
"""A simple test server which sends a file once a client connects.
use_sendfile decides whether using sendfile() or plain send().
If keep_sending is True restart sending file when EOF is reached.
"""
sock = socket.socket()
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.bind((HOST, PORT))
sock.listen(1)
conn, addr = sock.accept()
sock.close()
file = open(BIGFILE, 'rb')
def on_exit(signum, fram):
file.close()
conn.close()
sys.exit(0)
signal.signal(signal.SIGTERM, on_exit)
signal.signal(signal.SIGINT, on_exit)
if not use_sendfile:
while 1:
chunk = file.read(BUFFER_SIZE)
if not chunk:
# EOF
if keep_sending:
file.seek(0)
continue
else:
break
conn.sendall(chunk)
else:
offset = 0
sockno = conn.fileno()
fileno = file.fileno()
while 1:
try:
sent = sendfile(sockno, fileno, offset, BUFFER_SIZE)
except OSError:
err = sys.exc_info()[1]
if err.errno in (errno.EAGAIN, errno.EBUSY):
continue
raise
else:
if not sent:
# EOF
if keep_sending:
offset = 0
continue
else:
break
else:
offset += sent
示例29
def main():
parser = optparse.OptionParser()
parser.add_option('-k', '--keepfile', action="store_true", default=False,
help="do not remove test file on exit")
options, args = parser.parse_args()
if not options.keepfile:
atexit.register(lambda: safe_remove(BIGFILE))
if not os.path.exists(BIGFILE) or os.path.getsize(BIGFILE) < BIGFILE_SIZE:
print_("creating big file...")
create_file(BIGFILE, BIGFILE_SIZE)
print_("starting benchmark...")
# CPU time: use sendfile()
server = Process(target=start_server, kwargs={"use_sendfile": True})
server.start()
time.sleep(0.1)
t1 = timeit.Timer(setup="from __main__ import Client; client = Client()",
stmt="client.retr()").timeit(number=1)
server.terminate()
server.join()
# CPU time: use send()
server = Process(target=start_server, kwargs={"use_sendfile": False})
server.start()
time.sleep(0.1)
t2 = timeit.Timer(setup="from __main__ import Client; client = Client()",
stmt="client.retr()").timeit(number=1)
server.terminate()
server.join()
# MB/sec: use sendfile()
server = Process(target=start_server, kwargs={"use_sendfile": True,
"keep_sending": True})
server.start()
time.sleep(0.1)
client = Client()
bytes1 = client.retr_for_1_sec()
server.terminate()
server.join()
# MB/sec: use sendfile()
server = Process(target=start_server, kwargs={"use_sendfile": False,
"keep_sending": True})
server.start()
time.sleep(0.1)
client = Client()
bytes2 = client.retr_for_1_sec()
server.terminate()
server.join()
print_(" ")
print_("send()", hilite=True)
print_(" cpu: %7.2f usec/pass" % (1000000 * t2 / 100000))
print_(" rate: %7.2f MB/sec" % round(bytes2 / 1024.0 / 1024.0, 2))
print_("")
print_("sendfile()", hilite=True)
print_(" cpu: %7.2f usec/pass" % (1000000 * t1 / 100000))
print_(" rate: %7.2f MB/sec" % round(bytes1 / 1024.0 / 1024.0, 2))
