1
1
# Copyright (C) 2005, 2006 Canonical Ltd
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
5
5
# the Free Software Foundation; either version 2 of the License, or
6
6
# (at your option) any later version.
8
8
# This program is distributed in the hope that it will be useful,
9
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
11
# GNU General Public License for more details.
13
13
# You should have received a copy of the GNU General Public License
14
14
# along with this program; if not, write to the Free Software
15
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
164
164
return (scheme, username, password, host, port, path)
167
class _CoalescedOffset(object):
168
"""A data container for keeping track of coalesced offsets."""
170
__slots__ = ['start', 'length', 'ranges']
172
def __init__(self, start, length, ranges):
177
def __cmp__(self, other):
178
return cmp((self.start, self.length, self.ranges),
179
(other.start, other.length, other.ranges))
167
182
class Transport(object):
168
183
"""This class encapsulates methods for retrieving or putting a file
169
184
from/to a storage location.
353
379
:offsets: A list of (offset, size) tuples.
354
380
:return: A list or generator of (offset, data) tuples
356
def do_combined_read(combined_offsets):
358
for offset, size in combined_offsets:
360
mutter('readv coalesced %d reads.', len(combined_offsets))
361
offset = combined_offsets[0][0]
363
data = fp.read(total_size)
365
for offset, size in combined_offsets:
366
yield offset, data[pos:pos + size]
371
385
fp = self.get(relpath)
372
pending_offsets = deque(offsets)
373
combined_offsets = []
374
while len(pending_offsets):
375
offset, size = pending_offsets.popleft()
376
if not combined_offsets:
377
combined_offsets = [[offset, size]]
386
return self._seek_and_read(fp, offsets)
388
def _seek_and_read(self, fp, offsets):
389
"""An implementation of readv that uses fp.seek and fp.read.
391
This uses _coalesce_offsets to issue larger reads and fewer seeks.
393
:param fp: A file-like object that supports seek() and read(size)
394
:param offsets: A list of offsets to be read from the given file.
395
:return: yield (pos, data) tuples for each request
397
# We are going to iterate multiple times, we need a list
398
offsets = list(offsets)
399
sorted_offsets = sorted(offsets)
401
# turn the list of offsets into a stack
402
offset_stack = iter(offsets)
403
cur_offset_and_size = offset_stack.next()
404
coalesced = self._coalesce_offsets(sorted_offsets,
405
limit=self._max_readv_combine,
406
fudge_factor=self._bytes_to_read_before_seek)
408
# Cache the results, but only until they have been fulfilled
410
for c_offset in coalesced:
411
# TODO: jam 20060724 it might be faster to not issue seek if
412
# we are already at the right location. This should be
414
fp.seek(c_offset.start)
415
data = fp.read(c_offset.length)
416
for suboffset, subsize in c_offset.ranges:
417
key = (c_offset.start+suboffset, subsize)
418
data_map[key] = data[suboffset:suboffset+subsize]
420
# Now that we've read some data, see if we can yield anything back
421
while cur_offset_and_size in data_map:
422
this_data = data_map.pop(cur_offset_and_size)
423
yield cur_offset_and_size[0], this_data
424
cur_offset_and_size = offset_stack.next()
427
def _coalesce_offsets(offsets, limit, fudge_factor):
428
"""Yield coalesced offsets.
430
With a long list of neighboring requests, combine them
431
into a single large request, while retaining the original
433
Turns [(15, 10), (25, 10)] => [(15, 20, [(0, 10), (10, 10)])]
435
:param offsets: A list of (start, length) pairs
436
:param limit: Only combine a maximum of this many pairs
437
Some transports penalize multiple reads more than
438
others, and sometimes it is better to return early.
440
:param fudge_factor: All transports have some level of 'it is
441
better to read some more data and throw it away rather
442
than seek', so collapse if we are 'close enough'
443
:return: yield _CoalescedOffset objects, which have members for wher
444
to start, how much to read, and how to split those
448
cur = _CoalescedOffset(None, None, [])
450
for start, size in offsets:
452
if (last_end is not None
453
and start <= last_end + fudge_factor
454
and start >= cur.start
455
and (limit <= 0 or len(cur.ranges) < limit)):
456
cur.length = end - cur.start
457
cur.ranges.append((start-cur.start, size))
379
if (len (combined_offsets) < 50 and
380
combined_offsets[-1][0] + combined_offsets[-1][1] == offset):
382
combined_offsets.append([offset, size])
384
# incompatible, or over the threshold issue a read and yield
385
pending_offsets.appendleft((offset, size))
386
for result in do_combined_read(combined_offsets):
388
combined_offsets = []
389
# whatever is left is a single coalesced request
390
if len(combined_offsets):
391
for result in do_combined_read(combined_offsets):
459
if cur.start is not None:
461
cur = _CoalescedOffset(start, size, [(0, size)])
464
if cur.start is not None:
394
469
def get_multi(self, relpaths, pb=None):
395
470
"""Get a list of file-like objects, one for each entry in relpaths.
added
removed
bzrlib/transport/__init__.py
