1984
1977
:param key: The key of the record. Currently keys are always serialised
1985
1978
using just the trailing component.
1986
1979
:param dense_lines: The bytes of lines but in a denser form. For
1987
instance, if lines is a list of 1000 bytestrings each ending in \n,
1988
dense_lines may be a list with one line in it, containing all the
1989
1000's lines and their \n's. Using dense_lines if it is already
1990
known is a win because the string join to create bytes in this
1991
function spends less time resizing the final string.
1980
instance, if lines is a list of 1000 bytestrings each ending in
1981
\\n, dense_lines may be a list with one line in it, containing all
1982
the 1000's lines and their \\n's. Using dense_lines if it is
1983
already known is a win because the string join to create bytes in
1984
this function spends less time resizing the final string.
1992
1985
:return: (len, a StringIO instance with the raw data ready to read.)
1994
1987
chunks = ["version %s %d %s\n" % (key[-1], len(lines), digest)]
2785
2778
return key[:-1], key[-1]
2788
class _KeyRefs(object):
2790
def __init__(self, track_new_keys=False):
2791
# dict mapping 'key' to 'set of keys referring to that key'
2794
# set remembering all new keys
2795
self.new_keys = set()
2797
self.new_keys = None
2803
self.new_keys.clear()
2805
def add_references(self, key, refs):
2806
# Record the new references
2807
for referenced in refs:
2809
needed_by = self.refs[referenced]
2811
needed_by = self.refs[referenced] = set()
2813
# Discard references satisfied by the new key
2816
def get_new_keys(self):
2817
return self.new_keys
2819
def get_unsatisfied_refs(self):
2820
return self.refs.iterkeys()
2822
def _satisfy_refs_for_key(self, key):
2826
# No keys depended on this key. That's ok.
2829
def add_key(self, key):
2830
# satisfy refs for key, and remember that we've seen this key.
2831
self._satisfy_refs_for_key(key)
2832
if self.new_keys is not None:
2833
self.new_keys.add(key)
2835
def satisfy_refs_for_keys(self, keys):
2837
self._satisfy_refs_for_key(key)
2839
def get_referrers(self):
2841
for referrers in self.refs.itervalues():
2842
result.update(referrers)
2846
2781
class _KnitGraphIndex(object):
2847
2782
"""A KnitVersionedFiles index layered on GraphIndex."""
3280
class _DirectPackAccess(object):
3281
"""Access to data in one or more packs with less translation."""
3283
def __init__(self, index_to_packs, reload_func=None, flush_func=None):
3284
"""Create a _DirectPackAccess object.
3286
:param index_to_packs: A dict mapping index objects to the transport
3287
and file names for obtaining data.
3288
:param reload_func: A function to call if we determine that the pack
3289
files have moved and we need to reload our caches. See
3290
bzrlib.repo_fmt.pack_repo.AggregateIndex for more details.
3292
self._container_writer = None
3293
self._write_index = None
3294
self._indices = index_to_packs
3295
self._reload_func = reload_func
3296
self._flush_func = flush_func
3298
def add_raw_records(self, key_sizes, raw_data):
3299
"""Add raw knit bytes to a storage area.
3301
The data is spooled to the container writer in one bytes-record per
3304
:param sizes: An iterable of tuples containing the key and size of each
3306
:param raw_data: A bytestring containing the data.
3307
:return: A list of memos to retrieve the record later. Each memo is an
3308
opaque index memo. For _DirectPackAccess the memo is (index, pos,
3309
length), where the index field is the write_index object supplied
3310
to the PackAccess object.
3312
if type(raw_data) is not str:
3313
raise AssertionError(
3314
'data must be plain bytes was %s' % type(raw_data))
3317
for key, size in key_sizes:
3318
p_offset, p_length = self._container_writer.add_bytes_record(
3319
raw_data[offset:offset+size], [])
3321
result.append((self._write_index, p_offset, p_length))
3325
"""Flush pending writes on this access object.
3327
This will flush any buffered writes to a NewPack.
3329
if self._flush_func is not None:
3332
def get_raw_records(self, memos_for_retrieval):
3333
"""Get the raw bytes for a records.
3335
:param memos_for_retrieval: An iterable containing the (index, pos,
3336
length) memo for retrieving the bytes. The Pack access method
3337
looks up the pack to use for a given record in its index_to_pack
3339
:return: An iterator over the bytes of the records.
3341
# first pass, group into same-index requests
3343
current_index = None
3344
for (index, offset, length) in memos_for_retrieval:
3345
if current_index == index:
3346
current_list.append((offset, length))
3348
if current_index is not None:
3349
request_lists.append((current_index, current_list))
3350
current_index = index
3351
current_list = [(offset, length)]
3352
# handle the last entry
3353
if current_index is not None:
3354
request_lists.append((current_index, current_list))
3355
for index, offsets in request_lists:
3357
transport, path = self._indices[index]
3359
# A KeyError here indicates that someone has triggered an index
3360
# reload, and this index has gone missing, we need to start
3362
if self._reload_func is None:
3363
# If we don't have a _reload_func there is nothing that can
3366
raise errors.RetryWithNewPacks(index,
3367
reload_occurred=True,
3368
exc_info=sys.exc_info())
3370
reader = pack.make_readv_reader(transport, path, offsets)
3371
for names, read_func in reader.iter_records():
3372
yield read_func(None)
3373
except errors.NoSuchFile:
3374
# A NoSuchFile error indicates that a pack file has gone
3375
# missing on disk, we need to trigger a reload, and start over.
3376
if self._reload_func is None:
3378
raise errors.RetryWithNewPacks(transport.abspath(path),
3379
reload_occurred=False,
3380
exc_info=sys.exc_info())
3382
def set_writer(self, writer, index, transport_packname):
3383
"""Set a writer to use for adding data."""
3384
if index is not None:
3385
self._indices[index] = transport_packname
3386
self._container_writer = writer
3387
self._write_index = index
3389
def reload_or_raise(self, retry_exc):
3390
"""Try calling the reload function, or re-raise the original exception.
3392
This should be called after _DirectPackAccess raises a
3393
RetryWithNewPacks exception. This function will handle the common logic
3394
of determining when the error is fatal versus being temporary.
3395
It will also make sure that the original exception is raised, rather
3396
than the RetryWithNewPacks exception.
3398
If this function returns, then the calling function should retry
3399
whatever operation was being performed. Otherwise an exception will
3402
:param retry_exc: A RetryWithNewPacks exception.
3405
if self._reload_func is None:
3407
elif not self._reload_func():
3408
# The reload claimed that nothing changed
3409
if not retry_exc.reload_occurred:
3410
# If there wasn't an earlier reload, then we really were
3411
# expecting to find changes. We didn't find them, so this is a
3415
exc_class, exc_value, exc_traceback = retry_exc.exc_info
3416
raise exc_class, exc_value, exc_traceback
3419
# Deprecated, use PatienceSequenceMatcher instead
3420
KnitSequenceMatcher = patiencediff.PatienceSequenceMatcher
3423
3215
def annotate_knit(knit, revision_id):
3424
3216
"""Annotate a knit with no cached annotations.
added
removed
bzrlib/knit.py
