1977
1981
:param key: The key of the record. Currently keys are always serialised
1978
1982
using just the trailing component.
1979
1983
:param dense_lines: The bytes of lines but in a denser form. For
1980
instance, if lines is a list of 1000 bytestrings each ending in
1981
\\n, dense_lines may be a list with one line in it, containing all
1982
the 1000's lines and their \\n's. Using dense_lines if it is
1983
already known is a win because the string join to create bytes in
1984
this function spends less time resizing the final string.
1984
instance, if lines is a list of 1000 bytestrings each ending in \n,
1985
dense_lines may be a list with one line in it, containing all the
1986
1000's lines and their \n's. Using dense_lines if it is already
1987
known is a win because the string join to create bytes in this
1988
function spends less time resizing the final string.
1985
1989
:return: (len, a StringIO instance with the raw data ready to read.)
1987
1991
chunks = ["version %s %d %s\n" % (key[-1], len(lines), digest)]
2778
2782
return key[:-1], key[-1]
2785
class _KeyRefs(object):
2787
def __init__(self, track_new_keys=False):
2788
# dict mapping 'key' to 'set of keys referring to that key'
2791
# set remembering all new keys
2792
self.new_keys = set()
2794
self.new_keys = None
2800
self.new_keys.clear()
2802
def add_references(self, key, refs):
2803
# Record the new references
2804
for referenced in refs:
2806
needed_by = self.refs[referenced]
2808
needed_by = self.refs[referenced] = set()
2810
# Discard references satisfied by the new key
2813
def get_new_keys(self):
2814
return self.new_keys
2816
def get_unsatisfied_refs(self):
2817
return self.refs.iterkeys()
2819
def _satisfy_refs_for_key(self, key):
2823
# No keys depended on this key. That's ok.
2826
def add_key(self, key):
2827
# satisfy refs for key, and remember that we've seen this key.
2828
self._satisfy_refs_for_key(key)
2829
if self.new_keys is not None:
2830
self.new_keys.add(key)
2832
def satisfy_refs_for_keys(self, keys):
2834
self._satisfy_refs_for_key(key)
2836
def get_referrers(self):
2838
for referrers in self.refs.itervalues():
2839
result.update(referrers)
2781
2843
class _KnitGraphIndex(object):
2782
2844
"""A KnitVersionedFiles index layered on GraphIndex."""
3272
class _DirectPackAccess(object):
3273
"""Access to data in one or more packs with less translation."""
3275
def __init__(self, index_to_packs, reload_func=None, flush_func=None):
3276
"""Create a _DirectPackAccess object.
3278
:param index_to_packs: A dict mapping index objects to the transport
3279
and file names for obtaining data.
3280
:param reload_func: A function to call if we determine that the pack
3281
files have moved and we need to reload our caches. See
3282
bzrlib.repo_fmt.pack_repo.AggregateIndex for more details.
3284
self._container_writer = None
3285
self._write_index = None
3286
self._indices = index_to_packs
3287
self._reload_func = reload_func
3288
self._flush_func = flush_func
3290
def add_raw_records(self, key_sizes, raw_data):
3291
"""Add raw knit bytes to a storage area.
3293
The data is spooled to the container writer in one bytes-record per
3296
:param sizes: An iterable of tuples containing the key and size of each
3298
:param raw_data: A bytestring containing the data.
3299
:return: A list of memos to retrieve the record later. Each memo is an
3300
opaque index memo. For _DirectPackAccess the memo is (index, pos,
3301
length), where the index field is the write_index object supplied
3302
to the PackAccess object.
3304
if type(raw_data) is not str:
3305
raise AssertionError(
3306
'data must be plain bytes was %s' % type(raw_data))
3309
for key, size in key_sizes:
3310
p_offset, p_length = self._container_writer.add_bytes_record(
3311
raw_data[offset:offset+size], [])
3313
result.append((self._write_index, p_offset, p_length))
3317
"""Flush pending writes on this access object.
3319
This will flush any buffered writes to a NewPack.
3321
if self._flush_func is not None:
3324
def get_raw_records(self, memos_for_retrieval):
3325
"""Get the raw bytes for a records.
3327
:param memos_for_retrieval: An iterable containing the (index, pos,
3328
length) memo for retrieving the bytes. The Pack access method
3329
looks up the pack to use for a given record in its index_to_pack
3331
:return: An iterator over the bytes of the records.
3333
# first pass, group into same-index requests
3335
current_index = None
3336
for (index, offset, length) in memos_for_retrieval:
3337
if current_index == index:
3338
current_list.append((offset, length))
3340
if current_index is not None:
3341
request_lists.append((current_index, current_list))
3342
current_index = index
3343
current_list = [(offset, length)]
3344
# handle the last entry
3345
if current_index is not None:
3346
request_lists.append((current_index, current_list))
3347
for index, offsets in request_lists:
3349
transport, path = self._indices[index]
3351
# A KeyError here indicates that someone has triggered an index
3352
# reload, and this index has gone missing, we need to start
3354
if self._reload_func is None:
3355
# If we don't have a _reload_func there is nothing that can
3358
raise errors.RetryWithNewPacks(index,
3359
reload_occurred=True,
3360
exc_info=sys.exc_info())
3362
reader = pack.make_readv_reader(transport, path, offsets)
3363
for names, read_func in reader.iter_records():
3364
yield read_func(None)
3365
except errors.NoSuchFile:
3366
# A NoSuchFile error indicates that a pack file has gone
3367
# missing on disk, we need to trigger a reload, and start over.
3368
if self._reload_func is None:
3370
raise errors.RetryWithNewPacks(transport.abspath(path),
3371
reload_occurred=False,
3372
exc_info=sys.exc_info())
3374
def set_writer(self, writer, index, transport_packname):
3375
"""Set a writer to use for adding data."""
3376
if index is not None:
3377
self._indices[index] = transport_packname
3378
self._container_writer = writer
3379
self._write_index = index
3381
def reload_or_raise(self, retry_exc):
3382
"""Try calling the reload function, or re-raise the original exception.
3384
This should be called after _DirectPackAccess raises a
3385
RetryWithNewPacks exception. This function will handle the common logic
3386
of determining when the error is fatal versus being temporary.
3387
It will also make sure that the original exception is raised, rather
3388
than the RetryWithNewPacks exception.
3390
If this function returns, then the calling function should retry
3391
whatever operation was being performed. Otherwise an exception will
3394
:param retry_exc: A RetryWithNewPacks exception.
3397
if self._reload_func is None:
3399
elif not self._reload_func():
3400
# The reload claimed that nothing changed
3401
if not retry_exc.reload_occurred:
3402
# If there wasn't an earlier reload, then we really were
3403
# expecting to find changes. We didn't find them, so this is a
3407
exc_class, exc_value, exc_traceback = retry_exc.exc_info
3408
raise exc_class, exc_value, exc_traceback
3411
# Deprecated, use PatienceSequenceMatcher instead
3412
KnitSequenceMatcher = patiencediff.PatienceSequenceMatcher
3215
3415
def annotate_knit(knit, revision_id):
3216
3416
"""Annotate a knit with no cached annotations.
added
removed
bzrlib/knit.py
