1975
1986
:param key: The key of the record. Currently keys are always serialised
1976
1987
using just the trailing component.
1977
1988
:param dense_lines: The bytes of lines but in a denser form. For
1978
instance, if lines is a list of 1000 bytestrings each ending in
1979
\\n, dense_lines may be a list with one line in it, containing all
1980
the 1000's lines and their \\n's. Using dense_lines if it is
1981
already known is a win because the string join to create bytes in
1982
this function spends less time resizing the final string.
1989
instance, if lines is a list of 1000 bytestrings each ending in \n,
1990
dense_lines may be a list with one line in it, containing all the
1991
1000's lines and their \n's. Using dense_lines if it is already
1992
known is a win because the string join to create bytes in this
1993
function spends less time resizing the final string.
1983
1994
:return: (len, a StringIO instance with the raw data ready to read.)
1985
1996
chunks = ["version %s %d %s\n" % (key[-1], len(lines), digest)]
2776
2787
return key[:-1], key[-1]
2790
class _KeyRefs(object):
2792
def __init__(self, track_new_keys=False):
2793
# dict mapping 'key' to 'set of keys referring to that key'
2796
# set remembering all new keys
2797
self.new_keys = set()
2799
self.new_keys = None
2805
self.new_keys.clear()
2807
def add_references(self, key, refs):
2808
# Record the new references
2809
for referenced in refs:
2811
needed_by = self.refs[referenced]
2813
needed_by = self.refs[referenced] = set()
2815
# Discard references satisfied by the new key
2818
def get_new_keys(self):
2819
return self.new_keys
2821
def get_unsatisfied_refs(self):
2822
return self.refs.iterkeys()
2824
def _satisfy_refs_for_key(self, key):
2828
# No keys depended on this key. That's ok.
2831
def add_key(self, key):
2832
# satisfy refs for key, and remember that we've seen this key.
2833
self._satisfy_refs_for_key(key)
2834
if self.new_keys is not None:
2835
self.new_keys.add(key)
2837
def satisfy_refs_for_keys(self, keys):
2839
self._satisfy_refs_for_key(key)
2841
def get_referrers(self):
2843
for referrers in self.refs.itervalues():
2844
result.update(referrers)
2779
2848
class _KnitGraphIndex(object):
2780
2849
"""A KnitVersionedFiles index layered on GraphIndex."""
3282
class _DirectPackAccess(object):
3283
"""Access to data in one or more packs with less translation."""
3285
def __init__(self, index_to_packs, reload_func=None, flush_func=None):
3286
"""Create a _DirectPackAccess object.
3288
:param index_to_packs: A dict mapping index objects to the transport
3289
and file names for obtaining data.
3290
:param reload_func: A function to call if we determine that the pack
3291
files have moved and we need to reload our caches. See
3292
bzrlib.repo_fmt.pack_repo.AggregateIndex for more details.
3294
self._container_writer = None
3295
self._write_index = None
3296
self._indices = index_to_packs
3297
self._reload_func = reload_func
3298
self._flush_func = flush_func
3300
def add_raw_records(self, key_sizes, raw_data):
3301
"""Add raw knit bytes to a storage area.
3303
The data is spooled to the container writer in one bytes-record per
3306
:param sizes: An iterable of tuples containing the key and size of each
3308
:param raw_data: A bytestring containing the data.
3309
:return: A list of memos to retrieve the record later. Each memo is an
3310
opaque index memo. For _DirectPackAccess the memo is (index, pos,
3311
length), where the index field is the write_index object supplied
3312
to the PackAccess object.
3314
if type(raw_data) is not str:
3315
raise AssertionError(
3316
'data must be plain bytes was %s' % type(raw_data))
3319
for key, size in key_sizes:
3320
p_offset, p_length = self._container_writer.add_bytes_record(
3321
raw_data[offset:offset+size], [])
3323
result.append((self._write_index, p_offset, p_length))
3327
"""Flush pending writes on this access object.
3329
This will flush any buffered writes to a NewPack.
3331
if self._flush_func is not None:
3334
def get_raw_records(self, memos_for_retrieval):
3335
"""Get the raw bytes for a records.
3337
:param memos_for_retrieval: An iterable containing the (index, pos,
3338
length) memo for retrieving the bytes. The Pack access method
3339
looks up the pack to use for a given record in its index_to_pack
3341
:return: An iterator over the bytes of the records.
3343
# first pass, group into same-index requests
3345
current_index = None
3346
for (index, offset, length) in memos_for_retrieval:
3347
if current_index == index:
3348
current_list.append((offset, length))
3350
if current_index is not None:
3351
request_lists.append((current_index, current_list))
3352
current_index = index
3353
current_list = [(offset, length)]
3354
# handle the last entry
3355
if current_index is not None:
3356
request_lists.append((current_index, current_list))
3357
for index, offsets in request_lists:
3359
transport, path = self._indices[index]
3361
# A KeyError here indicates that someone has triggered an index
3362
# reload, and this index has gone missing, we need to start
3364
if self._reload_func is None:
3365
# If we don't have a _reload_func there is nothing that can
3368
raise errors.RetryWithNewPacks(index,
3369
reload_occurred=True,
3370
exc_info=sys.exc_info())
3372
reader = pack.make_readv_reader(transport, path, offsets)
3373
for names, read_func in reader.iter_records():
3374
yield read_func(None)
3375
except errors.NoSuchFile:
3376
# A NoSuchFile error indicates that a pack file has gone
3377
# missing on disk, we need to trigger a reload, and start over.
3378
if self._reload_func is None:
3380
raise errors.RetryWithNewPacks(transport.abspath(path),
3381
reload_occurred=False,
3382
exc_info=sys.exc_info())
3384
def set_writer(self, writer, index, transport_packname):
3385
"""Set a writer to use for adding data."""
3386
if index is not None:
3387
self._indices[index] = transport_packname
3388
self._container_writer = writer
3389
self._write_index = index
3391
def reload_or_raise(self, retry_exc):
3392
"""Try calling the reload function, or re-raise the original exception.
3394
This should be called after _DirectPackAccess raises a
3395
RetryWithNewPacks exception. This function will handle the common logic
3396
of determining when the error is fatal versus being temporary.
3397
It will also make sure that the original exception is raised, rather
3398
than the RetryWithNewPacks exception.
3400
If this function returns, then the calling function should retry
3401
whatever operation was being performed. Otherwise an exception will
3404
:param retry_exc: A RetryWithNewPacks exception.
3407
if self._reload_func is None:
3409
elif not self._reload_func():
3410
# The reload claimed that nothing changed
3411
if not retry_exc.reload_occurred:
3412
# If there wasn't an earlier reload, then we really were
3413
# expecting to find changes. We didn't find them, so this is a
3417
exc_class, exc_value, exc_traceback = retry_exc.exc_info
3418
raise exc_class, exc_value, exc_traceback
3213
3421
def annotate_knit(knit, revision_id):
3214
3422
"""Annotate a knit with no cached annotations.
added
removed
bzrlib/knit.py
