1975
1981
:param key: The key of the record. Currently keys are always serialised
1976
1982
using just the trailing component.
1977
1983
:param dense_lines: The bytes of lines but in a denser form. For
1978
instance, if lines is a list of 1000 bytestrings each ending in
1979
\\n, dense_lines may be a list with one line in it, containing all
1980
the 1000's lines and their \\n's. Using dense_lines if it is
1981
already known is a win because the string join to create bytes in
1982
this function spends less time resizing the final string.
1984
instance, if lines is a list of 1000 bytestrings each ending in \n,
1985
dense_lines may be a list with one line in it, containing all the
1986
1000's lines and their \n's. Using dense_lines if it is already
1987
known is a win because the string join to create bytes in this
1988
function spends less time resizing the final string.
1983
1989
:return: (len, a StringIO instance with the raw data ready to read.)
1985
1991
chunks = ["version %s %d %s\n" % (key[-1], len(lines), digest)]
2776
2782
return key[:-1], key[-1]
2785
class _KeyRefs(object):
2787
def __init__(self, track_new_keys=False):
2788
# dict mapping 'key' to 'set of keys referring to that key'
2791
# set remembering all new keys
2792
self.new_keys = set()
2794
self.new_keys = None
2800
self.new_keys.clear()
2802
def add_references(self, key, refs):
2803
# Record the new references
2804
for referenced in refs:
2806
needed_by = self.refs[referenced]
2808
needed_by = self.refs[referenced] = set()
2810
# Discard references satisfied by the new key
2813
def get_new_keys(self):
2814
return self.new_keys
2816
def get_unsatisfied_refs(self):
2817
return self.refs.iterkeys()
2819
def _satisfy_refs_for_key(self, key):
2823
# No keys depended on this key. That's ok.
2826
def add_key(self, key):
2827
# satisfy refs for key, and remember that we've seen this key.
2828
self._satisfy_refs_for_key(key)
2829
if self.new_keys is not None:
2830
self.new_keys.add(key)
2832
def satisfy_refs_for_keys(self, keys):
2834
self._satisfy_refs_for_key(key)
2836
def get_referrers(self):
2838
for referrers in self.refs.itervalues():
2839
result.update(referrers)
2779
2843
class _KnitGraphIndex(object):
2780
2844
"""A KnitVersionedFiles index layered on GraphIndex."""
3277
class _DirectPackAccess(object):
3278
"""Access to data in one or more packs with less translation."""
3280
def __init__(self, index_to_packs, reload_func=None, flush_func=None):
3281
"""Create a _DirectPackAccess object.
3283
:param index_to_packs: A dict mapping index objects to the transport
3284
and file names for obtaining data.
3285
:param reload_func: A function to call if we determine that the pack
3286
files have moved and we need to reload our caches. See
3287
bzrlib.repo_fmt.pack_repo.AggregateIndex for more details.
3289
self._container_writer = None
3290
self._write_index = None
3291
self._indices = index_to_packs
3292
self._reload_func = reload_func
3293
self._flush_func = flush_func
3295
def add_raw_records(self, key_sizes, raw_data):
3296
"""Add raw knit bytes to a storage area.
3298
The data is spooled to the container writer in one bytes-record per
3301
:param sizes: An iterable of tuples containing the key and size of each
3303
:param raw_data: A bytestring containing the data.
3304
:return: A list of memos to retrieve the record later. Each memo is an
3305
opaque index memo. For _DirectPackAccess the memo is (index, pos,
3306
length), where the index field is the write_index object supplied
3307
to the PackAccess object.
3309
if type(raw_data) is not str:
3310
raise AssertionError(
3311
'data must be plain bytes was %s' % type(raw_data))
3314
for key, size in key_sizes:
3315
p_offset, p_length = self._container_writer.add_bytes_record(
3316
raw_data[offset:offset+size], [])
3318
result.append((self._write_index, p_offset, p_length))
3322
"""Flush pending writes on this access object.
3324
This will flush any buffered writes to a NewPack.
3326
if self._flush_func is not None:
3329
def get_raw_records(self, memos_for_retrieval):
3330
"""Get the raw bytes for a records.
3332
:param memos_for_retrieval: An iterable containing the (index, pos,
3333
length) memo for retrieving the bytes. The Pack access method
3334
looks up the pack to use for a given record in its index_to_pack
3336
:return: An iterator over the bytes of the records.
3338
# first pass, group into same-index requests
3340
current_index = None
3341
for (index, offset, length) in memos_for_retrieval:
3342
if current_index == index:
3343
current_list.append((offset, length))
3345
if current_index is not None:
3346
request_lists.append((current_index, current_list))
3347
current_index = index
3348
current_list = [(offset, length)]
3349
# handle the last entry
3350
if current_index is not None:
3351
request_lists.append((current_index, current_list))
3352
for index, offsets in request_lists:
3354
transport, path = self._indices[index]
3356
# A KeyError here indicates that someone has triggered an index
3357
# reload, and this index has gone missing, we need to start
3359
if self._reload_func is None:
3360
# If we don't have a _reload_func there is nothing that can
3363
raise errors.RetryWithNewPacks(index,
3364
reload_occurred=True,
3365
exc_info=sys.exc_info())
3367
reader = pack.make_readv_reader(transport, path, offsets)
3368
for names, read_func in reader.iter_records():
3369
yield read_func(None)
3370
except errors.NoSuchFile:
3371
# A NoSuchFile error indicates that a pack file has gone
3372
# missing on disk, we need to trigger a reload, and start over.
3373
if self._reload_func is None:
3375
raise errors.RetryWithNewPacks(transport.abspath(path),
3376
reload_occurred=False,
3377
exc_info=sys.exc_info())
3379
def set_writer(self, writer, index, transport_packname):
3380
"""Set a writer to use for adding data."""
3381
if index is not None:
3382
self._indices[index] = transport_packname
3383
self._container_writer = writer
3384
self._write_index = index
3386
def reload_or_raise(self, retry_exc):
3387
"""Try calling the reload function, or re-raise the original exception.
3389
This should be called after _DirectPackAccess raises a
3390
RetryWithNewPacks exception. This function will handle the common logic
3391
of determining when the error is fatal versus being temporary.
3392
It will also make sure that the original exception is raised, rather
3393
than the RetryWithNewPacks exception.
3395
If this function returns, then the calling function should retry
3396
whatever operation was being performed. Otherwise an exception will
3399
:param retry_exc: A RetryWithNewPacks exception.
3402
if self._reload_func is None:
3404
elif not self._reload_func():
3405
# The reload claimed that nothing changed
3406
if not retry_exc.reload_occurred:
3407
# If there wasn't an earlier reload, then we really were
3408
# expecting to find changes. We didn't find them, so this is a
3412
exc_class, exc_value, exc_traceback = retry_exc.exc_info
3413
raise exc_class, exc_value, exc_traceback
3213
3416
def annotate_knit(knit, revision_id):
3214
3417
"""Annotate a knit with no cached annotations.
added
removed
bzrlib/knit.py
