287
320
self.storage_kind)
323
class LazyKnitContentFactory(ContentFactory):
324
"""A ContentFactory which can either generate full text or a wire form.
326
:seealso ContentFactory:
329
def __init__(self, key, parents, generator, first):
330
"""Create a LazyKnitContentFactory.
332
:param key: The key of the record.
333
:param parents: The parents of the record.
334
:param generator: A _ContentMapGenerator containing the record for this
336
:param first: Is this the first content object returned from generator?
337
if it is, its storage kind is knit-delta-closure, otherwise it is
338
knit-delta-closure-ref
341
self.parents = parents
343
self._generator = generator
344
self.storage_kind = "knit-delta-closure"
346
self.storage_kind = self.storage_kind + "-ref"
349
def get_bytes_as(self, storage_kind):
350
if storage_kind == self.storage_kind:
352
return self._generator._wire_bytes()
354
# all the keys etc are contained in the bytes returned in the
357
if storage_kind in ('chunked', 'fulltext'):
358
chunks = self._generator._get_one_work(self.key).text()
359
if storage_kind == 'chunked':
362
return ''.join(chunks)
363
raise errors.UnavailableRepresentation(self.key, storage_kind,
367
def knit_delta_closure_to_records(storage_kind, bytes, line_end):
368
"""Convert a network record to a iterator over stream records.
370
:param storage_kind: The storage kind of the record.
371
Must be 'knit-delta-closure'.
372
:param bytes: The bytes of the record on the network.
374
generator = _NetworkContentMapGenerator(bytes, line_end)
375
return generator.get_record_stream()
378
def knit_network_to_record(storage_kind, bytes, line_end):
379
"""Convert a network record to a record object.
381
:param storage_kind: The storage kind of the record.
382
:param bytes: The bytes of the record on the network.
385
line_end = bytes.find('\n', start)
386
key = tuple(bytes[start:line_end].split('\x00'))
388
line_end = bytes.find('\n', start)
389
parent_line = bytes[start:line_end]
390
if parent_line == 'None:':
394
[tuple(segment.split('\x00')) for segment in parent_line.split('\t')
397
noeol = bytes[start] == 'N'
398
if 'ft' in storage_kind:
401
method = 'line-delta'
402
build_details = (method, noeol)
404
raw_record = bytes[start:]
405
annotated = 'annotated' in storage_kind
406
return [KnitContentFactory(key, parents, build_details, None, raw_record,
407
annotated, network_bytes=bytes)]
290
410
class KnitContent(object):
291
411
"""Content of a knit version to which deltas can be applied.
293
413
This is always stored in memory as a list of lines with \n at the end,
294
plus a flag saying if the final ending is really there or not, because that
414
plus a flag saying if the final ending is really there or not, because that
295
415
corresponds to the on-disk knit representation.
760
909
# indexes can't directly store that, so we give them
761
910
# an empty tuple instead.
912
line_bytes = ''.join(lines)
763
913
return self._add(key, lines, parents,
764
parent_texts, left_matching_blocks, nostore_sha, random_id)
914
parent_texts, left_matching_blocks, nostore_sha, random_id,
915
line_bytes=line_bytes)
917
def _add_text(self, key, parents, text, nostore_sha=None, random_id=False):
918
"""See VersionedFiles._add_text()."""
919
self._index._check_write_ok()
920
self._check_add(key, None, random_id, check_content=False)
921
if text.__class__ is not str:
922
raise errors.BzrBadParameterUnicode("text")
924
# The caller might pass None if there is no graph data, but kndx
925
# indexes can't directly store that, so we give them
926
# an empty tuple instead.
928
return self._add(key, None, parents,
929
None, None, nostore_sha, random_id,
766
932
def _add(self, key, lines, parents, parent_texts,
767
left_matching_blocks, nostore_sha, random_id):
933
left_matching_blocks, nostore_sha, random_id,
768
935
"""Add a set of lines on top of version specified by parents.
770
937
Any versions not present will be converted into ghosts.
939
:param lines: A list of strings where each one is a single line (has a
940
single newline at the end of the string) This is now optional
941
(callers can pass None). It is left in its location for backwards
942
compatibility. It should ''.join(lines) must == line_bytes
943
:param line_bytes: A single string containing the content
945
We pass both lines and line_bytes because different routes bring the
946
values to this function. And for memory efficiency, we don't want to
947
have to split/join on-demand.
772
949
# first thing, if the content is something we don't need to store, find
774
line_bytes = ''.join(lines)
775
951
digest = sha_string(line_bytes)
776
952
if nostore_sha == digest:
777
953
raise errors.ExistingContent
986
1179
if not self.get_parent_map([key]):
987
1180
raise RevisionNotPresent(key, self)
988
1181
return cached_version
989
text_map, contents_map = self._get_content_maps([key])
990
return contents_map[key]
992
def _get_content_maps(self, keys, nonlocal_keys=None):
993
"""Produce maps of text and KnitContents
995
:param keys: The keys to produce content maps for.
996
:param nonlocal_keys: An iterable of keys(possibly intersecting keys)
997
which are known to not be in this knit, but rather in one of the
999
:return: (text_map, content_map) where text_map contains the texts for
1000
the requested versions and content_map contains the KnitContents.
1002
# FUTURE: This function could be improved for the 'extract many' case
1003
# by tracking each component and only doing the copy when the number of
1004
# children than need to apply delta's to it is > 1 or it is part of the
1007
multiple_versions = len(keys) != 1
1008
record_map = self._get_record_map(keys, allow_missing=True)
1013
if nonlocal_keys is None:
1014
nonlocal_keys = set()
1016
nonlocal_keys = frozenset(nonlocal_keys)
1017
missing_keys = set(nonlocal_keys)
1018
for source in self._fallback_vfs:
1019
if not missing_keys:
1021
for record in source.get_record_stream(missing_keys,
1023
if record.storage_kind == 'absent':
1025
missing_keys.remove(record.key)
1026
lines = osutils.chunks_to_lines(record.get_bytes_as('chunked'))
1027
text_map[record.key] = lines
1028
content_map[record.key] = PlainKnitContent(lines, record.key)
1029
if record.key in keys:
1030
final_content[record.key] = content_map[record.key]
1032
if key in nonlocal_keys:
1037
while cursor is not None:
1039
record, record_details, digest, next = record_map[cursor]
1041
raise RevisionNotPresent(cursor, self)
1042
components.append((cursor, record, record_details, digest))
1044
if cursor in content_map:
1045
# no need to plan further back
1046
components.append((cursor, None, None, None))
1050
for (component_id, record, record_details,
1051
digest) in reversed(components):
1052
if component_id in content_map:
1053
content = content_map[component_id]
1055
content, delta = self._factory.parse_record(key[-1],
1056
record, record_details, content,
1057
copy_base_content=multiple_versions)
1058
if multiple_versions:
1059
content_map[component_id] = content
1061
final_content[key] = content
1063
# digest here is the digest from the last applied component.
1064
text = content.text()
1065
actual_sha = sha_strings(text)
1066
if actual_sha != digest:
1067
raise SHA1KnitCorrupt(self, actual_sha, digest, key, text)
1068
text_map[key] = text
1069
return text_map, final_content
1182
generator = _VFContentMapGenerator(self, [key])
1183
return generator._get_content(key)
1071
1185
def get_parent_map(self, keys):
1072
1186
"""Get a map of the graph parents of keys.
1149
1289
This should be revisited if _get_content_maps() can ever cross
1150
1290
file-id boundaries.
1292
The keys for a given file_id are kept in the same relative order.
1293
Ordering between file_ids is not, though prefix_order will return the
1294
order that the key was first seen.
1152
1296
:param keys: An iterable of key tuples
1153
:return: A dict of {prefix: [key_list]}
1297
:return: (split_map, prefix_order)
1298
split_map A dictionary mapping prefix => keys
1299
prefix_order The order that we saw the various prefixes
1155
1301
split_by_prefix = {}
1156
1303
for key in keys:
1157
1304
if len(key) == 1:
1158
split_by_prefix.setdefault('', []).append(key)
1160
split_by_prefix.setdefault(key[0], []).append(key)
1161
return split_by_prefix
1309
if prefix in split_by_prefix:
1310
split_by_prefix[prefix].append(key)
1312
split_by_prefix[prefix] = [key]
1313
prefix_order.append(prefix)
1314
return split_by_prefix, prefix_order
1316
def _group_keys_for_io(self, keys, non_local_keys, positions,
1317
_min_buffer_size=_STREAM_MIN_BUFFER_SIZE):
1318
"""For the given keys, group them into 'best-sized' requests.
1320
The idea is to avoid making 1 request per file, but to never try to
1321
unpack an entire 1.5GB source tree in a single pass. Also when
1322
possible, we should try to group requests to the same pack file
1325
:return: list of (keys, non_local) tuples that indicate what keys
1326
should be fetched next.
1328
# TODO: Ideally we would group on 2 factors. We want to extract texts
1329
# from the same pack file together, and we want to extract all
1330
# the texts for a given build-chain together. Ultimately it
1331
# probably needs a better global view.
1332
total_keys = len(keys)
1333
prefix_split_keys, prefix_order = self._split_by_prefix(keys)
1334
prefix_split_non_local_keys, _ = self._split_by_prefix(non_local_keys)
1336
cur_non_local = set()
1340
for prefix in prefix_order:
1341
keys = prefix_split_keys[prefix]
1342
non_local = prefix_split_non_local_keys.get(prefix, [])
1344
this_size = self._index._get_total_build_size(keys, positions)
1345
cur_size += this_size
1346
cur_keys.extend(keys)
1347
cur_non_local.update(non_local)
1348
if cur_size > _min_buffer_size:
1349
result.append((cur_keys, cur_non_local))
1350
sizes.append(cur_size)
1352
cur_non_local = set()
1355
result.append((cur_keys, cur_non_local))
1356
sizes.append(cur_size)
1163
1359
def get_record_stream(self, keys, ordering, include_delta_closure):
1164
1360
"""Get a stream of records for keys.
1448
1647
elif record.storage_kind == 'chunked':
1449
1648
self.add_lines(record.key, parents,
1450
1649
osutils.chunks_to_lines(record.get_bytes_as('chunked')))
1451
elif record.storage_kind == 'fulltext':
1452
self.add_lines(record.key, parents,
1453
split_lines(record.get_bytes_as('fulltext')))
1455
# Not a fulltext, and not suitable for direct insertion as a
1651
# Not suitable for direct insertion as a
1456
1652
# delta, either because it's not the right format, or this
1457
1653
# KnitVersionedFiles doesn't permit deltas (_max_delta_chain ==
1458
1654
# 0) or because it depends on a base only present in the
1459
1655
# fallback kvfs.
1460
adapter_key = record.storage_kind, 'fulltext'
1461
adapter = get_adapter(adapter_key)
1462
lines = split_lines(adapter.get_bytes(
1463
record, record.get_bytes_as(record.storage_kind)))
1656
self._access.flush()
1658
# Try getting a fulltext directly from the record.
1659
bytes = record.get_bytes_as('fulltext')
1660
except errors.UnavailableRepresentation:
1661
adapter_key = record.storage_kind, 'fulltext'
1662
adapter = get_adapter(adapter_key)
1663
bytes = adapter.get_bytes(record)
1664
lines = split_lines(bytes)
1465
1666
self.add_lines(record.key, parents, lines)
1466
1667
except errors.RevisionAlreadyPresent:
1468
1669
# Add any records whose basis parent is now available.
1469
added_keys = [record.key]
1471
key = added_keys.pop(0)
1472
if key in buffered_index_entries:
1473
index_entries = buffered_index_entries[key]
1474
self._index.add_records(index_entries)
1476
[index_entry[0] for index_entry in index_entries])
1477
del buffered_index_entries[key]
1478
# If there were any deltas which had a missing basis parent, error.
1671
added_keys = [record.key]
1673
key = added_keys.pop(0)
1674
if key in buffered_index_entries:
1675
index_entries = buffered_index_entries[key]
1676
self._index.add_records(index_entries)
1678
[index_entry[0] for index_entry in index_entries])
1679
del buffered_index_entries[key]
1479
1680
if buffered_index_entries:
1480
from pprint import pformat
1481
raise errors.BzrCheckError(
1482
"record_stream refers to compression parents not in %r:\n%s"
1483
% (self, pformat(sorted(buffered_index_entries.keys()))))
1681
# There were index entries buffered at the end of the stream,
1682
# So these need to be added (if the index supports holding such
1683
# entries for later insertion)
1684
for key in buffered_index_entries:
1685
index_entries = buffered_index_entries[key]
1686
self._index.add_records(index_entries,
1687
missing_compression_parents=True)
1689
def get_missing_compression_parent_keys(self):
1690
"""Return an iterable of keys of missing compression parents.
1692
Check this after calling insert_record_stream to find out if there are
1693
any missing compression parents. If there are, the records that
1694
depend on them are not able to be inserted safely. For atomic
1695
KnitVersionedFiles built on packs, the transaction should be aborted or
1696
suspended - commit will fail at this point. Nonatomic knits will error
1697
earlier because they have no staging area to put pending entries into.
1699
return self._index.get_missing_compression_parents()
1485
1701
def iter_lines_added_or_present_in_keys(self, keys, pb=None):
1486
1702
"""Iterate over the lines in the versioned files from keys.
1992
class _ContentMapGenerator(object):
1993
"""Generate texts or expose raw deltas for a set of texts."""
1995
def _get_content(self, key):
1996
"""Get the content object for key."""
1997
# Note that _get_content is only called when the _ContentMapGenerator
1998
# has been constructed with just one key requested for reconstruction.
1999
if key in self.nonlocal_keys:
2000
record = self.get_record_stream().next()
2001
# Create a content object on the fly
2002
lines = osutils.chunks_to_lines(record.get_bytes_as('chunked'))
2003
return PlainKnitContent(lines, record.key)
2005
# local keys we can ask for directly
2006
return self._get_one_work(key)
2008
def get_record_stream(self):
2009
"""Get a record stream for the keys requested during __init__."""
2010
for record in self._work():
2014
"""Produce maps of text and KnitContents as dicts.
2016
:return: (text_map, content_map) where text_map contains the texts for
2017
the requested versions and content_map contains the KnitContents.
2019
# NB: By definition we never need to read remote sources unless texts
2020
# are requested from them: we don't delta across stores - and we
2021
# explicitly do not want to to prevent data loss situations.
2022
if self.global_map is None:
2023
self.global_map = self.vf.get_parent_map(self.keys)
2024
nonlocal_keys = self.nonlocal_keys
2026
missing_keys = set(nonlocal_keys)
2027
# Read from remote versioned file instances and provide to our caller.
2028
for source in self.vf._fallback_vfs:
2029
if not missing_keys:
2031
# Loop over fallback repositories asking them for texts - ignore
2032
# any missing from a particular fallback.
2033
for record in source.get_record_stream(missing_keys,
2035
if record.storage_kind == 'absent':
2036
# Not in thie particular stream, may be in one of the
2037
# other fallback vfs objects.
2039
missing_keys.remove(record.key)
2042
if self._raw_record_map is None:
2043
raise AssertionError('_raw_record_map should have been filled')
2045
for key in self.keys:
2046
if key in self.nonlocal_keys:
2048
yield LazyKnitContentFactory(key, self.global_map[key], self, first)
2051
def _get_one_work(self, requested_key):
2052
# Now, if we have calculated everything already, just return the
2054
if requested_key in self._contents_map:
2055
return self._contents_map[requested_key]
2056
# To simplify things, parse everything at once - code that wants one text
2057
# probably wants them all.
2058
# FUTURE: This function could be improved for the 'extract many' case
2059
# by tracking each component and only doing the copy when the number of
2060
# children than need to apply delta's to it is > 1 or it is part of the
2062
multiple_versions = len(self.keys) != 1
2063
if self._record_map is None:
2064
self._record_map = self.vf._raw_map_to_record_map(
2065
self._raw_record_map)
2066
record_map = self._record_map
2067
# raw_record_map is key:
2068
# Have read and parsed records at this point.
2069
for key in self.keys:
2070
if key in self.nonlocal_keys:
2075
while cursor is not None:
2077
record, record_details, digest, next = record_map[cursor]
2079
raise RevisionNotPresent(cursor, self)
2080
components.append((cursor, record, record_details, digest))
2082
if cursor in self._contents_map:
2083
# no need to plan further back
2084
components.append((cursor, None, None, None))
2088
for (component_id, record, record_details,
2089
digest) in reversed(components):
2090
if component_id in self._contents_map:
2091
content = self._contents_map[component_id]
2093
content, delta = self._factory.parse_record(key[-1],
2094
record, record_details, content,
2095
copy_base_content=multiple_versions)
2096
if multiple_versions:
2097
self._contents_map[component_id] = content
2099
# digest here is the digest from the last applied component.
2100
text = content.text()
2101
actual_sha = sha_strings(text)
2102
if actual_sha != digest:
2103
raise SHA1KnitCorrupt(self, actual_sha, digest, key, text)
2104
if multiple_versions:
2105
return self._contents_map[requested_key]
2109
def _wire_bytes(self):
2110
"""Get the bytes to put on the wire for 'key'.
2112
The first collection of bytes asked for returns the serialised
2113
raw_record_map and the additional details (key, parent) for key.
2114
Subsequent calls return just the additional details (key, parent).
2115
The wire storage_kind given for the first key is 'knit-delta-closure',
2116
For subsequent keys it is 'knit-delta-closure-ref'.
2118
:param key: A key from the content generator.
2119
:return: Bytes to put on the wire.
2122
# kind marker for dispatch on the far side,
2123
lines.append('knit-delta-closure')
2125
if self.vf._factory.annotated:
2126
lines.append('annotated')
2129
# then the list of keys
2130
lines.append('\t'.join(['\x00'.join(key) for key in self.keys
2131
if key not in self.nonlocal_keys]))
2132
# then the _raw_record_map in serialised form:
2134
# for each item in the map:
2136
# 1 line with parents if the key is to be yielded (None: for None, '' for ())
2137
# one line with method
2138
# one line with noeol
2139
# one line with next ('' for None)
2140
# one line with byte count of the record bytes
2142
for key, (record_bytes, (method, noeol), next) in \
2143
self._raw_record_map.iteritems():
2144
key_bytes = '\x00'.join(key)
2145
parents = self.global_map.get(key, None)
2147
parent_bytes = 'None:'
2149
parent_bytes = '\t'.join('\x00'.join(key) for key in parents)
2150
method_bytes = method
2156
next_bytes = '\x00'.join(next)
2159
map_byte_list.append('%s\n%s\n%s\n%s\n%s\n%d\n%s' % (
2160
key_bytes, parent_bytes, method_bytes, noeol_bytes, next_bytes,
2161
len(record_bytes), record_bytes))
2162
map_bytes = ''.join(map_byte_list)
2163
lines.append(map_bytes)
2164
bytes = '\n'.join(lines)
2168
class _VFContentMapGenerator(_ContentMapGenerator):
2169
"""Content map generator reading from a VersionedFiles object."""
2171
def __init__(self, versioned_files, keys, nonlocal_keys=None,
2172
global_map=None, raw_record_map=None):
2173
"""Create a _ContentMapGenerator.
2175
:param versioned_files: The versioned files that the texts are being
2177
:param keys: The keys to produce content maps for.
2178
:param nonlocal_keys: An iterable of keys(possibly intersecting keys)
2179
which are known to not be in this knit, but rather in one of the
2181
:param global_map: The result of get_parent_map(keys) (or a supermap).
2182
This is required if get_record_stream() is to be used.
2183
:param raw_record_map: A unparsed raw record map to use for answering
2186
# The vf to source data from
2187
self.vf = versioned_files
2189
self.keys = list(keys)
2190
# Keys known to be in fallback vfs objects
2191
if nonlocal_keys is None:
2192
self.nonlocal_keys = set()
2194
self.nonlocal_keys = frozenset(nonlocal_keys)
2195
# Parents data for keys to be returned in get_record_stream
2196
self.global_map = global_map
2197
# The chunked lists for self.keys in text form
2199
# A cache of KnitContent objects used in extracting texts.
2200
self._contents_map = {}
2201
# All the knit records needed to assemble the requested keys as full
2203
self._record_map = None
2204
if raw_record_map is None:
2205
self._raw_record_map = self.vf._get_record_map_unparsed(keys,
2208
self._raw_record_map = raw_record_map
2209
# the factory for parsing records
2210
self._factory = self.vf._factory
2213
class _NetworkContentMapGenerator(_ContentMapGenerator):
2214
"""Content map generator sourced from a network stream."""
2216
def __init__(self, bytes, line_end):
2217
"""Construct a _NetworkContentMapGenerator from a bytes block."""
2219
self.global_map = {}
2220
self._raw_record_map = {}
2221
self._contents_map = {}
2222
self._record_map = None
2223
self.nonlocal_keys = []
2224
# Get access to record parsing facilities
2225
self.vf = KnitVersionedFiles(None, None)
2228
line_end = bytes.find('\n', start)
2229
line = bytes[start:line_end]
2230
start = line_end + 1
2231
if line == 'annotated':
2232
self._factory = KnitAnnotateFactory()
2234
self._factory = KnitPlainFactory()
2235
# list of keys to emit in get_record_stream
2236
line_end = bytes.find('\n', start)
2237
line = bytes[start:line_end]
2238
start = line_end + 1
2240
tuple(segment.split('\x00')) for segment in line.split('\t')
2242
# now a loop until the end. XXX: It would be nice if this was just a
2243
# bunch of the same records as get_record_stream(..., False) gives, but
2244
# there is a decent sized gap stopping that at the moment.
2248
line_end = bytes.find('\n', start)
2249
key = tuple(bytes[start:line_end].split('\x00'))
2250
start = line_end + 1
2251
# 1 line with parents (None: for None, '' for ())
2252
line_end = bytes.find('\n', start)
2253
line = bytes[start:line_end]
2258
[tuple(segment.split('\x00')) for segment in line.split('\t')
2260
self.global_map[key] = parents
2261
start = line_end + 1
2262
# one line with method
2263
line_end = bytes.find('\n', start)
2264
line = bytes[start:line_end]
2266
start = line_end + 1
2267
# one line with noeol
2268
line_end = bytes.find('\n', start)
2269
line = bytes[start:line_end]
2271
start = line_end + 1
2272
# one line with next ('' for None)
2273
line_end = bytes.find('\n', start)
2274
line = bytes[start:line_end]
2278
next = tuple(bytes[start:line_end].split('\x00'))
2279
start = line_end + 1
2280
# one line with byte count of the record bytes
2281
line_end = bytes.find('\n', start)
2282
line = bytes[start:line_end]
2284
start = line_end + 1
2286
record_bytes = bytes[start:start+count]
2287
start = start + count
2289
self._raw_record_map[key] = (record_bytes, (method, noeol), next)
2291
def get_record_stream(self):
2292
"""Get a record stream for for keys requested by the bytestream."""
2294
for key in self.keys:
2295
yield LazyKnitContentFactory(key, self.global_map[key], self, first)
2298
def _wire_bytes(self):
1769
2302
class _KndxIndex(object):
1770
2303
"""Manages knit index files
2161
2715
return index_memo[0][:-1], index_memo[1]
2162
2716
return keys.sort(key=get_sort_key)
2718
_get_total_build_size = _get_total_build_size
2164
2720
def _split_key(self, key):
2165
2721
"""Split key into a prefix and suffix."""
2166
2722
return key[:-1], key[-1]
2725
class _KeyRefs(object):
2728
# dict mapping 'key' to 'set of keys referring to that key'
2731
def add_references(self, key, refs):
2732
# Record the new references
2733
for referenced in refs:
2735
needed_by = self.refs[referenced]
2737
needed_by = self.refs[referenced] = set()
2739
# Discard references satisfied by the new key
2742
def get_unsatisfied_refs(self):
2743
return self.refs.iterkeys()
2745
def add_key(self, key):
2749
# No keys depended on this key. That's ok.
2752
def add_keys(self, keys):
2756
def get_referrers(self):
2758
for referrers in self.refs.itervalues():
2759
result.update(referrers)
2169
2763
class _KnitGraphIndex(object):
2170
2764
"""A KnitVersionedFiles index layered on GraphIndex."""
2172
2766
def __init__(self, graph_index, is_locked, deltas=False, parents=True,
2767
add_callback=None, track_external_parent_refs=False):
2174
2768
"""Construct a KnitGraphIndex on a graph_index.
2176
2770
:param graph_index: An implementation of bzrlib.index.GraphIndex.
2177
2771
:param is_locked: A callback to check whether the object should answer
2179
2773
:param deltas: Allow delta-compressed records.
2180
:param parents: If True, record knits parents, if not do not record
2774
:param parents: If True, record knits parents, if not do not record
2182
2776
:param add_callback: If not None, allow additions to the index and call
2183
2777
this callback with a list of added GraphIndex nodes:
2184
2778
[(node, value, node_refs), ...]
2185
2779
:param is_locked: A callback, returns True if the index is locked and
2781
:param track_external_parent_refs: If True, record all external parent
2782
references parents from added records. These can be retrieved
2783
later by calling get_missing_parents().
2188
2785
self._add_callback = add_callback
2189
2786
self._graph_index = graph_index
2262
2875
for key, (value, node_refs) in keys.iteritems():
2263
2876
result.append((key, value))
2264
2877
self._add_callback(result)
2878
if missing_compression_parents:
2879
# This may appear to be incorrect (it does not check for
2880
# compression parents that are in the existing graph index),
2881
# but such records won't have been buffered, so this is
2882
# actually correct: every entry when
2883
# missing_compression_parents==True either has a missing parent, or
2884
# a parent that is one of the keys in records.
2885
compression_parents.difference_update(keys)
2886
self._missing_compression_parents.update(compression_parents)
2887
# Adding records may have satisfied missing compression parents.
2888
self._missing_compression_parents.difference_update(keys)
2890
def scan_unvalidated_index(self, graph_index):
2891
"""Inform this _KnitGraphIndex that there is an unvalidated index.
2893
This allows this _KnitGraphIndex to keep track of any missing
2894
compression parents we may want to have filled in to make those
2897
:param graph_index: A GraphIndex
2900
new_missing = graph_index.external_references(ref_list_num=1)
2901
new_missing.difference_update(self.get_parent_map(new_missing))
2902
self._missing_compression_parents.update(new_missing)
2903
if self._key_dependencies is not None:
2904
# Add parent refs from graph_index (and discard parent refs that
2905
# the graph_index has).
2906
for node in graph_index.iter_all_entries():
2907
self._key_dependencies.add_references(node[1], node[3][0])
2909
def get_missing_compression_parents(self):
2910
"""Return the keys of missing compression parents.
2912
Missing compression parents occur when a record stream was missing
2913
basis texts, or a index was scanned that had missing basis texts.
2915
return frozenset(self._missing_compression_parents)
2917
def get_missing_parents(self):
2918
"""Return the keys of missing parents."""
2919
# If updating this, you should also update
2920
# groupcompress._GCGraphIndex.get_missing_parents
2921
# We may have false positives, so filter those out.
2922
self._key_dependencies.add_keys(
2923
self.get_parent_map(self._key_dependencies.get_unsatisfied_refs()))
2924
return frozenset(self._key_dependencies.get_unsatisfied_refs())
2266
2926
def _check_read(self):
2267
2927
"""raise if reads are not permitted."""
2268
2928
if not self._is_locked():
added
removed
bzrlib/knit.py
