265
269
def __init__(self, name, revision_index, inventory_index, text_index,
266
270
signature_index, upload_transport, pack_transport, index_transport,
267
pack_collection, chk_index=None):
268
272
"""Create a ResumedPack object."""
269
273
ExistingPack.__init__(self, pack_transport, name, revision_index,
270
inventory_index, text_index, signature_index,
274
inventory_index, text_index, signature_index)
272
275
self.upload_transport = upload_transport
273
276
self.index_transport = index_transport
274
277
self.index_sizes = [None, None, None, None]
301
301
self.upload_transport.delete(self.file_name())
302
302
indices = [self.revision_index, self.inventory_index, self.text_index,
303
303
self.signature_index]
304
if self.chk_index is not None:
305
indices.append(self.chk_index)
306
304
for index in indices:
307
305
index._transport.delete(index._name)
309
307
def finish(self):
310
308
self._check_references()
311
index_types = ['revision', 'inventory', 'text', 'signature']
312
if self.chk_index is not None:
313
index_types.append('chk')
314
for index_type in index_types:
309
new_name = '../packs/' + self.file_name()
310
self.upload_transport.rename(self.file_name(), new_name)
311
for index_type in ['revision', 'inventory', 'text', 'signature']:
315
312
old_name = self.index_name(index_type, self.name)
316
313
new_name = '../indices/' + old_name
317
314
self.upload_transport.rename(old_name, new_name)
318
315
self._replace_index_with_readonly(index_type)
319
new_name = '../packs/' + self.file_name()
320
self.upload_transport.rename(self.file_name(), new_name)
321
316
self._state = 'finished'
323
318
def _get_external_refs(self, index):
324
"""Return compression parents for this index that are not present.
326
This returns any compression parents that are referenced by this index,
327
which are not contained *in* this index. They may be present elsewhere.
329
319
return index.external_references(1)
1504
1486
'containing %d revisions. Packing %d files into %d affecting %d'
1505
1487
' revisions', self, total_packs, total_revisions, num_old_packs,
1506
1488
num_new_packs, num_revs_affected)
1507
result = self._execute_pack_operations(pack_operations,
1489
self._execute_pack_operations(pack_operations,
1508
1490
reload_func=self._restart_autopack)
1509
1491
mutter('Auto-packing repository %s completed', self)
1512
1494
def _execute_pack_operations(self, pack_operations, _packer_class=Packer,
1513
1495
reload_func=None):
1558
1539
def _already_packed(self):
1559
1540
"""Is the collection already packed?"""
1560
return not (self.repo._format.pack_compresses or (len(self._names) > 1))
1541
return len(self._names) < 2
1562
def pack(self, hint=None):
1563
1544
"""Pack the pack collection totally."""
1564
1545
self.ensure_loaded()
1565
1546
total_packs = len(self._names)
1566
1547
if self._already_packed():
1548
# This is arguably wrong because we might not be optimal, but for
1549
# now lets leave it in. (e.g. reconcile -> one pack. But not
1568
1552
total_revisions = self.revision_index.combined_index.key_count()
1569
1553
# XXX: the following may want to be a class, to pack with a given
1571
1555
mutter('Packing repository %s, which has %d pack files, '
1572
'containing %d revisions with hint %r.', self, total_packs,
1573
total_revisions, hint)
1556
'containing %d revisions into 1 packs.', self, total_packs,
1574
1558
# determine which packs need changing
1559
pack_distribution = [1]
1575
1560
pack_operations = [[0, []]]
1576
1561
for pack in self.all_packs():
1577
if hint is None or pack.name in hint:
1578
# Either no hint was provided (so we are packing everything),
1579
# or this pack was included in the hint.
1580
pack_operations[-1][0] += pack.get_revision_count()
1581
pack_operations[-1][1].append(pack)
1562
pack_operations[-1][0] += pack.get_revision_count()
1563
pack_operations[-1][1].append(pack)
1582
1564
self._execute_pack_operations(pack_operations, OptimisingPacker)
1584
1566
def plan_autopack_combinations(self, existing_packs, pack_distribution):
1698
1680
inv_index = self._make_index(name, '.iix', resume=True)
1699
1681
txt_index = self._make_index(name, '.tix', resume=True)
1700
1682
sig_index = self._make_index(name, '.six', resume=True)
1701
if self.chk_index is not None:
1702
chk_index = self._make_index(name, '.cix', resume=True)
1705
result = self.resumed_pack_factory(name, rev_index, inv_index,
1706
txt_index, sig_index, self._upload_transport,
1707
self._pack_transport, self._index_transport, self,
1708
chk_index=chk_index)
1683
result = ResumedPack(name, rev_index, inv_index, txt_index,
1684
sig_index, self._upload_transport, self._pack_transport,
1685
self._index_transport, self)
1709
1686
except errors.NoSuchFile, e:
1710
1687
raise errors.UnresumableWriteGroup(self.repo, [name], str(e))
1711
1688
self.add_pack_to_memory(result)
2170
2150
self.revisions = KnitVersionedFiles(
2171
2151
_KnitGraphIndex(self._pack_collection.revision_index.combined_index,
2172
2152
add_callback=self._pack_collection.revision_index.add_callback,
2173
deltas=False, parents=True, is_locked=self.is_locked,
2174
track_external_parent_refs=True),
2153
deltas=False, parents=True, is_locked=self.is_locked),
2175
2154
data_access=self._pack_collection.revision_index.data_access,
2176
2155
max_delta_chain=0)
2177
2156
self.signatures = KnitVersionedFiles(
2222
2201
% (self._format, self.bzrdir.transport.base))
2224
2203
def _abort_write_group(self):
2225
self.revisions._index._key_dependencies.refs.clear()
2226
2204
self._pack_collection._abort_write_group()
2228
def _get_source(self, to_format):
2229
if to_format.network_name() == self._format.network_name():
2230
return KnitPackStreamSource(self, to_format)
2231
return super(KnitPackRepository, self)._get_source(to_format)
2206
def _find_inconsistent_revision_parents(self):
2207
"""Find revisions with incorrectly cached parents.
2209
:returns: an iterator yielding tuples of (revison-id, parents-in-index,
2210
parents-in-revision).
2212
if not self.is_locked():
2213
raise errors.ObjectNotLocked(self)
2214
pb = ui.ui_factory.nested_progress_bar()
2217
revision_nodes = self._pack_collection.revision_index \
2218
.combined_index.iter_all_entries()
2219
index_positions = []
2220
# Get the cached index values for all revisions, and also the
2221
# location in each index of the revision text so we can perform
2223
for index, key, value, refs in revision_nodes:
2224
node = (index, key, value, refs)
2225
index_memo = self.revisions._index._node_to_position(node)
2226
if index_memo[0] != index:
2227
raise AssertionError('%r != %r' % (index_memo[0], index))
2228
index_positions.append((index_memo, key[0],
2229
tuple(parent[0] for parent in refs[0])))
2230
pb.update("Reading revision index", 0, 0)
2231
index_positions.sort()
2233
pb.update("Checking cached revision graph", 0,
2234
len(index_positions))
2235
for offset in xrange(0, len(index_positions), 1000):
2236
pb.update("Checking cached revision graph", offset)
2237
to_query = index_positions[offset:offset + batch_size]
2240
rev_ids = [item[1] for item in to_query]
2241
revs = self.get_revisions(rev_ids)
2242
for revision, item in zip(revs, to_query):
2243
index_parents = item[2]
2244
rev_parents = tuple(revision.parent_ids)
2245
if index_parents != rev_parents:
2246
result.append((revision.revision_id, index_parents,
2233
2252
def _make_parents_provider(self):
2234
2253
return graph.CachingParentsProvider(self)
2242
2261
self._pack_collection._start_write_group()
2244
2263
def _commit_write_group(self):
2245
self.revisions._index._key_dependencies.refs.clear()
2246
2264
return self._pack_collection._commit_write_group()
2248
2266
def suspend_write_group(self):
2249
2267
# XXX check self._write_group is self.get_transaction()?
2250
2268
tokens = self._pack_collection._suspend_write_group()
2251
self.revisions._index._key_dependencies.refs.clear()
2252
2269
self._write_group = None
2255
2272
def _resume_write_group(self, tokens):
2256
2273
self._start_write_group()
2258
self._pack_collection._resume_write_group(tokens)
2259
except errors.UnresumableWriteGroup:
2260
self._abort_write_group()
2262
for pack in self._pack_collection._resumed_packs:
2263
self.revisions._index.scan_unvalidated_index(pack.revision_index)
2274
self._pack_collection._resume_write_group(tokens)
2265
2276
def get_transaction(self):
2266
2277
if self._write_lock_count:
2341
2353
transaction = self._transaction
2342
2354
self._transaction = None
2343
2355
transaction.finish()
2356
for repo in self._fallback_repositories:
2345
2359
self.control_files.unlock()
2347
if not self.is_locked():
2348
2360
for repo in self._fallback_repositories:
2352
class KnitPackStreamSource(StreamSource):
2353
"""A StreamSource used to transfer data between same-format KnitPack repos.
2355
This source assumes:
2356
1) Same serialization format for all objects
2357
2) Same root information
2358
3) XML format inventories
2359
4) Atomic inserts (so we can stream inventory texts before text
2364
def __init__(self, from_repository, to_format):
2365
super(KnitPackStreamSource, self).__init__(from_repository, to_format)
2366
self._text_keys = None
2367
self._text_fetch_order = 'unordered'
2369
def _get_filtered_inv_stream(self, revision_ids):
2370
from_repo = self.from_repository
2371
parent_ids = from_repo._find_parent_ids_of_revisions(revision_ids)
2372
parent_keys = [(p,) for p in parent_ids]
2373
find_text_keys = from_repo._find_text_key_references_from_xml_inventory_lines
2374
parent_text_keys = set(find_text_keys(
2375
from_repo._inventory_xml_lines_for_keys(parent_keys)))
2376
content_text_keys = set()
2377
knit = KnitVersionedFiles(None, None)
2378
factory = KnitPlainFactory()
2379
def find_text_keys_from_content(record):
2380
if record.storage_kind not in ('knit-delta-gz', 'knit-ft-gz'):
2381
raise ValueError("Unknown content storage kind for"
2382
" inventory text: %s" % (record.storage_kind,))
2383
# It's a knit record, it has a _raw_record field (even if it was
2384
# reconstituted from a network stream).
2385
raw_data = record._raw_record
2386
# read the entire thing
2387
revision_id = record.key[-1]
2388
content, _ = knit._parse_record(revision_id, raw_data)
2389
if record.storage_kind == 'knit-delta-gz':
2390
line_iterator = factory.get_linedelta_content(content)
2391
elif record.storage_kind == 'knit-ft-gz':
2392
line_iterator = factory.get_fulltext_content(content)
2393
content_text_keys.update(find_text_keys(
2394
[(line, revision_id) for line in line_iterator]))
2395
revision_keys = [(r,) for r in revision_ids]
2396
def _filtered_inv_stream():
2397
source_vf = from_repo.inventories
2398
stream = source_vf.get_record_stream(revision_keys,
2400
for record in stream:
2401
if record.storage_kind == 'absent':
2402
raise errors.NoSuchRevision(from_repo, record.key)
2403
find_text_keys_from_content(record)
2405
self._text_keys = content_text_keys - parent_text_keys
2406
return ('inventories', _filtered_inv_stream())
2408
def _get_text_stream(self):
2409
# Note: We know we don't have to handle adding root keys, because both
2410
# the source and target are the identical network name.
2411
text_stream = self.from_repository.texts.get_record_stream(
2412
self._text_keys, self._text_fetch_order, False)
2413
return ('texts', text_stream)
2415
def get_stream(self, search):
2416
revision_ids = search.get_keys()
2417
for stream_info in self._fetch_revision_texts(revision_ids):
2419
self._revision_keys = [(rev_id,) for rev_id in revision_ids]
2420
yield self._get_filtered_inv_stream(revision_ids)
2421
yield self._get_text_stream()
2425
2364
class RepositoryFormatPack(MetaDirRepositoryFormat):
2426
2365
"""Format logic for pack structured repositories.
2561
2503
_matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
2505
def check_conversion_target(self, target_format):
2506
if not target_format.rich_root_data:
2507
raise errors.BadConversionTarget(
2508
'Does not support rich root data.', target_format)
2509
if not getattr(target_format, 'supports_tree_reference', False):
2510
raise errors.BadConversionTarget(
2511
'Does not support nested trees', target_format)
2563
2513
def get_format_string(self):
2564
2514
"""See RepositoryFormat.get_format_string()."""
2565
2515
return "Bazaar pack repository format 1 with subtree support (needs bzr 0.92)\n"
2836
2812
_matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
2814
def check_conversion_target(self, target_format):
2815
if not target_format.rich_root_data:
2816
raise errors.BadConversionTarget(
2817
'Does not support rich root data.', target_format)
2818
if not getattr(target_format, 'supports_tree_reference', False):
2819
raise errors.BadConversionTarget(
2820
'Does not support nested trees', target_format)
2838
2822
def get_format_string(self):
2839
2823
"""See RepositoryFormat.get_format_string()."""
2840
2824
return ("Bazaar development format 2 with subtree support "