222
216
'signature': ('.six', 3),
225
def __init__(self, upload_transport, index_transport, pack_transport,
226
upload_suffix='', file_mode=None, index_builder_class=None,
219
def __init__(self, pack_collection, upload_suffix='', file_mode=None):
228
220
"""Create a NewPack instance.
230
:param upload_transport: A writable transport for the pack to be
231
incrementally uploaded to.
232
:param index_transport: A writable transport for the pack's indices to
233
be written to when the pack is finished.
234
:param pack_transport: A writable transport for the pack to be renamed
235
to when the upload is complete. This *must* be the same as
236
upload_transport.clone('../packs').
222
:param pack_collection: A PackCollection into which this is being inserted.
237
223
:param upload_suffix: An optional suffix to be given to any temporary
238
224
files created during the pack creation. e.g '.autopack'
239
:param file_mode: An optional file mode to create the new files with.
240
:param index_builder_class: Required keyword parameter - the class of
241
index builder to use.
242
:param index_class: Required keyword parameter - the class of index
225
:param file_mode: Unix permissions for newly created file.
245
227
# The relative locations of the packs are constrained, but all are
246
228
# passed in because the caller has them, so as to avoid object churn.
229
index_builder_class = pack_collection._index_builder_class
247
230
Pack.__init__(self,
248
231
# Revisions: parents list, no text compression.
249
232
index_builder_class(reference_lists=1),
335
319
raise AssertionError(self._state)
321
def _check_references(self):
322
"""Make sure our external references are present.
324
Packs are allowed to have deltas whose base is not in the pack, but it
325
must be present somewhere in this collection. It is not allowed to
326
have deltas based on a fallback repository.
327
(See <https://bugs.launchpad.net/bzr/+bug/288751>)
330
for (index_name, external_refs, index) in [
332
self.text_index._external_references(),
333
self._pack_collection.text_index.combined_index),
335
self.inventory_index._external_references(),
336
self._pack_collection.inventory_index.combined_index),
338
missing = external_refs.difference(
339
k for (idx, k, v, r) in
340
index.iter_entries(external_refs))
342
missing_items[index_name] = sorted(list(missing))
344
from pprint import pformat
345
raise errors.BzrCheckError(
346
"Newly created pack file %r has delta references to "
347
"items not in its repository:\n%s"
348
% (self, pformat(missing_items)))
337
350
def data_inserted(self):
338
351
"""True if data has been added to this pack."""
339
352
return bool(self.get_revision_count() or
562
588
def _extra_init(self):
563
589
"""A template hook to allow extending the constructor trivially."""
591
def _pack_map_and_index_list(self, index_attribute):
592
"""Convert a list of packs to an index pack map and index list.
594
:param index_attribute: The attribute that the desired index is found
596
:return: A tuple (map, list) where map contains the dict from
597
index:pack_tuple, and list contains the indices in the preferred
602
for pack_obj in self.packs:
603
index = getattr(pack_obj, index_attribute)
604
indices.append(index)
605
pack_map[index] = pack_obj
606
return pack_map, indices
608
def _index_contents(self, indices, key_filter=None):
609
"""Get an iterable of the index contents from a pack_map.
611
:param indices: The list of indices to query
612
:param key_filter: An optional filter to limit the keys returned.
614
all_index = CombinedGraphIndex(indices)
615
if key_filter is None:
616
return all_index.iter_all_entries()
618
return all_index.iter_entries(key_filter)
565
620
def pack(self, pb=None):
566
621
"""Create a new pack by reading data from other packs.
604
660
def open_pack(self):
605
661
"""Open a pack for the pack we are creating."""
606
return NewPack(self._pack_collection._upload_transport,
607
self._pack_collection._index_transport,
608
self._pack_collection._pack_transport, upload_suffix=self.suffix,
609
file_mode=self._pack_collection.repo.bzrdir._get_file_mode(),
610
index_builder_class=self._pack_collection._index_builder_class,
611
index_class=self._pack_collection._index_class)
662
return NewPack(self._pack_collection, upload_suffix=self.suffix,
663
file_mode=self._pack_collection.repo.bzrdir._get_file_mode())
665
def _update_pack_order(self, entries, index_to_pack_map):
666
"""Determine how we want our packs to be ordered.
668
This changes the sort order of the self.packs list so that packs unused
669
by 'entries' will be at the end of the list, so that future requests
670
can avoid probing them. Used packs will be at the front of the
671
self.packs list, in the order of their first use in 'entries'.
673
:param entries: A list of (index, ...) tuples
674
:param index_to_pack_map: A mapping from index objects to pack objects.
678
for entry in entries:
680
if index not in seen_indexes:
681
packs.append(index_to_pack_map[index])
682
seen_indexes.add(index)
683
if len(packs) == len(self.packs):
684
if 'pack' in debug.debug_flags:
685
mutter('Not changing pack list, all packs used.')
687
seen_packs = set(packs)
688
for pack in self.packs:
689
if pack not in seen_packs:
692
if 'pack' in debug.debug_flags:
693
old_names = [p.access_tuple()[1] for p in self.packs]
694
new_names = [p.access_tuple()[1] for p in packs]
695
mutter('Reordering packs\nfrom: %s\n to: %s',
696
old_names, new_names)
613
699
def _copy_revision_texts(self):
614
700
"""Copy revision data to the new pack."""
1226
1329
existing_packs.append((revision_count, pack))
1227
1330
pack_operations = self.plan_autopack_combinations(
1228
1331
existing_packs, pack_distribution)
1229
self._execute_pack_operations(pack_operations)
1332
num_new_packs = len(pack_operations)
1333
num_old_packs = sum([len(po[1]) for po in pack_operations])
1334
num_revs_affected = sum([po[0] for po in pack_operations])
1335
mutter('Auto-packing repository %s, which has %d pack files, '
1336
'containing %d revisions. Packing %d files into %d affecting %d'
1337
' revisions', self, total_packs, total_revisions, num_old_packs,
1338
num_new_packs, num_revs_affected)
1339
self._execute_pack_operations(pack_operations,
1340
reload_func=self._restart_autopack)
1232
def _execute_pack_operations(self, pack_operations, _packer_class=Packer):
1343
def _execute_pack_operations(self, pack_operations, _packer_class=Packer,
1233
1345
"""Execute a series of pack operations.
1235
1347
:param pack_operations: A list of [revision_count, packs_to_combine].
1498
1622
self._packs_by_name = {}
1499
1623
self._packs_at_load = None
1501
def _make_index_map(self, index_suffix):
1502
"""Return information on existing indices.
1504
:param suffix: Index suffix added to pack name.
1506
:returns: (pack_map, indices) where indices is a list of GraphIndex
1507
objects, and pack_map is a mapping from those objects to the
1508
pack tuple they describe.
1510
# TODO: stop using this; it creates new indices unnecessarily.
1511
self.ensure_loaded()
1512
suffix_map = {'.rix': 'revision_index',
1513
'.six': 'signature_index',
1514
'.iix': 'inventory_index',
1515
'.tix': 'text_index',
1517
return self._packs_list_to_pack_map_and_index_list(self.all_packs(),
1518
suffix_map[index_suffix])
1520
def _packs_list_to_pack_map_and_index_list(self, packs, index_attribute):
1521
"""Convert a list of packs to an index pack map and index list.
1523
:param packs: The packs list to process.
1524
:param index_attribute: The attribute that the desired index is found
1526
:return: A tuple (map, list) where map contains the dict from
1527
index:pack_tuple, and lsit contains the indices in the same order
1533
index = getattr(pack, index_attribute)
1534
indices.append(index)
1535
pack_map[index] = (pack.pack_transport, pack.file_name())
1536
return pack_map, indices
1538
def _index_contents(self, pack_map, key_filter=None):
1539
"""Get an iterable of the index contents from a pack_map.
1541
:param pack_map: A map from indices to pack details.
1542
:param key_filter: An optional filter to limit the
1545
indices = [index for index in pack_map.iterkeys()]
1546
all_index = CombinedGraphIndex(indices)
1547
if key_filter is None:
1548
return all_index.iter_all_entries()
1550
return all_index.iter_entries(key_filter)
1552
1625
def _unlock_names(self):
1553
1626
"""Release the mutex around the pack-names index."""
1554
1627
self.repo.control_files.unlock()
1556
def _save_pack_names(self, clear_obsolete_packs=False):
1557
"""Save the list of packs.
1559
This will take out the mutex around the pack names list for the
1560
duration of the method call. If concurrent updates have been made, a
1561
three-way merge between the current list and the current in memory list
1564
:param clear_obsolete_packs: If True, clear out the contents of the
1565
obsolete_packs directory.
1569
builder = self._index_builder_class()
1570
# load the disk nodes across
1572
for index, key, value in self._iter_disk_pack_index():
1573
disk_nodes.add((key, value))
1574
# do a two-way diff against our original content
1575
current_nodes = set()
1576
for name, sizes in self._names.iteritems():
1578
((name, ), ' '.join(str(size) for size in sizes)))
1579
deleted_nodes = self._packs_at_load - current_nodes
1580
new_nodes = current_nodes - self._packs_at_load
1581
disk_nodes.difference_update(deleted_nodes)
1582
disk_nodes.update(new_nodes)
1583
# TODO: handle same-name, index-size-changes here -
1584
# e.g. use the value from disk, not ours, *unless* we're the one
1586
for key, value in disk_nodes:
1587
builder.add_node(key, value)
1588
self.transport.put_file('pack-names', builder.finish(),
1589
mode=self.repo.bzrdir._get_file_mode())
1590
# move the baseline forward
1591
self._packs_at_load = disk_nodes
1592
if clear_obsolete_packs:
1593
self._clear_obsolete_packs()
1595
self._unlock_names()
1596
# synchronise the memory packs list with what we just wrote:
1629
def _diff_pack_names(self):
1630
"""Read the pack names from disk, and compare it to the one in memory.
1632
:return: (disk_nodes, deleted_nodes, new_nodes)
1633
disk_nodes The final set of nodes that should be referenced
1634
deleted_nodes Nodes which have been removed from when we started
1635
new_nodes Nodes that are newly introduced
1637
# load the disk nodes across
1639
for index, key, value in self._iter_disk_pack_index():
1640
disk_nodes.add((key, value))
1642
# do a two-way diff against our original content
1643
current_nodes = set()
1644
for name, sizes in self._names.iteritems():
1646
((name, ), ' '.join(str(size) for size in sizes)))
1648
# Packs no longer present in the repository, which were present when we
1649
# locked the repository
1650
deleted_nodes = self._packs_at_load - current_nodes
1651
# Packs which this process is adding
1652
new_nodes = current_nodes - self._packs_at_load
1654
# Update the disk_nodes set to include the ones we are adding, and
1655
# remove the ones which were removed by someone else
1656
disk_nodes.difference_update(deleted_nodes)
1657
disk_nodes.update(new_nodes)
1659
return disk_nodes, deleted_nodes, new_nodes
1661
def _syncronize_pack_names_from_disk_nodes(self, disk_nodes):
1662
"""Given the correct set of pack files, update our saved info.
1664
:return: (removed, added, modified)
1665
removed pack names removed from self._names
1666
added pack names added to self._names
1667
modified pack names that had changed value
1672
## self._packs_at_load = disk_nodes
1597
1673
new_names = dict(disk_nodes)
1598
1674
# drop no longer present nodes
1599
1675
for pack in self.all_packs():
1600
1676
if (pack.name,) not in new_names:
1677
removed.append(pack.name)
1601
1678
self._remove_pack_from_memory(pack)
1602
1679
# add new nodes/refresh existing ones
1603
1680
for key, value in disk_nodes:
1617
1694
self._remove_pack_from_memory(self.get_pack_by_name(name))
1618
1695
self._names[name] = sizes
1619
1696
self.get_pack_by_name(name)
1697
modified.append(name)
1622
1700
self._names[name] = sizes
1623
1701
self.get_pack_by_name(name)
1703
return removed, added, modified
1705
def _save_pack_names(self, clear_obsolete_packs=False):
1706
"""Save the list of packs.
1708
This will take out the mutex around the pack names list for the
1709
duration of the method call. If concurrent updates have been made, a
1710
three-way merge between the current list and the current in memory list
1713
:param clear_obsolete_packs: If True, clear out the contents of the
1714
obsolete_packs directory.
1718
builder = self._index_builder_class()
1719
disk_nodes, deleted_nodes, new_nodes = self._diff_pack_names()
1720
# TODO: handle same-name, index-size-changes here -
1721
# e.g. use the value from disk, not ours, *unless* we're the one
1723
for key, value in disk_nodes:
1724
builder.add_node(key, value)
1725
self.transport.put_file('pack-names', builder.finish(),
1726
mode=self.repo.bzrdir._get_file_mode())
1727
# move the baseline forward
1728
self._packs_at_load = disk_nodes
1729
if clear_obsolete_packs:
1730
self._clear_obsolete_packs()
1732
self._unlock_names()
1733
# synchronise the memory packs list with what we just wrote:
1734
self._syncronize_pack_names_from_disk_nodes(disk_nodes)
1736
def reload_pack_names(self):
1737
"""Sync our pack listing with what is present in the repository.
1739
This should be called when we find out that something we thought was
1740
present is now missing. This happens when another process re-packs the
1743
# This is functionally similar to _save_pack_names, but we don't write
1744
# out the new value.
1745
disk_nodes, _, _ = self._diff_pack_names()
1746
self._packs_at_load = disk_nodes
1748
modified) = self._syncronize_pack_names_from_disk_nodes(disk_nodes)
1749
if removed or added or modified:
1753
def _restart_autopack(self):
1754
"""Reload the pack names list, and restart the autopack code."""
1755
if not self.reload_pack_names():
1756
# Re-raise the original exception, because something went missing
1757
# and a restart didn't find it
1759
raise errors.RetryAutopack(self.repo, False, sys.exc_info())
1625
1761
def _clear_obsolete_packs(self):
1626
1762
"""Delete everything from the obsolete-packs directory.
2244
2385
" (deprecated)")
2388
class RepositoryFormatKnitPack6(RepositoryFormatPack):
2389
"""A repository with stacking and btree indexes,
2390
without rich roots or subtrees.
2392
This is equivalent to pack-1.6 with B+Tree indices.
2395
repository_class = KnitPackRepository
2396
_commit_builder_class = PackCommitBuilder
2397
supports_external_lookups = True
2398
# What index classes to use
2399
index_builder_class = BTreeBuilder
2400
index_class = BTreeGraphIndex
2403
def _serializer(self):
2404
return xml5.serializer_v5
2406
def _get_matching_bzrdir(self):
2407
return bzrdir.format_registry.make_bzrdir('1.9')
2409
def _ignore_setting_bzrdir(self, format):
2412
_matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
2414
def get_format_string(self):
2415
"""See RepositoryFormat.get_format_string()."""
2416
return "Bazaar RepositoryFormatKnitPack6 (bzr 1.9)\n"
2418
def get_format_description(self):
2419
"""See RepositoryFormat.get_format_description()."""
2420
return "Packs 6 (uses btree indexes, requires bzr 1.9)"
2422
def check_conversion_target(self, target_format):
2426
class RepositoryFormatKnitPack6RichRoot(RepositoryFormatPack):
2427
"""A repository with rich roots, no subtrees, stacking and btree indexes.
2429
1.6-rich-root with B+Tree indices.
2432
repository_class = KnitPackRepository
2433
_commit_builder_class = PackRootCommitBuilder
2434
rich_root_data = True
2435
supports_tree_reference = False # no subtrees
2436
supports_external_lookups = True
2437
# What index classes to use
2438
index_builder_class = BTreeBuilder
2439
index_class = BTreeGraphIndex
2442
def _serializer(self):
2443
return xml6.serializer_v6
2445
def _get_matching_bzrdir(self):
2446
return bzrdir.format_registry.make_bzrdir(
2449
def _ignore_setting_bzrdir(self, format):
2452
_matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
2454
def check_conversion_target(self, target_format):
2455
if not target_format.rich_root_data:
2456
raise errors.BadConversionTarget(
2457
'Does not support rich root data.', target_format)
2459
def get_format_string(self):
2460
"""See RepositoryFormat.get_format_string()."""
2461
return "Bazaar RepositoryFormatKnitPack6RichRoot (bzr 1.9)\n"
2463
def get_format_description(self):
2464
return "Packs 6 rich-root (uses btree indexes, requires bzr 1.9)"
2247
2467
class RepositoryFormatPackDevelopment2(RepositoryFormatPack):
2248
2468
"""A no-subtrees development repository.
added
removed
bzrlib/repofmt/pack_repo.py
