78
83
"""Base class for all repository formats that are VersionedFiles-based."""
80
85
supports_full_versioned_files = True
86
supports_versioned_directories = True
87
supports_unreferenced_revisions = True
82
89
# Should commit add an inventory, or an inventory delta to the repository.
83
90
_commit_inv_deltas = True
102
109
# the default CommitBuilder does not manage trees whose root is versioned.
103
110
_versioned_root = False
105
def __init__(self, repository, parents, config, timestamp=None,
112
def __init__(self, repository, parents, config_stack, timestamp=None,
106
113
timezone=None, committer=None, revprops=None,
107
114
revision_id=None, lossy=False):
108
115
super(VersionedFileCommitBuilder, self).__init__(repository,
109
parents, config, timestamp, timezone, committer, revprops,
116
parents, config_stack, timestamp, timezone, committer, revprops,
110
117
revision_id, lossy)
112
119
basis_id = self.parents[0]
193
200
revision_id=self._new_revision_id,
194
201
properties=self._revprops)
195
202
rev.parent_ids = self.parents
196
self.repository.add_revision(self._new_revision_id, rev,
197
self.new_inventory, self._config)
203
if self._config_stack.get('create_signatures') == _mod_config.SIGN_ALWAYS:
204
testament = Testament(rev, self.revision_tree())
205
plaintext = testament.as_short_text()
206
self.repository.store_revision_signature(
207
gpg.GPGStrategy(self._config_stack), plaintext,
208
self._new_revision_id)
209
self.repository._add_revision(rev)
198
210
self._ensure_fallback_inventories()
199
211
self.repository.commit_write_group()
200
212
return self._new_revision_id
279
291
def _get_delta(self, ie, basis_inv, path):
280
292
"""Get a delta against the basis inventory for ie."""
281
if ie.file_id not in basis_inv:
293
if not basis_inv.has_id(ie.file_id):
283
295
result = (None, path, ie.file_id, ie)
284
296
self._basis_delta.append(result)
397
409
# this masks when a change may have occurred against the basis.
398
410
# To match this we always issue a delta, because the revision
399
411
# of the root will always be changing.
400
if ie.file_id in basis_inv:
412
if basis_inv.has_id(ie.file_id):
401
413
delta = (basis_inv.id2path(ie.file_id), path,
418
430
return None, False, None
419
431
# XXX: Friction: parent_candidates should return a list not a dict
420
432
# so that we don't have to walk the inventories again.
421
parent_candiate_entries = ie.parent_candidates(parent_invs)
422
head_set = self._heads(ie.file_id, parent_candiate_entries.keys())
433
parent_candidate_entries = ie.parent_candidates(parent_invs)
434
head_set = self._heads(ie.file_id, parent_candidate_entries.keys())
424
436
for inv in parent_invs:
425
if ie.file_id in inv:
437
if inv.has_id(ie.file_id):
426
438
old_rev = inv[ie.file_id].revision
427
439
if old_rev in head_set:
428
440
heads.append(inv[ie.file_id].revision)
442
454
# There is a single head, look it up for comparison
443
parent_entry = parent_candiate_entries[heads[0]]
455
parent_entry = parent_candidate_entries[heads[0]]
444
456
# if the non-content specific data has changed, we'll be writing a
446
458
if (parent_entry.parent_id != ie.parent_id or
558
570
:param iter_changes: An iter_changes iterator with the changes to apply
559
571
to basis_revision_id. The iterator must not include any items with
560
572
a current kind of None - missing items must be either filtered out
561
or errored-on beefore record_iter_changes sees the item.
573
or errored-on before record_iter_changes sees the item.
562
574
:param _entry_factory: Private method to bind entry_factory locally for
564
576
:return: A generator of (file_id, relpath, fs_hash) tuples for use with
918
930
if not self._format.supports_external_lookups:
919
931
raise errors.UnstackableRepositoryFormat(self._format, self.base)
932
# This can raise an exception, so should be done before we lock the
933
# fallback repository.
934
self._check_fallback_repository(repository)
920
935
if self.is_locked():
921
936
# This repository will call fallback.unlock() when we transition to
922
937
# the unlocked state, so we make sure to increment the lock count
923
938
repository.lock_read()
924
self._check_fallback_repository(repository)
925
939
self._fallback_repositories.append(repository)
926
940
self.texts.add_fallback_versioned_files(repository.texts)
927
941
self.inventories.add_fallback_versioned_files(repository.inventories)
1024
1038
self.inventories._access.flush()
1027
def add_revision(self, revision_id, rev, inv=None, config=None):
1041
def add_revision(self, revision_id, rev, inv=None):
1028
1042
"""Add rev to the revision store as revision_id.
1030
1044
:param revision_id: the revision id to use.
1031
1045
:param rev: The revision object.
1032
1046
:param inv: The inventory for the revision. if None, it will be looked
1033
1047
up in the inventory storer
1034
:param config: If None no digital signature will be created.
1035
If supplied its signature_needed method will be used
1036
to determine if a signature should be made.
1038
1049
# TODO: jam 20070210 Shouldn't we check rev.revision_id and
1039
1050
# rev.parent_ids?
1040
1051
_mod_revision.check_not_reserved_id(revision_id)
1041
if config is not None and config.signature_needed():
1043
inv = self.get_inventory(revision_id)
1044
tree = InventoryRevisionTree(self, inv, revision_id)
1045
testament = Testament(rev, tree)
1046
plaintext = testament.as_short_text()
1047
self.store_revision_signature(
1048
gpg.GPGStrategy(config), plaintext, revision_id)
1049
1052
# check inventory present
1050
1053
if not self.inventories.get_parent_map([(revision_id,)]):
1051
1054
if inv is None:
1084
1087
keys = {'chk_bytes':set(), 'inventories':set(), 'texts':set()}
1085
1088
kinds = ['chk_bytes', 'texts']
1086
1089
count = len(checker.pending_keys)
1087
bar.update("inventories", 0, 2)
1090
bar.update(gettext("inventories"), 0, 2)
1088
1091
current_keys = checker.pending_keys
1089
1092
checker.pending_keys = {}
1090
1093
# Accumulate current checks.
1179
1182
'sha1 mismatch: %s has sha1 %s expected %s referenced by %s' %
1180
1183
(record.key, sha1, item_data[1], item_data[2]))
1186
def _eliminate_revisions_not_present(self, revision_ids):
1187
"""Check every revision id in revision_ids to see if we have it.
1189
Returns a set of the present revisions.
1192
graph = self.get_graph()
1193
parent_map = graph.get_parent_map(revision_ids)
1194
# The old API returned a list, should this actually be a set?
1195
return parent_map.keys()
1182
1197
def __init__(self, _format, a_bzrdir, control_files):
1183
1198
"""Instantiate a VersionedFileRepository.
1185
1200
:param _format: The format of the repository on disk.
1186
:param a_bzrdir: The BzrDir of the repository.
1201
:param controldir: The ControlDir of the repository.
1187
1202
:param control_files: Control files to use for locking, etc.
1189
1204
# In the future we will have a single api for all stores for
1191
1206
# this construct will accept instances of those things.
1192
1207
super(VersionedFileRepository, self).__init__(_format, a_bzrdir,
1209
self._transport = control_files._transport
1210
self.base = self._transport.base
1195
1212
self._reconcile_does_inventory_gc = True
1196
1213
self._reconcile_fixes_text_parents = False
1201
1218
# rather copying them?
1202
1219
self._safe_to_return_from_cache = False
1221
def fetch(self, source, revision_id=None, find_ghosts=False,
1223
"""Fetch the content required to construct revision_id from source.
1225
If revision_id is None and fetch_spec is None, then all content is
1228
fetch() may not be used when the repository is in a write group -
1229
either finish the current write group before using fetch, or use
1230
fetch before starting the write group.
1232
:param find_ghosts: Find and copy revisions in the source that are
1233
ghosts in the target (and not reachable directly by walking out to
1234
the first-present revision in target from revision_id).
1235
:param revision_id: If specified, all the content needed for this
1236
revision ID will be copied to the target. Fetch will determine for
1237
itself which content needs to be copied.
1238
:param fetch_spec: If specified, a SearchResult or
1239
PendingAncestryResult that describes which revisions to copy. This
1240
allows copying multiple heads at once. Mutually exclusive with
1243
if fetch_spec is not None and revision_id is not None:
1244
raise AssertionError(
1245
"fetch_spec and revision_id are mutually exclusive.")
1246
if self.is_in_write_group():
1247
raise errors.InternalBzrError(
1248
"May not fetch while in a write group.")
1249
# fast path same-url fetch operations
1250
# TODO: lift out to somewhere common with RemoteRepository
1251
# <https://bugs.launchpad.net/bzr/+bug/401646>
1252
if (self.has_same_location(source)
1253
and fetch_spec is None
1254
and self._has_same_fallbacks(source)):
1255
# check that last_revision is in 'from' and then return a
1257
if (revision_id is not None and
1258
not _mod_revision.is_null(revision_id)):
1259
self.get_revision(revision_id)
1261
inter = InterRepository.get(source, self)
1262
if (fetch_spec is not None and
1263
not getattr(inter, "supports_fetch_spec", False)):
1264
raise errors.UnsupportedOperation(
1265
"fetch_spec not supported for %r" % inter)
1266
return inter.fetch(revision_id=revision_id,
1267
find_ghosts=find_ghosts, fetch_spec=fetch_spec)
1204
1269
@needs_read_lock
1205
1270
def gather_stats(self, revid=None, committers=None):
1206
1271
"""See Repository.gather_stats()."""
1215
1280
# result['size'] = t
1218
def get_commit_builder(self, branch, parents, config, timestamp=None,
1283
def get_commit_builder(self, branch, parents, config_stack, timestamp=None,
1219
1284
timezone=None, committer=None, revprops=None,
1220
1285
revision_id=None, lossy=False):
1221
1286
"""Obtain a CommitBuilder for this repository.
1223
1288
:param branch: Branch to commit to.
1224
1289
:param parents: Revision ids of the parents of the new revision.
1225
:param config: Configuration to use.
1290
:param config_stack: Configuration stack to use.
1226
1291
:param timestamp: Optional timestamp recorded for commit.
1227
1292
:param timezone: Optional timezone for timestamp.
1228
1293
:param committer: Optional committer to set for commit.
1235
1300
raise errors.BzrError("Cannot commit directly to a stacked branch"
1236
1301
" in pre-2a formats. See "
1237
1302
"https://bugs.launchpad.net/bzr/+bug/375013 for details.")
1238
result = self._commit_builder_class(self, parents, config,
1303
result = self._commit_builder_class(self, parents, config_stack,
1239
1304
timestamp, timezone, committer, revprops, revision_id,
1241
1306
self.start_write_group()
1497
1562
text_keys[(file_id, revision_id)] = callable_data
1498
1563
for record in self.texts.get_record_stream(text_keys, 'unordered', True):
1499
1564
if record.storage_kind == 'absent':
1500
raise errors.RevisionNotPresent(record.key, self)
1565
raise errors.RevisionNotPresent(record.key[1], record.key[0])
1501
1566
yield text_keys[record.key], record.get_bytes_as('chunked')
1503
1568
def _generate_text_key_index(self, text_key_references=None,
1553
1618
batch_size = 10 # should be ~150MB on a 55K path tree
1554
1619
batch_count = len(revision_order) / batch_size + 1
1555
1620
processed_texts = 0
1556
pb.update("Calculating text parents", processed_texts, text_count)
1621
pb.update(gettext("Calculating text parents"), processed_texts, text_count)
1557
1622
for offset in xrange(batch_count):
1558
1623
to_query = revision_order[offset * batch_size:(offset + 1) *
1562
1627
for revision_id in to_query:
1563
1628
parent_ids = ancestors[revision_id]
1564
1629
for text_key in revision_keys[revision_id]:
1565
pb.update("Calculating text parents", processed_texts)
1630
pb.update(gettext("Calculating text parents"), processed_texts)
1566
1631
processed_texts += 1
1567
1632
candidate_parents = []
1568
1633
for parent_id in parent_ids:
1638
1703
num_file_ids = len(file_ids)
1639
1704
for file_id, altered_versions in file_ids.iteritems():
1640
1705
if pb is not None:
1641
pb.update("Fetch texts", count, num_file_ids)
1706
pb.update(gettext("Fetch texts"), count, num_file_ids)
1643
1708
yield ("file", file_id, altered_versions)
1681
1746
if ((None in revision_ids)
1682
1747
or (_mod_revision.NULL_REVISION in revision_ids)):
1683
1748
raise ValueError('cannot get null revision inventory')
1684
return self._iter_inventories(revision_ids, ordering)
1749
for inv, revid in self._iter_inventories(revision_ids, ordering):
1751
raise errors.NoSuchRevision(self, revid)
1686
1754
def _iter_inventories(self, revision_ids, ordering):
1687
1755
"""single-document based inventory iteration."""
1688
1756
inv_xmls = self._iter_inventory_xmls(revision_ids, ordering)
1689
1757
for text, revision_id in inv_xmls:
1690
yield self._deserialise_inventory(revision_id, text)
1759
yield None, revision_id
1761
yield self._deserialise_inventory(revision_id, text), revision_id
1692
1763
def _iter_inventory_xmls(self, revision_ids, ordering):
1693
1764
if ordering is None:
1712
1783
yield ''.join(chunks), record.key[-1]
1714
raise errors.NoSuchRevision(self, record.key)
1785
yield None, record.key[-1]
1715
1786
if order_as_requested:
1716
1787
# Yield as many results as we can while preserving order.
1717
1788
while next_key in text_chunks:
1746
1817
def _get_inventory_xml(self, revision_id):
1747
1818
"""Get serialized inventory as a string."""
1748
1819
texts = self._iter_inventory_xmls([revision_id], 'unordered')
1750
text, revision_id = texts.next()
1751
except StopIteration:
1752
raise errors.HistoryMissing(self, 'inventory', revision_id)
1820
text, revision_id = texts.next()
1822
raise errors.NoSuchRevision(self, revision_id)
1755
1825
@needs_read_lock
1830
1900
"""Return the graph walker for text revisions."""
1831
1901
return graph.Graph(self.texts)
1903
def revision_ids_to_search_result(self, result_set):
1904
"""Convert a set of revision ids to a graph SearchResult."""
1905
result_parents = set()
1906
for parents in self.get_graph().get_parent_map(
1907
result_set).itervalues():
1908
result_parents.update(parents)
1909
included_keys = result_set.intersection(result_parents)
1910
start_keys = result_set.difference(included_keys)
1911
exclude_keys = result_parents.difference(result_set)
1912
result = vf_search.SearchResult(start_keys, exclude_keys,
1913
len(result_set), result_set)
1833
1916
def _get_versioned_file_checker(self, text_key_references=None,
1834
1917
ancestors=None):
1835
1918
"""Return an object suitable for checking versioned files.
2449
2532
self.text_index.iterkeys()])
2450
2533
# text keys is now grouped by file_id
2451
2534
n_versions = len(self.text_index)
2452
progress_bar.update('loading text store', 0, n_versions)
2535
progress_bar.update(gettext('loading text store'), 0, n_versions)
2453
2536
parent_map = self.repository.texts.get_parent_map(self.text_index)
2454
2537
# On unlistable transports this could well be empty/error...
2455
2538
text_keys = self.repository.texts.keys()
2456
2539
unused_keys = frozenset(text_keys) - set(self.text_index)
2457
2540
for num, key in enumerate(self.text_index.iterkeys()):
2458
progress_bar.update('checking text graph', num, n_versions)
2541
progress_bar.update(gettext('checking text graph'), num, n_versions)
2459
2542
correct_parents = self.calculate_file_version_parents(key)
2461
2544
knit_parents = parent_map[key]
2482
2567
content is copied.
2485
ui.ui_factory.warn_experimental_format_fetch(self)
2570
if self.target._format.experimental:
2571
ui.ui_factory.show_user_warning('experimental_format_fetch',
2572
from_format=self.source._format,
2573
to_format=self.target._format)
2486
2574
from bzrlib.fetch import RepoFetcher
2487
2575
# See <https://launchpad.net/bugs/456077> asking for a warning here
2488
2576
if self.source._format.network_name() != self.target._format.network_name():
2549
2637
searcher.stop_searching_any(stop_revs)
2550
2638
if searcher_exhausted:
2552
return searcher.get_result()
2640
(started_keys, excludes, included_keys) = searcher.get_state()
2641
return vf_search.SearchResult(started_keys, excludes,
2642
len(included_keys), included_keys)
2554
2644
@needs_read_lock
2555
2645
def search_missing_revision_ids(self,
2902
2992
for offset in range(0, len(revision_ids), batch_size):
2903
2993
self.target.start_write_group()
2905
pb.update('Transferring revisions', offset,
2995
pb.update(gettext('Transferring revisions'), offset,
2906
2996
len(revision_ids))
2907
2997
batch = revision_ids[offset:offset+batch_size]
2908
2998
basis_id = self._fetch_batch(batch, basis_id, cache)
2927
3017
revision_ids = fetch_spec.get_keys()
2929
3019
revision_ids = None
2930
ui.ui_factory.warn_experimental_format_fetch(self)
3020
if self.source._format.experimental:
3021
ui.ui_factory.show_user_warning('experimental_format_fetch',
3022
from_format=self.source._format,
3023
to_format=self.target._format)
2931
3024
if (not self.source.supports_rich_root()
2932
3025
and self.target.supports_rich_root()):
2933
3026
self._converting_to_rich_root = True
3028
3121
_install_revision(repository, revision, revision_tree, signature,
3029
3122
inventory_cache)
3030
3123
if pb is not None:
3031
pb.update('Transferring revisions', n + 1, num_revisions)
3124
pb.update(gettext('Transferring revisions'), n + 1, num_revisions)
3033
3126
repository.abort_write_group()
3070
3163
# the parents inserted are not those commit would do - in particular
3071
3164
# they are not filtered by heads(). RBC, AB
3072
3165
for revision, tree in parent_trees.iteritems():
3073
if ie.file_id not in tree:
3166
if not tree.has_id(ie.file_id):
3075
3168
parent_id = tree.get_file_revision(ie.file_id)
3076
3169
if parent_id in text_parents:
added
removed
bzrlib/vf_repository.py
