~bzr-pqm/bzr/bzr.dev

5815.4.1 by Jelmer Vernooij
Split versionedfile-specific stuff out into VersionedFileRepository.
1
# Copyright (C) 2005-2011 Canonical Ltd
2
#
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
7
#
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
# GNU General Public License for more details.
12
#
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16
6379.6.7 by Jelmer Vernooij
Move importing from future until after doc string, otherwise the doc string will disappear.
17
"""Repository formats built around versioned files."""
18
6379.6.3 by Jelmer Vernooij
Use absolute_import.
19
from __future__ import absolute_import
20
5815.4.1 by Jelmer Vernooij
Split versionedfile-specific stuff out into VersionedFileRepository.
21
22
from bzrlib.lazy_import import lazy_import
23
lazy_import(globals(), """
5852.1.8 by Jelmer Vernooij
Simplify revision limiting.
24
import itertools
25
5815.4.1 by Jelmer Vernooij
Split versionedfile-specific stuff out into VersionedFileRepository.
26
from bzrlib import (
5850.1.3 by Jelmer Vernooij
Add VersionedFileCheck.
27
    check,
6351.3.2 by Jelmer Vernooij
Convert some gpg options to config stacks.
28
    config as _mod_config,
5815.4.1 by Jelmer Vernooij
Split versionedfile-specific stuff out into VersionedFileRepository.
29
    debug,
30
    fetch as _mod_fetch,
31
    fifo_cache,
32
    gpg,
33
    graph,
34
    inventory_delta,
35
    lru_cache,
36
    osutils,
37
    revision as _mod_revision,
38
    serializer as _mod_serializer,
39
    static_tuple,
5863.4.1 by Jelmer Vernooij
Move interrepository implementation to vf_repository.
40
    symbol_versioning,
5815.4.1 by Jelmer Vernooij
Split versionedfile-specific stuff out into VersionedFileRepository.
41
    tsort,
42
    ui,
43
    versionedfile,
6341.1.4 by Jelmer Vernooij
Move more functionality to vf_search.
44
    vf_search,
5815.4.1 by Jelmer Vernooij
Split versionedfile-specific stuff out into VersionedFileRepository.
45
    )
46
47
from bzrlib.recordcounter import RecordCounter
48
from bzrlib.revisiontree import InventoryRevisionTree
49
from bzrlib.testament import Testament
6138.4.1 by Jonathan Riddell
add gettext to progress bar strings
50
from bzrlib.i18n import gettext
5815.4.1 by Jelmer Vernooij
Split versionedfile-specific stuff out into VersionedFileRepository.
51
""")
52
53
from bzrlib import (
54
    errors,
55
    )
56
from bzrlib.decorators import (
57
    needs_read_lock,
58
    needs_write_lock,
59
    only_raises,
60
    )
61
from bzrlib.inventory import (
62
    Inventory,
5815.4.2 by Jelmer Vernooij
split out versionedfile-specific stuff from commitbuilder.
63
    InventoryDirectory,
64
    ROOT_ID,
65
    entry_factory,
5815.4.1 by Jelmer Vernooij
Split versionedfile-specific stuff out into VersionedFileRepository.
66
    )
67
68
from bzrlib.repository import (
5815.4.2 by Jelmer Vernooij
split out versionedfile-specific stuff from commitbuilder.
69
    CommitBuilder,
5815.4.1 by Jelmer Vernooij
Split versionedfile-specific stuff out into VersionedFileRepository.
70
    InterRepository,
71
    MetaDirRepository,
6349.2.1 by Jelmer Vernooij
Add BzrDirMetaComponentFormat.
72
    RepositoryFormatMetaDir,
5815.4.1 by Jelmer Vernooij
Split versionedfile-specific stuff out into VersionedFileRepository.
73
    Repository,
5815.4.5 by Jelmer Vernooij
Use MetaDirVersionedFileRepositoryFormat (a Soyuz worthy name).
74
    RepositoryFormat,
5815.4.1 by Jelmer Vernooij
Split versionedfile-specific stuff out into VersionedFileRepository.
75
    )
76
77
from bzrlib.trace import (
6015.23.15 by John Arbash Meinel
Clean out more of the cruft that got left by accident.
78
    mutter
5815.4.1 by Jelmer Vernooij
Split versionedfile-specific stuff out into VersionedFileRepository.
79
    )
80
81
5815.4.5 by Jelmer Vernooij
Use MetaDirVersionedFileRepositoryFormat (a Soyuz worthy name).
82
class VersionedFileRepositoryFormat(RepositoryFormat):
5815.4.16 by Jelmer Vernooij
Move supports_versioned_files setting to VersionedFileRepositoryFormat.
83
    """Base class for all repository formats that are VersionedFiles-based."""
84
85
    supports_full_versioned_files = True
5993.3.2 by Jelmer Vernooij
Add Repository.supports_versioned_directories.
86
    supports_versioned_directories = True
6217.4.2 by Jelmer Vernooij
s/invisible/unreferenced.
87
    supports_unreferenced_revisions = True
5815.4.5 by Jelmer Vernooij
Use MetaDirVersionedFileRepositoryFormat (a Soyuz worthy name).
88
89
    # Should commit add an inventory, or an inventory delta to the repository.
90
    _commit_inv_deltas = True
91
    # What order should fetch operations request streams in?
92
    # The default is unordered as that is the cheapest for an origin to
93
    # provide.
94
    _fetch_order = 'unordered'
95
    # Does this repository format use deltas that can be fetched as-deltas ?
96
    # (E.g. knits, where the knit deltas can be transplanted intact.
97
    # We default to False, which will ensure that enough data to get
98
    # a full text out of any fetch stream will be grabbed.
99
    _fetch_uses_deltas = False
100
101
5815.4.2 by Jelmer Vernooij
split out versionedfile-specific stuff from commitbuilder.
102
class VersionedFileCommitBuilder(CommitBuilder):
103
    """Commit builder implementation for versioned files based repositories.
104
    """
105
106
    # this commit builder supports the record_entry_contents interface
107
    supports_record_entry_contents = True
108
109
    # the default CommitBuilder does not manage trees whose root is versioned.
110
    _versioned_root = False
111
6351.3.2 by Jelmer Vernooij
Convert some gpg options to config stacks.
112
    def __init__(self, repository, parents, config_stack, timestamp=None,
5815.4.2 by Jelmer Vernooij
split out versionedfile-specific stuff from commitbuilder.
113
                 timezone=None, committer=None, revprops=None,
114
                 revision_id=None, lossy=False):
115
        super(VersionedFileCommitBuilder, self).__init__(repository,
6351.3.2 by Jelmer Vernooij
Convert some gpg options to config stacks.
116
            parents, config_stack, timestamp, timezone, committer, revprops,
5815.4.2 by Jelmer Vernooij
split out versionedfile-specific stuff from commitbuilder.
117
            revision_id, lossy)
5847.3.2 by Jelmer Vernooij
Move root checking to VersionedFileRepository.
118
        try:
119
            basis_id = self.parents[0]
120
        except IndexError:
121
            basis_id = _mod_revision.NULL_REVISION
122
        self.basis_delta_revision = basis_id
5815.4.2 by Jelmer Vernooij
split out versionedfile-specific stuff from commitbuilder.
123
        self.new_inventory = Inventory(None)
124
        self._basis_delta = []
125
        self.__heads = graph.HeadsCache(repository.get_graph()).heads
126
        # memo'd check for no-op commits.
127
        self._any_changes = False
128
        # API compatibility, older code that used CommitBuilder did not call
129
        # .record_delete(), which means the delta that is computed would not be
130
        # valid. Callers that will call record_delete() should call
131
        # .will_record_deletes() to indicate that.
132
        self._recording_deletes = False
133
134
    def will_record_deletes(self):
135
        """Tell the commit builder that deletes are being notified.
136
137
        This enables the accumulation of an inventory delta; for the resulting
138
        commit to be valid, deletes against the basis MUST be recorded via
139
        builder.record_delete().
140
        """
141
        self._recording_deletes = True
142
143
    def any_changes(self):
144
        """Return True if any entries were changed.
145
146
        This includes merge-only changes. It is the core for the --unchanged
147
        detection in commit.
148
149
        :return: True if any changes have occured.
150
        """
151
        return self._any_changes
152
153
    def _ensure_fallback_inventories(self):
154
        """Ensure that appropriate inventories are available.
155
156
        This only applies to repositories that are stacked, and is about
157
        enusring the stacking invariants. Namely, that for any revision that is
158
        present, we either have all of the file content, or we have the parent
159
        inventory and the delta file content.
160
        """
161
        if not self.repository._fallback_repositories:
162
            return
163
        if not self.repository._format.supports_chks:
164
            raise errors.BzrError("Cannot commit directly to a stacked branch"
165
                " in pre-2a formats. See "
166
                "https://bugs.launchpad.net/bzr/+bug/375013 for details.")
167
        # This is a stacked repo, we need to make sure we have the parent
168
        # inventories for the parents.
169
        parent_keys = [(p,) for p in self.parents]
170
        parent_map = self.repository.inventories._index.get_parent_map(parent_keys)
171
        missing_parent_keys = set([pk for pk in parent_keys
172
                                       if pk not in parent_map])
173
        fallback_repos = list(reversed(self.repository._fallback_repositories))
174
        missing_keys = [('inventories', pk[0])
175
                        for pk in missing_parent_keys]
176
        resume_tokens = []
177
        while missing_keys and fallback_repos:
178
            fallback_repo = fallback_repos.pop()
179
            source = fallback_repo._get_source(self.repository._format)
180
            sink = self.repository._get_sink()
181
            stream = source.get_stream_for_missing_keys(missing_keys)
182
            missing_keys = sink.insert_stream_without_locking(stream,
183
                self.repository._format)
184
        if missing_keys:
185
            raise errors.BzrError('Unable to fill in parent inventories for a'
186
                                  ' stacked branch')
187
188
    def commit(self, message):
189
        """Make the actual commit.
190
191
        :return: The revision id of the recorded revision.
192
        """
193
        self._validate_unicode_text(message, 'commit message')
194
        rev = _mod_revision.Revision(
195
                       timestamp=self._timestamp,
196
                       timezone=self._timezone,
197
                       committer=self._committer,
198
                       message=message,
199
                       inventory_sha1=self.inv_sha1,
200
                       revision_id=self._new_revision_id,
201
                       properties=self._revprops)
202
        rev.parent_ids = self.parents
6421.2.1 by Jelmer Vernooij
Move revision signing to CommitBuilder.
203
        if self._config_stack.get('create_signatures') == _mod_config.SIGN_ALWAYS:
204
            testament = Testament(rev, self.revision_tree())
205
            plaintext = testament.as_short_text()
206
            self.repository.store_revision_signature(
6421.2.2 by Jelmer Vernooij
Fix config name.
207
                gpg.GPGStrategy(self._config_stack), plaintext,
208
                self._new_revision_id)
6421.2.1 by Jelmer Vernooij
Move revision signing to CommitBuilder.
209
        self.repository._add_revision(rev)
5815.4.2 by Jelmer Vernooij
split out versionedfile-specific stuff from commitbuilder.
210
        self._ensure_fallback_inventories()
211
        self.repository.commit_write_group()
212
        return self._new_revision_id
213
214
    def abort(self):
215
        """Abort the commit that is being built.
216
        """
217
        self.repository.abort_write_group()
218
219
    def revision_tree(self):
220
        """Return the tree that was just committed.
221
222
        After calling commit() this can be called to get a
223
        RevisionTree representing the newly committed tree. This is
224
        preferred to calling Repository.revision_tree() because that may
225
        require deserializing the inventory, while we already have a copy in
226
        memory.
227
        """
228
        if self.new_inventory is None:
229
            self.new_inventory = self.repository.get_inventory(
230
                self._new_revision_id)
231
        return InventoryRevisionTree(self.repository, self.new_inventory,
232
            self._new_revision_id)
233
234
    def finish_inventory(self):
235
        """Tell the builder that the inventory is finished.
236
237
        :return: The inventory id in the repository, which can be used with
238
            repository.get_inventory.
239
        """
240
        if self.new_inventory is None:
241
            # an inventory delta was accumulated without creating a new
242
            # inventory.
243
            basis_id = self.basis_delta_revision
244
            # We ignore the 'inventory' returned by add_inventory_by_delta
245
            # because self.new_inventory is used to hint to the rest of the
246
            # system what code path was taken
247
            self.inv_sha1, _ = self.repository.add_inventory_by_delta(
248
                basis_id, self._basis_delta, self._new_revision_id,
249
                self.parents)
250
        else:
251
            if self.new_inventory.root is None:
252
                raise AssertionError('Root entry should be supplied to'
253
                    ' record_entry_contents, as of bzr 0.10.')
254
                self.new_inventory.add(InventoryDirectory(ROOT_ID, '', None))
255
            self.new_inventory.revision_id = self._new_revision_id
256
            self.inv_sha1 = self.repository.add_inventory(
257
                self._new_revision_id,
258
                self.new_inventory,
259
                self.parents
260
                )
261
        return self._new_revision_id
262
263
    def _check_root(self, ie, parent_invs, tree):
264
        """Helper for record_entry_contents.
265
266
        :param ie: An entry being added.
267
        :param parent_invs: The inventories of the parent revisions of the
268
            commit.
269
        :param tree: The tree that is being committed.
270
        """
271
        # In this revision format, root entries have no knit or weave When
272
        # serializing out to disk and back in root.revision is always
273
        # _new_revision_id
274
        ie.revision = self._new_revision_id
275
276
    def _require_root_change(self, tree):
277
        """Enforce an appropriate root object change.
278
279
        This is called once when record_iter_changes is called, if and only if
280
        the root was not in the delta calculated by record_iter_changes.
281
282
        :param tree: The tree which is being committed.
283
        """
284
        if len(self.parents) == 0:
285
            raise errors.RootMissing()
286
        entry = entry_factory['directory'](tree.path2id(''), '',
287
            None)
288
        entry.revision = self._new_revision_id
289
        self._basis_delta.append(('', '', entry.file_id, entry))
290
291
    def _get_delta(self, ie, basis_inv, path):
292
        """Get a delta against the basis inventory for ie."""
5967.7.1 by Martin Pool
Deprecate __contains__ on Tree and Inventory
293
        if not basis_inv.has_id(ie.file_id):
5815.4.2 by Jelmer Vernooij
split out versionedfile-specific stuff from commitbuilder.
294
            # add
295
            result = (None, path, ie.file_id, ie)
296
            self._basis_delta.append(result)
297
            return result
298
        elif ie != basis_inv[ie.file_id]:
299
            # common but altered
300
            # TODO: avoid tis id2path call.
301
            result = (basis_inv.id2path(ie.file_id), path, ie.file_id, ie)
302
            self._basis_delta.append(result)
303
            return result
304
        else:
305
            # common, unaltered
306
            return None
307
308
    def _heads(self, file_id, revision_ids):
309
        """Calculate the graph heads for revision_ids in the graph of file_id.
310
311
        This can use either a per-file graph or a global revision graph as we
312
        have an identity relationship between the two graphs.
313
        """
314
        return self.__heads(revision_ids)
315
316
    def get_basis_delta(self):
317
        """Return the complete inventory delta versus the basis inventory.
318
319
        This has been built up with the calls to record_delete and
320
        record_entry_contents. The client must have already called
321
        will_record_deletes() to indicate that they will be generating a
322
        complete delta.
323
324
        :return: An inventory delta, suitable for use with apply_delta, or
325
            Repository.add_inventory_by_delta, etc.
326
        """
327
        if not self._recording_deletes:
328
            raise AssertionError("recording deletes not activated.")
329
        return self._basis_delta
330
331
    def record_delete(self, path, file_id):
332
        """Record that a delete occured against a basis tree.
333
334
        This is an optional API - when used it adds items to the basis_delta
335
        being accumulated by the commit builder. It cannot be called unless the
336
        method will_record_deletes() has been called to inform the builder that
337
        a delta is being supplied.
338
339
        :param path: The path of the thing deleted.
340
        :param file_id: The file id that was deleted.
341
        """
342
        if not self._recording_deletes:
343
            raise AssertionError("recording deletes not activated.")
344
        delta = (path, None, file_id, None)
345
        self._basis_delta.append(delta)
346
        self._any_changes = True
347
        return delta
348
349
    def record_entry_contents(self, ie, parent_invs, path, tree,
350
        content_summary):
351
        """Record the content of ie from tree into the commit if needed.
352
353
        Side effect: sets ie.revision when unchanged
354
355
        :param ie: An inventory entry present in the commit.
356
        :param parent_invs: The inventories of the parent revisions of the
357
            commit.
358
        :param path: The path the entry is at in the tree.
359
        :param tree: The tree which contains this entry and should be used to
360
            obtain content.
361
        :param content_summary: Summary data from the tree about the paths
362
            content - stat, length, exec, sha/link target. This is only
363
            accessed when the entry has a revision of None - that is when it is
364
            a candidate to commit.
365
        :return: A tuple (change_delta, version_recorded, fs_hash).
366
            change_delta is an inventory_delta change for this entry against
367
            the basis tree of the commit, or None if no change occured against
368
            the basis tree.
369
            version_recorded is True if a new version of the entry has been
370
            recorded. For instance, committing a merge where a file was only
371
            changed on the other side will return (delta, False).
372
            fs_hash is either None, or the hash details for the path (currently
373
            a tuple of the contents sha1 and the statvalue returned by
374
            tree.get_file_with_stat()).
375
        """
376
        if self.new_inventory.root is None:
377
            if ie.parent_id is not None:
378
                raise errors.RootMissing()
379
            self._check_root(ie, parent_invs, tree)
380
        if ie.revision is None:
381
            kind = content_summary[0]
382
        else:
383
            # ie is carried over from a prior commit
384
            kind = ie.kind
385
        # XXX: repository specific check for nested tree support goes here - if
386
        # the repo doesn't want nested trees we skip it ?
387
        if (kind == 'tree-reference' and
388
            not self.repository._format.supports_tree_reference):
389
            # mismatch between commit builder logic and repository:
390
            # this needs the entry creation pushed down into the builder.
391
            raise NotImplementedError('Missing repository subtree support.')
392
        self.new_inventory.add(ie)
393
394
        # TODO: slow, take it out of the inner loop.
395
        try:
396
            basis_inv = parent_invs[0]
397
        except IndexError:
398
            basis_inv = Inventory(root_id=None)
399
400
        # ie.revision is always None if the InventoryEntry is considered
401
        # for committing. We may record the previous parents revision if the
402
        # content is actually unchanged against a sole head.
403
        if ie.revision is not None:
404
            if not self._versioned_root and path == '':
405
                # repositories that do not version the root set the root's
406
                # revision to the new commit even when no change occurs (more
407
                # specifically, they do not record a revision on the root; and
408
                # the rev id is assigned to the root during deserialisation -
409
                # this masks when a change may have occurred against the basis.
410
                # To match this we always issue a delta, because the revision
411
                # of the root will always be changing.
5967.7.1 by Martin Pool
Deprecate __contains__ on Tree and Inventory
412
                if basis_inv.has_id(ie.file_id):
5815.4.2 by Jelmer Vernooij
split out versionedfile-specific stuff from commitbuilder.
413
                    delta = (basis_inv.id2path(ie.file_id), path,
414
                        ie.file_id, ie)
415
                else:
416
                    # add
417
                    delta = (None, path, ie.file_id, ie)
418
                self._basis_delta.append(delta)
419
                return delta, False, None
420
            else:
421
                # we don't need to commit this, because the caller already
422
                # determined that an existing revision of this file is
423
                # appropriate. If it's not being considered for committing then
424
                # it and all its parents to the root must be unaltered so
425
                # no-change against the basis.
426
                if ie.revision == self._new_revision_id:
427
                    raise AssertionError("Impossible situation, a skipped "
428
                        "inventory entry (%r) claims to be modified in this "
429
                        "commit (%r).", (ie, self._new_revision_id))
430
                return None, False, None
431
        # XXX: Friction: parent_candidates should return a list not a dict
432
        #      so that we don't have to walk the inventories again.
6090.2.1 by Jelmer Vernooij
Fix some typos.
433
        parent_candidate_entries = ie.parent_candidates(parent_invs)
434
        head_set = self._heads(ie.file_id, parent_candidate_entries.keys())
5815.4.2 by Jelmer Vernooij
split out versionedfile-specific stuff from commitbuilder.
435
        heads = []
436
        for inv in parent_invs:
5967.7.1 by Martin Pool
Deprecate __contains__ on Tree and Inventory
437
            if inv.has_id(ie.file_id):
5815.4.2 by Jelmer Vernooij
split out versionedfile-specific stuff from commitbuilder.
438
                old_rev = inv[ie.file_id].revision
439
                if old_rev in head_set:
440
                    heads.append(inv[ie.file_id].revision)
441
                    head_set.remove(inv[ie.file_id].revision)
442
443
        store = False
444
        # now we check to see if we need to write a new record to the
445
        # file-graph.
446
        # We write a new entry unless there is one head to the ancestors, and
447
        # the kind-derived content is unchanged.
448
449
        # Cheapest check first: no ancestors, or more the one head in the
450
        # ancestors, we write a new node.
451
        if len(heads) != 1:
452
            store = True
453
        if not store:
454
            # There is a single head, look it up for comparison
6090.2.1 by Jelmer Vernooij
Fix some typos.
455
            parent_entry = parent_candidate_entries[heads[0]]
5815.4.2 by Jelmer Vernooij
split out versionedfile-specific stuff from commitbuilder.
456
            # if the non-content specific data has changed, we'll be writing a
457
            # node:
458
            if (parent_entry.parent_id != ie.parent_id or
459
                parent_entry.name != ie.name):
460
                store = True
461
        # now we need to do content specific checks:
462
        if not store:
463
            # if the kind changed the content obviously has
464
            if kind != parent_entry.kind:
465
                store = True
466
        # Stat cache fingerprint feedback for the caller - None as we usually
467
        # don't generate one.
468
        fingerprint = None
469
        if kind == 'file':
470
            if content_summary[2] is None:
471
                raise ValueError("Files must not have executable = None")
472
            if not store:
473
                # We can't trust a check of the file length because of content
474
                # filtering...
475
                if (# if the exec bit has changed we have to store:
476
                    parent_entry.executable != content_summary[2]):
477
                    store = True
478
                elif parent_entry.text_sha1 == content_summary[3]:
479
                    # all meta and content is unchanged (using a hash cache
480
                    # hit to check the sha)
481
                    ie.revision = parent_entry.revision
482
                    ie.text_size = parent_entry.text_size
483
                    ie.text_sha1 = parent_entry.text_sha1
484
                    ie.executable = parent_entry.executable
485
                    return self._get_delta(ie, basis_inv, path), False, None
486
                else:
487
                    # Either there is only a hash change(no hash cache entry,
488
                    # or same size content change), or there is no change on
489
                    # this file at all.
490
                    # Provide the parent's hash to the store layer, so that the
491
                    # content is unchanged we will not store a new node.
492
                    nostore_sha = parent_entry.text_sha1
493
            if store:
494
                # We want to record a new node regardless of the presence or
495
                # absence of a content change in the file.
496
                nostore_sha = None
497
            ie.executable = content_summary[2]
498
            file_obj, stat_value = tree.get_file_with_stat(ie.file_id, path)
499
            try:
500
                text = file_obj.read()
501
            finally:
502
                file_obj.close()
503
            try:
504
                ie.text_sha1, ie.text_size = self._add_text_to_weave(
505
                    ie.file_id, text, heads, nostore_sha)
506
                # Let the caller know we generated a stat fingerprint.
507
                fingerprint = (ie.text_sha1, stat_value)
508
            except errors.ExistingContent:
509
                # Turns out that the file content was unchanged, and we were
510
                # only going to store a new node if it was changed. Carry over
511
                # the entry.
512
                ie.revision = parent_entry.revision
513
                ie.text_size = parent_entry.text_size
514
                ie.text_sha1 = parent_entry.text_sha1
515
                ie.executable = parent_entry.executable
516
                return self._get_delta(ie, basis_inv, path), False, None
517
        elif kind == 'directory':
518
            if not store:
519
                # all data is meta here, nothing specific to directory, so
520
                # carry over:
521
                ie.revision = parent_entry.revision
522
                return self._get_delta(ie, basis_inv, path), False, None
523
            self._add_text_to_weave(ie.file_id, '', heads, None)
524
        elif kind == 'symlink':
525
            current_link_target = content_summary[3]
526
            if not store:
527
                # symlink target is not generic metadata, check if it has
528
                # changed.
529
                if current_link_target != parent_entry.symlink_target:
530
                    store = True
531
            if not store:
532
                # unchanged, carry over.
533
                ie.revision = parent_entry.revision
534
                ie.symlink_target = parent_entry.symlink_target
535
                return self._get_delta(ie, basis_inv, path), False, None
536
            ie.symlink_target = current_link_target
537
            self._add_text_to_weave(ie.file_id, '', heads, None)
538
        elif kind == 'tree-reference':
539
            if not store:
540
                if content_summary[3] != parent_entry.reference_revision:
541
                    store = True
542
            if not store:
543
                # unchanged, carry over.
544
                ie.reference_revision = parent_entry.reference_revision
545
                ie.revision = parent_entry.revision
546
                return self._get_delta(ie, basis_inv, path), False, None
547
            ie.reference_revision = content_summary[3]
548
            if ie.reference_revision is None:
549
                raise AssertionError("invalid content_summary for nested tree: %r"
550
                    % (content_summary,))
551
            self._add_text_to_weave(ie.file_id, '', heads, None)
552
        else:
553
            raise NotImplementedError('unknown kind')
554
        ie.revision = self._new_revision_id
5847.3.2 by Jelmer Vernooij
Move root checking to VersionedFileRepository.
555
        # The initial commit adds a root directory, but this in itself is not
556
        # a worthwhile commit.
557
        if (self.basis_delta_revision != _mod_revision.NULL_REVISION or
558
            path != ""):
559
            self._any_changes = True
5815.4.2 by Jelmer Vernooij
split out versionedfile-specific stuff from commitbuilder.
560
        return self._get_delta(ie, basis_inv, path), True, fingerprint
561
562
    def record_iter_changes(self, tree, basis_revision_id, iter_changes,
563
        _entry_factory=entry_factory):
564
        """Record a new tree via iter_changes.
565
566
        :param tree: The tree to obtain text contents from for changed objects.
567
        :param basis_revision_id: The revision id of the tree the iter_changes
568
            has been generated against. Currently assumed to be the same
569
            as self.parents[0] - if it is not, errors may occur.
570
        :param iter_changes: An iter_changes iterator with the changes to apply
571
            to basis_revision_id. The iterator must not include any items with
572
            a current kind of None - missing items must be either filtered out
6090.2.1 by Jelmer Vernooij
Fix some typos.
573
            or errored-on before record_iter_changes sees the item.
5815.4.2 by Jelmer Vernooij
split out versionedfile-specific stuff from commitbuilder.
574
        :param _entry_factory: Private method to bind entry_factory locally for
575
            performance.
576
        :return: A generator of (file_id, relpath, fs_hash) tuples for use with
577
            tree._observed_sha1.
578
        """
579
        # Create an inventory delta based on deltas between all the parents and
580
        # deltas between all the parent inventories. We use inventory delta's 
581
        # between the inventory objects because iter_changes masks
582
        # last-changed-field only changes.
583
        # Working data:
584
        # file_id -> change map, change is fileid, paths, changed, versioneds,
585
        # parents, names, kinds, executables
586
        merged_ids = {}
587
        # {file_id -> revision_id -> inventory entry, for entries in parent
588
        # trees that are not parents[0]
589
        parent_entries = {}
590
        ghost_basis = False
591
        try:
592
            revtrees = list(self.repository.revision_trees(self.parents))
593
        except errors.NoSuchRevision:
594
            # one or more ghosts, slow path.
595
            revtrees = []
596
            for revision_id in self.parents:
597
                try:
598
                    revtrees.append(self.repository.revision_tree(revision_id))
599
                except errors.NoSuchRevision:
600
                    if not revtrees:
601
                        basis_revision_id = _mod_revision.NULL_REVISION
602
                        ghost_basis = True
603
                    revtrees.append(self.repository.revision_tree(
604
                        _mod_revision.NULL_REVISION))
605
        # The basis inventory from a repository 
606
        if revtrees:
6405.2.5 by Jelmer Vernooij
Add root_inventory.
607
            basis_tree = revtrees[0]
5815.4.2 by Jelmer Vernooij
split out versionedfile-specific stuff from commitbuilder.
608
        else:
6405.2.5 by Jelmer Vernooij
Add root_inventory.
609
            basis_tree = self.repository.revision_tree(
610
                _mod_revision.NULL_REVISION)
611
        basis_inv = basis_tree.root_inventory
5815.4.2 by Jelmer Vernooij
split out versionedfile-specific stuff from commitbuilder.
612
        if len(self.parents) > 0:
613
            if basis_revision_id != self.parents[0] and not ghost_basis:
614
                raise Exception(
615
                    "arbitrary basis parents not yet supported with merges")
616
            for revtree in revtrees[1:]:
6405.2.5 by Jelmer Vernooij
Add root_inventory.
617
                for change in revtree.root_inventory._make_delta(basis_inv):
5815.4.2 by Jelmer Vernooij
split out versionedfile-specific stuff from commitbuilder.
618
                    if change[1] is None:
619
                        # Not present in this parent.
620
                        continue
621
                    if change[2] not in merged_ids:
622
                        if change[0] is not None:
623
                            basis_entry = basis_inv[change[2]]
624
                            merged_ids[change[2]] = [
625
                                # basis revid
626
                                basis_entry.revision,
627
                                # new tree revid
628
                                change[3].revision]
629
                            parent_entries[change[2]] = {
630
                                # basis parent
631
                                basis_entry.revision:basis_entry,
632
                                # this parent 
633
                                change[3].revision:change[3],
634
                                }
635
                        else:
636
                            merged_ids[change[2]] = [change[3].revision]
637
                            parent_entries[change[2]] = {change[3].revision:change[3]}
638
                    else:
639
                        merged_ids[change[2]].append(change[3].revision)
640
                        parent_entries[change[2]][change[3].revision] = change[3]
641
        else:
642
            merged_ids = {}
643
        # Setup the changes from the tree:
644
        # changes maps file_id -> (change, [parent revision_ids])
645
        changes= {}
646
        for change in iter_changes:
647
            # This probably looks up in basis_inv way to much.
648
            if change[1][0] is not None:
649
                head_candidate = [basis_inv[change[0]].revision]
650
            else:
651
                head_candidate = []
652
            changes[change[0]] = change, merged_ids.get(change[0],
653
                head_candidate)
654
        unchanged_merged = set(merged_ids) - set(changes)
655
        # Extend the changes dict with synthetic changes to record merges of
656
        # texts.
657
        for file_id in unchanged_merged:
658
            # Record a merged version of these items that did not change vs the
659
            # basis. This can be either identical parallel changes, or a revert
660
            # of a specific file after a merge. The recorded content will be
661
            # that of the current tree (which is the same as the basis), but
662
            # the per-file graph will reflect a merge.
663
            # NB:XXX: We are reconstructing path information we had, this
664
            # should be preserved instead.
665
            # inv delta  change: (file_id, (path_in_source, path_in_target),
666
            #   changed_content, versioned, parent, name, kind,
667
            #   executable)
668
            try:
669
                basis_entry = basis_inv[file_id]
670
            except errors.NoSuchId:
671
                # a change from basis->some_parents but file_id isn't in basis
672
                # so was new in the merge, which means it must have changed
673
                # from basis -> current, and as it hasn't the add was reverted
674
                # by the user. So we discard this change.
675
                pass
676
            else:
677
                change = (file_id,
678
                    (basis_inv.id2path(file_id), tree.id2path(file_id)),
679
                    False, (True, True),
680
                    (basis_entry.parent_id, basis_entry.parent_id),
681
                    (basis_entry.name, basis_entry.name),
682
                    (basis_entry.kind, basis_entry.kind),
683
                    (basis_entry.executable, basis_entry.executable))
684
                changes[file_id] = (change, merged_ids[file_id])
685
        # changes contains tuples with the change and a set of inventory
686
        # candidates for the file.
687
        # inv delta is:
688
        # old_path, new_path, file_id, new_inventory_entry
689
        seen_root = False # Is the root in the basis delta?
690
        inv_delta = self._basis_delta
691
        modified_rev = self._new_revision_id
692
        for change, head_candidates in changes.values():
693
            if change[3][1]: # versioned in target.
694
                # Several things may be happening here:
695
                # We may have a fork in the per-file graph
696
                #  - record a change with the content from tree
697
                # We may have a change against < all trees  
698
                #  - carry over the tree that hasn't changed
699
                # We may have a change against all trees
700
                #  - record the change with the content from tree
701
                kind = change[6][1]
702
                file_id = change[0]
703
                entry = _entry_factory[kind](file_id, change[5][1],
704
                    change[4][1])
705
                head_set = self._heads(change[0], set(head_candidates))
706
                heads = []
707
                # Preserve ordering.
708
                for head_candidate in head_candidates:
709
                    if head_candidate in head_set:
710
                        heads.append(head_candidate)
711
                        head_set.remove(head_candidate)
712
                carried_over = False
713
                if len(heads) == 1:
714
                    # Could be a carry-over situation:
715
                    parent_entry_revs = parent_entries.get(file_id, None)
716
                    if parent_entry_revs:
717
                        parent_entry = parent_entry_revs.get(heads[0], None)
718
                    else:
719
                        parent_entry = None
720
                    if parent_entry is None:
721
                        # The parent iter_changes was called against is the one
722
                        # that is the per-file head, so any change is relevant
723
                        # iter_changes is valid.
724
                        carry_over_possible = False
725
                    else:
726
                        # could be a carry over situation
727
                        # A change against the basis may just indicate a merge,
728
                        # we need to check the content against the source of the
729
                        # merge to determine if it was changed after the merge
730
                        # or carried over.
731
                        if (parent_entry.kind != entry.kind or
732
                            parent_entry.parent_id != entry.parent_id or
733
                            parent_entry.name != entry.name):
734
                            # Metadata common to all entries has changed
735
                            # against per-file parent
736
                            carry_over_possible = False
737
                        else:
738
                            carry_over_possible = True
739
                        # per-type checks for changes against the parent_entry
740
                        # are done below.
741
                else:
742
                    # Cannot be a carry-over situation
743
                    carry_over_possible = False
744
                # Populate the entry in the delta
745
                if kind == 'file':
746
                    # XXX: There is still a small race here: If someone reverts the content of a file
747
                    # after iter_changes examines and decides it has changed,
748
                    # we will unconditionally record a new version even if some
749
                    # other process reverts it while commit is running (with
750
                    # the revert happening after iter_changes did its
751
                    # examination).
752
                    if change[7][1]:
753
                        entry.executable = True
754
                    else:
755
                        entry.executable = False
756
                    if (carry_over_possible and
757
                        parent_entry.executable == entry.executable):
758
                            # Check the file length, content hash after reading
759
                            # the file.
760
                            nostore_sha = parent_entry.text_sha1
761
                    else:
762
                        nostore_sha = None
763
                    file_obj, stat_value = tree.get_file_with_stat(file_id, change[1][1])
764
                    try:
765
                        text = file_obj.read()
766
                    finally:
767
                        file_obj.close()
768
                    try:
769
                        entry.text_sha1, entry.text_size = self._add_text_to_weave(
770
                            file_id, text, heads, nostore_sha)
771
                        yield file_id, change[1][1], (entry.text_sha1, stat_value)
772
                    except errors.ExistingContent:
773
                        # No content change against a carry_over parent
774
                        # Perhaps this should also yield a fs hash update?
775
                        carried_over = True
776
                        entry.text_size = parent_entry.text_size
777
                        entry.text_sha1 = parent_entry.text_sha1
778
                elif kind == 'symlink':
779
                    # Wants a path hint?
780
                    entry.symlink_target = tree.get_symlink_target(file_id)
781
                    if (carry_over_possible and
782
                        parent_entry.symlink_target == entry.symlink_target):
783
                        carried_over = True
784
                    else:
785
                        self._add_text_to_weave(change[0], '', heads, None)
786
                elif kind == 'directory':
787
                    if carry_over_possible:
788
                        carried_over = True
789
                    else:
790
                        # Nothing to set on the entry.
791
                        # XXX: split into the Root and nonRoot versions.
792
                        if change[1][1] != '' or self.repository.supports_rich_root():
793
                            self._add_text_to_weave(change[0], '', heads, None)
794
                elif kind == 'tree-reference':
795
                    if not self.repository._format.supports_tree_reference:
796
                        # This isn't quite sane as an error, but we shouldn't
797
                        # ever see this code path in practice: tree's don't
798
                        # permit references when the repo doesn't support tree
799
                        # references.
800
                        raise errors.UnsupportedOperation(tree.add_reference,
801
                            self.repository)
802
                    reference_revision = tree.get_reference_revision(change[0])
803
                    entry.reference_revision = reference_revision
804
                    if (carry_over_possible and
805
                        parent_entry.reference_revision == reference_revision):
806
                        carried_over = True
807
                    else:
808
                        self._add_text_to_weave(change[0], '', heads, None)
809
                else:
810
                    raise AssertionError('unknown kind %r' % kind)
811
                if not carried_over:
812
                    entry.revision = modified_rev
813
                else:
814
                    entry.revision = parent_entry.revision
815
            else:
816
                entry = None
817
            new_path = change[1][1]
818
            inv_delta.append((change[1][0], new_path, change[0], entry))
819
            if new_path == '':
820
                seen_root = True
821
        self.new_inventory = None
5847.3.2 by Jelmer Vernooij
Move root checking to VersionedFileRepository.
822
        # The initial commit adds a root directory, but this in itself is not
823
        # a worthwhile commit.
824
        if ((len(inv_delta) > 0 and basis_revision_id != _mod_revision.NULL_REVISION) or
825
            (len(inv_delta) > 1 and basis_revision_id == _mod_revision.NULL_REVISION)):
5815.4.2 by Jelmer Vernooij
split out versionedfile-specific stuff from commitbuilder.
826
            # This should perhaps be guarded by a check that the basis we
827
            # commit against is the basis for the commit and if not do a delta
828
            # against the basis.
829
            self._any_changes = True
830
        if not seen_root:
831
            # housekeeping root entry changes do not affect no-change commits.
832
            self._require_root_change(tree)
833
        self.basis_delta_revision = basis_revision_id
834
835
    def _add_text_to_weave(self, file_id, new_text, parents, nostore_sha):
836
        parent_keys = tuple([(file_id, parent) for parent in parents])
837
        return self.repository.texts._add_text(
838
            (file_id, self._new_revision_id), parent_keys, new_text,
839
            nostore_sha=nostore_sha, random_id=self.random_revid)[0:2]
840
841
842
class VersionedFileRootCommitBuilder(VersionedFileCommitBuilder):
843
    """This commitbuilder actually records the root id"""
844
845
    # the root entry gets versioned properly by this builder.
846
    _versioned_root = True
847
848
    def _check_root(self, ie, parent_invs, tree):
849
        """Helper for record_entry_contents.
850
851
        :param ie: An entry being added.
852
        :param parent_invs: The inventories of the parent revisions of the
853
            commit.
854
        :param tree: The tree that is being committed.
855
        """
856
857
    def _require_root_change(self, tree):
858
        """Enforce an appropriate root object change.
859
860
        This is called once when record_iter_changes is called, if and only if
861
        the root was not in the delta calculated by record_iter_changes.
862
863
        :param tree: The tree which is being committed.
864
        """
865
        # versioned roots do not change unless the tree found a change.
866
867
5815.4.1 by Jelmer Vernooij
Split versionedfile-specific stuff out into VersionedFileRepository.
868
class VersionedFileRepository(Repository):
869
    """Repository holding history for one or more branches.
870
871
    The repository holds and retrieves historical information including
872
    revisions and file history.  It's normally accessed only by the Branch,
873
    which views a particular line of development through that history.
874
875
    The Repository builds on top of some byte storage facilies (the revisions,
876
    signatures, inventories, texts and chk_bytes attributes) and a Transport,
877
    which respectively provide byte storage and a means to access the (possibly
878
    remote) disk.
879
880
    The byte storage facilities are addressed via tuples, which we refer to
881
    as 'keys' throughout the code base. Revision_keys, inventory_keys and
882
    signature_keys are all 1-tuples: (revision_id,). text_keys are two-tuples:
883
    (file_id, revision_id). chk_bytes uses CHK keys - a 1-tuple with a single
884
    byte string made up of a hash identifier and a hash value.
885
    We use this interface because it allows low friction with the underlying
886
    code that implements disk indices, network encoding and other parts of
887
    bzrlib.
888
889
    :ivar revisions: A bzrlib.versionedfile.VersionedFiles instance containing
890
        the serialised revisions for the repository. This can be used to obtain
891
        revision graph information or to access raw serialised revisions.
892
        The result of trying to insert data into the repository via this store
893
        is undefined: it should be considered read-only except for implementors
894
        of repositories.
895
    :ivar signatures: A bzrlib.versionedfile.VersionedFiles instance containing
896
        the serialised signatures for the repository. This can be used to
897
        obtain access to raw serialised signatures.  The result of trying to
898
        insert data into the repository via this store is undefined: it should
899
        be considered read-only except for implementors of repositories.
900
    :ivar inventories: A bzrlib.versionedfile.VersionedFiles instance containing
901
        the serialised inventories for the repository. This can be used to
902
        obtain unserialised inventories.  The result of trying to insert data
903
        into the repository via this store is undefined: it should be
904
        considered read-only except for implementors of repositories.
905
    :ivar texts: A bzrlib.versionedfile.VersionedFiles instance containing the
906
        texts of files and directories for the repository. This can be used to
907
        obtain file texts or file graphs. Note that Repository.iter_file_bytes
908
        is usually a better interface for accessing file texts.
909
        The result of trying to insert data into the repository via this store
910
        is undefined: it should be considered read-only except for implementors
911
        of repositories.
912
    :ivar chk_bytes: A bzrlib.versionedfile.VersionedFiles instance containing
913
        any data the repository chooses to store or have indexed by its hash.
914
        The result of trying to insert data into the repository via this store
915
        is undefined: it should be considered read-only except for implementors
916
        of repositories.
917
    :ivar _transport: Transport for file access to repository, typically
918
        pointing to .bzr/repository.
919
    """
920
5815.4.2 by Jelmer Vernooij
split out versionedfile-specific stuff from commitbuilder.
921
    # What class to use for a CommitBuilder. Often it's simpler to change this
922
    # in a Repository class subclass rather than to override
923
    # get_commit_builder.
924
    _commit_builder_class = VersionedFileCommitBuilder
925
5815.4.1 by Jelmer Vernooij
Split versionedfile-specific stuff out into VersionedFileRepository.
926
    def add_fallback_repository(self, repository):
927
        """Add a repository to use for looking up data not held locally.
928
929
        :param repository: A repository.
930
        """
931
        if not self._format.supports_external_lookups:
932
            raise errors.UnstackableRepositoryFormat(self._format, self.base)
6015.29.1 by John Arbash Meinel
Merge 2.3 into 2.4, and fix up the conflict for resolving bug #835035
933
        # This can raise an exception, so should be done before we lock the
934
        # fallback repository.
935
        self._check_fallback_repository(repository)
5815.4.1 by Jelmer Vernooij
Split versionedfile-specific stuff out into VersionedFileRepository.
936
        if self.is_locked():
937
            # This repository will call fallback.unlock() when we transition to
938
            # the unlocked state, so we make sure to increment the lock count
939
            repository.lock_read()
940
        self._fallback_repositories.append(repository)
941
        self.texts.add_fallback_versioned_files(repository.texts)
942
        self.inventories.add_fallback_versioned_files(repository.inventories)
943
        self.revisions.add_fallback_versioned_files(repository.revisions)
944
        self.signatures.add_fallback_versioned_files(repository.signatures)
945
        if self.chk_bytes is not None:
946
            self.chk_bytes.add_fallback_versioned_files(repository.chk_bytes)
947
948
    @only_raises(errors.LockNotHeld, errors.LockBroken)
949
    def unlock(self):
950
        super(VersionedFileRepository, self).unlock()
951
        if self.control_files._lock_count == 0:
952
            self._inventory_entry_cache.clear()
953
954
    def add_inventory(self, revision_id, inv, parents):
955
        """Add the inventory inv to the repository as revision_id.
956
957
        :param parents: The revision ids of the parents that revision_id
958
                        is known to have and are in the repository already.
959
960
        :returns: The validator(which is a sha1 digest, though what is sha'd is
961
            repository format specific) of the serialized inventory.
962
        """
963
        if not self.is_in_write_group():
964
            raise AssertionError("%r not in write group" % (self,))
965
        _mod_revision.check_not_reserved_id(revision_id)
966
        if not (inv.revision_id is None or inv.revision_id == revision_id):
967
            raise AssertionError(
968
                "Mismatch between inventory revision"
969
                " id and insertion revid (%r, %r)"
970
                % (inv.revision_id, revision_id))
971
        if inv.root is None:
972
            raise errors.RootMissing()
973
        return self._add_inventory_checked(revision_id, inv, parents)
974
975
    def _add_inventory_checked(self, revision_id, inv, parents):
976
        """Add inv to the repository after checking the inputs.
977
978
        This function can be overridden to allow different inventory styles.
979
980
        :seealso: add_inventory, for the contract.
981
        """
982
        inv_lines = self._serializer.write_inventory_to_lines(inv)
983
        return self._inventory_add_lines(revision_id, parents,
984
            inv_lines, check_content=False)
985
986
    def add_inventory_by_delta(self, basis_revision_id, delta, new_revision_id,
987
                               parents, basis_inv=None, propagate_caches=False):
988
        """Add a new inventory expressed as a delta against another revision.
989
990
        See the inventory developers documentation for the theory behind
991
        inventory deltas.
992
993
        :param basis_revision_id: The inventory id the delta was created
994
            against. (This does not have to be a direct parent.)
995
        :param delta: The inventory delta (see Inventory.apply_delta for
996
            details).
997
        :param new_revision_id: The revision id that the inventory is being
998
            added for.
999
        :param parents: The revision ids of the parents that revision_id is
1000
            known to have and are in the repository already. These are supplied
1001
            for repositories that depend on the inventory graph for revision
1002
            graph access, as well as for those that pun ancestry with delta
1003
            compression.
1004
        :param basis_inv: The basis inventory if it is already known,
1005
            otherwise None.
1006
        :param propagate_caches: If True, the caches for this inventory are
1007
          copied to and updated for the result if possible.
1008
1009
        :returns: (validator, new_inv)
1010
            The validator(which is a sha1 digest, though what is sha'd is
1011
            repository format specific) of the serialized inventory, and the
1012
            resulting inventory.
1013
        """
1014
        if not self.is_in_write_group():
1015
            raise AssertionError("%r not in write group" % (self,))
1016
        _mod_revision.check_not_reserved_id(new_revision_id)
1017
        basis_tree = self.revision_tree(basis_revision_id)
1018
        basis_tree.lock_read()
1019
        try:
1020
            # Note that this mutates the inventory of basis_tree, which not all
1021
            # inventory implementations may support: A better idiom would be to
1022
            # return a new inventory, but as there is no revision tree cache in
1023
            # repository this is safe for now - RBC 20081013
1024
            if basis_inv is None:
6405.2.5 by Jelmer Vernooij
Add root_inventory.
1025
                basis_inv = basis_tree.root_inventory
5815.4.1 by Jelmer Vernooij
Split versionedfile-specific stuff out into VersionedFileRepository.
1026
            basis_inv.apply_delta(delta)
1027
            basis_inv.revision_id = new_revision_id
1028
            return (self.add_inventory(new_revision_id, basis_inv, parents),
1029
                    basis_inv)
1030
        finally:
1031
            basis_tree.unlock()
1032
1033
    def _inventory_add_lines(self, revision_id, parents, lines,
1034
        check_content=True):
1035
        """Store lines in inv_vf and return the sha1 of the inventory."""
1036
        parents = [(parent,) for parent in parents]
1037
        result = self.inventories.add_lines((revision_id,), parents, lines,
1038
            check_content=check_content)[0]
1039
        self.inventories._access.flush()
1040
        return result
1041
6421.2.1 by Jelmer Vernooij
Move revision signing to CommitBuilder.
1042
    def add_revision(self, revision_id, rev, inv=None):
5815.4.1 by Jelmer Vernooij
Split versionedfile-specific stuff out into VersionedFileRepository.
1043
        """Add rev to the revision store as revision_id.
1044
1045
        :param revision_id: the revision id to use.
1046
        :param rev: The revision object.
1047
        :param inv: The inventory for the revision. if None, it will be looked
1048
                    up in the inventory storer
1049
        """
1050
        # TODO: jam 20070210 Shouldn't we check rev.revision_id and
1051
        #       rev.parent_ids?
1052
        _mod_revision.check_not_reserved_id(revision_id)
1053
        # check inventory present
1054
        if not self.inventories.get_parent_map([(revision_id,)]):
1055
            if inv is None:
1056
                raise errors.WeaveRevisionNotPresent(revision_id,
1057
                                                     self.inventories)
1058
            else:
1059
                # yes, this is not suitable for adding with ghosts.
1060
                rev.inventory_sha1 = self.add_inventory(revision_id, inv,
1061
                                                        rev.parent_ids)
1062
        else:
1063
            key = (revision_id,)
1064
            rev.inventory_sha1 = self.inventories.get_sha1s([key])[key]
1065
        self._add_revision(rev)
1066
1067
    def _add_revision(self, revision):
1068
        text = self._serializer.write_revision_to_string(revision)
1069
        key = (revision.revision_id,)
1070
        parents = tuple((parent,) for parent in revision.parent_ids)
1071
        self.revisions.add_lines(key, parents, osutils.split_lines(text))
1072
1073
    def _check_inventories(self, checker):
1074
        """Check the inventories found from the revision scan.
1075
        
1076
        This is responsible for verifying the sha1 of inventories and
1077
        creating a pending_keys set that covers data referenced by inventories.
1078
        """
1079
        bar = ui.ui_factory.nested_progress_bar()
1080
        try:
1081
            self._do_check_inventories(checker, bar)
1082
        finally:
1083
            bar.finished()
1084
1085
    def _do_check_inventories(self, checker, bar):
1086
        """Helper for _check_inventories."""
1087
        revno = 0
1088
        keys = {'chk_bytes':set(), 'inventories':set(), 'texts':set()}
1089
        kinds = ['chk_bytes', 'texts']
1090
        count = len(checker.pending_keys)
6138.4.1 by Jonathan Riddell
add gettext to progress bar strings
1091
        bar.update(gettext("inventories"), 0, 2)
5815.4.1 by Jelmer Vernooij
Split versionedfile-specific stuff out into VersionedFileRepository.
1092
        current_keys = checker.pending_keys
1093
        checker.pending_keys = {}
1094
        # Accumulate current checks.
1095
        for key in current_keys:
1096
            if key[0] != 'inventories' and key[0] not in kinds:
1097
                checker._report_items.append('unknown key type %r' % (key,))
1098
            keys[key[0]].add(key[1:])
1099
        if keys['inventories']:
1100
            # NB: output order *should* be roughly sorted - topo or
1101
            # inverse topo depending on repository - either way decent
1102
            # to just delta against. However, pre-CHK formats didn't
1103
            # try to optimise inventory layout on disk. As such the
1104
            # pre-CHK code path does not use inventory deltas.
1105
            last_object = None
1106
            for record in self.inventories.check(keys=keys['inventories']):
1107
                if record.storage_kind == 'absent':
1108
                    checker._report_items.append(
1109
                        'Missing inventory {%s}' % (record.key,))
1110
                else:
1111
                    last_object = self._check_record('inventories', record,
1112
                        checker, last_object,
1113
                        current_keys[('inventories',) + record.key])
1114
            del keys['inventories']
1115
        else:
1116
            return
6138.4.1 by Jonathan Riddell
add gettext to progress bar strings
1117
        bar.update(gettext("texts"), 1)
5815.4.1 by Jelmer Vernooij
Split versionedfile-specific stuff out into VersionedFileRepository.
1118
        while (checker.pending_keys or keys['chk_bytes']
1119
            or keys['texts']):
1120
            # Something to check.
1121
            current_keys = checker.pending_keys
1122
            checker.pending_keys = {}
1123
            # Accumulate current checks.
1124
            for key in current_keys:
1125
                if key[0] not in kinds:
1126
                    checker._report_items.append('unknown key type %r' % (key,))
1127
                keys[key[0]].add(key[1:])
1128
            # Check the outermost kind only - inventories || chk_bytes || texts
1129
            for kind in kinds:
1130
                if keys[kind]:
1131
                    last_object = None
1132
                    for record in getattr(self, kind).check(keys=keys[kind]):
1133
                        if record.storage_kind == 'absent':
1134
                            checker._report_items.append(
1135
                                'Missing %s {%s}' % (kind, record.key,))
1136
                        else:
1137
                            last_object = self._check_record(kind, record,
1138
                                checker, last_object, current_keys[(kind,) + record.key])
1139
                    keys[kind] = set()
1140
                    break
1141
1142
    def _check_record(self, kind, record, checker, last_object, item_data):
1143
        """Check a single text from this repository."""
1144
        if kind == 'inventories':
1145
            rev_id = record.key[0]
1146
            inv = self._deserialise_inventory(rev_id,
1147
                record.get_bytes_as('fulltext'))
1148
            if last_object is not None:
1149
                delta = inv._make_delta(last_object)
1150
                for old_path, path, file_id, ie in delta:
1151
                    if ie is None:
1152
                        continue
1153
                    ie.check(checker, rev_id, inv)
1154
            else:
1155
                for path, ie in inv.iter_entries():
1156
                    ie.check(checker, rev_id, inv)
1157
            if self._format.fast_deltas:
1158
                return inv
1159
        elif kind == 'chk_bytes':
1160
            # No code written to check chk_bytes for this repo format.
1161
            checker._report_items.append(
1162
                'unsupported key type chk_bytes for %s' % (record.key,))
1163
        elif kind == 'texts':
1164
            self._check_text(record, checker, item_data)
1165
        else:
1166
            checker._report_items.append(
1167
                'unknown key type %s for %s' % (kind, record.key))
1168
1169
    def _check_text(self, record, checker, item_data):
1170
        """Check a single text."""
1171
        # Check it is extractable.
1172
        # TODO: check length.
1173
        if record.storage_kind == 'chunked':
1174
            chunks = record.get_bytes_as(record.storage_kind)
1175
            sha1 = osutils.sha_strings(chunks)
1176
            length = sum(map(len, chunks))
1177
        else:
1178
            content = record.get_bytes_as('fulltext')
1179
            sha1 = osutils.sha_string(content)
1180
            length = len(content)
1181
        if item_data and sha1 != item_data[1]:
1182
            checker._report_items.append(
1183
                'sha1 mismatch: %s has sha1 %s expected %s referenced by %s' %
1184
                (record.key, sha1, item_data[1], item_data[2]))
1185
6039.2.1 by Jelmer Vernooij
Move Repository._eliminate_revisions_not_present to VFRepository.
1186
    @needs_read_lock
1187
    def _eliminate_revisions_not_present(self, revision_ids):
1188
        """Check every revision id in revision_ids to see if we have it.
1189
1190
        Returns a set of the present revisions.
1191
        """
1192
        result = []
1193
        graph = self.get_graph()
1194
        parent_map = graph.get_parent_map(revision_ids)
1195
        # The old API returned a list, should this actually be a set?
1196
        return parent_map.keys()
1197
5815.4.1 by Jelmer Vernooij
Split versionedfile-specific stuff out into VersionedFileRepository.
1198
    def __init__(self, _format, a_bzrdir, control_files):
1199
        """Instantiate a VersionedFileRepository.
1200
1201
        :param _format: The format of the repository on disk.
6207.3.3 by jelmer at samba
Fix tests and the like.
1202
        :param controldir: The ControlDir of the repository.
5815.4.1 by Jelmer Vernooij
Split versionedfile-specific stuff out into VersionedFileRepository.
1203
        :param control_files: Control files to use for locking, etc.
1204
        """
1205
        # In the future we will have a single api for all stores for
1206
        # getting file texts, inventories and revisions, then
1207
        # this construct will accept instances of those things.
1208
        super(VersionedFileRepository, self).__init__(_format, a_bzrdir,
1209
            control_files)
6155.1.1 by Jelmer Vernooij
Allow repositories to not have a control_files attribute.
1210
        self._transport = control_files._transport
1211
        self.base = self._transport.base
5815.4.1 by Jelmer Vernooij
Split versionedfile-specific stuff out into VersionedFileRepository.
1212
        # for tests
1213
        self._reconcile_does_inventory_gc = True
1214
        self._reconcile_fixes_text_parents = False
1215
        self._reconcile_backsup_inventory = True
1216
        # An InventoryEntry cache, used during deserialization
1217
        self._inventory_entry_cache = fifo_cache.FIFOCache(10*1024)
1218
        # Is it safe to return inventory entries directly from the entry cache,
1219
        # rather copying them?
1220
        self._safe_to_return_from_cache = False
1221
6341.1.1 by Jelmer Vernooij
Make fetch_spec specific to bzr format repositories.
1222
    def fetch(self, source, revision_id=None, find_ghosts=False,
1223
            fetch_spec=None):
1224
        """Fetch the content required to construct revision_id from source.
1225
1226
        If revision_id is None and fetch_spec is None, then all content is
1227
        copied.
1228
1229
        fetch() may not be used when the repository is in a write group -
1230
        either finish the current write group before using fetch, or use
1231
        fetch before starting the write group.
1232
1233
        :param find_ghosts: Find and copy revisions in the source that are
1234
            ghosts in the target (and not reachable directly by walking out to
1235
            the first-present revision in target from revision_id).
1236
        :param revision_id: If specified, all the content needed for this
1237
            revision ID will be copied to the target.  Fetch will determine for
1238
            itself which content needs to be copied.
1239
        :param fetch_spec: If specified, a SearchResult or
1240
            PendingAncestryResult that describes which revisions to copy.  This
1241
            allows copying multiple heads at once.  Mutually exclusive with
1242
            revision_id.
1243
        """
1244
        if fetch_spec is not None and revision_id is not None:
1245
            raise AssertionError(
1246
                "fetch_spec and revision_id are mutually exclusive.")
1247
        if self.is_in_write_group():
1248
            raise errors.InternalBzrError(
1249
                "May not fetch while in a write group.")
1250
        # fast path same-url fetch operations
1251
        # TODO: lift out to somewhere common with RemoteRepository
1252
        # <https://bugs.launchpad.net/bzr/+bug/401646>
1253
        if (self.has_same_location(source)
1254
            and fetch_spec is None
1255
            and self._has_same_fallbacks(source)):
1256
            # check that last_revision is in 'from' and then return a
1257
            # no-operation.
1258
            if (revision_id is not None and
1259
                not _mod_revision.is_null(revision_id)):
1260
                self.get_revision(revision_id)
1261
            return 0, []
1262
        inter = InterRepository.get(source, self)
1263
        if (fetch_spec is not None and
1264
            not getattr(inter, "supports_fetch_spec", False)):
1265
            raise errors.UnsupportedOperation(
1266
                "fetch_spec not supported for %r" % inter)
1267
        return inter.fetch(revision_id=revision_id,
1268
            find_ghosts=find_ghosts, fetch_spec=fetch_spec)
1269
5815.4.21 by Jelmer Vernooij
Fix lock.
1270
    @needs_read_lock
1271
    def gather_stats(self, revid=None, committers=None):
5815.4.18 by Jelmer Vernooij
Move vf-specific gather_stats bits to vf_repository.
1272
        """See Repository.gather_stats()."""
5815.4.21 by Jelmer Vernooij
Fix lock.
1273
        result = super(VersionedFileRepository, self).gather_stats(revid, committers)
5815.4.18 by Jelmer Vernooij
Move vf-specific gather_stats bits to vf_repository.
1274
        # now gather global repository information
1275
        # XXX: This is available for many repos regardless of listability.
1276
        if self.user_transport.listable():
1277
            # XXX: do we want to __define len__() ?
1278
            # Maybe the versionedfiles object should provide a different
1279
            # method to get the number of keys.
1280
            result['revisions'] = len(self.revisions.keys())
1281
            # result['size'] = t
1282
        return result
1283
6351.3.2 by Jelmer Vernooij
Convert some gpg options to config stacks.
1284
    def get_commit_builder(self, branch, parents, config_stack, timestamp=None,
5815.4.2 by Jelmer Vernooij
split out versionedfile-specific stuff from commitbuilder.
1285
                           timezone=None, committer=None, revprops=None,
1286
                           revision_id=None, lossy=False):
1287
        """Obtain a CommitBuilder for this repository.
1288
1289
        :param branch: Branch to commit to.
1290
        :param parents: Revision ids of the parents of the new revision.
6351.3.2 by Jelmer Vernooij
Convert some gpg options to config stacks.
1291
        :param config_stack: Configuration stack to use.
5815.4.2 by Jelmer Vernooij
split out versionedfile-specific stuff from commitbuilder.
1292
        :param timestamp: Optional timestamp recorded for commit.
1293
        :param timezone: Optional timezone for timestamp.
1294
        :param committer: Optional committer to set for commit.
1295
        :param revprops: Optional dictionary of revision properties.
1296
        :param revision_id: Optional revision id.
1297
        :param lossy: Whether to discard data that can not be natively
1298
            represented, when pushing to a foreign VCS
1299
        """
1300
        if self._fallback_repositories and not self._format.supports_chks:
1301
            raise errors.BzrError("Cannot commit directly to a stacked branch"
1302
                " in pre-2a formats. See "
1303
                "https://bugs.launchpad.net/bzr/+bug/375013 for details.")
6351.3.2 by Jelmer Vernooij
Convert some gpg options to config stacks.
1304
        result = self._commit_builder_class(self, parents, config_stack,
5815.4.2 by Jelmer Vernooij
split out versionedfile-specific stuff from commitbuilder.
1305
            timestamp, timezone, committer, revprops, revision_id,
1306
            lossy)
1307
        self.start_write_group()
1308
        return result
1309
5815.4.1 by Jelmer Vernooij
Split versionedfile-specific stuff out into VersionedFileRepository.
1310
    def get_missing_parent_inventories(self, check_for_missing_texts=True):
1311
        """Return the keys of missing inventory parents for revisions added in
1312
        this write group.
1313
1314
        A revision is not complete if the inventory delta for that revision
1315
        cannot be calculated.  Therefore if the parent inventories of a
1316
        revision are not present, the revision is incomplete, and e.g. cannot
1317
        be streamed by a smart server.  This method finds missing inventory
1318
        parents for revisions added in this write group.
1319
        """
1320
        if not self._format.supports_external_lookups:
1321
            # This is only an issue for stacked repositories
1322
            return set()
1323
        if not self.is_in_write_group():
1324
            raise AssertionError('not in a write group')
1325
1326
        # XXX: We assume that every added revision already has its
1327
        # corresponding inventory, so we only check for parent inventories that
1328
        # might be missing, rather than all inventories.
1329
        parents = set(self.revisions._index.get_missing_parents())
1330
        parents.discard(_mod_revision.NULL_REVISION)
1331
        unstacked_inventories = self.inventories._index
1332
        present_inventories = unstacked_inventories.get_parent_map(
1333
            key[-1:] for key in parents)
1334
        parents.difference_update(present_inventories)
1335
        if len(parents) == 0:
1336
            # No missing parent inventories.
1337
            return set()
1338
        if not check_for_missing_texts:
1339
            return set(('inventories', rev_id) for (rev_id,) in parents)
1340
        # Ok, now we have a list of missing inventories.  But these only matter
1341
        # if the inventories that reference them are missing some texts they
1342
        # appear to introduce.
1343
        # XXX: Texts referenced by all added inventories need to be present,
1344
        # but at the moment we're only checking for texts referenced by
1345
        # inventories at the graph's edge.
1346
        key_deps = self.revisions._index._key_dependencies
1347
        key_deps.satisfy_refs_for_keys(present_inventories)
1348
        referrers = frozenset(r[0] for r in key_deps.get_referrers())
1349
        file_ids = self.fileids_altered_by_revision_ids(referrers)
1350
        missing_texts = set()
1351
        for file_id, version_ids in file_ids.iteritems():
1352
            missing_texts.update(
1353
                (file_id, version_id) for version_id in version_ids)
1354
        present_texts = self.texts.get_parent_map(missing_texts)
1355
        missing_texts.difference_update(present_texts)
1356
        if not missing_texts:
1357
            # No texts are missing, so all revisions and their deltas are
1358
            # reconstructable.
1359
            return set()
1360
        # Alternatively the text versions could be returned as the missing
1361
        # keys, but this is likely to be less data.
1362
        missing_keys = set(('inventories', rev_id) for (rev_id,) in parents)
1363
        return missing_keys
1364
1365
    @needs_read_lock
1366
    def has_revisions(self, revision_ids):
1367
        """Probe to find out the presence of multiple revisions.
1368
1369
        :param revision_ids: An iterable of revision_ids.
1370
        :return: A set of the revision_ids that were present.
1371
        """
1372
        parent_map = self.revisions.get_parent_map(
1373
            [(rev_id,) for rev_id in revision_ids])
1374
        result = set()
1375
        if _mod_revision.NULL_REVISION in revision_ids:
1376
            result.add(_mod_revision.NULL_REVISION)
1377
        result.update([key[0] for key in parent_map])
1378
        return result
1379
1380
    @needs_read_lock
1381
    def get_revision_reconcile(self, revision_id):
1382
        """'reconcile' helper routine that allows access to a revision always.
1383
1384
        This variant of get_revision does not cross check the weave graph
1385
        against the revision one as get_revision does: but it should only
1386
        be used by reconcile, or reconcile-alike commands that are correcting
1387
        or testing the revision graph.
1388
        """
1389
        return self._get_revisions([revision_id])[0]
1390
1391
    @needs_read_lock
1392
    def get_revisions(self, revision_ids):
1393
        """Get many revisions at once.
1394
        
1395
        Repositories that need to check data on every revision read should 
1396
        subclass this method.
1397
        """
1398
        return self._get_revisions(revision_ids)
1399
1400
    @needs_read_lock
1401
    def _get_revisions(self, revision_ids):
1402
        """Core work logic to get many revisions without sanity checks."""
1403
        revs = {}
1404
        for revid, rev in self._iter_revisions(revision_ids):
1405
            if rev is None:
1406
                raise errors.NoSuchRevision(self, revid)
1407
            revs[revid] = rev
1408
        return [revs[revid] for revid in revision_ids]
1409
1410
    def _iter_revisions(self, revision_ids):
1411
        """Iterate over revision objects.
1412
1413
        :param revision_ids: An iterable of revisions to examine. None may be
1414
            passed to request all revisions known to the repository. Note that
1415
            not all repositories can find unreferenced revisions; for those
1416
            repositories only referenced ones will be returned.
1417
        :return: An iterator of (revid, revision) tuples. Absent revisions (
1418
            those asked for but not available) are returned as (revid, None).
1419
        """
1420
        if revision_ids is None:
1421
            revision_ids = self.all_revision_ids()
1422
        else:
1423
            for rev_id in revision_ids:
1424
                if not rev_id or not isinstance(rev_id, basestring):
1425
                    raise errors.InvalidRevisionId(revision_id=rev_id, branch=self)
1426
        keys = [(key,) for key in revision_ids]
1427
        stream = self.revisions.get_record_stream(keys, 'unordered', True)
1428
        for record in stream:
1429
            revid = record.key[0]
1430
            if record.storage_kind == 'absent':
1431
                yield (revid, None)
1432
            else:
1433
                text = record.get_bytes_as('fulltext')
1434
                rev = self._serializer.read_revision_from_string(text)
1435
                yield (revid, rev)
1436
1437
    @needs_write_lock
1438
    def add_signature_text(self, revision_id, signature):
1439
        """Store a signature text for a revision.
1440
1441
        :param revision_id: Revision id of the revision
1442
        :param signature: Signature text.
1443
        """
1444
        self.signatures.add_lines((revision_id,), (),
1445
            osutils.split_lines(signature))
1446
1447
    def find_text_key_references(self):
1448
        """Find the text key references within the repository.
1449
1450
        :return: A dictionary mapping text keys ((fileid, revision_id) tuples)
1451
            to whether they were referred to by the inventory of the
1452
            revision_id that they contain. The inventory texts from all present
1453
            revision ids are assessed to generate this report.
1454
        """
1455
        revision_keys = self.revisions.keys()
1456
        w = self.inventories
1457
        pb = ui.ui_factory.nested_progress_bar()
1458
        try:
1459
            return self._serializer._find_text_key_references(
1460
                w.iter_lines_added_or_present_in_keys(revision_keys, pb=pb))
1461
        finally:
1462
            pb.finished()
1463
1464
    def _inventory_xml_lines_for_keys(self, keys):
1465
        """Get a line iterator of the sort needed for findind references.
1466
1467
        Not relevant for non-xml inventory repositories.
1468
1469
        Ghosts in revision_keys are ignored.
1470
1471
        :param revision_keys: The revision keys for the inventories to inspect.
1472
        :return: An iterator over (inventory line, revid) for the fulltexts of
1473
            all of the xml inventories specified by revision_keys.
1474
        """
1475
        stream = self.inventories.get_record_stream(keys, 'unordered', True)
1476
        for record in stream:
1477
            if record.storage_kind != 'absent':
1478
                chunks = record.get_bytes_as('chunked')
1479
                revid = record.key[-1]
1480
                lines = osutils.chunks_to_lines(chunks)
1481
                for line in lines:
1482
                    yield line, revid
1483
1484
    def _find_file_ids_from_xml_inventory_lines(self, line_iterator,
1485
        revision_keys):
1486
        """Helper routine for fileids_altered_by_revision_ids.
1487
1488
        This performs the translation of xml lines to revision ids.
1489
1490
        :param line_iterator: An iterator of lines, origin_version_id
1491
        :param revision_keys: The revision ids to filter for. This should be a
1492
            set or other type which supports efficient __contains__ lookups, as
1493
            the revision key from each parsed line will be looked up in the
1494
            revision_keys filter.
1495
        :return: a dictionary mapping altered file-ids to an iterable of
5891.1.3 by Andrew Bennetts
Move docstring formatting fixes.
1496
            revision_ids. Each altered file-ids has the exact revision_ids that
1497
            altered it listed explicitly.
5815.4.1 by Jelmer Vernooij
Split versionedfile-specific stuff out into VersionedFileRepository.
1498
        """
1499
        seen = set(self._serializer._find_text_key_references(
1500
                line_iterator).iterkeys())
1501
        parent_keys = self._find_parent_keys_of_revisions(revision_keys)
1502
        parent_seen = set(self._serializer._find_text_key_references(
1503
            self._inventory_xml_lines_for_keys(parent_keys)))
1504
        new_keys = seen - parent_seen
1505
        result = {}
1506
        setdefault = result.setdefault
1507
        for key in new_keys:
1508
            setdefault(key[0], set()).add(key[-1])
1509
        return result
1510
1511
    def _find_parent_keys_of_revisions(self, revision_keys):
1512
        """Similar to _find_parent_ids_of_revisions, but used with keys.
1513
1514
        :param revision_keys: An iterable of revision_keys.
1515
        :return: The parents of all revision_keys that are not already in
1516
            revision_keys
1517
        """
1518
        parent_map = self.revisions.get_parent_map(revision_keys)
1519
        parent_keys = set()
1520
        map(parent_keys.update, parent_map.itervalues())
1521
        parent_keys.difference_update(revision_keys)
1522
        parent_keys.discard(_mod_revision.NULL_REVISION)
1523
        return parent_keys
1524
1525
    def fileids_altered_by_revision_ids(self, revision_ids, _inv_weave=None):
1526
        """Find the file ids and versions affected by revisions.
1527
1528
        :param revisions: an iterable containing revision ids.
1529
        :param _inv_weave: The inventory weave from this repository or None.
1530
            If None, the inventory weave will be opened automatically.
1531
        :return: a dictionary mapping altered file-ids to an iterable of
5891.1.3 by Andrew Bennetts
Move docstring formatting fixes.
1532
            revision_ids. Each altered file-ids has the exact revision_ids that
1533
            altered it listed explicitly.
5815.4.1 by Jelmer Vernooij
Split versionedfile-specific stuff out into VersionedFileRepository.
1534
        """
1535
        selected_keys = set((revid,) for revid in revision_ids)
1536
        w = _inv_weave or self.inventories
1537
        return self._find_file_ids_from_xml_inventory_lines(
1538
            w.iter_lines_added_or_present_in_keys(
1539
                selected_keys, pb=None),
1540
            selected_keys)
1541
1542
    def iter_files_bytes(self, desired_files):
1543
        """Iterate through file versions.
1544
1545
        Files will not necessarily be returned in the order they occur in
1546
        desired_files.  No specific order is guaranteed.
1547
1548
        Yields pairs of identifier, bytes_iterator.  identifier is an opaque
1549
        value supplied by the caller as part of desired_files.  It should
1550
        uniquely identify the file version in the caller's context.  (Examples:
1551
        an index number or a TreeTransform trans_id.)
1552
1553
        bytes_iterator is an iterable of bytestrings for the file.  The
1554
        kind of iterable and length of the bytestrings are unspecified, but for
1555
        this implementation, it is a list of bytes produced by
1556
        VersionedFile.get_record_stream().
1557
1558
        :param desired_files: a list of (file_id, revision_id, identifier)
1559
            triples
1560
        """
1561
        text_keys = {}
1562
        for file_id, revision_id, callable_data in desired_files:
1563
            text_keys[(file_id, revision_id)] = callable_data
1564
        for record in self.texts.get_record_stream(text_keys, 'unordered', True):
1565
            if record.storage_kind == 'absent':
6280.10.6 by Jelmer Vernooij
Convert to iter_files_bytes_bz2.
1566
                raise errors.RevisionNotPresent(record.key[1], record.key[0])
5815.4.1 by Jelmer Vernooij
Split versionedfile-specific stuff out into VersionedFileRepository.
1567
            yield text_keys[record.key], record.get_bytes_as('chunked')
1568
1569
    def _generate_text_key_index(self, text_key_references=None,
1570
        ancestors=None):
1571
        """Generate a new text key index for the repository.
1572
1573
        This is an expensive function that will take considerable time to run.
1574
1575
        :return: A dict mapping text keys ((file_id, revision_id) tuples) to a
1576
            list of parents, also text keys. When a given key has no parents,
1577
            the parents list will be [NULL_REVISION].
1578
        """
1579
        # All revisions, to find inventory parents.
1580
        if ancestors is None:
1581
            graph = self.get_graph()
1582
            ancestors = graph.get_parent_map(self.all_revision_ids())
1583
        if text_key_references is None:
1584
            text_key_references = self.find_text_key_references()
1585
        pb = ui.ui_factory.nested_progress_bar()
1586
        try:
1587
            return self._do_generate_text_key_index(ancestors,
1588
                text_key_references, pb)
1589
        finally:
1590
            pb.finished()
1591
1592
    def _do_generate_text_key_index(self, ancestors, text_key_references, pb):
1593
        """Helper for _generate_text_key_index to avoid deep nesting."""
1594
        revision_order = tsort.topo_sort(ancestors)
1595
        invalid_keys = set()
1596
        revision_keys = {}
1597
        for revision_id in revision_order:
1598
            revision_keys[revision_id] = set()
1599
        text_count = len(text_key_references)
1600
        # a cache of the text keys to allow reuse; costs a dict of all the
1601
        # keys, but saves a 2-tuple for every child of a given key.
1602
        text_key_cache = {}
1603
        for text_key, valid in text_key_references.iteritems():
1604
            if not valid:
1605
                invalid_keys.add(text_key)
1606
            else:
1607
                revision_keys[text_key[1]].add(text_key)
1608
            text_key_cache[text_key] = text_key
1609
        del text_key_references
1610
        text_index = {}
1611
        text_graph = graph.Graph(graph.DictParentsProvider(text_index))
1612
        NULL_REVISION = _mod_revision.NULL_REVISION
1613
        # Set a cache with a size of 10 - this suffices for bzr.dev but may be
1614
        # too small for large or very branchy trees. However, for 55K path
1615
        # trees, it would be easy to use too much memory trivially. Ideally we
1616
        # could gauge this by looking at available real memory etc, but this is
1617
        # always a tricky proposition.
1618
        inventory_cache = lru_cache.LRUCache(10)
1619
        batch_size = 10 # should be ~150MB on a 55K path tree
1620
        batch_count = len(revision_order) / batch_size + 1
1621
        processed_texts = 0
6138.4.1 by Jonathan Riddell
add gettext to progress bar strings
1622
        pb.update(gettext("Calculating text parents"), processed_texts, text_count)
5815.4.1 by Jelmer Vernooij
Split versionedfile-specific stuff out into VersionedFileRepository.
1623
        for offset in xrange(batch_count):
1624
            to_query = revision_order[offset * batch_size:(offset + 1) *
1625
                batch_size]
1626
            if not to_query:
1627
                break
1628
            for revision_id in to_query:
1629
                parent_ids = ancestors[revision_id]
1630
                for text_key in revision_keys[revision_id]:
6138.4.1 by Jonathan Riddell
add gettext to progress bar strings
1631
                    pb.update(gettext("Calculating text parents"), processed_texts)
5815.4.1 by Jelmer Vernooij
Split versionedfile-specific stuff out into VersionedFileRepository.
1632
                    processed_texts += 1
1633
                    candidate_parents = []
1634
                    for parent_id in parent_ids:
1635
                        parent_text_key = (text_key[0], parent_id)
1636
                        try:
1637
                            check_parent = parent_text_key not in \
1638
                                revision_keys[parent_id]
1639
                        except KeyError:
1640
                            # the parent parent_id is a ghost:
1641
                            check_parent = False
1642
                            # truncate the derived graph against this ghost.
1643
                            parent_text_key = None
1644
                        if check_parent:
1645
                            # look at the parent commit details inventories to
1646
                            # determine possible candidates in the per file graph.
1647
                            # TODO: cache here.
1648
                            try:
1649
                                inv = inventory_cache[parent_id]
1650
                            except KeyError:
6405.2.7 by Jelmer Vernooij
Fix more tests.
1651
                                inv = self.revision_tree(parent_id).root_inventory
5815.4.1 by Jelmer Vernooij
Split versionedfile-specific stuff out into VersionedFileRepository.
1652
                                inventory_cache[parent_id] = inv
1653
                            try:
1654
                                parent_entry = inv[text_key[0]]
1655
                            except (KeyError, errors.NoSuchId):
1656
                                parent_entry = None
1657
                            if parent_entry is not None:
1658
                                parent_text_key = (
1659
                                    text_key[0], parent_entry.revision)
1660
                            else:
1661
                                parent_text_key = None
1662
                        if parent_text_key is not None:
1663
                            candidate_parents.append(
1664
                                text_key_cache[parent_text_key])
1665
                    parent_heads = text_graph.heads(candidate_parents)
1666
                    new_parents = list(parent_heads)
1667
                    new_parents.sort(key=lambda x:candidate_parents.index(x))
1668
                    if new_parents == []:
1669
                        new_parents = [NULL_REVISION]
1670
                    text_index[text_key] = new_parents
1671
1672
        for text_key in invalid_keys:
1673
            text_index[text_key] = [NULL_REVISION]
1674
        return text_index
1675
1676
    def item_keys_introduced_by(self, revision_ids, _files_pb=None):
1677
        """Get an iterable listing the keys of all the data introduced by a set
1678
        of revision IDs.
1679
1680
        The keys will be ordered so that the corresponding items can be safely
1681
        fetched and inserted in that order.
1682
1683
        :returns: An iterable producing tuples of (knit-kind, file-id,
1684
            versions).  knit-kind is one of 'file', 'inventory', 'signatures',
1685
            'revisions'.  file-id is None unless knit-kind is 'file'.
1686
        """
1687
        for result in self._find_file_keys_to_fetch(revision_ids, _files_pb):
1688
            yield result
1689
        del _files_pb
1690
        for result in self._find_non_file_keys_to_fetch(revision_ids):
1691
            yield result
1692
1693
    def _find_file_keys_to_fetch(self, revision_ids, pb):
1694
        # XXX: it's a bit weird to control the inventory weave caching in this
1695
        # generator.  Ideally the caching would be done in fetch.py I think.  Or
1696
        # maybe this generator should explicitly have the contract that it
1697
        # should not be iterated until the previously yielded item has been
1698
        # processed?
1699
        inv_w = self.inventories
1700
1701
        # file ids that changed
1702
        file_ids = self.fileids_altered_by_revision_ids(revision_ids, inv_w)
1703
        count = 0
1704
        num_file_ids = len(file_ids)
1705
        for file_id, altered_versions in file_ids.iteritems():
1706
            if pb is not None:
6138.4.1 by Jonathan Riddell
add gettext to progress bar strings
1707
                pb.update(gettext("Fetch texts"), count, num_file_ids)
5815.4.1 by Jelmer Vernooij
Split versionedfile-specific stuff out into VersionedFileRepository.
1708
            count += 1
1709
            yield ("file", file_id, altered_versions)
1710
1711
    def _find_non_file_keys_to_fetch(self, revision_ids):
1712
        # inventory
1713
        yield ("inventory", None, revision_ids)
1714
1715
        # signatures
1716
        # XXX: Note ATM no callers actually pay attention to this return
1717
        #      instead they just use the list of revision ids and ignore
1718
        #      missing sigs. Consider removing this work entirely
1719
        revisions_with_signatures = set(self.signatures.get_parent_map(
1720
            [(r,) for r in revision_ids]))
1721
        revisions_with_signatures = set(
1722
            [r for (r,) in revisions_with_signatures])
1723
        revisions_with_signatures.intersection_update(revision_ids)
1724
        yield ("signatures", None, revisions_with_signatures)
1725
1726
        # revisions
1727
        yield ("revisions", None, revision_ids)
1728
1729
    @needs_read_lock
1730
    def get_inventory(self, revision_id):
1731
        """Get Inventory object by revision id."""
1732
        return self.iter_inventories([revision_id]).next()
1733
1734
    def iter_inventories(self, revision_ids, ordering=None):
1735
        """Get many inventories by revision_ids.
1736
1737
        This will buffer some or all of the texts used in constructing the
1738
        inventories in memory, but will only parse a single inventory at a
1739
        time.
1740
1741
        :param revision_ids: The expected revision ids of the inventories.
1742
        :param ordering: optional ordering, e.g. 'topological'.  If not
1743
            specified, the order of revision_ids will be preserved (by
1744
            buffering if necessary).
1745
        :return: An iterator of inventories.
1746
        """
1747
        if ((None in revision_ids)
1748
            or (_mod_revision.NULL_REVISION in revision_ids)):
1749
            raise ValueError('cannot get null revision inventory')
6282.6.8 by Jelmer Vernooij
Adjust _iter_inventories contract slightly, don't raise NoSuchRevision immediately
1750
        for inv, revid in self._iter_inventories(revision_ids, ordering):
1751
            if inv is None:
6282.6.9 by Jelmer Vernooij
More tests.
1752
                raise errors.NoSuchRevision(self, revid)
6282.6.8 by Jelmer Vernooij
Adjust _iter_inventories contract slightly, don't raise NoSuchRevision immediately
1753
            yield inv
5815.4.1 by Jelmer Vernooij
Split versionedfile-specific stuff out into VersionedFileRepository.
1754
1755
    def _iter_inventories(self, revision_ids, ordering):
1756
        """single-document based inventory iteration."""
1757
        inv_xmls = self._iter_inventory_xmls(revision_ids, ordering)
1758
        for text, revision_id in inv_xmls:
6282.6.8 by Jelmer Vernooij
Adjust _iter_inventories contract slightly, don't raise NoSuchRevision immediately
1759
            if text is None:
1760
                yield None, revision_id
1761
            else:
1762
                yield self._deserialise_inventory(revision_id, text), revision_id
5815.4.1 by Jelmer Vernooij
Split versionedfile-specific stuff out into VersionedFileRepository.
1763
1764
    def _iter_inventory_xmls(self, revision_ids, ordering):
1765
        if ordering is None:
1766
            order_as_requested = True
1767
            ordering = 'unordered'
1768
        else:
1769
            order_as_requested = False
1770
        keys = [(revision_id,) for revision_id in revision_ids]
1771
        if not keys:
1772
            return
1773
        if order_as_requested:
1774
            key_iter = iter(keys)
1775
            next_key = key_iter.next()
1776
        stream = self.inventories.get_record_stream(keys, ordering, True)
1777
        text_chunks = {}
1778
        for record in stream:
1779
            if record.storage_kind != 'absent':
1780
                chunks = record.get_bytes_as('chunked')
1781
                if order_as_requested:
1782
                    text_chunks[record.key] = chunks
1783
                else:
1784
                    yield ''.join(chunks), record.key[-1]
1785
            else:
6282.6.8 by Jelmer Vernooij
Adjust _iter_inventories contract slightly, don't raise NoSuchRevision immediately
1786
                yield None, record.key[-1]
5815.4.1 by Jelmer Vernooij
Split versionedfile-specific stuff out into VersionedFileRepository.
1787
            if order_as_requested:
1788
                # Yield as many results as we can while preserving order.
1789
                while next_key in text_chunks:
1790
                    chunks = text_chunks.pop(next_key)
1791
                    yield ''.join(chunks), next_key[-1]
1792
                    try:
1793
                        next_key = key_iter.next()
1794
                    except StopIteration:
1795
                        # We still want to fully consume the get_record_stream,
1796
                        # just in case it is not actually finished at this point
1797
                        next_key = None
1798
                        break
1799
1800
    def _deserialise_inventory(self, revision_id, xml):
1801
        """Transform the xml into an inventory object.
1802
1803
        :param revision_id: The expected revision id of the inventory.
1804
        :param xml: A serialised inventory.
1805
        """
1806
        result = self._serializer.read_inventory_from_string(xml, revision_id,
1807
                    entry_cache=self._inventory_entry_cache,
1808
                    return_from_cache=self._safe_to_return_from_cache)
1809
        if result.revision_id != revision_id:
1810
            raise AssertionError('revision id mismatch %s != %s' % (
1811
                result.revision_id, revision_id))
1812
        return result
1813
1814
    def get_serializer_format(self):
1815
        return self._serializer.format_num
1816
1817
    @needs_read_lock
1818
    def _get_inventory_xml(self, revision_id):
1819
        """Get serialized inventory as a string."""
1820
        texts = self._iter_inventory_xmls([revision_id], 'unordered')
6282.6.8 by Jelmer Vernooij
Adjust _iter_inventories contract slightly, don't raise NoSuchRevision immediately
1821
        text, revision_id = texts.next()
1822
        if text is None:
6282.6.19 by Jelmer Vernooij
More test fixes.
1823
            raise errors.NoSuchRevision(self, revision_id)
5815.4.1 by Jelmer Vernooij
Split versionedfile-specific stuff out into VersionedFileRepository.
1824
        return text
1825
1826
    @needs_read_lock
1827
    def revision_tree(self, revision_id):
1828
        """Return Tree for a revision on this branch.
1829
1830
        `revision_id` may be NULL_REVISION for the empty tree revision.
1831
        """
1832
        revision_id = _mod_revision.ensure_null(revision_id)
1833
        # TODO: refactor this to use an existing revision object
1834
        # so we don't need to read it in twice.
1835
        if revision_id == _mod_revision.NULL_REVISION:
1836
            return InventoryRevisionTree(self,
1837
                Inventory(root_id=None), _mod_revision.NULL_REVISION)
1838
        else:
1839
            inv = self.get_inventory(revision_id)
1840
            return InventoryRevisionTree(self, inv, revision_id)
1841
1842
    def revision_trees(self, revision_ids):
1843
        """Return Trees for revisions in this repository.
1844
1845
        :param revision_ids: a sequence of revision-ids;
1846
          a revision-id may not be None or 'null:'
1847
        """
1848
        inventories = self.iter_inventories(revision_ids)
1849
        for inv in inventories:
1850
            yield InventoryRevisionTree(self, inv, inv.revision_id)
1851
1852
    def _filtered_revision_trees(self, revision_ids, file_ids):
1853
        """Return Tree for a revision on this branch with only some files.
1854
1855
        :param revision_ids: a sequence of revision-ids;
1856
          a revision-id may not be None or 'null:'
1857
        :param file_ids: if not None, the result is filtered
1858
          so that only those file-ids, their parents and their
1859
          children are included.
1860
        """
1861
        inventories = self.iter_inventories(revision_ids)
1862
        for inv in inventories:
1863
            # Should we introduce a FilteredRevisionTree class rather
1864
            # than pre-filter the inventory here?
1865
            filtered_inv = inv.filter(file_ids)
1866
            yield InventoryRevisionTree(self, filtered_inv, filtered_inv.revision_id)
1867
1868
    def get_parent_map(self, revision_ids):
1869
        """See graph.StackedParentsProvider.get_parent_map"""
1870
        # revisions index works in keys; this just works in revisions
1871
        # therefore wrap and unwrap
1872
        query_keys = []
1873
        result = {}
1874
        for revision_id in revision_ids:
1875
            if revision_id == _mod_revision.NULL_REVISION:
1876
                result[revision_id] = ()
1877
            elif revision_id is None:
1878
                raise ValueError('get_parent_map(None) is not valid')
1879
            else:
1880
                query_keys.append((revision_id ,))
1881
        for ((revision_id,), parent_keys) in \
1882
                self.revisions.get_parent_map(query_keys).iteritems():
1883
            if parent_keys:
1884
                result[revision_id] = tuple([parent_revid
1885
                    for (parent_revid,) in parent_keys])
1886
            else:
1887
                result[revision_id] = (_mod_revision.NULL_REVISION,)
1888
        return result
1889
1890
    @needs_read_lock
1891
    def get_known_graph_ancestry(self, revision_ids):
1892
        """Return the known graph for a set of revision ids and their ancestors.
1893
        """
1894
        st = static_tuple.StaticTuple
1895
        revision_keys = [st(r_id).intern() for r_id in revision_ids]
1896
        known_graph = self.revisions.get_known_graph_ancestry(revision_keys)
1897
        return graph.GraphThunkIdsToKeys(known_graph)
1898
5815.5.5 by Jelmer Vernooij
Add more tests.
1899
    @needs_read_lock
5815.5.8 by Jelmer Vernooij
Use traditional (fileid, revision) entries in file graph.
1900
    def get_file_graph(self):
1901
        """Return the graph walker for text revisions."""
1902
        return graph.Graph(self.texts)
5815.5.5 by Jelmer Vernooij
Add more tests.
1903
6341.1.4 by Jelmer Vernooij
Move more functionality to vf_search.
1904
    def revision_ids_to_search_result(self, result_set):
1905
        """Convert a set of revision ids to a graph SearchResult."""
1906
        result_parents = set()
1907
        for parents in self.get_graph().get_parent_map(
1908
            result_set).itervalues():
1909
            result_parents.update(parents)
1910
        included_keys = result_set.intersection(result_parents)
1911
        start_keys = result_set.difference(included_keys)
1912
        exclude_keys = result_parents.difference(result_set)
1913
        result = vf_search.SearchResult(start_keys, exclude_keys,
1914
            len(result_set), result_set)
1915
        return result
1916
5815.4.1 by Jelmer Vernooij
Split versionedfile-specific stuff out into VersionedFileRepository.
1917
    def _get_versioned_file_checker(self, text_key_references=None,
1918
        ancestors=None):
1919
        """Return an object suitable for checking versioned files.
1920
        
1921
        :param text_key_references: if non-None, an already built
1922
            dictionary mapping text keys ((fileid, revision_id) tuples)
1923
            to whether they were referred to by the inventory of the
1924
            revision_id that they contain. If None, this will be
1925
            calculated.
1926
        :param ancestors: Optional result from
1927
            self.get_graph().get_parent_map(self.all_revision_ids()) if already
1928
            available.
1929
        """
1930
        return _VersionedFileChecker(self,
1931
            text_key_references=text_key_references, ancestors=ancestors)
1932
1933
    @needs_read_lock
1934
    def has_signature_for_revision_id(self, revision_id):
1935
        """Query for a revision signature for revision_id in the repository."""
1936
        if not self.has_revision(revision_id):
1937
            raise errors.NoSuchRevision(self, revision_id)
1938
        sig_present = (1 == len(
1939
            self.signatures.get_parent_map([(revision_id,)])))
1940
        return sig_present
1941
1942
    @needs_read_lock
1943
    def get_signature_text(self, revision_id):
1944
        """Return the text for a signature."""
1945
        stream = self.signatures.get_record_stream([(revision_id,)],
1946
            'unordered', True)
1947
        record = stream.next()
1948
        if record.storage_kind == 'absent':
1949
            raise errors.NoSuchRevision(self, revision_id)
1950
        return record.get_bytes_as('fulltext')
1951
5850.1.6 by Jelmer Vernooij
Reintroduce double indirection, raise NotImplementedError from _check rather than
1952
    @needs_read_lock
1953
    def _check(self, revision_ids, callback_refs, check_repo):
5850.1.3 by Jelmer Vernooij
Add VersionedFileCheck.
1954
        result = check.VersionedFileCheck(self, check_repo=check_repo)
1955
        result.check(callback_refs)
1956
        return result
1957
5815.4.1 by Jelmer Vernooij
Split versionedfile-specific stuff out into VersionedFileRepository.
1958
    def _find_inconsistent_revision_parents(self, revisions_iterator=None):
1959
        """Find revisions with different parent lists in the revision object
1960
        and in the index graph.
1961
1962
        :param revisions_iterator: None, or an iterator of (revid,
1963
            Revision-or-None). This iterator controls the revisions checked.
1964
        :returns: an iterator yielding tuples of (revison-id, parents-in-index,
1965
            parents-in-revision).
1966
        """
1967
        if not self.is_locked():
1968
            raise AssertionError()
1969
        vf = self.revisions
1970
        if revisions_iterator is None:
1971
            revisions_iterator = self._iter_revisions(None)
1972
        for revid, revision in revisions_iterator:
1973
            if revision is None:
1974
                pass
1975
            parent_map = vf.get_parent_map([(revid,)])
1976
            parents_according_to_index = tuple(parent[-1] for parent in
1977
                parent_map[(revid,)])
1978
            parents_according_to_revision = tuple(revision.parent_ids)
1979
            if parents_according_to_index != parents_according_to_revision:
1980
                yield (revid, parents_according_to_index,
1981
                    parents_according_to_revision)
1982
1983
    def _check_for_inconsistent_revision_parents(self):
1984
        inconsistencies = list(self._find_inconsistent_revision_parents())
1985
        if inconsistencies:
1986
            raise errors.BzrCheckError(
1987
                "Revision knit has inconsistent parents.")
1988
1989
    def _get_sink(self):
1990
        """Return a sink for streaming into this repository."""
1991
        return StreamSink(self)
1992
1993
    def _get_source(self, to_format):
1994
        """Return a source for streaming from this repository."""
1995
        return StreamSource(self, to_format)
1996
1997
1998
class MetaDirVersionedFileRepository(MetaDirRepository,
1999
                                     VersionedFileRepository):
2000
    """Repositories in a meta-dir, that work via versioned file objects."""
2001
2002
    def __init__(self, _format, a_bzrdir, control_files):
2003
        super(MetaDirVersionedFileRepository, self).__init__(_format, a_bzrdir,
2004
            control_files)
2005
2006
6349.2.1 by Jelmer Vernooij
Add BzrDirMetaComponentFormat.
2007
class MetaDirVersionedFileRepositoryFormat(RepositoryFormatMetaDir,
5815.4.5 by Jelmer Vernooij
Use MetaDirVersionedFileRepositoryFormat (a Soyuz worthy name).
2008
        VersionedFileRepositoryFormat):
2009
    """Base class for repository formats using versioned files in metadirs."""
2010
2011
5815.4.1 by Jelmer Vernooij
Split versionedfile-specific stuff out into VersionedFileRepository.
2012
class StreamSink(object):
2013
    """An object that can insert a stream into a repository.
2014
2015
    This interface handles the complexity of reserialising inventories and
2016
    revisions from different formats, and allows unidirectional insertion into
2017
    stacked repositories without looking for the missing basis parents
2018
    beforehand.
2019
    """
2020
2021
    def __init__(self, target_repo):
2022
        self.target_repo = target_repo
2023
2024
    def insert_stream(self, stream, src_format, resume_tokens):
2025
        """Insert a stream's content into the target repository.
2026
2027
        :param src_format: a bzr repository format.
2028
2029
        :return: a list of resume tokens and an  iterable of keys additional
2030
            items required before the insertion can be completed.
2031
        """
2032
        self.target_repo.lock_write()
2033
        try:
2034
            if resume_tokens:
2035
                self.target_repo.resume_write_group(resume_tokens)
2036
                is_resume = True
2037
            else:
2038
                self.target_repo.start_write_group()
2039
                is_resume = False
2040
            try:
2041
                # locked_insert_stream performs a commit|suspend.
2042
                missing_keys = self.insert_stream_without_locking(stream,
2043
                                    src_format, is_resume)
2044
                if missing_keys:
2045
                    # suspend the write group and tell the caller what we is
2046
                    # missing. We know we can suspend or else we would not have
2047
                    # entered this code path. (All repositories that can handle
2048
                    # missing keys can handle suspending a write group).
2049
                    write_group_tokens = self.target_repo.suspend_write_group()
2050
                    return write_group_tokens, missing_keys
2051
                hint = self.target_repo.commit_write_group()
2052
                to_serializer = self.target_repo._format._serializer
2053
                src_serializer = src_format._serializer
2054
                if (to_serializer != src_serializer and
2055
                    self.target_repo._format.pack_compresses):
2056
                    self.target_repo.pack(hint=hint)
2057
                return [], set()
2058
            except:
2059
                self.target_repo.abort_write_group(suppress_errors=True)
2060
                raise
2061
        finally:
2062
            self.target_repo.unlock()
2063
2064
    def insert_stream_without_locking(self, stream, src_format,
2065
                                      is_resume=False):
2066
        """Insert a stream's content into the target repository.
2067
2068
        This assumes that you already have a locked repository and an active
2069
        write group.
2070
2071
        :param src_format: a bzr repository format.
2072
        :param is_resume: Passed down to get_missing_parent_inventories to
2073
            indicate if we should be checking for missing texts at the same
2074
            time.
2075
2076
        :return: A set of keys that are missing.
2077
        """
2078
        if not self.target_repo.is_write_locked():
2079
            raise errors.ObjectNotLocked(self)
2080
        if not self.target_repo.is_in_write_group():
2081
            raise errors.BzrError('you must already be in a write group')
2082
        to_serializer = self.target_repo._format._serializer
2083
        src_serializer = src_format._serializer
2084
        new_pack = None
2085
        if to_serializer == src_serializer:
2086
            # If serializers match and the target is a pack repository, set the
2087
            # write cache size on the new pack.  This avoids poor performance
2088
            # on transports where append is unbuffered (such as
2089
            # RemoteTransport).  This is safe to do because nothing should read
2090
            # back from the target repository while a stream with matching
2091
            # serialization is being inserted.
2092
            # The exception is that a delta record from the source that should
2093
            # be a fulltext may need to be expanded by the target (see
2094
            # test_fetch_revisions_with_deltas_into_pack); but we take care to
2095
            # explicitly flush any buffered writes first in that rare case.
2096
            try:
2097
                new_pack = self.target_repo._pack_collection._new_pack
2098
            except AttributeError:
2099
                # Not a pack repository
2100
                pass
2101
            else:
2102
                new_pack.set_write_cache_size(1024*1024)
2103
        for substream_type, substream in stream:
2104
            if 'stream' in debug.debug_flags:
2105
                mutter('inserting substream: %s', substream_type)
2106
            if substream_type == 'texts':
2107
                self.target_repo.texts.insert_record_stream(substream)
2108
            elif substream_type == 'inventories':
2109
                if src_serializer == to_serializer:
2110
                    self.target_repo.inventories.insert_record_stream(
2111
                        substream)
2112
                else:
2113
                    self._extract_and_insert_inventories(
2114
                        substream, src_serializer)
2115
            elif substream_type == 'inventory-deltas':
2116
                self._extract_and_insert_inventory_deltas(
2117
                    substream, src_serializer)
2118
            elif substream_type == 'chk_bytes':
2119
                # XXX: This doesn't support conversions, as it assumes the
2120
                #      conversion was done in the fetch code.
2121
                self.target_repo.chk_bytes.insert_record_stream(substream)
2122
            elif substream_type == 'revisions':
2123
                # This may fallback to extract-and-insert more often than
2124
                # required if the serializers are different only in terms of
2125
                # the inventory.
2126
                if src_serializer == to_serializer:
2127
                    self.target_repo.revisions.insert_record_stream(substream)
2128
                else:
2129
                    self._extract_and_insert_revisions(substream,
2130
                        src_serializer)
2131
            elif substream_type == 'signatures':
2132
                self.target_repo.signatures.insert_record_stream(substream)
2133
            else:
2134
                raise AssertionError('kaboom! %s' % (substream_type,))
2135
        # Done inserting data, and the missing_keys calculations will try to
2136
        # read back from the inserted data, so flush the writes to the new pack
2137
        # (if this is pack format).
2138
        if new_pack is not None:
2139
            new_pack._write_data('', flush=True)
2140
        # Find all the new revisions (including ones from resume_tokens)
2141
        missing_keys = self.target_repo.get_missing_parent_inventories(
2142
            check_for_missing_texts=is_resume)
2143
        try:
2144
            for prefix, versioned_file in (
2145
                ('texts', self.target_repo.texts),
2146
                ('inventories', self.target_repo.inventories),
2147
                ('revisions', self.target_repo.revisions),
2148
                ('signatures', self.target_repo.signatures),
2149
                ('chk_bytes', self.target_repo.chk_bytes),
2150
                ):
2151
                if versioned_file is None:
2152
                    continue
2153
                # TODO: key is often going to be a StaticTuple object
2154
                #       I don't believe we can define a method by which
2155
                #       (prefix,) + StaticTuple will work, though we could
2156
                #       define a StaticTuple.sq_concat that would allow you to
2157
                #       pass in either a tuple or a StaticTuple as the second
2158
                #       object, so instead we could have:
2159
                #       StaticTuple(prefix) + key here...
2160
                missing_keys.update((prefix,) + key for key in
2161
                    versioned_file.get_missing_compression_parent_keys())
2162
        except NotImplementedError:
2163
            # cannot even attempt suspending, and missing would have failed
2164
            # during stream insertion.
2165
            missing_keys = set()
2166
        return missing_keys
2167
2168
    def _extract_and_insert_inventory_deltas(self, substream, serializer):
2169
        target_rich_root = self.target_repo._format.rich_root_data
2170
        target_tree_refs = self.target_repo._format.supports_tree_reference
2171
        for record in substream:
2172
            # Insert the delta directly
2173
            inventory_delta_bytes = record.get_bytes_as('fulltext')
2174
            deserialiser = inventory_delta.InventoryDeltaDeserializer()
2175
            try:
2176
                parse_result = deserialiser.parse_text_bytes(
2177
                    inventory_delta_bytes)
2178
            except inventory_delta.IncompatibleInventoryDelta, err:
2179
                mutter("Incompatible delta: %s", err.msg)
2180
                raise errors.IncompatibleRevision(self.target_repo._format)
2181
            basis_id, new_id, rich_root, tree_refs, inv_delta = parse_result
2182
            revision_id = new_id
2183
            parents = [key[0] for key in record.parents]
2184
            self.target_repo.add_inventory_by_delta(
2185
                basis_id, inv_delta, revision_id, parents)
2186
2187
    def _extract_and_insert_inventories(self, substream, serializer,
2188
            parse_delta=None):
2189
        """Generate a new inventory versionedfile in target, converting data.
2190
2191
        The inventory is retrieved from the source, (deserializing it), and
2192
        stored in the target (reserializing it in a different format).
2193
        """
2194
        target_rich_root = self.target_repo._format.rich_root_data
2195
        target_tree_refs = self.target_repo._format.supports_tree_reference
2196
        for record in substream:
2197
            # It's not a delta, so it must be a fulltext in the source
2198
            # serializer's format.
2199
            bytes = record.get_bytes_as('fulltext')
2200
            revision_id = record.key[0]
2201
            inv = serializer.read_inventory_from_string(bytes, revision_id)
2202
            parents = [key[0] for key in record.parents]
2203
            self.target_repo.add_inventory(revision_id, inv, parents)
2204
            # No need to keep holding this full inv in memory when the rest of
2205
            # the substream is likely to be all deltas.
2206
            del inv
2207
2208
    def _extract_and_insert_revisions(self, substream, serializer):
2209
        for record in substream:
2210
            bytes = record.get_bytes_as('fulltext')
2211
            revision_id = record.key[0]
2212
            rev = serializer.read_revision_from_string(bytes)
2213
            if rev.revision_id != revision_id:
2214
                raise AssertionError('wtf: %s != %s' % (rev, revision_id))
2215
            self.target_repo.add_revision(revision_id, rev)
2216
2217
    def finished(self):
2218
        if self.target_repo._format._fetch_reconcile:
2219
            self.target_repo.reconcile()
2220
2221
2222
class StreamSource(object):
2223
    """A source of a stream for fetching between repositories."""
2224
2225
    def __init__(self, from_repository, to_format):
2226
        """Create a StreamSource streaming from from_repository."""
2227
        self.from_repository = from_repository
2228
        self.to_format = to_format
2229
        self._record_counter = RecordCounter()
2230
2231
    def delta_on_metadata(self):
2232
        """Return True if delta's are permitted on metadata streams.
2233
2234
        That is on revisions and signatures.
2235
        """
2236
        src_serializer = self.from_repository._format._serializer
2237
        target_serializer = self.to_format._serializer
2238
        return (self.to_format._fetch_uses_deltas and
2239
            src_serializer == target_serializer)
2240
2241
    def _fetch_revision_texts(self, revs):
2242
        # fetch signatures first and then the revision texts
2243
        # may need to be a InterRevisionStore call here.
2244
        from_sf = self.from_repository.signatures
2245
        # A missing signature is just skipped.
2246
        keys = [(rev_id,) for rev_id in revs]
2247
        signatures = versionedfile.filter_absent(from_sf.get_record_stream(
2248
            keys,
2249
            self.to_format._fetch_order,
2250
            not self.to_format._fetch_uses_deltas))
2251
        # If a revision has a delta, this is actually expanded inside the
2252
        # insert_record_stream code now, which is an alternate fix for
2253
        # bug #261339
2254
        from_rf = self.from_repository.revisions
2255
        revisions = from_rf.get_record_stream(
2256
            keys,
2257
            self.to_format._fetch_order,
2258
            not self.delta_on_metadata())
2259
        return [('signatures', signatures), ('revisions', revisions)]
2260
2261
    def _generate_root_texts(self, revs):
2262
        """This will be called by get_stream between fetching weave texts and
2263
        fetching the inventory weave.
2264
        """
2265
        if self._rich_root_upgrade():
2266
            return _mod_fetch.Inter1and2Helper(
2267
                self.from_repository).generate_root_texts(revs)
2268
        else:
2269
            return []
2270
2271
    def get_stream(self, search):
2272
        phase = 'file'
2273
        revs = search.get_keys()
2274
        graph = self.from_repository.get_graph()
2275
        revs = tsort.topo_sort(graph.get_parent_map(revs))
2276
        data_to_fetch = self.from_repository.item_keys_introduced_by(revs)
2277
        text_keys = []
2278
        for knit_kind, file_id, revisions in data_to_fetch:
2279
            if knit_kind != phase:
2280
                phase = knit_kind
2281
                # Make a new progress bar for this phase
2282
            if knit_kind == "file":
2283
                # Accumulate file texts
2284
                text_keys.extend([(file_id, revision) for revision in
2285
                    revisions])
2286
            elif knit_kind == "inventory":
2287
                # Now copy the file texts.
2288
                from_texts = self.from_repository.texts
2289
                yield ('texts', from_texts.get_record_stream(
2290
                    text_keys, self.to_format._fetch_order,
2291
                    not self.to_format._fetch_uses_deltas))
2292
                # Cause an error if a text occurs after we have done the
2293
                # copy.
2294
                text_keys = None
2295
                # Before we process the inventory we generate the root
2296
                # texts (if necessary) so that the inventories references
2297
                # will be valid.
2298
                for _ in self._generate_root_texts(revs):
2299
                    yield _
2300
                # we fetch only the referenced inventories because we do not
2301
                # know for unselected inventories whether all their required
2302
                # texts are present in the other repository - it could be
2303
                # corrupt.
2304
                for info in self._get_inventory_stream(revs):
2305
                    yield info
2306
            elif knit_kind == "signatures":
2307
                # Nothing to do here; this will be taken care of when
2308
                # _fetch_revision_texts happens.
2309
                pass
2310
            elif knit_kind == "revisions":
2311
                for record in self._fetch_revision_texts(revs):
2312
                    yield record
2313
            else:
2314
                raise AssertionError("Unknown knit kind %r" % knit_kind)
2315
2316
    def get_stream_for_missing_keys(self, missing_keys):
2317
        # missing keys can only occur when we are byte copying and not
2318
        # translating (because translation means we don't send
2319
        # unreconstructable deltas ever).
2320
        keys = {}
2321
        keys['texts'] = set()
2322
        keys['revisions'] = set()
2323
        keys['inventories'] = set()
2324
        keys['chk_bytes'] = set()
2325
        keys['signatures'] = set()
2326
        for key in missing_keys:
2327
            keys[key[0]].add(key[1:])
2328
        if len(keys['revisions']):
2329
            # If we allowed copying revisions at this point, we could end up
2330
            # copying a revision without copying its required texts: a
2331
            # violation of the requirements for repository integrity.
2332
            raise AssertionError(
2333
                'cannot copy revisions to fill in missing deltas %s' % (
2334
                    keys['revisions'],))
2335
        for substream_kind, keys in keys.iteritems():
2336
            vf = getattr(self.from_repository, substream_kind)
2337
            if vf is None and keys:
2338
                    raise AssertionError(
2339
                        "cannot fill in keys for a versioned file we don't"
2340
                        " have: %s needs %s" % (substream_kind, keys))
2341
            if not keys:
2342
                # No need to stream something we don't have
2343
                continue
2344
            if substream_kind == 'inventories':
2345
                # Some missing keys are genuinely ghosts, filter those out.
2346
                present = self.from_repository.inventories.get_parent_map(keys)
2347
                revs = [key[0] for key in present]
2348
                # Get the inventory stream more-or-less as we do for the
2349
                # original stream; there's no reason to assume that records
2350
                # direct from the source will be suitable for the sink.  (Think
2351
                # e.g. 2a -> 1.9-rich-root).
2352
                for info in self._get_inventory_stream(revs, missing=True):
2353
                    yield info
2354
                continue
2355
2356
            # Ask for full texts always so that we don't need more round trips
2357
            # after this stream.
2358
            # Some of the missing keys are genuinely ghosts, so filter absent
2359
            # records. The Sink is responsible for doing another check to
2360
            # ensure that ghosts don't introduce missing data for future
2361
            # fetches.
2362
            stream = versionedfile.filter_absent(vf.get_record_stream(keys,
2363
                self.to_format._fetch_order, True))
2364
            yield substream_kind, stream
2365
2366
    def inventory_fetch_order(self):
2367
        if self._rich_root_upgrade():
2368
            return 'topological'
2369
        else:
2370
            return self.to_format._fetch_order
2371
2372
    def _rich_root_upgrade(self):
2373
        return (not self.from_repository._format.rich_root_data and
2374
            self.to_format.rich_root_data)
2375
2376
    def _get_inventory_stream(self, revision_ids, missing=False):
2377
        from_format = self.from_repository._format
2378
        if (from_format.supports_chks and self.to_format.supports_chks and
2379
            from_format.network_name() == self.to_format.network_name()):
2380
            raise AssertionError(
2381
                "this case should be handled by GroupCHKStreamSource")
2382
        elif 'forceinvdeltas' in debug.debug_flags:
2383
            return self._get_convertable_inventory_stream(revision_ids,
2384
                    delta_versus_null=missing)
2385
        elif from_format.network_name() == self.to_format.network_name():
2386
            # Same format.
2387
            return self._get_simple_inventory_stream(revision_ids,
2388
                    missing=missing)
2389
        elif (not from_format.supports_chks and not self.to_format.supports_chks
2390
                and from_format._serializer == self.to_format._serializer):
2391
            # Essentially the same format.
2392
            return self._get_simple_inventory_stream(revision_ids,
2393
                    missing=missing)
2394
        else:
2395
            # Any time we switch serializations, we want to use an
2396
            # inventory-delta based approach.
2397
            return self._get_convertable_inventory_stream(revision_ids,
2398
                    delta_versus_null=missing)
2399
2400
    def _get_simple_inventory_stream(self, revision_ids, missing=False):
2401
        # NB: This currently reopens the inventory weave in source;
2402
        # using a single stream interface instead would avoid this.
2403
        from_weave = self.from_repository.inventories
2404
        if missing:
2405
            delta_closure = True
2406
        else:
2407
            delta_closure = not self.delta_on_metadata()
2408
        yield ('inventories', from_weave.get_record_stream(
2409
            [(rev_id,) for rev_id in revision_ids],
2410
            self.inventory_fetch_order(), delta_closure))
2411
2412
    def _get_convertable_inventory_stream(self, revision_ids,
2413
                                          delta_versus_null=False):
2414
        # The two formats are sufficiently different that there is no fast
2415
        # path, so we need to send just inventorydeltas, which any
2416
        # sufficiently modern client can insert into any repository.
2417
        # The StreamSink code expects to be able to
2418
        # convert on the target, so we need to put bytes-on-the-wire that can
2419
        # be converted.  That means inventory deltas (if the remote is <1.19,
2420
        # RemoteStreamSink will fallback to VFS to insert the deltas).
2421
        yield ('inventory-deltas',
2422
           self._stream_invs_as_deltas(revision_ids,
2423
                                       delta_versus_null=delta_versus_null))
2424
2425
    def _stream_invs_as_deltas(self, revision_ids, delta_versus_null=False):
2426
        """Return a stream of inventory-deltas for the given rev ids.
2427
2428
        :param revision_ids: The list of inventories to transmit
2429
        :param delta_versus_null: Don't try to find a minimal delta for this
2430
            entry, instead compute the delta versus the NULL_REVISION. This
2431
            effectively streams a complete inventory. Used for stuff like
2432
            filling in missing parents, etc.
2433
        """
2434
        from_repo = self.from_repository
2435
        revision_keys = [(rev_id,) for rev_id in revision_ids]
2436
        parent_map = from_repo.inventories.get_parent_map(revision_keys)
2437
        # XXX: possibly repos could implement a more efficient iter_inv_deltas
2438
        # method...
2439
        inventories = self.from_repository.iter_inventories(
2440
            revision_ids, 'topological')
2441
        format = from_repo._format
2442
        invs_sent_so_far = set([_mod_revision.NULL_REVISION])
2443
        inventory_cache = lru_cache.LRUCache(50)
2444
        null_inventory = from_repo.revision_tree(
6405.2.6 by Jelmer Vernooij
Lots of test fixes.
2445
            _mod_revision.NULL_REVISION).root_inventory
5815.4.1 by Jelmer Vernooij
Split versionedfile-specific stuff out into VersionedFileRepository.
2446
        # XXX: ideally the rich-root/tree-refs flags would be per-revision, not
2447
        # per-repo (e.g.  streaming a non-rich-root revision out of a rich-root
2448
        # repo back into a non-rich-root repo ought to be allowed)
2449
        serializer = inventory_delta.InventoryDeltaSerializer(
2450
            versioned_root=format.rich_root_data,
2451
            tree_references=format.supports_tree_reference)
2452
        for inv in inventories:
2453
            key = (inv.revision_id,)
2454
            parent_keys = parent_map.get(key, ())
2455
            delta = None
2456
            if not delta_versus_null and parent_keys:
2457
                # The caller did not ask for complete inventories and we have
2458
                # some parents that we can delta against.  Make a delta against
2459
                # each parent so that we can find the smallest.
2460
                parent_ids = [parent_key[0] for parent_key in parent_keys]
2461
                for parent_id in parent_ids:
2462
                    if parent_id not in invs_sent_so_far:
2463
                        # We don't know that the remote side has this basis, so
2464
                        # we can't use it.
2465
                        continue
2466
                    if parent_id == _mod_revision.NULL_REVISION:
2467
                        parent_inv = null_inventory
2468
                    else:
2469
                        parent_inv = inventory_cache.get(parent_id, None)
2470
                        if parent_inv is None:
2471
                            parent_inv = from_repo.get_inventory(parent_id)
2472
                    candidate_delta = inv._make_delta(parent_inv)
2473
                    if (delta is None or
2474
                        len(delta) > len(candidate_delta)):
2475
                        delta = candidate_delta
2476
                        basis_id = parent_id
2477
            if delta is None:
2478
                # Either none of the parents ended up being suitable, or we
2479
                # were asked to delta against NULL
2480
                basis_id = _mod_revision.NULL_REVISION
2481
                delta = inv._make_delta(null_inventory)
2482
            invs_sent_so_far.add(inv.revision_id)
2483
            inventory_cache[inv.revision_id] = inv
2484
            delta_serialized = ''.join(
2485
                serializer.delta_to_lines(basis_id, key[-1], delta))
2486
            yield versionedfile.FulltextContentFactory(
2487
                key, parent_keys, None, delta_serialized)
2488
2489
2490
class _VersionedFileChecker(object):
2491
2492
    def __init__(self, repository, text_key_references=None, ancestors=None):
2493
        self.repository = repository
2494
        self.text_index = self.repository._generate_text_key_index(
2495
            text_key_references=text_key_references, ancestors=ancestors)
2496
2497
    def calculate_file_version_parents(self, text_key):
2498
        """Calculate the correct parents for a file version according to
2499
        the inventories.
2500
        """
2501
        parent_keys = self.text_index[text_key]
2502
        if parent_keys == [_mod_revision.NULL_REVISION]:
2503
            return ()
2504
        return tuple(parent_keys)
2505
2506
    def check_file_version_parents(self, texts, progress_bar=None):
2507
        """Check the parents stored in a versioned file are correct.
2508
2509
        It also detects file versions that are not referenced by their
2510
        corresponding revision's inventory.
2511
2512
        :returns: A tuple of (wrong_parents, dangling_file_versions).
2513
            wrong_parents is a dict mapping {revision_id: (stored_parents,
2514
            correct_parents)} for each revision_id where the stored parents
2515
            are not correct.  dangling_file_versions is a set of (file_id,
2516
            revision_id) tuples for versions that are present in this versioned
2517
            file, but not used by the corresponding inventory.
2518
        """
2519
        local_progress = None
2520
        if progress_bar is None:
2521
            local_progress = ui.ui_factory.nested_progress_bar()
2522
            progress_bar = local_progress
2523
        try:
2524
            return self._check_file_version_parents(texts, progress_bar)
2525
        finally:
2526
            if local_progress:
2527
                local_progress.finished()
2528
2529
    def _check_file_version_parents(self, texts, progress_bar):
2530
        """See check_file_version_parents."""
2531
        wrong_parents = {}
2532
        self.file_ids = set([file_id for file_id, _ in
2533
            self.text_index.iterkeys()])
2534
        # text keys is now grouped by file_id
2535
        n_versions = len(self.text_index)
6138.4.1 by Jonathan Riddell
add gettext to progress bar strings
2536
        progress_bar.update(gettext('loading text store'), 0, n_versions)
5815.4.1 by Jelmer Vernooij
Split versionedfile-specific stuff out into VersionedFileRepository.
2537
        parent_map = self.repository.texts.get_parent_map(self.text_index)
2538
        # On unlistable transports this could well be empty/error...
2539
        text_keys = self.repository.texts.keys()
2540
        unused_keys = frozenset(text_keys) - set(self.text_index)
2541
        for num, key in enumerate(self.text_index.iterkeys()):
6138.4.1 by Jonathan Riddell
add gettext to progress bar strings
2542
            progress_bar.update(gettext('checking text graph'), num, n_versions)
5815.4.1 by Jelmer Vernooij
Split versionedfile-specific stuff out into VersionedFileRepository.
2543
            correct_parents = self.calculate_file_version_parents(key)
2544
            try:
2545
                knit_parents = parent_map[key]
2546
            except errors.RevisionNotPresent:
2547
                # Missing text!
2548
                knit_parents = None
2549
            if correct_parents != knit_parents:
2550
                wrong_parents[key] = (knit_parents, correct_parents)
2551
        return wrong_parents, unused_keys
2552
2553
5863.4.1 by Jelmer Vernooij
Move interrepository implementation to vf_repository.
2554
class InterVersionedFileRepository(InterRepository):
2555
2556
    _walk_to_common_revisions_batch_size = 50
2557
6341.1.1 by Jelmer Vernooij
Make fetch_spec specific to bzr format repositories.
2558
    supports_fetch_spec = True
2559
5863.4.1 by Jelmer Vernooij
Move interrepository implementation to vf_repository.
2560
    @needs_write_lock
2561
    def fetch(self, revision_id=None, find_ghosts=False,
2562
            fetch_spec=None):
2563
        """Fetch the content required to construct revision_id.
2564
2565
        The content is copied from self.source to self.target.
2566
2567
        :param revision_id: if None all content is copied, if NULL_REVISION no
2568
                            content is copied.
2569
        :return: None.
2570
        """
6047.1.1 by Jelmer Vernooij
Use show_user_warning rather than custom warning methods on UIFactory.
2571
        if self.target._format.experimental:
2572
            ui.ui_factory.show_user_warning('experimental_format_fetch',
2573
                from_format=self.source._format,
2574
                to_format=self.target._format)
5863.4.1 by Jelmer Vernooij
Move interrepository implementation to vf_repository.
2575
        from bzrlib.fetch import RepoFetcher
2576
        # See <https://launchpad.net/bugs/456077> asking for a warning here
2577
        if self.source._format.network_name() != self.target._format.network_name():
2578
            ui.ui_factory.show_user_warning('cross_format_fetch',
2579
                from_format=self.source._format,
2580
                to_format=self.target._format)
2581
        f = RepoFetcher(to_repository=self.target,
2582
                               from_repository=self.source,
2583
                               last_revision=revision_id,
2584
                               fetch_spec=fetch_spec,
2585
                               find_ghosts=find_ghosts)
2586
2587
    def _walk_to_common_revisions(self, revision_ids, if_present_ids=None):
2588
        """Walk out from revision_ids in source to revisions target has.
2589
2590
        :param revision_ids: The start point for the search.
2591
        :return: A set of revision ids.
2592
        """
2593
        target_graph = self.target.get_graph()
2594
        revision_ids = frozenset(revision_ids)
2595
        if if_present_ids:
2596
            all_wanted_revs = revision_ids.union(if_present_ids)
2597
        else:
2598
            all_wanted_revs = revision_ids
2599
        missing_revs = set()
2600
        source_graph = self.source.get_graph()
2601
        # ensure we don't pay silly lookup costs.
2602
        searcher = source_graph._make_breadth_first_searcher(all_wanted_revs)
2603
        null_set = frozenset([_mod_revision.NULL_REVISION])
2604
        searcher_exhausted = False
2605
        while True:
2606
            next_revs = set()
2607
            ghosts = set()
2608
            # Iterate the searcher until we have enough next_revs
2609
            while len(next_revs) < self._walk_to_common_revisions_batch_size:
2610
                try:
2611
                    next_revs_part, ghosts_part = searcher.next_with_ghosts()
2612
                    next_revs.update(next_revs_part)
2613
                    ghosts.update(ghosts_part)
2614
                except StopIteration:
2615
                    searcher_exhausted = True
2616
                    break
2617
            # If there are ghosts in the source graph, and the caller asked for
2618
            # them, make sure that they are present in the target.
2619
            # We don't care about other ghosts as we can't fetch them and
2620
            # haven't been asked to.
2621
            ghosts_to_check = set(revision_ids.intersection(ghosts))
2622
            revs_to_get = set(next_revs).union(ghosts_to_check)
2623
            if revs_to_get:
2624
                have_revs = set(target_graph.get_parent_map(revs_to_get))
2625
                # we always have NULL_REVISION present.
2626
                have_revs = have_revs.union(null_set)
2627
                # Check if the target is missing any ghosts we need.
2628
                ghosts_to_check.difference_update(have_revs)
2629
                if ghosts_to_check:
2630
                    # One of the caller's revision_ids is a ghost in both the
2631
                    # source and the target.
2632
                    raise errors.NoSuchRevision(
2633
                        self.source, ghosts_to_check.pop())
2634
                missing_revs.update(next_revs - have_revs)
2635
                # Because we may have walked past the original stop point, make
2636
                # sure everything is stopped
2637
                stop_revs = searcher.find_seen_ancestors(have_revs)
2638
                searcher.stop_searching_any(stop_revs)
2639
            if searcher_exhausted:
2640
                break
6341.1.5 by Jelmer Vernooij
Fix get_state().
2641
        (started_keys, excludes, included_keys) = searcher.get_state()
2642
        return vf_search.SearchResult(started_keys, excludes,
2643
            len(included_keys), included_keys)
5863.4.1 by Jelmer Vernooij
Move interrepository implementation to vf_repository.
2644
2645
    @needs_read_lock
2646
    def search_missing_revision_ids(self,
2647
            revision_id=symbol_versioning.DEPRECATED_PARAMETER,
5852.1.7 by Jelmer Vernooij
merge bzr.dev.
2648
            find_ghosts=True, revision_ids=None, if_present_ids=None,
2649
            limit=None):
5863.4.1 by Jelmer Vernooij
Move interrepository implementation to vf_repository.
2650
        """Return the revision ids that source has that target does not.
2651
2652
        :param revision_id: only return revision ids included by this
2653
            revision_id.
2654
        :param revision_ids: return revision ids included by these
2655
            revision_ids.  NoSuchRevision will be raised if any of these
2656
            revisions are not present.
2657
        :param if_present_ids: like revision_ids, but will not cause
2658
            NoSuchRevision if any of these are absent, instead they will simply
2659
            not be in the result.  This is useful for e.g. finding revisions
2660
            to fetch for tags, which may reference absent revisions.
2661
        :param find_ghosts: If True find missing revisions in deep history
2662
            rather than just finding the surface difference.
2663
        :return: A bzrlib.graph.SearchResult.
2664
        """
2665
        if symbol_versioning.deprecated_passed(revision_id):
2666
            symbol_versioning.warn(
2667
                'search_missing_revision_ids(revision_id=...) was '
2668
                'deprecated in 2.4.  Use revision_ids=[...] instead.',
2669
                DeprecationWarning, stacklevel=2)
2670
            if revision_ids is not None:
2671
                raise AssertionError(
2672
                    'revision_ids is mutually exclusive with revision_id')
2673
            if revision_id is not None:
2674
                revision_ids = [revision_id]
2675
        del revision_id
2676
        # stop searching at found target revisions.
2677
        if not find_ghosts and (revision_ids is not None or if_present_ids is
2678
                not None):
5852.1.7 by Jelmer Vernooij
merge bzr.dev.
2679
            result = self._walk_to_common_revisions(revision_ids,
5863.4.1 by Jelmer Vernooij
Move interrepository implementation to vf_repository.
2680
                    if_present_ids=if_present_ids)
5852.1.8 by Jelmer Vernooij
Simplify revision limiting.
2681
            if limit is None:
2682
                return result
5852.1.7 by Jelmer Vernooij
merge bzr.dev.
2683
            result_set = result.get_keys()
2684
        else:
2685
            # generic, possibly worst case, slow code path.
2686
            target_ids = set(self.target.all_revision_ids())
2687
            source_ids = self._present_source_revisions_for(
2688
                revision_ids, if_present_ids)
2689
            result_set = set(source_ids).difference(target_ids)
2690
        if limit is not None:
5852.1.8 by Jelmer Vernooij
Simplify revision limiting.
2691
            topo_ordered = self.source.get_graph().iter_topo_order(result_set)
2692
            result_set = set(itertools.islice(topo_ordered, limit))
5863.4.1 by Jelmer Vernooij
Move interrepository implementation to vf_repository.
2693
        return self.source.revision_ids_to_search_result(result_set)
2694
2695
    def _present_source_revisions_for(self, revision_ids, if_present_ids=None):
2696
        """Returns set of all revisions in ancestry of revision_ids present in
2697
        the source repo.
2698
2699
        :param revision_ids: if None, all revisions in source are returned.
2700
        :param if_present_ids: like revision_ids, but if any/all of these are
2701
            absent no error is raised.
2702
        """
2703
        if revision_ids is not None or if_present_ids is not None:
2704
            # First, ensure all specified revisions exist.  Callers expect
2705
            # NoSuchRevision when they pass absent revision_ids here.
2706
            if revision_ids is None:
2707
                revision_ids = set()
2708
            if if_present_ids is None:
2709
                if_present_ids = set()
2710
            revision_ids = set(revision_ids)
2711
            if_present_ids = set(if_present_ids)
2712
            all_wanted_ids = revision_ids.union(if_present_ids)
2713
            graph = self.source.get_graph()
2714
            present_revs = set(graph.get_parent_map(all_wanted_ids))
2715
            missing = revision_ids.difference(present_revs)
2716
            if missing:
2717
                raise errors.NoSuchRevision(self.source, missing.pop())
2718
            found_ids = all_wanted_ids.intersection(present_revs)
2719
            source_ids = [rev_id for (rev_id, parents) in
2720
                          graph.iter_ancestry(found_ids)
2721
                          if rev_id != _mod_revision.NULL_REVISION
2722
                          and parents is not None]
2723
        else:
2724
            source_ids = self.source.all_revision_ids()
2725
        return set(source_ids)
2726
2727
    @classmethod
2728
    def _get_repo_format_to_test(self):
2729
        return None
2730
2731
    @classmethod
2732
    def is_compatible(cls, source, target):
2733
        # The default implementation is compatible with everything
2734
        return (source._format.supports_full_versioned_files and
2735
                target._format.supports_full_versioned_files)
2736
2737
2738
class InterDifferingSerializer(InterVersionedFileRepository):
5815.4.1 by Jelmer Vernooij
Split versionedfile-specific stuff out into VersionedFileRepository.
2739
2740
    @classmethod
2741
    def _get_repo_format_to_test(self):
2742
        return None
2743
2744
    @staticmethod
2745
    def is_compatible(source, target):
5815.4.19 by Jelmer Vernooij
Fix test failures.
2746
        if not source._format.supports_full_versioned_files:
2747
            return False
2748
        if not target._format.supports_full_versioned_files:
2749
            return False
5815.4.1 by Jelmer Vernooij
Split versionedfile-specific stuff out into VersionedFileRepository.
2750
        # This is redundant with format.check_conversion_target(), however that
2751
        # raises an exception, and we just want to say "False" as in we won't
2752
        # support converting between these formats.
2753
        if 'IDS_never' in debug.debug_flags:
2754
            return False
2755
        if source.supports_rich_root() and not target.supports_rich_root():
2756
            return False
2757
        if (source._format.supports_tree_reference
2758
            and not target._format.supports_tree_reference):
2759
            return False
2760
        if target._fallback_repositories and target._format.supports_chks:
2761
            # IDS doesn't know how to copy CHKs for the parent inventories it
2762
            # adds to stacked repos.
2763
            return False
2764
        if 'IDS_always' in debug.debug_flags:
2765
            return True
2766
        # Only use this code path for local source and target.  IDS does far
2767
        # too much IO (both bandwidth and roundtrips) over a network.
2768
        if not source.bzrdir.transport.base.startswith('file:///'):
2769
            return False
2770
        if not target.bzrdir.transport.base.startswith('file:///'):
2771
            return False
2772
        return True
2773
2774
    def _get_trees(self, revision_ids, cache):
2775
        possible_trees = []
2776
        for rev_id in revision_ids:
2777
            if rev_id in cache:
2778
                possible_trees.append((rev_id, cache[rev_id]))
2779
            else:
2780
                # Not cached, but inventory might be present anyway.
2781
                try:
2782
                    tree = self.source.revision_tree(rev_id)
2783
                except errors.NoSuchRevision:
2784
                    # Nope, parent is ghost.
2785
                    pass
2786
                else:
2787
                    cache[rev_id] = tree
2788
                    possible_trees.append((rev_id, tree))
2789
        return possible_trees
2790
2791
    def _get_delta_for_revision(self, tree, parent_ids, possible_trees):
2792
        """Get the best delta and base for this revision.
2793
2794
        :return: (basis_id, delta)
2795
        """
2796
        deltas = []
2797
        # Generate deltas against each tree, to find the shortest.
6405.2.10 by Jelmer Vernooij
Fix more tests.
2798
        # FIXME: Support nested trees
5815.4.1 by Jelmer Vernooij
Split versionedfile-specific stuff out into VersionedFileRepository.
2799
        texts_possibly_new_in_tree = set()
2800
        for basis_id, basis_tree in possible_trees:
6405.2.10 by Jelmer Vernooij
Fix more tests.
2801
            delta = tree.root_inventory._make_delta(basis_tree.root_inventory)
5815.4.1 by Jelmer Vernooij
Split versionedfile-specific stuff out into VersionedFileRepository.
2802
            for old_path, new_path, file_id, new_entry in delta:
2803
                if new_path is None:
2804
                    # This file_id isn't present in the new rev, so we don't
2805
                    # care about it.
2806
                    continue
2807
                if not new_path:
2808
                    # Rich roots are handled elsewhere...
2809
                    continue
2810
                kind = new_entry.kind
2811
                if kind != 'directory' and kind != 'file':
2812
                    # No text record associated with this inventory entry.
2813
                    continue
2814
                # This is a directory or file that has changed somehow.
2815
                texts_possibly_new_in_tree.add((file_id, new_entry.revision))
2816
            deltas.append((len(delta), basis_id, delta))
2817
        deltas.sort()
2818
        return deltas[0][1:]
2819
2820
    def _fetch_parent_invs_for_stacking(self, parent_map, cache):
2821
        """Find all parent revisions that are absent, but for which the
2822
        inventory is present, and copy those inventories.
2823
2824
        This is necessary to preserve correctness when the source is stacked
2825
        without fallbacks configured.  (Note that in cases like upgrade the
2826
        source may be not have _fallback_repositories even though it is
2827
        stacked.)
2828
        """
2829
        parent_revs = set()
2830
        for parents in parent_map.values():
2831
            parent_revs.update(parents)
2832
        present_parents = self.source.get_parent_map(parent_revs)
2833
        absent_parents = set(parent_revs).difference(present_parents)
2834
        parent_invs_keys_for_stacking = self.source.inventories.get_parent_map(
2835
            (rev_id,) for rev_id in absent_parents)
2836
        parent_inv_ids = [key[-1] for key in parent_invs_keys_for_stacking]
2837
        for parent_tree in self.source.revision_trees(parent_inv_ids):
2838
            current_revision_id = parent_tree.get_revision_id()
2839
            parents_parents_keys = parent_invs_keys_for_stacking[
2840
                (current_revision_id,)]
2841
            parents_parents = [key[-1] for key in parents_parents_keys]
2842
            basis_id = _mod_revision.NULL_REVISION
2843
            basis_tree = self.source.revision_tree(basis_id)
6405.2.10 by Jelmer Vernooij
Fix more tests.
2844
            delta = parent_tree.root_inventory._make_delta(
2845
                basis_tree.root_inventory)
5815.4.1 by Jelmer Vernooij
Split versionedfile-specific stuff out into VersionedFileRepository.
2846
            self.target.add_inventory_by_delta(
2847
                basis_id, delta, current_revision_id, parents_parents)
2848
            cache[current_revision_id] = parent_tree
2849
2850
    def _fetch_batch(self, revision_ids, basis_id, cache):
2851
        """Fetch across a few revisions.
2852
2853
        :param revision_ids: The revisions to copy
2854
        :param basis_id: The revision_id of a tree that must be in cache, used
2855
            as a basis for delta when no other base is available
2856
        :param cache: A cache of RevisionTrees that we can use.
2857
        :return: The revision_id of the last converted tree. The RevisionTree
2858
            for it will be in cache
2859
        """
2860
        # Walk though all revisions; get inventory deltas, copy referenced
2861
        # texts that delta references, insert the delta, revision and
2862
        # signature.
2863
        root_keys_to_create = set()
2864
        text_keys = set()
2865
        pending_deltas = []
2866
        pending_revisions = []
2867
        parent_map = self.source.get_parent_map(revision_ids)
2868
        self._fetch_parent_invs_for_stacking(parent_map, cache)
2869
        self.source._safe_to_return_from_cache = True
2870
        for tree in self.source.revision_trees(revision_ids):
2871
            # Find a inventory delta for this revision.
2872
            # Find text entries that need to be copied, too.
2873
            current_revision_id = tree.get_revision_id()
2874
            parent_ids = parent_map.get(current_revision_id, ())
2875
            parent_trees = self._get_trees(parent_ids, cache)
2876
            possible_trees = list(parent_trees)
2877
            if len(possible_trees) == 0:
2878
                # There either aren't any parents, or the parents are ghosts,
2879
                # so just use the last converted tree.
2880
                possible_trees.append((basis_id, cache[basis_id]))
2881
            basis_id, delta = self._get_delta_for_revision(tree, parent_ids,
2882
                                                           possible_trees)
2883
            revision = self.source.get_revision(current_revision_id)
2884
            pending_deltas.append((basis_id, delta,
2885
                current_revision_id, revision.parent_ids))
2886
            if self._converting_to_rich_root:
2887
                self._revision_id_to_root_id[current_revision_id] = \
2888
                    tree.get_root_id()
2889
            # Determine which texts are in present in this revision but not in
2890
            # any of the available parents.
2891
            texts_possibly_new_in_tree = set()
2892
            for old_path, new_path, file_id, entry in delta:
2893
                if new_path is None:
2894
                    # This file_id isn't present in the new rev
2895
                    continue
2896
                if not new_path:
2897
                    # This is the root
2898
                    if not self.target.supports_rich_root():
2899
                        # The target doesn't support rich root, so we don't
2900
                        # copy
2901
                        continue
2902
                    if self._converting_to_rich_root:
2903
                        # This can't be copied normally, we have to insert
2904
                        # it specially
2905
                        root_keys_to_create.add((file_id, entry.revision))
2906
                        continue
2907
                kind = entry.kind
2908
                texts_possibly_new_in_tree.add((file_id, entry.revision))
2909
            for basis_id, basis_tree in possible_trees:
6405.2.9 by Jelmer Vernooij
More test fixes.
2910
                basis_inv = basis_tree.root_inventory
5815.4.1 by Jelmer Vernooij
Split versionedfile-specific stuff out into VersionedFileRepository.
2911
                for file_key in list(texts_possibly_new_in_tree):
2912
                    file_id, file_revision = file_key
2913
                    try:
2914
                        entry = basis_inv[file_id]
2915
                    except errors.NoSuchId:
2916
                        continue
2917
                    if entry.revision == file_revision:
2918
                        texts_possibly_new_in_tree.remove(file_key)
2919
            text_keys.update(texts_possibly_new_in_tree)
2920
            pending_revisions.append(revision)
2921
            cache[current_revision_id] = tree
2922
            basis_id = current_revision_id
2923
        self.source._safe_to_return_from_cache = False
2924
        # Copy file texts
2925
        from_texts = self.source.texts
2926
        to_texts = self.target.texts
2927
        if root_keys_to_create:
2928
            root_stream = _mod_fetch._new_root_data_stream(
2929
                root_keys_to_create, self._revision_id_to_root_id, parent_map,
2930
                self.source)
2931
            to_texts.insert_record_stream(root_stream)
2932
        to_texts.insert_record_stream(from_texts.get_record_stream(
2933
            text_keys, self.target._format._fetch_order,
2934
            not self.target._format._fetch_uses_deltas))
2935
        # insert inventory deltas
2936
        for delta in pending_deltas:
2937
            self.target.add_inventory_by_delta(*delta)
2938
        if self.target._fallback_repositories:
2939
            # Make sure this stacked repository has all the parent inventories
2940
            # for the new revisions that we are about to insert.  We do this
2941
            # before adding the revisions so that no revision is added until
2942
            # all the inventories it may depend on are added.
2943
            # Note that this is overzealous, as we may have fetched these in an
2944
            # earlier batch.
2945
            parent_ids = set()
2946
            revision_ids = set()
2947
            for revision in pending_revisions:
2948
                revision_ids.add(revision.revision_id)
2949
                parent_ids.update(revision.parent_ids)
2950
            parent_ids.difference_update(revision_ids)
2951
            parent_ids.discard(_mod_revision.NULL_REVISION)
2952
            parent_map = self.source.get_parent_map(parent_ids)
2953
            # we iterate over parent_map and not parent_ids because we don't
2954
            # want to try copying any revision which is a ghost
2955
            for parent_tree in self.source.revision_trees(parent_map):
2956
                current_revision_id = parent_tree.get_revision_id()
2957
                parents_parents = parent_map[current_revision_id]
2958
                possible_trees = self._get_trees(parents_parents, cache)
2959
                if len(possible_trees) == 0:
2960
                    # There either aren't any parents, or the parents are
2961
                    # ghosts, so just use the last converted tree.
2962
                    possible_trees.append((basis_id, cache[basis_id]))
2963
                basis_id, delta = self._get_delta_for_revision(parent_tree,
2964
                    parents_parents, possible_trees)
2965
                self.target.add_inventory_by_delta(
2966
                    basis_id, delta, current_revision_id, parents_parents)
2967
        # insert signatures and revisions
2968
        for revision in pending_revisions:
2969
            try:
2970
                signature = self.source.get_signature_text(
2971
                    revision.revision_id)
2972
                self.target.add_signature_text(revision.revision_id,
2973
                    signature)
2974
            except errors.NoSuchRevision:
2975
                pass
2976
            self.target.add_revision(revision.revision_id, revision)
2977
        return basis_id
2978
2979
    def _fetch_all_revisions(self, revision_ids, pb):
2980
        """Fetch everything for the list of revisions.
2981
2982
        :param revision_ids: The list of revisions to fetch. Must be in
2983
            topological order.
2984
        :param pb: A ProgressTask
2985
        :return: None
2986
        """
2987
        basis_id, basis_tree = self._get_basis(revision_ids[0])
2988
        batch_size = 100
2989
        cache = lru_cache.LRUCache(100)
2990
        cache[basis_id] = basis_tree
2991
        del basis_tree # We don't want to hang on to it here
2992
        hints = []
2993
        a_graph = None
2994
2995
        for offset in range(0, len(revision_ids), batch_size):
2996
            self.target.start_write_group()
2997
            try:
6138.4.1 by Jonathan Riddell
add gettext to progress bar strings
2998
                pb.update(gettext('Transferring revisions'), offset,
5815.4.1 by Jelmer Vernooij
Split versionedfile-specific stuff out into VersionedFileRepository.
2999
                          len(revision_ids))
3000
                batch = revision_ids[offset:offset+batch_size]
3001
                basis_id = self._fetch_batch(batch, basis_id, cache)
3002
            except:
3003
                self.source._safe_to_return_from_cache = False
3004
                self.target.abort_write_group()
3005
                raise
3006
            else:
3007
                hint = self.target.commit_write_group()
3008
                if hint:
3009
                    hints.extend(hint)
3010
        if hints and self.target._format.pack_compresses:
3011
            self.target.pack(hint=hints)
6138.4.1 by Jonathan Riddell
add gettext to progress bar strings
3012
        pb.update(gettext('Transferring revisions'), len(revision_ids),
5815.4.1 by Jelmer Vernooij
Split versionedfile-specific stuff out into VersionedFileRepository.
3013
                  len(revision_ids))
3014
3015
    @needs_write_lock
3016
    def fetch(self, revision_id=None, find_ghosts=False,
3017
            fetch_spec=None):
3018
        """See InterRepository.fetch()."""
3019
        if fetch_spec is not None:
3020
            revision_ids = fetch_spec.get_keys()
3021
        else:
3022
            revision_ids = None
6047.1.1 by Jelmer Vernooij
Use show_user_warning rather than custom warning methods on UIFactory.
3023
        if self.source._format.experimental:
3024
            ui.ui_factory.show_user_warning('experimental_format_fetch',
3025
                from_format=self.source._format,
3026
                to_format=self.target._format)
5815.4.1 by Jelmer Vernooij
Split versionedfile-specific stuff out into VersionedFileRepository.
3027
        if (not self.source.supports_rich_root()
3028
            and self.target.supports_rich_root()):
3029
            self._converting_to_rich_root = True
3030
            self._revision_id_to_root_id = {}
3031
        else:
3032
            self._converting_to_rich_root = False
3033
        # See <https://launchpad.net/bugs/456077> asking for a warning here
3034
        if self.source._format.network_name() != self.target._format.network_name():
3035
            ui.ui_factory.show_user_warning('cross_format_fetch',
3036
                from_format=self.source._format,
3037
                to_format=self.target._format)
3038
        if revision_ids is None:
3039
            if revision_id:
3040
                search_revision_ids = [revision_id]
3041
            else:
3042
                search_revision_ids = None
3043
            revision_ids = self.target.search_missing_revision_ids(self.source,
3044
                revision_ids=search_revision_ids,
3045
                find_ghosts=find_ghosts).get_keys()
3046
        if not revision_ids:
3047
            return 0, 0
3048
        revision_ids = tsort.topo_sort(
3049
            self.source.get_graph().get_parent_map(revision_ids))
3050
        if not revision_ids:
3051
            return 0, 0
3052
        # Walk though all revisions; get inventory deltas, copy referenced
3053
        # texts that delta references, insert the delta, revision and
3054
        # signature.
3055
        pb = ui.ui_factory.nested_progress_bar()
3056
        try:
3057
            self._fetch_all_revisions(revision_ids, pb)
3058
        finally:
3059
            pb.finished()
3060
        return len(revision_ids), 0
3061
3062
    def _get_basis(self, first_revision_id):
3063
        """Get a revision and tree which exists in the target.
3064
3065
        This assumes that first_revision_id is selected for transmission
3066
        because all other ancestors are already present. If we can't find an
3067
        ancestor we fall back to NULL_REVISION since we know that is safe.
3068
3069
        :return: (basis_id, basis_tree)
3070
        """
3071
        first_rev = self.source.get_revision(first_revision_id)
3072
        try:
3073
            basis_id = first_rev.parent_ids[0]
3074
            # only valid as a basis if the target has it
3075
            self.target.get_revision(basis_id)
3076
            # Try to get a basis tree - if it's a ghost it will hit the
3077
            # NoSuchRevision case.
3078
            basis_tree = self.source.revision_tree(basis_id)
3079
        except (IndexError, errors.NoSuchRevision):
3080
            basis_id = _mod_revision.NULL_REVISION
3081
            basis_tree = self.source.revision_tree(basis_id)
3082
        return basis_id, basis_tree
3083
3084
5863.4.1 by Jelmer Vernooij
Move interrepository implementation to vf_repository.
3085
class InterSameDataRepository(InterVersionedFileRepository):
5815.4.19 by Jelmer Vernooij
Fix test failures.
3086
    """Code for converting between repositories that represent the same data.
3087
3088
    Data format and model must match for this to work.
3089
    """
3090
3091
    @classmethod
3092
    def _get_repo_format_to_test(self):
3093
        """Repository format for testing with.
3094
3095
        InterSameData can pull from subtree to subtree and from non-subtree to
3096
        non-subtree, so we test this with the richest repository format.
3097
        """
3098
        from bzrlib.repofmt import knitrepo
3099
        return knitrepo.RepositoryFormatKnit3()
3100
3101
    @staticmethod
3102
    def is_compatible(source, target):
3103
        return (
3104
            InterRepository._same_model(source, target) and
3105
            source._format.supports_full_versioned_files and
3106
            target._format.supports_full_versioned_files)
3107
3108
5863.4.1 by Jelmer Vernooij
Move interrepository implementation to vf_repository.
3109
InterRepository.register_optimiser(InterVersionedFileRepository)
5815.4.1 by Jelmer Vernooij
Split versionedfile-specific stuff out into VersionedFileRepository.
3110
InterRepository.register_optimiser(InterDifferingSerializer)
5815.4.19 by Jelmer Vernooij
Fix test failures.
3111
InterRepository.register_optimiser(InterSameDataRepository)
5815.4.1 by Jelmer Vernooij
Split versionedfile-specific stuff out into VersionedFileRepository.
3112
3113
3114
def install_revisions(repository, iterable, num_revisions=None, pb=None):
3115
    """Install all revision data into a repository.
3116
3117
    Accepts an iterable of revision, tree, signature tuples.  The signature
3118
    may be None.
3119
    """
3120
    repository.start_write_group()
3121
    try:
3122
        inventory_cache = lru_cache.LRUCache(10)
3123
        for n, (revision, revision_tree, signature) in enumerate(iterable):
3124
            _install_revision(repository, revision, revision_tree, signature,
3125
                inventory_cache)
3126
            if pb is not None:
6138.4.1 by Jonathan Riddell
add gettext to progress bar strings
3127
                pb.update(gettext('Transferring revisions'), n + 1, num_revisions)
5815.4.1 by Jelmer Vernooij
Split versionedfile-specific stuff out into VersionedFileRepository.
3128
    except:
3129
        repository.abort_write_group()
3130
        raise
3131
    else:
3132
        repository.commit_write_group()
3133
3134
3135
def _install_revision(repository, rev, revision_tree, signature,
3136
    inventory_cache):
3137
    """Install all revision data into a repository."""
3138
    present_parents = []
3139
    parent_trees = {}
3140
    for p_id in rev.parent_ids:
3141
        if repository.has_revision(p_id):
3142
            present_parents.append(p_id)
3143
            parent_trees[p_id] = repository.revision_tree(p_id)
3144
        else:
3145
            parent_trees[p_id] = repository.revision_tree(
3146
                                     _mod_revision.NULL_REVISION)
3147
6405.2.10 by Jelmer Vernooij
Fix more tests.
3148
    # FIXME: Support nested trees
3149
    inv = revision_tree.root_inventory
5815.4.1 by Jelmer Vernooij
Split versionedfile-specific stuff out into VersionedFileRepository.
3150
    entries = inv.iter_entries()
3151
    # backwards compatibility hack: skip the root id.
3152
    if not repository.supports_rich_root():
3153
        path, root = entries.next()
3154
        if root.revision != rev.revision_id:
3155
            raise errors.IncompatibleRevision(repr(repository))
3156
    text_keys = {}
3157
    for path, ie in entries:
3158
        text_keys[(ie.file_id, ie.revision)] = ie
3159
    text_parent_map = repository.texts.get_parent_map(text_keys)
3160
    missing_texts = set(text_keys) - set(text_parent_map)
3161
    # Add the texts that are not already present
3162
    for text_key in missing_texts:
3163
        ie = text_keys[text_key]
3164
        text_parents = []
3165
        # FIXME: TODO: The following loop overlaps/duplicates that done by
3166
        # commit to determine parents. There is a latent/real bug here where
3167
        # the parents inserted are not those commit would do - in particular
3168
        # they are not filtered by heads(). RBC, AB
3169
        for revision, tree in parent_trees.iteritems():
5967.7.1 by Martin Pool
Deprecate __contains__ on Tree and Inventory
3170
            if not tree.has_id(ie.file_id):
5815.4.1 by Jelmer Vernooij
Split versionedfile-specific stuff out into VersionedFileRepository.
3171
                continue
3172
            parent_id = tree.get_file_revision(ie.file_id)
3173
            if parent_id in text_parents:
3174
                continue
3175
            text_parents.append((ie.file_id, parent_id))
3176
        lines = revision_tree.get_file(ie.file_id).readlines()
3177
        repository.texts.add_lines(text_key, text_parents, lines)
3178
    try:
3179
        # install the inventory
3180
        if repository._format._commit_inv_deltas and len(rev.parent_ids):
3181
            # Cache this inventory
3182
            inventory_cache[rev.revision_id] = inv
3183
            try:
3184
                basis_inv = inventory_cache[rev.parent_ids[0]]
3185
            except KeyError:
3186
                repository.add_inventory(rev.revision_id, inv, present_parents)
3187
            else:
3188
                delta = inv._make_delta(basis_inv)
3189
                repository.add_inventory_by_delta(rev.parent_ids[0], delta,
3190
                    rev.revision_id, present_parents)
3191
        else:
3192
            repository.add_inventory(rev.revision_id, inv, present_parents)
3193
    except errors.RevisionAlreadyPresent:
3194
        pass
3195
    if signature is not None:
3196
        repository.add_signature_text(rev.revision_id, signature)
3197
    repository.add_revision(rev.revision_id, rev, inv)
3198
3199
3200
def install_revision(repository, rev, revision_tree):
3201
    """Install all revision data into a repository."""
3202
    install_revisions(repository, [(rev, revision_tree, None)])