~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/remote.py

  • Committer: Jelmer Vernooij
  • Date: 2012-01-06 22:44:57 UTC
  • mfrom: (6436 +trunk)
  • mto: (6437.3.11 2.5)
  • mto: This revision was merged to the branch mainline in revision 6444.
  • Revision ID: jelmer@samba.org-20120106224457-re0pcy0fz31xob77
Merge bzr.dev.

Show diffs side-by-side

added added

removed removed

Lines of Context:
14
14
# along with this program; if not, write to the Free Software
15
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16
16
 
 
17
from __future__ import absolute_import
 
18
 
17
19
import bz2
18
20
import zlib
19
21
 
21
23
    bencode,
22
24
    branch,
23
25
    bzrdir as _mod_bzrdir,
24
 
    config,
 
26
    config as _mod_config,
25
27
    controldir,
26
28
    debug,
27
29
    errors,
28
30
    gpg,
29
31
    graph,
 
32
    inventory_delta,
30
33
    lock,
31
34
    lockdir,
32
35
    osutils,
56
59
from bzrlib.repository import RepositoryWriteLockResult, _LazyListJoin
57
60
from bzrlib.serializer import format_registry as serializer_format_registry
58
61
from bzrlib.trace import mutter, note, warning, log_exception_quietly
 
62
from bzrlib.versionedfile import ChunkedContentFactory, FulltextContentFactory
59
63
 
60
64
 
61
65
_DEFAULT_SEARCH_DEPTH = 100
345
349
        _mod_bzrdir.BzrDirMetaFormat1._set_repository_format) #.im_func)
346
350
 
347
351
 
348
 
class RemoteControlStore(config.IniFileStore):
 
352
class RemoteControlStore(_mod_config.IniFileStore):
349
353
    """Control store which attempts to use HPSS calls to retrieve control store.
350
354
 
351
355
    Note that this is specific to bzr-based formats.
375
379
    def _ensure_real(self):
376
380
        self.bzrdir._ensure_real()
377
381
        if self._real_store is None:
378
 
            self._real_store = config.ControlStore(self.bzrdir)
 
382
            self._real_store = _mod_config.ControlStore(self.bzrdir)
379
383
 
380
384
    def external_url(self):
381
385
        return self.bzrdir.user_url
477
481
                warning('VFS BzrDir access triggered\n%s',
478
482
                    ''.join(traceback.format_stack()))
479
483
            self._real_bzrdir = _mod_bzrdir.BzrDir.open_from_transport(
480
 
                self.root_transport, _server_formats=False)
 
484
                self.root_transport, probers=[_mod_bzrdir.BzrProber])
481
485
            self._format._network_name = \
482
486
                self._real_bzrdir._format.network_name()
483
487
 
490
494
        self._next_open_branch_result = None
491
495
        return _mod_bzrdir.BzrDir.break_lock(self)
492
496
 
 
497
    def _vfs_checkout_metadir(self):
 
498
        self._ensure_real()
 
499
        return self._real_bzrdir.checkout_metadir()
 
500
 
 
501
    def checkout_metadir(self):
 
502
        """Retrieve the controldir format to use for checkouts of this one.
 
503
        """
 
504
        medium = self._client._medium
 
505
        if medium._is_remote_before((2, 5)):
 
506
            return self._vfs_checkout_metadir()
 
507
        path = self._path_for_remote_call(self._client)
 
508
        try:
 
509
            response = self._client.call('BzrDir.checkout_metadir',
 
510
                path)
 
511
        except errors.UnknownSmartMethod:
 
512
            medium._remember_remote_is_before((2, 5))
 
513
            return self._vfs_checkout_metadir()
 
514
        if len(response) != 3:
 
515
            raise errors.UnexpectedSmartServerResponse(response)
 
516
        control_name, repo_name, branch_name = response
 
517
        try:
 
518
            format = controldir.network_format_registry.get(control_name)
 
519
        except KeyError:
 
520
            raise errors.UnknownFormatError(kind='control',
 
521
                format=control_name)
 
522
        if repo_name:
 
523
            try:
 
524
                repo_format = _mod_repository.network_format_registry.get(
 
525
                    repo_name)
 
526
            except KeyError:
 
527
                raise errors.UnknownFormatError(kind='repository',
 
528
                    format=repo_name)
 
529
            format.repository_format = repo_format
 
530
        if branch_name:
 
531
            try:
 
532
                format.set_branch_format(
 
533
                    branch.network_format_registry.get(branch_name))
 
534
            except KeyError:
 
535
                raise errors.UnknownFormatError(kind='branch',
 
536
                    format=branch_name)
 
537
        return format
 
538
 
493
539
    def _vfs_cloning_metadir(self, require_stacking=False):
494
540
        self._ensure_real()
495
541
        return self._real_bzrdir.cloning_metadir(
1166
1212
        if response != ('ok', ):
1167
1213
            raise errors.UnexpectedSmartServerResponse(response)
1168
1214
        self._write_group_tokens = None
 
1215
        # Refresh data after writing to the repository.
 
1216
        self.refresh_data()
1169
1217
 
1170
1218
    def resume_write_group(self, tokens):
1171
1219
        if self._real_repository:
1720
1768
    def get_commit_builder(self, branch, parents, config, timestamp=None,
1721
1769
                           timezone=None, committer=None, revprops=None,
1722
1770
                           revision_id=None, lossy=False):
1723
 
        # FIXME: It ought to be possible to call this without immediately
1724
 
        # triggering _ensure_real.  For now it's the easiest thing to do.
1725
 
        self._ensure_real()
1726
 
        real_repo = self._real_repository
1727
 
        builder = real_repo.get_commit_builder(branch, parents,
1728
 
                config, timestamp=timestamp, timezone=timezone,
1729
 
                committer=committer, revprops=revprops,
1730
 
                revision_id=revision_id, lossy=lossy)
1731
 
        return builder
 
1771
        """Obtain a CommitBuilder for this repository.
 
1772
 
 
1773
        :param branch: Branch to commit to.
 
1774
        :param parents: Revision ids of the parents of the new revision.
 
1775
        :param config: Configuration to use.
 
1776
        :param timestamp: Optional timestamp recorded for commit.
 
1777
        :param timezone: Optional timezone for timestamp.
 
1778
        :param committer: Optional committer to set for commit.
 
1779
        :param revprops: Optional dictionary of revision properties.
 
1780
        :param revision_id: Optional revision id.
 
1781
        :param lossy: Whether to discard data that can not be natively
 
1782
            represented, when pushing to a foreign VCS
 
1783
        """
 
1784
        if self._fallback_repositories and not self._format.supports_chks:
 
1785
            raise errors.BzrError("Cannot commit directly to a stacked branch"
 
1786
                " in pre-2a formats. See "
 
1787
                "https://bugs.launchpad.net/bzr/+bug/375013 for details.")
 
1788
        if self._format.rich_root_data:
 
1789
            commit_builder_kls = vf_repository.VersionedFileRootCommitBuilder
 
1790
        else:
 
1791
            commit_builder_kls = vf_repository.VersionedFileCommitBuilder
 
1792
        result = commit_builder_kls(self, parents, config,
 
1793
            timestamp, timezone, committer, revprops, revision_id,
 
1794
            lossy)
 
1795
        self.start_write_group()
 
1796
        return result
1732
1797
 
1733
1798
    def add_fallback_repository(self, repository):
1734
1799
        """Add a repository to use for looking up data not held locally.
1779
1844
            delta, new_revision_id, parents, basis_inv=basis_inv,
1780
1845
            propagate_caches=propagate_caches)
1781
1846
 
1782
 
    def add_revision(self, rev_id, rev, inv=None, config=None):
1783
 
        self._ensure_real()
1784
 
        return self._real_repository.add_revision(
1785
 
            rev_id, rev, inv=inv, config=config)
 
1847
    def add_revision(self, revision_id, rev, inv=None):
 
1848
        _mod_revision.check_not_reserved_id(revision_id)
 
1849
        key = (revision_id,)
 
1850
        # check inventory present
 
1851
        if not self.inventories.get_parent_map([key]):
 
1852
            if inv is None:
 
1853
                raise errors.WeaveRevisionNotPresent(revision_id,
 
1854
                                                     self.inventories)
 
1855
            else:
 
1856
                # yes, this is not suitable for adding with ghosts.
 
1857
                rev.inventory_sha1 = self.add_inventory(revision_id, inv,
 
1858
                                                        rev.parent_ids)
 
1859
        else:
 
1860
            rev.inventory_sha1 = self.inventories.get_sha1s([key])[key]
 
1861
        self._add_revision(rev)
 
1862
 
 
1863
    def _add_revision(self, rev):
 
1864
        if self._real_repository is not None:
 
1865
            return self._real_repository._add_revision(rev)
 
1866
        text = self._serializer.write_revision_to_string(rev)
 
1867
        key = (rev.revision_id,)
 
1868
        parents = tuple((parent,) for parent in rev.parent_ids)
 
1869
        self._write_group_tokens, missing_keys = self._get_sink().insert_stream(
 
1870
            [('revisions', [FulltextContentFactory(key, parents, None, text)])],
 
1871
            self._format, self._write_group_tokens)
1786
1872
 
1787
1873
    @needs_read_lock
1788
1874
    def get_inventory(self, revision_id):
1789
1875
        return list(self.iter_inventories([revision_id]))[0]
1790
1876
 
 
1877
    def _iter_inventories_rpc(self, revision_ids, ordering):
 
1878
        if ordering is None:
 
1879
            ordering = 'unordered'
 
1880
        path = self.bzrdir._path_for_remote_call(self._client)
 
1881
        body = "\n".join(revision_ids)
 
1882
        response_tuple, response_handler = (
 
1883
            self._call_with_body_bytes_expecting_body(
 
1884
                "VersionedFileRepository.get_inventories",
 
1885
                (path, ordering), body))
 
1886
        if response_tuple[0] != "ok":
 
1887
            raise errors.UnexpectedSmartServerResponse(response_tuple)
 
1888
        deserializer = inventory_delta.InventoryDeltaDeserializer()
 
1889
        byte_stream = response_handler.read_streamed_body()
 
1890
        decoded = smart_repo._byte_stream_to_stream(byte_stream)
 
1891
        if decoded is None:
 
1892
            # no results whatsoever
 
1893
            return
 
1894
        src_format, stream = decoded
 
1895
        if src_format.network_name() != self._format.network_name():
 
1896
            raise AssertionError(
 
1897
                "Mismatched RemoteRepository and stream src %r, %r" % (
 
1898
                src_format.network_name(), self._format.network_name()))
 
1899
        # ignore the src format, it's not really relevant
 
1900
        prev_inv = Inventory(root_id=None,
 
1901
            revision_id=_mod_revision.NULL_REVISION)
 
1902
        # there should be just one substream, with inventory deltas
 
1903
        substream_kind, substream = stream.next()
 
1904
        if substream_kind != "inventory-deltas":
 
1905
            raise AssertionError(
 
1906
                 "Unexpected stream %r received" % substream_kind)
 
1907
        for record in substream:
 
1908
            (parent_id, new_id, versioned_root, tree_references, invdelta) = (
 
1909
                deserializer.parse_text_bytes(record.get_bytes_as("fulltext")))
 
1910
            if parent_id != prev_inv.revision_id:
 
1911
                raise AssertionError("invalid base %r != %r" % (parent_id,
 
1912
                    prev_inv.revision_id))
 
1913
            inv = prev_inv.create_by_apply_delta(invdelta, new_id)
 
1914
            yield inv, inv.revision_id
 
1915
            prev_inv = inv
 
1916
 
 
1917
    def _iter_inventories_vfs(self, revision_ids, ordering=None):
 
1918
        self._ensure_real()
 
1919
        return self._real_repository._iter_inventories(revision_ids, ordering)
 
1920
 
1791
1921
    def iter_inventories(self, revision_ids, ordering=None):
1792
 
        self._ensure_real()
1793
 
        return self._real_repository.iter_inventories(revision_ids, ordering)
 
1922
        """Get many inventories by revision_ids.
 
1923
 
 
1924
        This will buffer some or all of the texts used in constructing the
 
1925
        inventories in memory, but will only parse a single inventory at a
 
1926
        time.
 
1927
 
 
1928
        :param revision_ids: The expected revision ids of the inventories.
 
1929
        :param ordering: optional ordering, e.g. 'topological'.  If not
 
1930
            specified, the order of revision_ids will be preserved (by
 
1931
            buffering if necessary).
 
1932
        :return: An iterator of inventories.
 
1933
        """
 
1934
        if ((None in revision_ids)
 
1935
            or (_mod_revision.NULL_REVISION in revision_ids)):
 
1936
            raise ValueError('cannot get null revision inventory')
 
1937
        for inv, revid in self._iter_inventories(revision_ids, ordering):
 
1938
            if inv is None:
 
1939
                raise errors.NoSuchRevision(self, revid)
 
1940
            yield inv
 
1941
 
 
1942
    def _iter_inventories(self, revision_ids, ordering=None):
 
1943
        if len(revision_ids) == 0:
 
1944
            return
 
1945
        missing = set(revision_ids)
 
1946
        if ordering is None:
 
1947
            order_as_requested = True
 
1948
            invs = {}
 
1949
            order = list(revision_ids)
 
1950
            order.reverse()
 
1951
            next_revid = order.pop()
 
1952
        else:
 
1953
            order_as_requested = False
 
1954
            if ordering != 'unordered' and self._fallback_repositories:
 
1955
                raise ValueError('unsupported ordering %r' % ordering)
 
1956
        iter_inv_fns = [self._iter_inventories_rpc] + [
 
1957
            fallback._iter_inventories for fallback in
 
1958
            self._fallback_repositories]
 
1959
        try:
 
1960
            for iter_inv in iter_inv_fns:
 
1961
                request = [revid for revid in revision_ids if revid in missing]
 
1962
                for inv, revid in iter_inv(request, ordering):
 
1963
                    if inv is None:
 
1964
                        continue
 
1965
                    missing.remove(inv.revision_id)
 
1966
                    if ordering != 'unordered':
 
1967
                        invs[revid] = inv
 
1968
                    else:
 
1969
                        yield inv, revid
 
1970
                if order_as_requested:
 
1971
                    # Yield as many results as we can while preserving order.
 
1972
                    while next_revid in invs:
 
1973
                        inv = invs.pop(next_revid)
 
1974
                        yield inv, inv.revision_id
 
1975
                        try:
 
1976
                            next_revid = order.pop()
 
1977
                        except IndexError:
 
1978
                            # We still want to fully consume the stream, just
 
1979
                            # in case it is not actually finished at this point
 
1980
                            next_revid = None
 
1981
                            break
 
1982
        except errors.UnknownSmartMethod:
 
1983
            for inv, revid in self._iter_inventories_vfs(revision_ids, ordering):
 
1984
                yield inv, revid
 
1985
            return
 
1986
        # Report missing
 
1987
        if order_as_requested:
 
1988
            if next_revid is not None:
 
1989
                yield None, next_revid
 
1990
            while order:
 
1991
                revid = order.pop()
 
1992
                yield invs.get(revid), revid
 
1993
        else:
 
1994
            while missing:
 
1995
                yield None, missing.pop()
1794
1996
 
1795
1997
    @needs_read_lock
1796
1998
    def get_revision(self, revision_id):
1832
2034
        """
1833
2035
        if self._real_repository is not None:
1834
2036
            self._real_repository.refresh_data()
 
2037
        # Refresh the parents cache for this object
 
2038
        self._unstacked_provider.disable_cache()
 
2039
        self._unstacked_provider.enable_cache()
1835
2040
 
1836
2041
    def revision_ids_to_search_result(self, result_set):
1837
2042
        """Convert a set of revision ids to a graph SearchResult."""
2149
2354
 
2150
2355
    @needs_read_lock
2151
2356
    def _get_inventory_xml(self, revision_id):
 
2357
        # This call is used by older working tree formats,
 
2358
        # which stored a serialized basis inventory.
2152
2359
        self._ensure_real()
2153
2360
        return self._real_repository._get_inventory_xml(revision_id)
2154
2361
 
2193
2400
            revids.update(set(fallback.all_revision_ids()))
2194
2401
        return list(revids)
2195
2402
 
 
2403
    def _filtered_revision_trees(self, revision_ids, file_ids):
 
2404
        """Return Tree for a revision on this branch with only some files.
 
2405
 
 
2406
        :param revision_ids: a sequence of revision-ids;
 
2407
          a revision-id may not be None or 'null:'
 
2408
        :param file_ids: if not None, the result is filtered
 
2409
          so that only those file-ids, their parents and their
 
2410
          children are included.
 
2411
        """
 
2412
        inventories = self.iter_inventories(revision_ids)
 
2413
        for inv in inventories:
 
2414
            # Should we introduce a FilteredRevisionTree class rather
 
2415
            # than pre-filter the inventory here?
 
2416
            filtered_inv = inv.filter(file_ids)
 
2417
            yield InventoryRevisionTree(self, filtered_inv, filtered_inv.revision_id)
 
2418
 
2196
2419
    @needs_read_lock
2197
2420
    def get_deltas_for_revisions(self, revisions, specific_fileids=None):
2198
 
        self._ensure_real()
2199
 
        return self._real_repository.get_deltas_for_revisions(revisions,
2200
 
            specific_fileids=specific_fileids)
 
2421
        medium = self._client._medium
 
2422
        if medium._is_remote_before((1, 2)):
 
2423
            self._ensure_real()
 
2424
            for delta in self._real_repository.get_deltas_for_revisions(
 
2425
                    revisions, specific_fileids):
 
2426
                yield delta
 
2427
            return
 
2428
        # Get the revision-ids of interest
 
2429
        required_trees = set()
 
2430
        for revision in revisions:
 
2431
            required_trees.add(revision.revision_id)
 
2432
            required_trees.update(revision.parent_ids[:1])
 
2433
 
 
2434
        # Get the matching filtered trees. Note that it's more
 
2435
        # efficient to pass filtered trees to changes_from() rather
 
2436
        # than doing the filtering afterwards. changes_from() could
 
2437
        # arguably do the filtering itself but it's path-based, not
 
2438
        # file-id based, so filtering before or afterwards is
 
2439
        # currently easier.
 
2440
        if specific_fileids is None:
 
2441
            trees = dict((t.get_revision_id(), t) for
 
2442
                t in self.revision_trees(required_trees))
 
2443
        else:
 
2444
            trees = dict((t.get_revision_id(), t) for
 
2445
                t in self._filtered_revision_trees(required_trees,
 
2446
                specific_fileids))
 
2447
 
 
2448
        # Calculate the deltas
 
2449
        for revision in revisions:
 
2450
            if not revision.parent_ids:
 
2451
                old_tree = self.revision_tree(_mod_revision.NULL_REVISION)
 
2452
            else:
 
2453
                old_tree = trees[revision.parent_ids[0]]
 
2454
            yield trees[revision.revision_id].changes_from(old_tree)
2201
2455
 
2202
2456
    @needs_read_lock
2203
2457
    def get_revision_delta(self, revision_id, specific_fileids=None):
2946
3200
        return False
2947
3201
 
2948
3202
 
2949
 
class RemoteBranchStore(config.IniFileStore):
 
3203
class RemoteBranchStore(_mod_config.IniFileStore):
2950
3204
    """Branch store which attempts to use HPSS calls to retrieve branch store.
2951
3205
 
2952
3206
    Note that this is specific to bzr-based formats.
3004
3258
    def _ensure_real(self):
3005
3259
        self.branch._ensure_real()
3006
3260
        if self._real_store is None:
3007
 
            self._real_store = config.BranchStore(self.branch)
 
3261
            self._real_store = _mod_config.BranchStore(self.branch)
3008
3262
 
3009
3263
 
3010
3264
class RemoteBranch(branch.Branch, _RpcHelper, lock._RelockDebugMixin):
3179
3433
                self.bzrdir, self._client)
3180
3434
        return self._control_files
3181
3435
 
3182
 
    def _get_checkout_format(self, lightweight=False):
3183
 
        self._ensure_real()
3184
 
        if lightweight:
3185
 
            format = RemoteBzrDirFormat()
3186
 
            self.bzrdir._format._supply_sub_formats_to(format)
3187
 
            format.workingtree_format = self._real_branch._get_checkout_format(
3188
 
                lightweight=lightweight).workingtree_format
3189
 
            return format
3190
 
        else:
3191
 
            return self._real_branch._get_checkout_format(lightweight=False)
3192
 
 
3193
3436
    def get_physical_lock_status(self):
3194
3437
        """See Branch.get_physical_lock_status()."""
3195
3438
        try:
3720
3963
                value = section_obj.get(name, default)
3721
3964
        except errors.UnknownSmartMethod:
3722
3965
            value = self._vfs_get_option(name, section, default)
3723
 
        for hook in config.OldConfigHooks['get']:
 
3966
        for hook in _mod_config.OldConfigHooks['get']:
3724
3967
            hook(self, name, value)
3725
3968
        return value
3726
3969
 
3728
3971
        if len(response[0]) and response[0][0] != 'ok':
3729
3972
            raise errors.UnexpectedSmartServerResponse(response)
3730
3973
        lines = response[1].read_body_bytes().splitlines()
3731
 
        conf = config.ConfigObj(lines, encoding='utf-8')
3732
 
        for hook in config.OldConfigHooks['load']:
 
3974
        conf = _mod_config.ConfigObj(lines, encoding='utf-8')
 
3975
        for hook in _mod_config.OldConfigHooks['load']:
3733
3976
            hook(self)
3734
3977
        return conf
3735
3978