~bzr-pqm/bzr/bzr.dev

2018.18.1 by Martin Pool
Add stub Repository.tarball smart method
1
# Copyright (C) 2006, 2007 Canonical Ltd
2018.5.40 by Robert Collins
Implement a remote Repository.has_revision method.
2
#
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
7
#
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
# GNU General Public License for more details.
12
#
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
16
17
"""Server-side repository related request implmentations."""
18
2535.3.14 by Andrew Bennetts
Move serialising repo data stream to bytes into smart protocol.
19
from cStringIO import StringIO
2571.2.2 by Ian Clatworthy
use basename as poolie recommended
20
import os
2018.18.2 by Martin Pool
smart method Repository.tarball actually returns the tarball
21
import sys
22
import tempfile
23
import tarfile
2018.5.40 by Robert Collins
Implement a remote Repository.has_revision method.
24
25
from bzrlib import errors
26
from bzrlib.bzrdir import BzrDir
2535.3.31 by Andrew Bennetts
Fix imports broken by reverting the container-format merge. I didn't notice them earlier because of .pyc files :(
27
from bzrlib.pack import ContainerWriter
2432.4.5 by Robert Collins
Make using SuccessfulSmartServerResponse and FailedSmartServerResponse mandatory rather than optional in smart server logic.
28
from bzrlib.smart.request import (
29
    FailedSmartServerResponse,
30
    SmartServerRequest,
31
    SuccessfulSmartServerResponse,
32
    )
2018.5.40 by Robert Collins
Implement a remote Repository.has_revision method.
33
34
2018.5.56 by Robert Collins
Factor out code we expect to be common in SmartServerRequestHasRevision to SmartServerRepositoryRequest (Robert Collins, Vincent Ladeuil).
35
class SmartServerRepositoryRequest(SmartServerRequest):
36
    """Common base class for Repository requests."""
37
38
    def do(self, path, *args):
39
        """Execute a repository request.
40
        
2018.5.40 by Robert Collins
Implement a remote Repository.has_revision method.
41
        The repository must be at the exact path - no searching is done.
42
2018.5.56 by Robert Collins
Factor out code we expect to be common in SmartServerRequestHasRevision to SmartServerRepositoryRequest (Robert Collins, Vincent Ladeuil).
43
        The actual logic is delegated to self.do_repository_request.
44
2018.5.40 by Robert Collins
Implement a remote Repository.has_revision method.
45
        :param path: The path for the repository.
2018.5.56 by Robert Collins
Factor out code we expect to be common in SmartServerRequestHasRevision to SmartServerRepositoryRequest (Robert Collins, Vincent Ladeuil).
46
        :return: A smart server from self.do_repository_request().
47
        """
48
        transport = self._backing_transport.clone(path)
49
        bzrdir = BzrDir.open_from_transport(transport)
50
        repository = bzrdir.open_repository()
51
        return self.do_repository_request(repository, *args)
52
53
2018.5.67 by Wouter van Heyst
Implement RemoteRepository.get_revision_graph (Wouter van Heyst, Robert Collins)
54
class SmartServerRepositoryGetRevisionGraph(SmartServerRepositoryRequest):
55
    
56
    def do_repository_request(self, repository, revision_id):
57
        """Return the result of repository.get_revision_graph(revision_id).
58
        
59
        :param repository: The repository to query in.
60
        :param revision_id: The utf8 encoded revision_id to get a graph from.
61
        :return: A smart server response where the body contains an utf8
62
            encoded flattened list of the revision graph.
63
        """
2018.5.83 by Andrew Bennetts
Fix some test failures caused by the switch from unicode to UTF-8-encoded strs for revision IDs.
64
        if not revision_id:
65
            revision_id = None
2018.5.67 by Wouter van Heyst
Implement RemoteRepository.get_revision_graph (Wouter van Heyst, Robert Collins)
66
67
        lines = []
68
        try:
2018.5.83 by Andrew Bennetts
Fix some test failures caused by the switch from unicode to UTF-8-encoded strs for revision IDs.
69
            revision_graph = repository.get_revision_graph(revision_id)
2018.5.67 by Wouter van Heyst
Implement RemoteRepository.get_revision_graph (Wouter van Heyst, Robert Collins)
70
        except errors.NoSuchRevision:
2018.14.1 by Andrew Bennetts
Update to current hpss branch? Fix lots of test failures.
71
            # Note that we return an empty body, rather than omitting the body.
72
            # This way the client knows that it can always expect to find a body
73
            # in the response for this method, even in the error case.
2432.4.5 by Robert Collins
Make using SuccessfulSmartServerResponse and FailedSmartServerResponse mandatory rather than optional in smart server logic.
74
            return FailedSmartServerResponse(('nosuchrevision', revision_id), '')
2018.5.67 by Wouter van Heyst
Implement RemoteRepository.get_revision_graph (Wouter van Heyst, Robert Collins)
75
76
        for revision, parents in revision_graph.items():
2592.3.50 by Robert Collins
Merge bzr.dev.
77
            lines.append(' '.join((revision, ) + tuple(parents)))
2018.5.67 by Wouter van Heyst
Implement RemoteRepository.get_revision_graph (Wouter van Heyst, Robert Collins)
78
2432.4.5 by Robert Collins
Make using SuccessfulSmartServerResponse and FailedSmartServerResponse mandatory rather than optional in smart server logic.
79
        return SuccessfulSmartServerResponse(('ok', ), '\n'.join(lines))
2018.5.67 by Wouter van Heyst
Implement RemoteRepository.get_revision_graph (Wouter van Heyst, Robert Collins)
80
81
2018.5.56 by Robert Collins
Factor out code we expect to be common in SmartServerRequestHasRevision to SmartServerRepositoryRequest (Robert Collins, Vincent Ladeuil).
82
class SmartServerRequestHasRevision(SmartServerRepositoryRequest):
83
84
    def do_repository_request(self, repository, revision_id):
85
        """Return ok if a specific revision is in the repository at path.
86
87
        :param repository: The repository to query in.
2018.5.40 by Robert Collins
Implement a remote Repository.has_revision method.
88
        :param revision_id: The utf8 encoded revision_id to lookup.
89
        :return: A smart server response of ('ok', ) if the revision is
90
            present.
91
        """
2018.5.83 by Andrew Bennetts
Fix some test failures caused by the switch from unicode to UTF-8-encoded strs for revision IDs.
92
        if repository.has_revision(revision_id):
2432.4.5 by Robert Collins
Make using SuccessfulSmartServerResponse and FailedSmartServerResponse mandatory rather than optional in smart server logic.
93
            return SuccessfulSmartServerResponse(('yes', ))
2018.5.40 by Robert Collins
Implement a remote Repository.has_revision method.
94
        else:
2432.4.5 by Robert Collins
Make using SuccessfulSmartServerResponse and FailedSmartServerResponse mandatory rather than optional in smart server logic.
95
            return SuccessfulSmartServerResponse(('no', ))
2018.5.57 by Robert Collins
Implement RemoteRepository.is_shared (Robert Collins, Vincent Ladeuil).
96
97
2018.10.2 by v.ladeuil+lp at free
gather_stats server side and request registration
98
class SmartServerRepositoryGatherStats(SmartServerRepositoryRequest):
99
100
    def do_repository_request(self, repository, revid, committers):
101
        """Return the result of repository.gather_stats().
102
103
        :param repository: The repository to query in.
104
        :param revid: utf8 encoded rev id or an empty string to indicate None
105
        :param committers: 'yes' or 'no'.
106
107
        :return: A SmartServerResponse ('ok',), a encoded body looking like
108
              committers: 1
109
              firstrev: 1234.230 0
110
              latestrev: 345.700 3600
111
              revisions: 2
112
              size:45
113
114
              But containing only fields returned by the gather_stats() call
115
        """
116
        if revid == '':
117
            decoded_revision_id = None
118
        else:
2018.5.83 by Andrew Bennetts
Fix some test failures caused by the switch from unicode to UTF-8-encoded strs for revision IDs.
119
            decoded_revision_id = revid
2018.10.2 by v.ladeuil+lp at free
gather_stats server side and request registration
120
        if committers == 'yes':
121
            decoded_committers = True
122
        else:
123
            decoded_committers = None
124
        stats = repository.gather_stats(decoded_revision_id, decoded_committers)
125
126
        body = ''
127
        if stats.has_key('committers'):
128
            body += 'committers: %d\n' % stats['committers']
129
        if stats.has_key('firstrev'):
130
            body += 'firstrev: %.3f %d\n' % stats['firstrev']
131
        if stats.has_key('latestrev'):
132
             body += 'latestrev: %.3f %d\n' % stats['latestrev']
133
        if stats.has_key('revisions'):
134
            body += 'revisions: %d\n' % stats['revisions']
135
        if stats.has_key('size'):
136
            body += 'size: %d\n' % stats['size']
137
2432.4.5 by Robert Collins
Make using SuccessfulSmartServerResponse and FailedSmartServerResponse mandatory rather than optional in smart server logic.
138
        return SuccessfulSmartServerResponse(('ok', ), body)
2018.10.2 by v.ladeuil+lp at free
gather_stats server side and request registration
139
140
2018.5.57 by Robert Collins
Implement RemoteRepository.is_shared (Robert Collins, Vincent Ladeuil).
141
class SmartServerRepositoryIsShared(SmartServerRepositoryRequest):
142
143
    def do_repository_request(self, repository):
144
        """Return the result of repository.is_shared().
145
146
        :param repository: The repository to query in.
147
        :return: A smart server response of ('yes', ) if the repository is
148
            shared, and ('no', ) if it is not.
149
        """
150
        if repository.is_shared():
2432.4.5 by Robert Collins
Make using SuccessfulSmartServerResponse and FailedSmartServerResponse mandatory rather than optional in smart server logic.
151
            return SuccessfulSmartServerResponse(('yes', ))
2018.5.57 by Robert Collins
Implement RemoteRepository.is_shared (Robert Collins, Vincent Ladeuil).
152
        else:
2432.4.5 by Robert Collins
Make using SuccessfulSmartServerResponse and FailedSmartServerResponse mandatory rather than optional in smart server logic.
153
            return SuccessfulSmartServerResponse(('no', ))
2018.5.78 by Andrew Bennetts
Implement RemoteRepository.lock_write/unlock to expect and send tokens over the
154
155
156
class SmartServerRepositoryLockWrite(SmartServerRepositoryRequest):
157
2018.5.79 by Andrew Bennetts
Implement RemoteBranch.lock_write/unlock as smart operations.
158
    def do_repository_request(self, repository, token=''):
2018.5.78 by Andrew Bennetts
Implement RemoteRepository.lock_write/unlock to expect and send tokens over the
159
        # XXX: this probably should not have a token.
160
        if token == '':
161
            token = None
162
        try:
163
            token = repository.lock_write(token=token)
164
        except errors.LockContention, e:
2432.4.5 by Robert Collins
Make using SuccessfulSmartServerResponse and FailedSmartServerResponse mandatory rather than optional in smart server logic.
165
            return FailedSmartServerResponse(('LockContention',))
2018.5.95 by Andrew Bennetts
Add a Transport.is_readonly remote call, let {Branch,Repository}.lock_write remote call return UnlockableTransport, and miscellaneous test fixes.
166
        except errors.UnlockableTransport:
2432.4.5 by Robert Collins
Make using SuccessfulSmartServerResponse and FailedSmartServerResponse mandatory rather than optional in smart server logic.
167
            return FailedSmartServerResponse(('UnlockableTransport',))
2872.5.3 by Martin Pool
Pass back LockFailed from smart server lock methods
168
        except errors.LockFailed, e:
169
            return FailedSmartServerResponse(('LockFailed',
170
                str(e.lock), str(e.why)))
3015.2.7 by Robert Collins
In the RemoteServer repository methods handle repositories that cannot be remotely locked like pack repositories, and add a read lock in SmartServerRepositoryStreamKnitDataForRevisions.
171
        if token is not None:
172
            repository.leave_lock_in_place()
2018.5.78 by Andrew Bennetts
Implement RemoteRepository.lock_write/unlock to expect and send tokens over the
173
        repository.unlock()
174
        if token is None:
175
            token = ''
2432.4.5 by Robert Collins
Make using SuccessfulSmartServerResponse and FailedSmartServerResponse mandatory rather than optional in smart server logic.
176
        return SuccessfulSmartServerResponse(('ok', token))
2018.5.78 by Andrew Bennetts
Implement RemoteRepository.lock_write/unlock to expect and send tokens over the
177
178
179
class SmartServerRepositoryUnlock(SmartServerRepositoryRequest):
180
181
    def do_repository_request(self, repository, token):
182
        try:
183
            repository.lock_write(token=token)
184
        except errors.TokenMismatch, e:
2432.4.5 by Robert Collins
Make using SuccessfulSmartServerResponse and FailedSmartServerResponse mandatory rather than optional in smart server logic.
185
            return FailedSmartServerResponse(('TokenMismatch',))
2018.5.78 by Andrew Bennetts
Implement RemoteRepository.lock_write/unlock to expect and send tokens over the
186
        repository.dont_leave_lock_in_place()
187
        repository.unlock()
2432.4.5 by Robert Collins
Make using SuccessfulSmartServerResponse and FailedSmartServerResponse mandatory rather than optional in smart server logic.
188
        return SuccessfulSmartServerResponse(('ok',))
2018.5.78 by Andrew Bennetts
Implement RemoteRepository.lock_write/unlock to expect and send tokens over the
189
2018.18.1 by Martin Pool
Add stub Repository.tarball smart method
190
191
class SmartServerRepositoryTarball(SmartServerRepositoryRequest):
2018.18.11 by Martin Pool
merge hpss changes
192
    """Get the raw repository files as a tarball.
193
194
    The returned tarball contains a .bzr control directory which in turn
195
    contains a repository.
2018.18.1 by Martin Pool
Add stub Repository.tarball smart method
196
    
2018.18.2 by Martin Pool
smart method Repository.tarball actually returns the tarball
197
    This takes one parameter, compression, which currently must be 
198
    "", "gz", or "bz2".
2018.18.9 by Martin Pool
remote Repository.tarball builds a temporary directory and tars that
199
200
    This is used to implement the Repository.copy_content_into operation.
2018.18.1 by Martin Pool
Add stub Repository.tarball smart method
201
    """
202
2018.18.2 by Martin Pool
smart method Repository.tarball actually returns the tarball
203
    def do_repository_request(self, repository, compression):
2018.18.9 by Martin Pool
remote Repository.tarball builds a temporary directory and tars that
204
        from bzrlib import osutils
2018.18.2 by Martin Pool
smart method Repository.tarball actually returns the tarball
205
        repo_transport = repository.control_files._transport
2018.18.9 by Martin Pool
remote Repository.tarball builds a temporary directory and tars that
206
        tmp_dirname, tmp_repo = self._copy_to_tempdir(repository)
2018.18.5 by Martin Pool
Repository.tarball locks repository while running for consistency
207
        try:
2018.18.10 by Martin Pool
copy_content_into from Remote repositories by using temporary directories on both ends.
208
            controldir_name = tmp_dirname + '/.bzr'
209
            return self._tarfile_response(controldir_name, compression)
2018.18.9 by Martin Pool
remote Repository.tarball builds a temporary directory and tars that
210
        finally:
211
            osutils.rmtree(tmp_dirname)
212
213
    def _copy_to_tempdir(self, from_repo):
214
        tmp_dirname = tempfile.mkdtemp(prefix='tmpbzrclone')
215
        tmp_bzrdir = from_repo.bzrdir._format.initialize(tmp_dirname)
216
        tmp_repo = from_repo._format.initialize(tmp_bzrdir)
217
        from_repo.copy_content_into(tmp_repo)
218
        return tmp_dirname, tmp_repo
219
2018.18.10 by Martin Pool
copy_content_into from Remote repositories by using temporary directories on both ends.
220
    def _tarfile_response(self, tmp_dirname, compression):
2018.18.2 by Martin Pool
smart method Repository.tarball actually returns the tarball
221
        temp = tempfile.NamedTemporaryFile()
222
        try:
2557.1.1 by Martin Pool
[BUG 119330] Fix tempfile permissions error in smart server tar bundling (under windows) (Martin_)
223
            self._tarball_of_dir(tmp_dirname, compression, temp.file)
2018.18.2 by Martin Pool
smart method Repository.tarball actually returns the tarball
224
            # all finished; write the tempfile out to the network
225
            temp.seek(0)
2420.2.2 by Andrew Bennetts
Merge tarball branch that's already with PQM, resolving conflicts.
226
            return SuccessfulSmartServerResponse(('ok',), temp.read())
2018.18.9 by Martin Pool
remote Repository.tarball builds a temporary directory and tars that
227
            # FIXME: Don't read the whole thing into memory here; rather stream it
228
            # out from the file onto the network. mbp 20070411
2018.18.2 by Martin Pool
smart method Repository.tarball actually returns the tarball
229
        finally:
230
            temp.close()
2018.18.9 by Martin Pool
remote Repository.tarball builds a temporary directory and tars that
231
2557.1.1 by Martin Pool
[BUG 119330] Fix tempfile permissions error in smart server tar bundling (under windows) (Martin_)
232
    def _tarball_of_dir(self, dirname, compression, ofile):
2571.2.2 by Ian Clatworthy
use basename as poolie recommended
233
        filename = os.path.basename(ofile.name)
234
        tarball = tarfile.open(fileobj=ofile, name=filename,
2571.2.1 by Ian Clatworthy
fix #123485 - selftest broken under Python 2.5.1 because of tafile API change
235
            mode='w|' + compression)
2018.18.9 by Martin Pool
remote Repository.tarball builds a temporary directory and tars that
236
        try:
237
            # The tarball module only accepts ascii names, and (i guess)
238
            # packs them with their 8bit names.  We know all the files
239
            # within the repository have ASCII names so the should be safe
240
            # to pack in.
2018.18.10 by Martin Pool
copy_content_into from Remote repositories by using temporary directories on both ends.
241
            dirname = dirname.encode(sys.getfilesystemencoding())
2018.18.9 by Martin Pool
remote Repository.tarball builds a temporary directory and tars that
242
            # python's tarball module includes the whole path by default so
243
            # override it
2018.18.10 by Martin Pool
copy_content_into from Remote repositories by using temporary directories on both ends.
244
            assert dirname.endswith('.bzr')
245
            tarball.add(dirname, '.bzr') # recursive by default
2018.18.9 by Martin Pool
remote Repository.tarball builds a temporary directory and tars that
246
        finally:
247
            tarball.close()
2535.3.12 by Andrew Bennetts
Add a first cut of a get_data_stream method to Repository.
248
249
2535.3.49 by Andrew Bennetts
Rename 'Repository.fetch_revisions' smart request to 'Repository.stream_knit_data_for_revisions'.
250
class SmartServerRepositoryStreamKnitDataForRevisions(SmartServerRepositoryRequest):
2535.3.14 by Andrew Bennetts
Move serialising repo data stream to bytes into smart protocol.
251
252
    def do_repository_request(self, repository, *revision_ids):
3015.2.7 by Robert Collins
In the RemoteServer repository methods handle repositories that cannot be remotely locked like pack repositories, and add a read lock in SmartServerRepositoryStreamKnitDataForRevisions.
253
        repository.lock_read()
254
        try:
255
            return self._do_repository_request(repository, revision_ids)
256
        finally:
257
            repository.unlock()
258
259
    def _do_repository_request(self, repository, revision_ids):
2535.3.14 by Andrew Bennetts
Move serialising repo data stream to bytes into smart protocol.
260
        stream = repository.get_data_stream(revision_ids)
261
        filelike = StringIO()
262
        pack = ContainerWriter(filelike.write)
263
        pack.begin()
2535.3.40 by Andrew Bennetts
Tidy up more XXXs.
264
        try:
2535.3.50 by Andrew Bennetts
Use tuple names in data streams rather than concatenated strings.
265
            for name_tuple, bytes in stream:
266
                pack.add_bytes_record(bytes, [name_tuple])
2535.3.40 by Andrew Bennetts
Tidy up more XXXs.
267
        except errors.RevisionNotPresent, e:
268
            return FailedSmartServerResponse(('NoSuchRevision', e.revision_id))
2535.3.14 by Andrew Bennetts
Move serialising repo data stream to bytes into smart protocol.
269
        pack.end()
270
        return SuccessfulSmartServerResponse(('ok',), filelike.getvalue())
2535.3.15 by Andrew Bennetts
Add KnitVersionedFile.get_stream_as_bytes, start smart implementation of RemoteRepository.get_data_stream.
271