1
# Copyright (C) 2006 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
"""RemoteTransport client for the smart-server.
19
This module shouldn't be accessed directly. The classes defined here should be
20
imported from bzrlib.smart.
23
__all__ = ['RemoteTransport', 'RemoteTCPTransport', 'RemoteSSHTransport']
25
from cStringIO import StringIO
36
from bzrlib.smart import client, medium, protocol
38
# must do this otherwise urllib can't parse the urls properly :(
39
for scheme in ['ssh', 'bzr', 'bzr+loopback', 'bzr+ssh', 'bzr+http']:
40
transport.register_urlparse_netloc_protocol(scheme)
44
# Port 4155 is the default port for bzr://, registered with IANA.
45
BZR_DEFAULT_INTERFACE = '0.0.0.0'
46
BZR_DEFAULT_PORT = 4155
49
class _SmartStat(object):
51
def __init__(self, size, mode):
56
class RemoteTransport(transport.ConnectedTransport):
57
"""Connection to a smart server.
59
The connection holds references to the medium that can be used to send
60
requests to the server.
62
The connection has a notion of the current directory to which it's
63
connected; this is incorporated in filenames passed to the server.
65
This supports some higher-level RPC operations and can also be treated
66
like a Transport to do file-like operations.
68
The connection can be made over a tcp socket, an ssh pipe or a series of
69
http requests. There are concrete subclasses for each type:
70
RemoteTCPTransport, etc.
73
# IMPORTANT FOR IMPLEMENTORS: RemoteTransport MUST NOT be given encoding
74
# responsibilities: Put those on SmartClient or similar. This is vital for
75
# the ability to support multiple versions of the smart protocol over time:
76
# RemoteTransport is an adapter from the Transport object model to the
77
# SmartClient model, not an encoder.
79
# FIXME: the medium parameter should be private, only the tests requires
80
# it. It may be even clearer to define a TestRemoteTransport that handles
81
# the specific cases of providing a _client and/or a _medium, and leave
82
# RemoteTransport as an abstract class.
83
def __init__(self, url, _from_transport=None, medium=None, _client=None):
86
:param _from_transport: Another RemoteTransport instance that this
87
one is being cloned from. Attributes such as the medium will
90
:param medium: The medium to use for this RemoteTransport. This must be
91
supplied if _from_transport is None.
93
:param _client: Override the _SmartClient used by this transport. This
94
should only be used for testing purposes; normally this is
95
determined from the medium.
97
super(RemoteTransport, self).__init__(url,
98
_from_transport=_from_transport)
100
# The medium is the connection, except when we need to share it with
101
# other objects (RemoteBzrDir, RemoteRepository etc). In these cases
102
# what we want to share is really the shared connection.
104
if _from_transport is None:
105
# If no _from_transport is specified, we need to intialize the
109
medium, credentials = self._build_medium()
110
if 'hpss' in debug.debug_flags:
111
trace.mutter('hpss: Built a new medium: %s',
112
medium.__class__.__name__)
113
self._shared_connection = transport._SharedConnection(medium,
117
self._client = client._SmartClient(self.get_shared_medium())
119
self._client = _client
121
def _build_medium(self):
122
"""Create the medium if _from_transport does not provide one.
124
The medium is analogous to the connection for ConnectedTransport: it
125
allows connection sharing.
130
def is_readonly(self):
131
"""Smart server transport can do read/write file operations."""
132
resp = self._call2('Transport.is_readonly')
133
if resp == ('yes', ):
135
elif resp == ('no', ):
137
elif (resp == ('error', "Generic bzr smart protocol error: "
138
"bad request 'Transport.is_readonly'") or
139
resp == ('error', "Generic bzr smart protocol error: "
140
"bad request u'Transport.is_readonly'")):
141
# XXX: nasty hack: servers before 0.16 don't have a
142
# 'Transport.is_readonly' verb, so we do what clients before 0.16
146
self._translate_error(resp)
147
raise errors.UnexpectedSmartServerResponse(resp)
149
def get_smart_client(self):
150
return self._get_connection()
152
def get_smart_medium(self):
153
return self._get_connection()
155
def get_shared_medium(self):
156
return self._get_shared_connection()
158
def _remote_path(self, relpath):
159
"""Returns the Unicode version of the absolute path for relpath."""
160
return self._combine_paths(self._path, relpath)
162
def _call(self, method, *args):
163
resp = self._call2(method, *args)
164
self._translate_error(resp)
166
def _call2(self, method, *args):
167
"""Call a method on the remote server."""
168
return self._client.call(method, *args)
170
def _call_with_body_bytes(self, method, args, body):
171
"""Call a method on the remote server with body bytes."""
172
return self._client.call_with_body_bytes(method, args, body)
174
def has(self, relpath):
175
"""Indicate whether a remote file of the given name exists or not.
177
:see: Transport.has()
179
resp = self._call2('has', self._remote_path(relpath))
180
if resp == ('yes', ):
182
elif resp == ('no', ):
185
self._translate_error(resp)
187
def get(self, relpath):
188
"""Return file-like object reading the contents of a remote file.
190
:see: Transport.get_bytes()/get_file()
192
return StringIO(self.get_bytes(relpath))
194
def get_bytes(self, relpath):
195
remote = self._remote_path(relpath)
196
request = self.get_smart_medium().get_request()
197
smart_protocol = protocol.SmartClientRequestProtocolOne(request)
198
smart_protocol.call('get', remote)
199
resp = smart_protocol.read_response_tuple(True)
201
smart_protocol.cancel_read_body()
202
self._translate_error(resp, relpath)
203
return smart_protocol.read_body_bytes()
205
def _serialise_optional_mode(self, mode):
211
def mkdir(self, relpath, mode=None):
212
resp = self._call2('mkdir', self._remote_path(relpath),
213
self._serialise_optional_mode(mode))
214
self._translate_error(resp)
216
def open_write_stream(self, relpath, mode=None):
217
"""See Transport.open_write_stream."""
218
self.put_bytes(relpath, "", mode)
219
result = transport.AppendBasedFileStream(self, relpath)
220
transport._file_streams[self.abspath(relpath)] = result
223
def put_bytes(self, relpath, upload_contents, mode=None):
224
# FIXME: upload_file is probably not safe for non-ascii characters -
225
# should probably just pass all parameters as length-delimited
227
if type(upload_contents) is unicode:
228
# Although not strictly correct, we raise UnicodeEncodeError to be
229
# compatible with other transports.
230
raise UnicodeEncodeError(
231
'undefined', upload_contents, 0, 1,
232
'put_bytes must be given bytes, not unicode.')
233
resp = self._call_with_body_bytes('put',
234
(self._remote_path(relpath), self._serialise_optional_mode(mode)),
236
self._translate_error(resp)
238
def put_bytes_non_atomic(self, relpath, bytes, mode=None,
239
create_parent_dir=False,
241
"""See Transport.put_bytes_non_atomic."""
242
# FIXME: no encoding in the transport!
243
create_parent_str = 'F'
244
if create_parent_dir:
245
create_parent_str = 'T'
247
resp = self._call_with_body_bytes(
249
(self._remote_path(relpath), self._serialise_optional_mode(mode),
250
create_parent_str, self._serialise_optional_mode(dir_mode)),
252
self._translate_error(resp)
254
def put_file(self, relpath, upload_file, mode=None):
255
# its not ideal to seek back, but currently put_non_atomic_file depends
256
# on transports not reading before failing - which is a faulty
257
# assumption I think - RBC 20060915
258
pos = upload_file.tell()
260
return self.put_bytes(relpath, upload_file.read(), mode)
262
upload_file.seek(pos)
265
def put_file_non_atomic(self, relpath, f, mode=None,
266
create_parent_dir=False,
268
return self.put_bytes_non_atomic(relpath, f.read(), mode=mode,
269
create_parent_dir=create_parent_dir,
272
def append_file(self, relpath, from_file, mode=None):
273
return self.append_bytes(relpath, from_file.read(), mode)
275
def append_bytes(self, relpath, bytes, mode=None):
276
resp = self._call_with_body_bytes(
278
(self._remote_path(relpath), self._serialise_optional_mode(mode)),
280
if resp[0] == 'appended':
282
self._translate_error(resp)
284
def delete(self, relpath):
285
resp = self._call2('delete', self._remote_path(relpath))
286
self._translate_error(resp)
288
def external_url(self):
289
"""See bzrlib.transport.Transport.external_url."""
290
# the external path for RemoteTransports is the base
293
def readv(self, relpath, offsets):
297
offsets = list(offsets)
299
sorted_offsets = sorted(offsets)
300
# turn the list of offsets into a stack
301
offset_stack = iter(offsets)
302
cur_offset_and_size = offset_stack.next()
303
coalesced = list(self._coalesce_offsets(sorted_offsets,
304
limit=self._max_readv_combine,
305
fudge_factor=self._bytes_to_read_before_seek))
307
request = self.get_smart_medium().get_request()
308
smart_protocol = protocol.SmartClientRequestProtocolOne(request)
309
smart_protocol.call_with_body_readv_array(
310
('readv', self._remote_path(relpath)),
311
[(c.start, c.length) for c in coalesced])
312
resp = smart_protocol.read_response_tuple(True)
314
if resp[0] != 'readv':
315
# This should raise an exception
316
smart_protocol.cancel_read_body()
317
self._translate_error(resp)
320
# FIXME: this should know how many bytes are needed, for clarity.
321
data = smart_protocol.read_body_bytes()
322
# Cache the results, but only until they have been fulfilled
324
for c_offset in coalesced:
325
if len(data) < c_offset.length:
326
raise errors.ShortReadvError(relpath, c_offset.start,
327
c_offset.length, actual=len(data))
328
for suboffset, subsize in c_offset.ranges:
329
key = (c_offset.start+suboffset, subsize)
330
data_map[key] = data[suboffset:suboffset+subsize]
331
data = data[c_offset.length:]
333
# Now that we've read some data, see if we can yield anything back
334
while cur_offset_and_size in data_map:
335
this_data = data_map.pop(cur_offset_and_size)
336
yield cur_offset_and_size[0], this_data
337
cur_offset_and_size = offset_stack.next()
339
def rename(self, rel_from, rel_to):
341
self._remote_path(rel_from),
342
self._remote_path(rel_to))
344
def move(self, rel_from, rel_to):
346
self._remote_path(rel_from),
347
self._remote_path(rel_to))
349
def rmdir(self, relpath):
350
resp = self._call('rmdir', self._remote_path(relpath))
352
def _translate_error(self, resp, orig_path=None):
353
"""Raise an exception from a response"""
360
elif what == 'NoSuchFile':
361
if orig_path is not None:
362
error_path = orig_path
365
raise errors.NoSuchFile(error_path)
366
elif what == 'error':
367
raise errors.SmartProtocolError(unicode(resp[1]))
368
elif what == 'FileExists':
369
raise errors.FileExists(resp[1])
370
elif what == 'DirectoryNotEmpty':
371
raise errors.DirectoryNotEmpty(resp[1])
372
elif what == 'ShortReadvError':
373
raise errors.ShortReadvError(resp[1], int(resp[2]),
374
int(resp[3]), int(resp[4]))
375
elif what in ('UnicodeEncodeError', 'UnicodeDecodeError'):
376
encoding = str(resp[1]) # encoding must always be a string
380
reason = str(resp[5]) # reason must always be a string
381
if val.startswith('u:'):
382
val = val[2:].decode('utf-8')
383
elif val.startswith('s:'):
384
val = val[2:].decode('base64')
385
if what == 'UnicodeDecodeError':
386
raise UnicodeDecodeError(encoding, val, start, end, reason)
387
elif what == 'UnicodeEncodeError':
388
raise UnicodeEncodeError(encoding, val, start, end, reason)
389
elif what == "ReadOnlyError":
390
raise errors.TransportNotPossible('readonly transport')
391
elif what == "ReadError":
392
if orig_path is not None:
393
error_path = orig_path
396
raise errors.ReadError(error_path)
398
raise errors.SmartProtocolError('unexpected smart server error: %r' % (resp,))
400
def disconnect(self):
401
self.get_smart_medium().disconnect()
403
def delete_tree(self, relpath):
404
raise errors.TransportNotPossible('readonly transport')
406
def stat(self, relpath):
407
resp = self._call2('stat', self._remote_path(relpath))
408
if resp[0] == 'stat':
409
return _SmartStat(int(resp[1]), int(resp[2], 8))
411
self._translate_error(resp)
413
## def lock_read(self, relpath):
414
## """Lock the given file for shared (read) access.
415
## :return: A lock object, which should be passed to Transport.unlock()
417
## # The old RemoteBranch ignore lock for reading, so we will
418
## # continue that tradition and return a bogus lock object.
419
## class BogusLock(object):
420
## def __init__(self, path):
424
## return BogusLock(relpath)
429
def list_dir(self, relpath):
430
resp = self._call2('list_dir', self._remote_path(relpath))
431
if resp[0] == 'names':
432
return [name.encode('ascii') for name in resp[1:]]
434
self._translate_error(resp)
436
def iter_files_recursive(self):
437
resp = self._call2('iter_files_recursive', self._remote_path(''))
438
if resp[0] == 'names':
441
self._translate_error(resp)
444
class RemoteTCPTransport(RemoteTransport):
445
"""Connection to smart server over plain tcp.
447
This is essentially just a factory to get 'RemoteTransport(url,
448
SmartTCPClientMedium).
451
def _build_medium(self):
452
assert self.base.startswith('bzr://')
453
if self._port is None:
454
self._port = BZR_DEFAULT_PORT
455
return medium.SmartTCPClientMedium(self._host, self._port), None
458
class RemoteSSHTransport(RemoteTransport):
459
"""Connection to smart server over SSH.
461
This is essentially just a factory to get 'RemoteTransport(url,
462
SmartSSHClientMedium).
465
def _build_medium(self):
466
assert self.base.startswith('bzr+ssh://')
467
# ssh will prompt the user for a password if needed and if none is
468
# provided but it will not give it back, so no credentials can be
470
return medium.SmartSSHClientMedium(self._host, self._port,
471
self._user, self._password), None
474
class RemoteHTTPTransport(RemoteTransport):
475
"""Just a way to connect between a bzr+http:// url and http://.
477
This connection operates slightly differently than the RemoteSSHTransport.
478
It uses a plain http:// transport underneath, which defines what remote
479
.bzr/smart URL we are connected to. From there, all paths that are sent are
480
sent as relative paths, this way, the remote side can properly
481
de-reference them, since it is likely doing rewrite rules to translate an
482
HTTP path into a local path.
485
def __init__(self, base, _from_transport=None, http_transport=None):
486
assert base.startswith('bzr+http://')
488
if http_transport is None:
489
# FIXME: the password may be lost here because it appears in the
490
# url only for an intial construction (when the url came from the
492
http_url = base[len('bzr+'):]
493
self._http_transport = transport.get_transport(http_url)
495
self._http_transport = http_transport
496
super(RemoteHTTPTransport, self).__init__(
497
base, _from_transport=_from_transport)
499
def _build_medium(self):
500
# We let http_transport take care of the credentials
501
return self._http_transport.get_smart_medium(), None
503
def _remote_path(self, relpath):
504
"""After connecting, HTTP Transport only deals in relative URLs."""
505
# Adjust the relpath based on which URL this smart transport is
507
http_base = urlutils.normalize_url(self._http_transport.base)
508
url = urlutils.join(self.base[len('bzr+'):], relpath)
509
url = urlutils.normalize_url(url)
510
return urlutils.relative_url(http_base, url)
512
def clone(self, relative_url):
513
"""Make a new RemoteHTTPTransport related to me.
515
This is re-implemented rather than using the default
516
RemoteTransport.clone() because we must be careful about the underlying
519
Also, the cloned smart transport will POST to the same .bzr/smart
520
location as this transport (although obviously the relative paths in the
521
smart requests may be different). This is so that the server doesn't
522
have to handle .bzr/smart requests at arbitrary places inside .bzr
523
directories, just at the initial URL the user uses.
525
The exception is parent paths (i.e. relative_url of "..").
528
abs_url = self.abspath(relative_url)
531
# We either use the exact same http_transport (for child locations), or
532
# a clone of the underlying http_transport (for parent locations). This
533
# means we share the connection.
534
norm_base = urlutils.normalize_url(self.base)
535
norm_abs_url = urlutils.normalize_url(abs_url)
536
normalized_rel_url = urlutils.relative_url(norm_base, norm_abs_url)
537
if normalized_rel_url == ".." or normalized_rel_url.startswith("../"):
538
http_transport = self._http_transport.clone(normalized_rel_url)
540
http_transport = self._http_transport
541
return RemoteHTTPTransport(abs_url,
542
_from_transport=self,
543
http_transport=http_transport)
546
def get_test_permutations():
547
"""Return (transport, server) permutations for testing."""
548
### We may need a little more test framework support to construct an
549
### appropriate RemoteTransport in the future.
550
from bzrlib.smart import server
551
return [(RemoteTCPTransport, server.SmartTCPServer_for_testing)]
added
removed
bzrlib/transport/remote.py
