1
# Copyright (C) 2006 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
"""RemoteTransport client for the smart-server.
19
This module shouldn't be accessed directly. The classes defined here should be
20
imported from bzrlib.smart.
23
__all__ = ['RemoteTransport', 'RemoteTCPTransport', 'RemoteSSHTransport']
25
from cStringIO import StringIO
36
from bzrlib.smart import client, medium, protocol
38
# must do this otherwise urllib can't parse the urls properly :(
39
for scheme in ['ssh', 'bzr', 'bzr+loopback', 'bzr+ssh', 'bzr+http']:
40
transport.register_urlparse_netloc_protocol(scheme)
44
# Port 4155 is the default port for bzr://, registered with IANA.
45
BZR_DEFAULT_INTERFACE = '0.0.0.0'
46
BZR_DEFAULT_PORT = 4155
49
class _SmartStat(object):
51
def __init__(self, size, mode):
56
class RemoteTransport(transport.ConnectedTransport):
57
"""Connection to a smart server.
59
The connection holds references to the medium that can be used to send
60
requests to the server.
62
The connection has a notion of the current directory to which it's
63
connected; this is incorporated in filenames passed to the server.
65
This supports some higher-level RPC operations and can also be treated
66
like a Transport to do file-like operations.
68
The connection can be made over a tcp socket, an ssh pipe or a series of
69
http requests. There are concrete subclasses for each type:
70
RemoteTCPTransport, etc.
73
# IMPORTANT FOR IMPLEMENTORS: RemoteTransport MUST NOT be given encoding
74
# responsibilities: Put those on SmartClient or similar. This is vital for
75
# the ability to support multiple versions of the smart protocol over time:
76
# RemoteTransport is an adapter from the Transport object model to the
77
# SmartClient model, not an encoder.
79
# FIXME: the medium parameter should be private, only the tests requires
80
# it. It may be even clearer to define a TestRemoteTransport that handles
81
# the specific cases of providing a _client and/or a _medium, and leave
82
# RemoteTransport as an abstract class.
83
def __init__(self, url, _from_transport=None, medium=None, _client=None):
86
:param _from_transport: Another RemoteTransport instance that this
87
one is being cloned from. Attributes such as the medium will
90
:param medium: The medium to use for this RemoteTransport. This must be
91
supplied if _from_transport is None.
93
:param _client: Override the _SmartClient used by this transport. This
94
should only be used for testing purposes; normally this is
95
determined from the medium.
97
super(RemoteTransport, self).__init__(url,
98
_from_transport=_from_transport)
100
# The medium is the connection, except when we need to share it with
101
# other objects (RemoteBzrDir, RemoteRepository etc). In these cases
102
# what we want to share is really the shared connection.
104
if _from_transport is None:
105
# If no _from_transport is specified, we need to intialize the
109
medium, credentials = self._build_medium()
110
if 'hpss' in debug.debug_flags:
111
trace.mutter('hpss: Built a new medium: %s',
112
medium.__class__.__name__)
113
self._shared_connection = transport._SharedConnection(medium,
117
self._client = client._SmartClient(self.get_shared_medium())
119
self._client = _client
121
def _build_medium(self):
122
"""Create the medium if _from_transport does not provide one.
124
The medium is analogous to the connection for ConnectedTransport: it
125
allows connection sharing.
130
def close_file_stream(self, relpath):
131
"""See Transport.close_file_stream."""
132
del transport._file_streams[self.abspath(relpath)]
134
def is_readonly(self):
135
"""Smart server transport can do read/write file operations."""
136
resp = self._call2('Transport.is_readonly')
137
if resp == ('yes', ):
139
elif resp == ('no', ):
141
elif (resp == ('error', "Generic bzr smart protocol error: "
142
"bad request 'Transport.is_readonly'") or
143
resp == ('error', "Generic bzr smart protocol error: "
144
"bad request u'Transport.is_readonly'")):
145
# XXX: nasty hack: servers before 0.16 don't have a
146
# 'Transport.is_readonly' verb, so we do what clients before 0.16
150
self._translate_error(resp)
151
raise errors.UnexpectedSmartServerResponse(resp)
153
def get_smart_client(self):
154
return self._get_connection()
156
def get_smart_medium(self):
157
return self._get_connection()
159
def get_shared_medium(self):
160
return self._get_shared_connection()
162
def _remote_path(self, relpath):
163
"""Returns the Unicode version of the absolute path for relpath."""
164
return self._combine_paths(self._path, relpath)
166
def _call(self, method, *args):
167
resp = self._call2(method, *args)
168
self._translate_error(resp)
170
def _call2(self, method, *args):
171
"""Call a method on the remote server."""
172
return self._client.call(method, *args)
174
def _call_with_body_bytes(self, method, args, body):
175
"""Call a method on the remote server with body bytes."""
176
return self._client.call_with_body_bytes(method, args, body)
178
def has(self, relpath):
179
"""Indicate whether a remote file of the given name exists or not.
181
:see: Transport.has()
183
resp = self._call2('has', self._remote_path(relpath))
184
if resp == ('yes', ):
186
elif resp == ('no', ):
189
self._translate_error(resp)
191
def get(self, relpath):
192
"""Return file-like object reading the contents of a remote file.
194
:see: Transport.get_bytes()/get_file()
196
return StringIO(self.get_bytes(relpath))
198
def get_bytes(self, relpath):
199
remote = self._remote_path(relpath)
200
request = self.get_smart_medium().get_request()
201
smart_protocol = protocol.SmartClientRequestProtocolOne(request)
202
smart_protocol.call('get', remote)
203
resp = smart_protocol.read_response_tuple(True)
205
smart_protocol.cancel_read_body()
206
self._translate_error(resp, relpath)
207
return smart_protocol.read_body_bytes()
209
def _serialise_optional_mode(self, mode):
215
def mkdir(self, relpath, mode=None):
216
resp = self._call2('mkdir', self._remote_path(relpath),
217
self._serialise_optional_mode(mode))
218
self._translate_error(resp)
220
def open_file_stream(self, relpath):
221
"""See Transport.open_file_stream."""
222
def append_data(bytes):
223
self.append_bytes(relpath, bytes)
224
self.put_bytes(relpath, "")
225
transport._file_streams[self.abspath(relpath)] = append_data
228
def put_bytes(self, relpath, upload_contents, mode=None):
229
# FIXME: upload_file is probably not safe for non-ascii characters -
230
# should probably just pass all parameters as length-delimited
232
if type(upload_contents) is unicode:
233
# Although not strictly correct, we raise UnicodeEncodeError to be
234
# compatible with other transports.
235
raise UnicodeEncodeError(
236
'undefined', upload_contents, 0, 1,
237
'put_bytes must be given bytes, not unicode.')
238
resp = self._call_with_body_bytes('put',
239
(self._remote_path(relpath), self._serialise_optional_mode(mode)),
241
self._translate_error(resp)
243
def put_bytes_non_atomic(self, relpath, bytes, mode=None,
244
create_parent_dir=False,
246
"""See Transport.put_bytes_non_atomic."""
247
# FIXME: no encoding in the transport!
248
create_parent_str = 'F'
249
if create_parent_dir:
250
create_parent_str = 'T'
252
resp = self._call_with_body_bytes(
254
(self._remote_path(relpath), self._serialise_optional_mode(mode),
255
create_parent_str, self._serialise_optional_mode(dir_mode)),
257
self._translate_error(resp)
259
def put_file(self, relpath, upload_file, mode=None):
260
# its not ideal to seek back, but currently put_non_atomic_file depends
261
# on transports not reading before failing - which is a faulty
262
# assumption I think - RBC 20060915
263
pos = upload_file.tell()
265
return self.put_bytes(relpath, upload_file.read(), mode)
267
upload_file.seek(pos)
270
def put_file_non_atomic(self, relpath, f, mode=None,
271
create_parent_dir=False,
273
return self.put_bytes_non_atomic(relpath, f.read(), mode=mode,
274
create_parent_dir=create_parent_dir,
277
def append_file(self, relpath, from_file, mode=None):
278
return self.append_bytes(relpath, from_file.read(), mode)
280
def append_bytes(self, relpath, bytes, mode=None):
281
resp = self._call_with_body_bytes(
283
(self._remote_path(relpath), self._serialise_optional_mode(mode)),
285
if resp[0] == 'appended':
287
self._translate_error(resp)
289
def delete(self, relpath):
290
resp = self._call2('delete', self._remote_path(relpath))
291
self._translate_error(resp)
293
def external_url(self):
294
"""See bzrlib.transport.Transport.external_url."""
295
# the external path for RemoteTransports is the base
298
def readv(self, relpath, offsets):
302
offsets = list(offsets)
304
sorted_offsets = sorted(offsets)
305
# turn the list of offsets into a stack
306
offset_stack = iter(offsets)
307
cur_offset_and_size = offset_stack.next()
308
coalesced = list(self._coalesce_offsets(sorted_offsets,
309
limit=self._max_readv_combine,
310
fudge_factor=self._bytes_to_read_before_seek))
312
request = self.get_smart_medium().get_request()
313
smart_protocol = protocol.SmartClientRequestProtocolOne(request)
314
smart_protocol.call_with_body_readv_array(
315
('readv', self._remote_path(relpath)),
316
[(c.start, c.length) for c in coalesced])
317
resp = smart_protocol.read_response_tuple(True)
319
if resp[0] != 'readv':
320
# This should raise an exception
321
smart_protocol.cancel_read_body()
322
self._translate_error(resp)
325
# FIXME: this should know how many bytes are needed, for clarity.
326
data = smart_protocol.read_body_bytes()
327
# Cache the results, but only until they have been fulfilled
329
for c_offset in coalesced:
330
if len(data) < c_offset.length:
331
raise errors.ShortReadvError(relpath, c_offset.start,
332
c_offset.length, actual=len(data))
333
for suboffset, subsize in c_offset.ranges:
334
key = (c_offset.start+suboffset, subsize)
335
data_map[key] = data[suboffset:suboffset+subsize]
336
data = data[c_offset.length:]
338
# Now that we've read some data, see if we can yield anything back
339
while cur_offset_and_size in data_map:
340
this_data = data_map.pop(cur_offset_and_size)
341
yield cur_offset_and_size[0], this_data
342
cur_offset_and_size = offset_stack.next()
344
def rename(self, rel_from, rel_to):
346
self._remote_path(rel_from),
347
self._remote_path(rel_to))
349
def move(self, rel_from, rel_to):
351
self._remote_path(rel_from),
352
self._remote_path(rel_to))
354
def rmdir(self, relpath):
355
resp = self._call('rmdir', self._remote_path(relpath))
357
def _translate_error(self, resp, orig_path=None):
358
"""Raise an exception from a response"""
365
elif what == 'NoSuchFile':
366
if orig_path is not None:
367
error_path = orig_path
370
raise errors.NoSuchFile(error_path)
371
elif what == 'error':
372
raise errors.SmartProtocolError(unicode(resp[1]))
373
elif what == 'FileExists':
374
raise errors.FileExists(resp[1])
375
elif what == 'DirectoryNotEmpty':
376
raise errors.DirectoryNotEmpty(resp[1])
377
elif what == 'ShortReadvError':
378
raise errors.ShortReadvError(resp[1], int(resp[2]),
379
int(resp[3]), int(resp[4]))
380
elif what in ('UnicodeEncodeError', 'UnicodeDecodeError'):
381
encoding = str(resp[1]) # encoding must always be a string
385
reason = str(resp[5]) # reason must always be a string
386
if val.startswith('u:'):
387
val = val[2:].decode('utf-8')
388
elif val.startswith('s:'):
389
val = val[2:].decode('base64')
390
if what == 'UnicodeDecodeError':
391
raise UnicodeDecodeError(encoding, val, start, end, reason)
392
elif what == 'UnicodeEncodeError':
393
raise UnicodeEncodeError(encoding, val, start, end, reason)
394
elif what == "ReadOnlyError":
395
raise errors.TransportNotPossible('readonly transport')
396
elif what == "ReadError":
397
if orig_path is not None:
398
error_path = orig_path
401
raise errors.ReadError(error_path)
403
raise errors.SmartProtocolError('unexpected smart server error: %r' % (resp,))
405
def disconnect(self):
406
self.get_smart_medium().disconnect()
408
def delete_tree(self, relpath):
409
raise errors.TransportNotPossible('readonly transport')
411
def stat(self, relpath):
412
resp = self._call2('stat', self._remote_path(relpath))
413
if resp[0] == 'stat':
414
return _SmartStat(int(resp[1]), int(resp[2], 8))
416
self._translate_error(resp)
418
## def lock_read(self, relpath):
419
## """Lock the given file for shared (read) access.
420
## :return: A lock object, which should be passed to Transport.unlock()
422
## # The old RemoteBranch ignore lock for reading, so we will
423
## # continue that tradition and return a bogus lock object.
424
## class BogusLock(object):
425
## def __init__(self, path):
429
## return BogusLock(relpath)
434
def list_dir(self, relpath):
435
resp = self._call2('list_dir', self._remote_path(relpath))
436
if resp[0] == 'names':
437
return [name.encode('ascii') for name in resp[1:]]
439
self._translate_error(resp)
441
def iter_files_recursive(self):
442
resp = self._call2('iter_files_recursive', self._remote_path(''))
443
if resp[0] == 'names':
446
self._translate_error(resp)
449
class RemoteTCPTransport(RemoteTransport):
450
"""Connection to smart server over plain tcp.
452
This is essentially just a factory to get 'RemoteTransport(url,
453
SmartTCPClientMedium).
456
def _build_medium(self):
457
assert self.base.startswith('bzr://')
458
if self._port is None:
459
self._port = BZR_DEFAULT_PORT
460
return medium.SmartTCPClientMedium(self._host, self._port), None
463
class RemoteSSHTransport(RemoteTransport):
464
"""Connection to smart server over SSH.
466
This is essentially just a factory to get 'RemoteTransport(url,
467
SmartSSHClientMedium).
470
def _build_medium(self):
471
assert self.base.startswith('bzr+ssh://')
472
# ssh will prompt the user for a password if needed and if none is
473
# provided but it will not give it back, so no credentials can be
475
return medium.SmartSSHClientMedium(self._host, self._port,
476
self._user, self._password), None
479
class RemoteHTTPTransport(RemoteTransport):
480
"""Just a way to connect between a bzr+http:// url and http://.
482
This connection operates slightly differently than the RemoteSSHTransport.
483
It uses a plain http:// transport underneath, which defines what remote
484
.bzr/smart URL we are connected to. From there, all paths that are sent are
485
sent as relative paths, this way, the remote side can properly
486
de-reference them, since it is likely doing rewrite rules to translate an
487
HTTP path into a local path.
490
def __init__(self, base, _from_transport=None, http_transport=None):
491
assert base.startswith('bzr+http://')
493
if http_transport is None:
494
# FIXME: the password may be lost here because it appears in the
495
# url only for an intial construction (when the url came from the
497
http_url = base[len('bzr+'):]
498
self._http_transport = transport.get_transport(http_url)
500
self._http_transport = http_transport
501
super(RemoteHTTPTransport, self).__init__(
502
base, _from_transport=_from_transport)
504
def _build_medium(self):
505
# We let http_transport take care of the credentials
506
return self._http_transport.get_smart_medium(), None
508
def _remote_path(self, relpath):
509
"""After connecting, HTTP Transport only deals in relative URLs."""
510
# Adjust the relpath based on which URL this smart transport is
512
http_base = urlutils.normalize_url(self._http_transport.base)
513
url = urlutils.join(self.base[len('bzr+'):], relpath)
514
url = urlutils.normalize_url(url)
515
return urlutils.relative_url(http_base, url)
517
def clone(self, relative_url):
518
"""Make a new RemoteHTTPTransport related to me.
520
This is re-implemented rather than using the default
521
RemoteTransport.clone() because we must be careful about the underlying
524
Also, the cloned smart transport will POST to the same .bzr/smart
525
location as this transport (although obviously the relative paths in the
526
smart requests may be different). This is so that the server doesn't
527
have to handle .bzr/smart requests at arbitrary places inside .bzr
528
directories, just at the initial URL the user uses.
530
The exception is parent paths (i.e. relative_url of "..").
533
abs_url = self.abspath(relative_url)
536
# We either use the exact same http_transport (for child locations), or
537
# a clone of the underlying http_transport (for parent locations). This
538
# means we share the connection.
539
norm_base = urlutils.normalize_url(self.base)
540
norm_abs_url = urlutils.normalize_url(abs_url)
541
normalized_rel_url = urlutils.relative_url(norm_base, norm_abs_url)
542
if normalized_rel_url == ".." or normalized_rel_url.startswith("../"):
543
http_transport = self._http_transport.clone(normalized_rel_url)
545
http_transport = self._http_transport
546
return RemoteHTTPTransport(abs_url,
547
_from_transport=self,
548
http_transport=http_transport)
551
def get_test_permutations():
552
"""Return (transport, server) permutations for testing."""
553
### We may need a little more test framework support to construct an
554
### appropriate RemoteTransport in the future.
555
from bzrlib.smart import server
556
return [(RemoteTCPTransport, server.SmartTCPServer_for_testing)]
added
removed
bzrlib/transport/remote.py
