1
# Copyright (C) 2006 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
"""RemoteTransport client for the smart-server.
19
This module shouldn't be accessed directly. The classes defined here should be
20
imported from bzrlib.smart.
23
__all__ = ['RemoteTransport', 'RemoteTCPTransport', 'RemoteSSHTransport']
25
from cStringIO import StringIO
37
from bzrlib.smart import client, medium, protocol
39
# must do this otherwise urllib can't parse the urls properly :(
40
for scheme in ['ssh', 'bzr', 'bzr+loopback', 'bzr+ssh', 'bzr+http', 'bzr+https']:
41
transport.register_urlparse_netloc_protocol(scheme)
45
# Port 4155 is the default port for bzr://, registered with IANA.
46
BZR_DEFAULT_INTERFACE = '0.0.0.0'
47
BZR_DEFAULT_PORT = 4155
50
class _SmartStat(object):
52
def __init__(self, size, mode):
57
class RemoteTransport(transport.ConnectedTransport):
58
"""Connection to a smart server.
60
The connection holds references to the medium that can be used to send
61
requests to the server.
63
The connection has a notion of the current directory to which it's
64
connected; this is incorporated in filenames passed to the server.
66
This supports some higher-level RPC operations and can also be treated
67
like a Transport to do file-like operations.
69
The connection can be made over a tcp socket, an ssh pipe or a series of
70
http requests. There are concrete subclasses for each type:
71
RemoteTCPTransport, etc.
74
# IMPORTANT FOR IMPLEMENTORS: RemoteTransport MUST NOT be given encoding
75
# responsibilities: Put those on SmartClient or similar. This is vital for
76
# the ability to support multiple versions of the smart protocol over time:
77
# RemoteTransport is an adapter from the Transport object model to the
78
# SmartClient model, not an encoder.
80
# FIXME: the medium parameter should be private, only the tests requires
81
# it. It may be even clearer to define a TestRemoteTransport that handles
82
# the specific cases of providing a _client and/or a _medium, and leave
83
# RemoteTransport as an abstract class.
84
def __init__(self, url, _from_transport=None, medium=None, _client=None):
87
:param _from_transport: Another RemoteTransport instance that this
88
one is being cloned from. Attributes such as the medium will
91
:param medium: The medium to use for this RemoteTransport. This must be
92
supplied if _from_transport is None.
94
:param _client: Override the _SmartClient used by this transport. This
95
should only be used for testing purposes; normally this is
96
determined from the medium.
98
super(RemoteTransport, self).__init__(url,
99
_from_transport=_from_transport)
101
# The medium is the connection, except when we need to share it with
102
# other objects (RemoteBzrDir, RemoteRepository etc). In these cases
103
# what we want to share is really the shared connection.
105
if _from_transport is None:
106
# If no _from_transport is specified, we need to intialize the
110
medium, credentials = self._build_medium()
111
if 'hpss' in debug.debug_flags:
112
trace.mutter('hpss: Built a new medium: %s',
113
medium.__class__.__name__)
114
self._shared_connection = transport._SharedConnection(medium,
118
self._client = client._SmartClient(self.get_shared_medium())
120
self._client = _client
122
def _build_medium(self):
123
"""Create the medium if _from_transport does not provide one.
125
The medium is analogous to the connection for ConnectedTransport: it
126
allows connection sharing.
131
def is_readonly(self):
132
"""Smart server transport can do read/write file operations."""
133
resp = self._call2('Transport.is_readonly')
134
if resp == ('yes', ):
136
elif resp == ('no', ):
138
elif (resp == ('error', "Generic bzr smart protocol error: "
139
"bad request 'Transport.is_readonly'") or
140
resp == ('error', "Generic bzr smart protocol error: "
141
"bad request u'Transport.is_readonly'")):
142
# XXX: nasty hack: servers before 0.16 don't have a
143
# 'Transport.is_readonly' verb, so we do what clients before 0.16
147
self._translate_error(resp)
148
raise errors.UnexpectedSmartServerResponse(resp)
150
def get_smart_client(self):
151
return self._get_connection()
153
def get_smart_medium(self):
154
return self._get_connection()
156
def get_shared_medium(self):
157
return self._get_shared_connection()
159
def _remote_path(self, relpath):
160
"""Returns the Unicode version of the absolute path for relpath."""
161
return self._combine_paths(self._path, relpath)
163
def _call(self, method, *args):
164
resp = self._call2(method, *args)
165
self._translate_error(resp)
167
def _call2(self, method, *args):
168
"""Call a method on the remote server."""
169
return self._client.call(method, *args)
171
def _call_with_body_bytes(self, method, args, body):
172
"""Call a method on the remote server with body bytes."""
173
return self._client.call_with_body_bytes(method, args, body)
175
def has(self, relpath):
176
"""Indicate whether a remote file of the given name exists or not.
178
:see: Transport.has()
180
resp = self._call2('has', self._remote_path(relpath))
181
if resp == ('yes', ):
183
elif resp == ('no', ):
186
self._translate_error(resp)
188
def get(self, relpath):
189
"""Return file-like object reading the contents of a remote file.
191
:see: Transport.get_bytes()/get_file()
193
return StringIO(self.get_bytes(relpath))
195
def get_bytes(self, relpath):
196
remote = self._remote_path(relpath)
197
request = self.get_smart_medium().get_request()
198
smart_protocol = protocol.SmartClientRequestProtocolOne(request)
199
smart_protocol.call('get', remote)
200
resp = smart_protocol.read_response_tuple(True)
202
smart_protocol.cancel_read_body()
203
self._translate_error(resp, relpath)
204
return smart_protocol.read_body_bytes()
206
def _serialise_optional_mode(self, mode):
212
def mkdir(self, relpath, mode=None):
213
resp = self._call2('mkdir', self._remote_path(relpath),
214
self._serialise_optional_mode(mode))
215
self._translate_error(resp)
217
def open_write_stream(self, relpath, mode=None):
218
"""See Transport.open_write_stream."""
219
self.put_bytes(relpath, "", mode)
220
result = transport.AppendBasedFileStream(self, relpath)
221
transport._file_streams[self.abspath(relpath)] = result
224
def put_bytes(self, relpath, upload_contents, mode=None):
225
# FIXME: upload_file is probably not safe for non-ascii characters -
226
# should probably just pass all parameters as length-delimited
228
if type(upload_contents) is unicode:
229
# Although not strictly correct, we raise UnicodeEncodeError to be
230
# compatible with other transports.
231
raise UnicodeEncodeError(
232
'undefined', upload_contents, 0, 1,
233
'put_bytes must be given bytes, not unicode.')
234
resp = self._call_with_body_bytes('put',
235
(self._remote_path(relpath), self._serialise_optional_mode(mode)),
237
self._translate_error(resp)
238
return len(upload_contents)
240
def put_bytes_non_atomic(self, relpath, bytes, mode=None,
241
create_parent_dir=False,
243
"""See Transport.put_bytes_non_atomic."""
244
# FIXME: no encoding in the transport!
245
create_parent_str = 'F'
246
if create_parent_dir:
247
create_parent_str = 'T'
249
resp = self._call_with_body_bytes(
251
(self._remote_path(relpath), self._serialise_optional_mode(mode),
252
create_parent_str, self._serialise_optional_mode(dir_mode)),
254
self._translate_error(resp)
256
def put_file(self, relpath, upload_file, mode=None):
257
# its not ideal to seek back, but currently put_non_atomic_file depends
258
# on transports not reading before failing - which is a faulty
259
# assumption I think - RBC 20060915
260
pos = upload_file.tell()
262
return self.put_bytes(relpath, upload_file.read(), mode)
264
upload_file.seek(pos)
267
def put_file_non_atomic(self, relpath, f, mode=None,
268
create_parent_dir=False,
270
return self.put_bytes_non_atomic(relpath, f.read(), mode=mode,
271
create_parent_dir=create_parent_dir,
274
def append_file(self, relpath, from_file, mode=None):
275
return self.append_bytes(relpath, from_file.read(), mode)
277
def append_bytes(self, relpath, bytes, mode=None):
278
resp = self._call_with_body_bytes(
280
(self._remote_path(relpath), self._serialise_optional_mode(mode)),
282
if resp[0] == 'appended':
284
self._translate_error(resp)
286
def delete(self, relpath):
287
resp = self._call2('delete', self._remote_path(relpath))
288
self._translate_error(resp)
290
def external_url(self):
291
"""See bzrlib.transport.Transport.external_url."""
292
# the external path for RemoteTransports is the base
295
def _readv(self, relpath, offsets):
299
offsets = list(offsets)
301
sorted_offsets = sorted(offsets)
302
# turn the list of offsets into a stack
303
offset_stack = iter(offsets)
304
cur_offset_and_size = offset_stack.next()
305
coalesced = list(self._coalesce_offsets(sorted_offsets,
306
limit=self._max_readv_combine,
307
fudge_factor=self._bytes_to_read_before_seek))
309
request = self.get_smart_medium().get_request()
310
smart_protocol = protocol.SmartClientRequestProtocolOne(request)
311
smart_protocol.call_with_body_readv_array(
312
('readv', self._remote_path(relpath)),
313
[(c.start, c.length) for c in coalesced])
314
resp = smart_protocol.read_response_tuple(True)
316
if resp[0] != 'readv':
317
# This should raise an exception
318
smart_protocol.cancel_read_body()
319
self._translate_error(resp)
322
# FIXME: this should know how many bytes are needed, for clarity.
323
data = smart_protocol.read_body_bytes()
324
# Cache the results, but only until they have been fulfilled
326
for c_offset in coalesced:
327
if len(data) < c_offset.length:
328
raise errors.ShortReadvError(relpath, c_offset.start,
329
c_offset.length, actual=len(data))
330
for suboffset, subsize in c_offset.ranges:
331
key = (c_offset.start+suboffset, subsize)
332
data_map[key] = data[suboffset:suboffset+subsize]
333
data = data[c_offset.length:]
335
# Now that we've read some data, see if we can yield anything back
336
while cur_offset_and_size in data_map:
337
this_data = data_map.pop(cur_offset_and_size)
338
yield cur_offset_and_size[0], this_data
339
cur_offset_and_size = offset_stack.next()
341
def rename(self, rel_from, rel_to):
343
self._remote_path(rel_from),
344
self._remote_path(rel_to))
346
def move(self, rel_from, rel_to):
348
self._remote_path(rel_from),
349
self._remote_path(rel_to))
351
def rmdir(self, relpath):
352
resp = self._call('rmdir', self._remote_path(relpath))
354
def _translate_error(self, resp, orig_path=None):
355
"""Raise an exception from a response"""
362
elif what == 'NoSuchFile':
363
if orig_path is not None:
364
error_path = orig_path
367
raise errors.NoSuchFile(error_path)
368
elif what == 'error':
369
raise errors.SmartProtocolError(unicode(resp[1]))
370
elif what == 'FileExists':
371
raise errors.FileExists(resp[1])
372
elif what == 'DirectoryNotEmpty':
373
raise errors.DirectoryNotEmpty(resp[1])
374
elif what == 'ShortReadvError':
375
raise errors.ShortReadvError(resp[1], int(resp[2]),
376
int(resp[3]), int(resp[4]))
377
elif what in ('UnicodeEncodeError', 'UnicodeDecodeError'):
378
encoding = str(resp[1]) # encoding must always be a string
382
reason = str(resp[5]) # reason must always be a string
383
if val.startswith('u:'):
384
val = val[2:].decode('utf-8')
385
elif val.startswith('s:'):
386
val = val[2:].decode('base64')
387
if what == 'UnicodeDecodeError':
388
raise UnicodeDecodeError(encoding, val, start, end, reason)
389
elif what == 'UnicodeEncodeError':
390
raise UnicodeEncodeError(encoding, val, start, end, reason)
391
elif what == "ReadOnlyError":
392
raise errors.TransportNotPossible('readonly transport')
393
elif what == "ReadError":
394
if orig_path is not None:
395
error_path = orig_path
398
raise errors.ReadError(error_path)
400
raise errors.SmartProtocolError('unexpected smart server error: %r' % (resp,))
402
def disconnect(self):
403
self.get_smart_medium().disconnect()
405
def delete_tree(self, relpath):
406
raise errors.TransportNotPossible('readonly transport')
408
def stat(self, relpath):
409
resp = self._call2('stat', self._remote_path(relpath))
410
if resp[0] == 'stat':
411
return _SmartStat(int(resp[1]), int(resp[2], 8))
413
self._translate_error(resp)
415
## def lock_read(self, relpath):
416
## """Lock the given file for shared (read) access.
417
## :return: A lock object, which should be passed to Transport.unlock()
419
## # The old RemoteBranch ignore lock for reading, so we will
420
## # continue that tradition and return a bogus lock object.
421
## class BogusLock(object):
422
## def __init__(self, path):
426
## return BogusLock(relpath)
431
def list_dir(self, relpath):
432
resp = self._call2('list_dir', self._remote_path(relpath))
433
if resp[0] == 'names':
434
return [name.encode('ascii') for name in resp[1:]]
436
self._translate_error(resp)
438
def iter_files_recursive(self):
439
resp = self._call2('iter_files_recursive', self._remote_path(''))
440
if resp[0] == 'names':
443
self._translate_error(resp)
446
class RemoteTCPTransport(RemoteTransport):
447
"""Connection to smart server over plain tcp.
449
This is essentially just a factory to get 'RemoteTransport(url,
450
SmartTCPClientMedium).
453
def _build_medium(self):
454
assert self.base.startswith('bzr://')
455
return medium.SmartTCPClientMedium(self._host, self._port), None
458
class RemoteSSHTransport(RemoteTransport):
459
"""Connection to smart server over SSH.
461
This is essentially just a factory to get 'RemoteTransport(url,
462
SmartSSHClientMedium).
465
def _build_medium(self):
466
assert self.base.startswith('bzr+ssh://')
467
# ssh will prompt the user for a password if needed and if none is
468
# provided but it will not give it back, so no credentials can be
470
location_config = config.LocationConfig(self.base)
471
bzr_remote_path = location_config.get_bzr_remote_path()
472
return medium.SmartSSHClientMedium(self._host, self._port,
473
self._user, self._password, bzr_remote_path=bzr_remote_path), None
476
class RemoteHTTPTransport(RemoteTransport):
477
"""Just a way to connect between a bzr+http:// url and http://.
479
This connection operates slightly differently than the RemoteSSHTransport.
480
It uses a plain http:// transport underneath, which defines what remote
481
.bzr/smart URL we are connected to. From there, all paths that are sent are
482
sent as relative paths, this way, the remote side can properly
483
de-reference them, since it is likely doing rewrite rules to translate an
484
HTTP path into a local path.
487
def __init__(self, base, _from_transport=None, http_transport=None):
488
assert ( base.startswith('bzr+http://') or base.startswith('bzr+https://') )
490
if http_transport is None:
491
# FIXME: the password may be lost here because it appears in the
492
# url only for an intial construction (when the url came from the
494
http_url = base[len('bzr+'):]
495
self._http_transport = transport.get_transport(http_url)
497
self._http_transport = http_transport
498
super(RemoteHTTPTransport, self).__init__(
499
base, _from_transport=_from_transport)
501
def _build_medium(self):
502
# We let http_transport take care of the credentials
503
return self._http_transport.get_smart_medium(), None
505
def _remote_path(self, relpath):
506
"""After connecting, HTTP Transport only deals in relative URLs."""
507
# Adjust the relpath based on which URL this smart transport is
509
http_base = urlutils.normalize_url(self._http_transport.base)
510
url = urlutils.join(self.base[len('bzr+'):], relpath)
511
url = urlutils.normalize_url(url)
512
return urlutils.relative_url(http_base, url)
514
def clone(self, relative_url):
515
"""Make a new RemoteHTTPTransport related to me.
517
This is re-implemented rather than using the default
518
RemoteTransport.clone() because we must be careful about the underlying
521
Also, the cloned smart transport will POST to the same .bzr/smart
522
location as this transport (although obviously the relative paths in the
523
smart requests may be different). This is so that the server doesn't
524
have to handle .bzr/smart requests at arbitrary places inside .bzr
525
directories, just at the initial URL the user uses.
527
The exception is parent paths (i.e. relative_url of "..").
530
abs_url = self.abspath(relative_url)
533
# We either use the exact same http_transport (for child locations), or
534
# a clone of the underlying http_transport (for parent locations). This
535
# means we share the connection.
536
norm_base = urlutils.normalize_url(self.base)
537
norm_abs_url = urlutils.normalize_url(abs_url)
538
normalized_rel_url = urlutils.relative_url(norm_base, norm_abs_url)
539
if normalized_rel_url == ".." or normalized_rel_url.startswith("../"):
540
http_transport = self._http_transport.clone(normalized_rel_url)
542
http_transport = self._http_transport
543
return RemoteHTTPTransport(abs_url,
544
_from_transport=self,
545
http_transport=http_transport)
548
def get_test_permutations():
549
"""Return (transport, server) permutations for testing."""
550
### We may need a little more test framework support to construct an
551
### appropriate RemoteTransport in the future.
552
from bzrlib.smart import server
553
return [(RemoteTCPTransport, server.SmartTCPServer_for_testing)]
added
removed
bzrlib/transport/remote.py
