1
# Copyright (C) 2006 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
"""RemoteTransport client for the smart-server.
19
This module shouldn't be accessed directly. The classes defined here should be
20
imported from bzrlib.smart.
23
__all__ = ['RemoteTransport', 'RemoteTCPTransport', 'RemoteSSHTransport']
25
from cStringIO import StringIO
35
from bzrlib.smart import client, medium
36
from bzrlib.symbol_versioning import (deprecated_method, one_four)
39
class _SmartStat(object):
41
def __init__(self, size, mode):
46
class RemoteTransport(transport.ConnectedTransport):
47
"""Connection to a smart server.
49
The connection holds references to the medium that can be used to send
50
requests to the server.
52
The connection has a notion of the current directory to which it's
53
connected; this is incorporated in filenames passed to the server.
55
This supports some higher-level RPC operations and can also be treated
56
like a Transport to do file-like operations.
58
The connection can be made over a tcp socket, an ssh pipe or a series of
59
http requests. There are concrete subclasses for each type:
60
RemoteTCPTransport, etc.
63
# IMPORTANT FOR IMPLEMENTORS: RemoteTransport MUST NOT be given encoding
64
# responsibilities: Put those on SmartClient or similar. This is vital for
65
# the ability to support multiple versions of the smart protocol over time:
66
# RemoteTransport is an adapter from the Transport object model to the
67
# SmartClient model, not an encoder.
69
# FIXME: the medium parameter should be private, only the tests requires
70
# it. It may be even clearer to define a TestRemoteTransport that handles
71
# the specific cases of providing a _client and/or a _medium, and leave
72
# RemoteTransport as an abstract class.
73
def __init__(self, url, _from_transport=None, medium=None, _client=None):
76
:param _from_transport: Another RemoteTransport instance that this
77
one is being cloned from. Attributes such as the medium will
80
:param medium: The medium to use for this RemoteTransport. If None,
81
the medium from the _from_transport is shared. If both this
82
and _from_transport are None, a new medium will be built.
83
_from_transport and medium cannot both be specified.
85
:param _client: Override the _SmartClient used by this transport. This
86
should only be used for testing purposes; normally this is
87
determined from the medium.
89
super(RemoteTransport, self).__init__(url,
90
_from_transport=_from_transport)
92
# The medium is the connection, except when we need to share it with
93
# other objects (RemoteBzrDir, RemoteRepository etc). In these cases
94
# what we want to share is really the shared connection.
96
if _from_transport is None:
97
# If no _from_transport is specified, we need to intialize the
101
medium, credentials = self._build_medium()
102
if 'hpss' in debug.debug_flags:
103
trace.mutter('hpss: Built a new medium: %s',
104
medium.__class__.__name__)
105
self._shared_connection = transport._SharedConnection(medium,
109
# No medium was specified, so share the medium from the
111
medium = self._shared_connection.connection
113
raise AssertionError(
114
"Both _from_transport (%r) and medium (%r) passed to "
115
"RemoteTransport.__init__, but these parameters are mutally "
116
"exclusive." % (_from_transport, medium))
119
self._client = client._SmartClient(medium)
121
self._client = _client
123
def _build_medium(self):
124
"""Create the medium if _from_transport does not provide one.
126
The medium is analogous to the connection for ConnectedTransport: it
127
allows connection sharing.
132
def is_readonly(self):
133
"""Smart server transport can do read/write file operations."""
135
resp = self._call2('Transport.is_readonly')
136
except errors.UnknownSmartMethod:
137
# XXX: nasty hack: servers before 0.16 don't have a
138
# 'Transport.is_readonly' verb, so we do what clients before 0.16
141
if resp == ('yes', ):
143
elif resp == ('no', ):
146
self._translate_error(resp)
147
raise errors.UnexpectedSmartServerResponse(resp)
149
def get_smart_client(self):
150
return self._get_connection()
152
def get_smart_medium(self):
153
return self._get_connection()
155
@deprecated_method(one_four)
156
def get_shared_medium(self):
157
return self._get_shared_connection()
159
def _remote_path(self, relpath):
160
"""Returns the Unicode version of the absolute path for relpath."""
161
return self._combine_paths(self._path, relpath)
163
def _call(self, method, *args):
165
resp = self._call2(method, *args)
166
except errors.ErrorFromSmartServer, err:
167
self._translate_error(err.error_tuple)
168
self._translate_error(resp)
170
def _call2(self, method, *args):
171
"""Call a method on the remote server."""
173
return self._client.call(method, *args)
174
except errors.ErrorFromSmartServer, err:
175
self._translate_error(err.error_tuple)
177
def _call_with_body_bytes(self, method, args, body):
178
"""Call a method on the remote server with body bytes."""
180
return self._client.call_with_body_bytes(method, args, body)
181
except errors.ErrorFromSmartServer, err:
182
self._translate_error(err.error_tuple)
184
def has(self, relpath):
185
"""Indicate whether a remote file of the given name exists or not.
187
:see: Transport.has()
189
resp = self._call2('has', self._remote_path(relpath))
190
if resp == ('yes', ):
192
elif resp == ('no', ):
195
self._translate_error(resp)
197
def get(self, relpath):
198
"""Return file-like object reading the contents of a remote file.
200
:see: Transport.get_bytes()/get_file()
202
return StringIO(self.get_bytes(relpath))
204
def get_bytes(self, relpath):
205
remote = self._remote_path(relpath)
207
resp, response_handler = self._client.call_expecting_body('get', remote)
208
except errors.ErrorFromSmartServer, err:
209
self._translate_error(err.error_tuple, relpath)
211
response_handler.cancel_read_body()
212
raise errors.UnexpectedSmartServerResponse(resp)
213
return response_handler.read_body_bytes()
215
def _serialise_optional_mode(self, mode):
221
def mkdir(self, relpath, mode=None):
222
resp = self._call2('mkdir', self._remote_path(relpath),
223
self._serialise_optional_mode(mode))
224
self._translate_error(resp)
226
def open_write_stream(self, relpath, mode=None):
227
"""See Transport.open_write_stream."""
228
self.put_bytes(relpath, "", mode)
229
result = transport.AppendBasedFileStream(self, relpath)
230
transport._file_streams[self.abspath(relpath)] = result
233
def put_bytes(self, relpath, upload_contents, mode=None):
234
# FIXME: upload_file is probably not safe for non-ascii characters -
235
# should probably just pass all parameters as length-delimited
237
if type(upload_contents) is unicode:
238
# Although not strictly correct, we raise UnicodeEncodeError to be
239
# compatible with other transports.
240
raise UnicodeEncodeError(
241
'undefined', upload_contents, 0, 1,
242
'put_bytes must be given bytes, not unicode.')
243
resp = self._call_with_body_bytes('put',
244
(self._remote_path(relpath), self._serialise_optional_mode(mode)),
246
self._translate_error(resp)
247
return len(upload_contents)
249
def put_bytes_non_atomic(self, relpath, bytes, mode=None,
250
create_parent_dir=False,
252
"""See Transport.put_bytes_non_atomic."""
253
# FIXME: no encoding in the transport!
254
create_parent_str = 'F'
255
if create_parent_dir:
256
create_parent_str = 'T'
258
resp = self._call_with_body_bytes(
260
(self._remote_path(relpath), self._serialise_optional_mode(mode),
261
create_parent_str, self._serialise_optional_mode(dir_mode)),
263
self._translate_error(resp)
265
def put_file(self, relpath, upload_file, mode=None):
266
# its not ideal to seek back, but currently put_non_atomic_file depends
267
# on transports not reading before failing - which is a faulty
268
# assumption I think - RBC 20060915
269
pos = upload_file.tell()
271
return self.put_bytes(relpath, upload_file.read(), mode)
273
upload_file.seek(pos)
276
def put_file_non_atomic(self, relpath, f, mode=None,
277
create_parent_dir=False,
279
return self.put_bytes_non_atomic(relpath, f.read(), mode=mode,
280
create_parent_dir=create_parent_dir,
283
def append_file(self, relpath, from_file, mode=None):
284
return self.append_bytes(relpath, from_file.read(), mode)
286
def append_bytes(self, relpath, bytes, mode=None):
287
resp = self._call_with_body_bytes(
289
(self._remote_path(relpath), self._serialise_optional_mode(mode)),
291
if resp[0] == 'appended':
293
self._translate_error(resp)
295
def delete(self, relpath):
296
resp = self._call2('delete', self._remote_path(relpath))
297
self._translate_error(resp)
299
def external_url(self):
300
"""See bzrlib.transport.Transport.external_url."""
301
# the external path for RemoteTransports is the base
304
def recommended_page_size(self):
305
"""Return the recommended page size for this transport."""
308
def _readv(self, relpath, offsets):
312
offsets = list(offsets)
314
sorted_offsets = sorted(offsets)
315
coalesced = list(self._coalesce_offsets(sorted_offsets,
316
limit=self._max_readv_combine,
317
fudge_factor=self._bytes_to_read_before_seek))
320
result = self._client.call_with_body_readv_array(
321
('readv', self._remote_path(relpath),),
322
[(c.start, c.length) for c in coalesced])
323
resp, response_handler = result
324
except errors.ErrorFromSmartServer, err:
325
self._translate_error(err.error_tuple)
327
if resp[0] != 'readv':
328
# This should raise an exception
329
response_handler.cancel_read_body()
330
raise errors.UnexpectedSmartServerResponse(resp)
332
return self._handle_response(offsets, coalesced, response_handler)
334
def _handle_response(self, offsets, coalesced, response_handler):
335
# turn the list of offsets into a stack
336
offset_stack = iter(offsets)
337
cur_offset_and_size = offset_stack.next()
338
# FIXME: this should know how many bytes are needed, for clarity.
339
data = response_handler.read_body_bytes()
340
# Cache the results, but only until they have been fulfilled
343
for c_offset in coalesced:
344
if len(data) < c_offset.length:
345
raise errors.ShortReadvError(relpath, c_offset.start,
346
c_offset.length, actual=len(data))
347
for suboffset, subsize in c_offset.ranges:
348
key = (c_offset.start+suboffset, subsize)
349
this_data = data[data_offset+suboffset:
350
data_offset+suboffset+subsize]
351
# Special case when the data is in-order, rather than packing
352
# into a map and then back out again. Benchmarking shows that
353
# this has 100% hit rate, but leave in the data_map work just
355
# TODO: Could we get away with using buffer() to avoid the
356
# memory copy? Callers would need to realize they may
357
# not have a real string.
358
if key == cur_offset_and_size:
359
yield cur_offset_and_size[0], this_data
360
cur_offset_and_size = offset_stack.next()
362
data_map[key] = this_data
363
data_offset += c_offset.length
365
# Now that we've read some data, see if we can yield anything back
366
while cur_offset_and_size in data_map:
367
this_data = data_map.pop(cur_offset_and_size)
368
yield cur_offset_and_size[0], this_data
369
cur_offset_and_size = offset_stack.next()
371
def rename(self, rel_from, rel_to):
373
self._remote_path(rel_from),
374
self._remote_path(rel_to))
376
def move(self, rel_from, rel_to):
378
self._remote_path(rel_from),
379
self._remote_path(rel_to))
381
def rmdir(self, relpath):
382
resp = self._call('rmdir', self._remote_path(relpath))
384
def _translate_error(self, resp, orig_path=None):
385
"""Raise an exception from a response"""
392
elif what == 'NoSuchFile':
393
if orig_path is not None:
394
error_path = orig_path
397
raise errors.NoSuchFile(error_path)
398
elif what == 'error':
399
raise errors.SmartProtocolError(unicode(resp[1]))
400
elif what == 'FileExists':
401
raise errors.FileExists(resp[1])
402
elif what == 'DirectoryNotEmpty':
403
raise errors.DirectoryNotEmpty(resp[1])
404
elif what == 'ShortReadvError':
405
raise errors.ShortReadvError(resp[1], int(resp[2]),
406
int(resp[3]), int(resp[4]))
407
elif what in ('UnicodeEncodeError', 'UnicodeDecodeError'):
408
encoding = str(resp[1]) # encoding must always be a string
412
reason = str(resp[5]) # reason must always be a string
413
if val.startswith('u:'):
414
val = val[2:].decode('utf-8')
415
elif val.startswith('s:'):
416
val = val[2:].decode('base64')
417
if what == 'UnicodeDecodeError':
418
raise UnicodeDecodeError(encoding, val, start, end, reason)
419
elif what == 'UnicodeEncodeError':
420
raise UnicodeEncodeError(encoding, val, start, end, reason)
421
elif what == "ReadOnlyError":
422
raise errors.TransportNotPossible('readonly transport')
423
elif what == "ReadError":
424
if orig_path is not None:
425
error_path = orig_path
428
raise errors.ReadError(error_path)
429
elif what == "PermissionDenied":
430
if orig_path is not None:
431
error_path = orig_path
434
raise errors.PermissionDenied(error_path)
436
raise errors.SmartProtocolError('unexpected smart server error: %r' % (resp,))
438
def disconnect(self):
439
self.get_smart_medium().disconnect()
441
def stat(self, relpath):
442
resp = self._call2('stat', self._remote_path(relpath))
443
if resp[0] == 'stat':
444
return _SmartStat(int(resp[1]), int(resp[2], 8))
446
self._translate_error(resp)
448
## def lock_read(self, relpath):
449
## """Lock the given file for shared (read) access.
450
## :return: A lock object, which should be passed to Transport.unlock()
452
## # The old RemoteBranch ignore lock for reading, so we will
453
## # continue that tradition and return a bogus lock object.
454
## class BogusLock(object):
455
## def __init__(self, path):
459
## return BogusLock(relpath)
464
def list_dir(self, relpath):
465
resp = self._call2('list_dir', self._remote_path(relpath))
466
if resp[0] == 'names':
467
return [name.encode('ascii') for name in resp[1:]]
469
self._translate_error(resp)
471
def iter_files_recursive(self):
472
resp = self._call2('iter_files_recursive', self._remote_path(''))
473
if resp[0] == 'names':
476
self._translate_error(resp)
479
class RemoteTCPTransport(RemoteTransport):
480
"""Connection to smart server over plain tcp.
482
This is essentially just a factory to get 'RemoteTransport(url,
483
SmartTCPClientMedium).
486
def _build_medium(self):
487
client_medium = medium.SmartTCPClientMedium(
488
self._host, self._port, self.base)
489
return client_medium, None
492
class RemoteTCPTransportV2Only(RemoteTransport):
493
"""Connection to smart server over plain tcp with the client hard-coded to
494
assume protocol v2 and remote server version <= 1.6.
496
This should only be used for testing.
499
def _build_medium(self):
500
client_medium = medium.SmartTCPClientMedium(
501
self._host, self._port, self.base)
502
client_medium._protocol_version = 2
503
client_medium._remember_remote_is_before((1, 6))
504
return client_medium, None
507
class RemoteSSHTransport(RemoteTransport):
508
"""Connection to smart server over SSH.
510
This is essentially just a factory to get 'RemoteTransport(url,
511
SmartSSHClientMedium).
514
def _build_medium(self):
515
location_config = config.LocationConfig(self.base)
516
bzr_remote_path = location_config.get_bzr_remote_path()
519
auth = config.AuthenticationConfig()
520
user = auth.get_user('ssh', self._host, self._port)
521
client_medium = medium.SmartSSHClientMedium(self._host, self._port,
522
user, self._password, self.base,
523
bzr_remote_path=bzr_remote_path)
524
return client_medium, (user, self._password)
527
class RemoteHTTPTransport(RemoteTransport):
528
"""Just a way to connect between a bzr+http:// url and http://.
530
This connection operates slightly differently than the RemoteSSHTransport.
531
It uses a plain http:// transport underneath, which defines what remote
532
.bzr/smart URL we are connected to. From there, all paths that are sent are
533
sent as relative paths, this way, the remote side can properly
534
de-reference them, since it is likely doing rewrite rules to translate an
535
HTTP path into a local path.
538
def __init__(self, base, _from_transport=None, http_transport=None):
539
if http_transport is None:
540
# FIXME: the password may be lost here because it appears in the
541
# url only for an intial construction (when the url came from the
543
http_url = base[len('bzr+'):]
544
self._http_transport = transport.get_transport(http_url)
546
self._http_transport = http_transport
547
super(RemoteHTTPTransport, self).__init__(
548
base, _from_transport=_from_transport)
550
def _build_medium(self):
551
# We let http_transport take care of the credentials
552
return self._http_transport.get_smart_medium(), None
554
def _remote_path(self, relpath):
555
"""After connecting, HTTP Transport only deals in relative URLs."""
556
# Adjust the relpath based on which URL this smart transport is
558
http_base = urlutils.normalize_url(self.get_smart_medium().base)
559
url = urlutils.join(self.base[len('bzr+'):], relpath)
560
url = urlutils.normalize_url(url)
561
return urlutils.relative_url(http_base, url)
563
def clone(self, relative_url):
564
"""Make a new RemoteHTTPTransport related to me.
566
This is re-implemented rather than using the default
567
RemoteTransport.clone() because we must be careful about the underlying
570
Also, the cloned smart transport will POST to the same .bzr/smart
571
location as this transport (although obviously the relative paths in the
572
smart requests may be different). This is so that the server doesn't
573
have to handle .bzr/smart requests at arbitrary places inside .bzr
574
directories, just at the initial URL the user uses.
577
abs_url = self.abspath(relative_url)
580
return RemoteHTTPTransport(abs_url,
581
_from_transport=self,
582
http_transport=self._http_transport)
585
def get_test_permutations():
586
"""Return (transport, server) permutations for testing."""
587
### We may need a little more test framework support to construct an
588
### appropriate RemoteTransport in the future.
589
from bzrlib.smart import server
590
return [(RemoteTCPTransport, server.SmartTCPServer_for_testing)]
added
removed
bzrlib/transport/remote.py
