1
# Copyright (C) 2006-2010 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
"""RemoteTransport client for the smart-server.
19
This module shouldn't be accessed directly. The classes defined here should be
20
imported from bzrlib.smart.
23
from __future__ import absolute_import
25
__all__ = ['RemoteTransport', 'RemoteTCPTransport', 'RemoteSSHTransport']
27
from cStringIO import StringIO
38
from bzrlib.smart import client, medium
41
class _SmartStat(object):
43
def __init__(self, size, mode):
48
class RemoteTransport(transport.ConnectedTransport):
49
"""Connection to a smart server.
51
The connection holds references to the medium that can be used to send
52
requests to the server.
54
The connection has a notion of the current directory to which it's
55
connected; this is incorporated in filenames passed to the server.
57
This supports some higher-level RPC operations and can also be treated
58
like a Transport to do file-like operations.
60
The connection can be made over a tcp socket, an ssh pipe or a series of
61
http requests. There are concrete subclasses for each type:
62
RemoteTCPTransport, etc.
65
# When making a readv request, cap it at requesting 5MB of data
66
_max_readv_bytes = 5*1024*1024
68
# IMPORTANT FOR IMPLEMENTORS: RemoteTransport MUST NOT be given encoding
69
# responsibilities: Put those on SmartClient or similar. This is vital for
70
# the ability to support multiple versions of the smart protocol over time:
71
# RemoteTransport is an adapter from the Transport object model to the
72
# SmartClient model, not an encoder.
74
# FIXME: the medium parameter should be private, only the tests requires
75
# it. It may be even clearer to define a TestRemoteTransport that handles
76
# the specific cases of providing a _client and/or a _medium, and leave
77
# RemoteTransport as an abstract class.
78
def __init__(self, url, _from_transport=None, medium=None, _client=None):
81
:param _from_transport: Another RemoteTransport instance that this
82
one is being cloned from. Attributes such as the medium will
85
:param medium: The medium to use for this RemoteTransport. If None,
86
the medium from the _from_transport is shared. If both this
87
and _from_transport are None, a new medium will be built.
88
_from_transport and medium cannot both be specified.
90
:param _client: Override the _SmartClient used by this transport. This
91
should only be used for testing purposes; normally this is
92
determined from the medium.
94
super(RemoteTransport, self).__init__(
95
url, _from_transport=_from_transport)
97
# The medium is the connection, except when we need to share it with
98
# other objects (RemoteBzrDir, RemoteRepository etc). In these cases
99
# what we want to share is really the shared connection.
101
if (_from_transport is not None
102
and isinstance(_from_transport, RemoteTransport)):
103
_client = _from_transport._client
104
elif _from_transport is None:
105
# If no _from_transport is specified, we need to intialize the
109
medium, credentials = self._build_medium()
110
if 'hpss' in debug.debug_flags:
111
trace.mutter('hpss: Built a new medium: %s',
112
medium.__class__.__name__)
113
self._shared_connection = transport._SharedConnection(medium,
117
# No medium was specified, so share the medium from the
119
medium = self._shared_connection.connection
121
raise AssertionError(
122
"Both _from_transport (%r) and medium (%r) passed to "
123
"RemoteTransport.__init__, but these parameters are mutally "
124
"exclusive." % (_from_transport, medium))
127
self._client = client._SmartClient(medium)
129
self._client = _client
131
def _build_medium(self):
132
"""Create the medium if _from_transport does not provide one.
134
The medium is analogous to the connection for ConnectedTransport: it
135
allows connection sharing.
140
def _report_activity(self, bytes, direction):
141
"""See Transport._report_activity.
143
Does nothing; the smart medium will report activity triggered by a
148
def is_readonly(self):
149
"""Smart server transport can do read/write file operations."""
151
resp = self._call2('Transport.is_readonly')
152
except errors.UnknownSmartMethod:
153
# XXX: nasty hack: servers before 0.16 don't have a
154
# 'Transport.is_readonly' verb, so we do what clients before 0.16
157
if resp == ('yes', ):
159
elif resp == ('no', ):
162
raise errors.UnexpectedSmartServerResponse(resp)
164
def get_smart_client(self):
165
return self._get_connection()
167
def get_smart_medium(self):
168
return self._get_connection()
170
def _remote_path(self, relpath):
171
"""Returns the Unicode version of the absolute path for relpath."""
172
return urlutils.URL._combine_paths(self._parsed_url.path, relpath)
174
def _call(self, method, *args):
175
resp = self._call2(method, *args)
176
self._ensure_ok(resp)
178
def _call2(self, method, *args):
179
"""Call a method on the remote server."""
181
return self._client.call(method, *args)
182
except errors.ErrorFromSmartServer, err:
183
# The first argument, if present, is always a path.
185
context = {'relpath': args[0]}
188
self._translate_error(err, **context)
190
def _call_with_body_bytes(self, method, args, body):
191
"""Call a method on the remote server with body bytes."""
193
return self._client.call_with_body_bytes(method, args, body)
194
except errors.ErrorFromSmartServer, err:
195
# The first argument, if present, is always a path.
197
context = {'relpath': args[0]}
200
self._translate_error(err, **context)
202
def has(self, relpath):
203
"""Indicate whether a remote file of the given name exists or not.
205
:see: Transport.has()
207
resp = self._call2('has', self._remote_path(relpath))
208
if resp == ('yes', ):
210
elif resp == ('no', ):
213
raise errors.UnexpectedSmartServerResponse(resp)
215
def get(self, relpath):
216
"""Return file-like object reading the contents of a remote file.
218
:see: Transport.get_bytes()/get_file()
220
return StringIO(self.get_bytes(relpath))
222
def get_bytes(self, relpath):
223
remote = self._remote_path(relpath)
225
resp, response_handler = self._client.call_expecting_body('get', remote)
226
except errors.ErrorFromSmartServer, err:
227
self._translate_error(err, relpath)
229
response_handler.cancel_read_body()
230
raise errors.UnexpectedSmartServerResponse(resp)
231
return response_handler.read_body_bytes()
233
def _serialise_optional_mode(self, mode):
239
def mkdir(self, relpath, mode=None):
240
resp = self._call2('mkdir', self._remote_path(relpath),
241
self._serialise_optional_mode(mode))
243
def open_write_stream(self, relpath, mode=None):
244
"""See Transport.open_write_stream."""
245
self.put_bytes(relpath, "", mode)
246
result = transport.AppendBasedFileStream(self, relpath)
247
transport._file_streams[self.abspath(relpath)] = result
250
def put_bytes(self, relpath, upload_contents, mode=None):
251
# FIXME: upload_file is probably not safe for non-ascii characters -
252
# should probably just pass all parameters as length-delimited
254
if type(upload_contents) is unicode:
255
# Although not strictly correct, we raise UnicodeEncodeError to be
256
# compatible with other transports.
257
raise UnicodeEncodeError(
258
'undefined', upload_contents, 0, 1,
259
'put_bytes must be given bytes, not unicode.')
260
resp = self._call_with_body_bytes('put',
261
(self._remote_path(relpath), self._serialise_optional_mode(mode)),
263
self._ensure_ok(resp)
264
return len(upload_contents)
266
def put_bytes_non_atomic(self, relpath, bytes, mode=None,
267
create_parent_dir=False,
269
"""See Transport.put_bytes_non_atomic."""
270
# FIXME: no encoding in the transport!
271
create_parent_str = 'F'
272
if create_parent_dir:
273
create_parent_str = 'T'
275
resp = self._call_with_body_bytes(
277
(self._remote_path(relpath), self._serialise_optional_mode(mode),
278
create_parent_str, self._serialise_optional_mode(dir_mode)),
280
self._ensure_ok(resp)
282
def put_file(self, relpath, upload_file, mode=None):
283
# its not ideal to seek back, but currently put_non_atomic_file depends
284
# on transports not reading before failing - which is a faulty
285
# assumption I think - RBC 20060915
286
pos = upload_file.tell()
288
return self.put_bytes(relpath, upload_file.read(), mode)
290
upload_file.seek(pos)
293
def put_file_non_atomic(self, relpath, f, mode=None,
294
create_parent_dir=False,
296
return self.put_bytes_non_atomic(relpath, f.read(), mode=mode,
297
create_parent_dir=create_parent_dir,
300
def append_file(self, relpath, from_file, mode=None):
301
return self.append_bytes(relpath, from_file.read(), mode)
303
def append_bytes(self, relpath, bytes, mode=None):
304
resp = self._call_with_body_bytes(
306
(self._remote_path(relpath), self._serialise_optional_mode(mode)),
308
if resp[0] == 'appended':
310
raise errors.UnexpectedSmartServerResponse(resp)
312
def delete(self, relpath):
313
resp = self._call2('delete', self._remote_path(relpath))
314
self._ensure_ok(resp)
316
def external_url(self):
317
"""See bzrlib.transport.Transport.external_url."""
318
# the external path for RemoteTransports is the base
321
def recommended_page_size(self):
322
"""Return the recommended page size for this transport."""
325
def _readv(self, relpath, offsets):
329
offsets = list(offsets)
331
sorted_offsets = sorted(offsets)
332
coalesced = list(self._coalesce_offsets(sorted_offsets,
333
limit=self._max_readv_combine,
334
fudge_factor=self._bytes_to_read_before_seek,
335
max_size=self._max_readv_bytes))
337
# now that we've coallesced things, avoid making enormous requests
342
if c.length + cur_len > self._max_readv_bytes:
343
requests.append(cur_request)
347
cur_request.append(c)
350
requests.append(cur_request)
351
if 'hpss' in debug.debug_flags:
352
trace.mutter('%s.readv %s offsets => %s coalesced'
353
' => %s requests (%s)',
354
self.__class__.__name__, len(offsets), len(coalesced),
355
len(requests), sum(map(len, requests)))
356
# Cache the results, but only until they have been fulfilled
358
# turn the list of offsets into a single stack to iterate
359
offset_stack = iter(offsets)
360
# using a list so it can be modified when passing down and coming back
361
next_offset = [offset_stack.next()]
362
for cur_request in requests:
364
result = self._client.call_with_body_readv_array(
365
('readv', self._remote_path(relpath),),
366
[(c.start, c.length) for c in cur_request])
367
resp, response_handler = result
368
except errors.ErrorFromSmartServer, err:
369
self._translate_error(err, relpath)
371
if resp[0] != 'readv':
372
# This should raise an exception
373
response_handler.cancel_read_body()
374
raise errors.UnexpectedSmartServerResponse(resp)
376
for res in self._handle_response(offset_stack, cur_request,
382
def _handle_response(self, offset_stack, coalesced, response_handler,
383
data_map, next_offset):
384
cur_offset_and_size = next_offset[0]
385
# FIXME: this should know how many bytes are needed, for clarity.
386
data = response_handler.read_body_bytes()
388
for c_offset in coalesced:
389
if len(data) < c_offset.length:
390
raise errors.ShortReadvError(relpath, c_offset.start,
391
c_offset.length, actual=len(data))
392
for suboffset, subsize in c_offset.ranges:
393
key = (c_offset.start+suboffset, subsize)
394
this_data = data[data_offset+suboffset:
395
data_offset+suboffset+subsize]
396
# Special case when the data is in-order, rather than packing
397
# into a map and then back out again. Benchmarking shows that
398
# this has 100% hit rate, but leave in the data_map work just
400
# TODO: Could we get away with using buffer() to avoid the
401
# memory copy? Callers would need to realize they may
402
# not have a real string.
403
if key == cur_offset_and_size:
404
yield cur_offset_and_size[0], this_data
405
cur_offset_and_size = next_offset[0] = offset_stack.next()
407
data_map[key] = this_data
408
data_offset += c_offset.length
410
# Now that we've read some data, see if we can yield anything back
411
while cur_offset_and_size in data_map:
412
this_data = data_map.pop(cur_offset_and_size)
413
yield cur_offset_and_size[0], this_data
414
cur_offset_and_size = next_offset[0] = offset_stack.next()
416
def rename(self, rel_from, rel_to):
418
self._remote_path(rel_from),
419
self._remote_path(rel_to))
421
def move(self, rel_from, rel_to):
423
self._remote_path(rel_from),
424
self._remote_path(rel_to))
426
def rmdir(self, relpath):
427
resp = self._call('rmdir', self._remote_path(relpath))
429
def _ensure_ok(self, resp):
431
raise errors.UnexpectedSmartServerResponse(resp)
433
def _translate_error(self, err, relpath=None):
434
remote._translate_error(err, path=relpath)
436
def disconnect(self):
437
m = self.get_smart_medium()
441
def stat(self, relpath):
442
resp = self._call2('stat', self._remote_path(relpath))
443
if resp[0] == 'stat':
444
return _SmartStat(int(resp[1]), int(resp[2], 8))
445
raise errors.UnexpectedSmartServerResponse(resp)
447
## def lock_read(self, relpath):
448
## """Lock the given file for shared (read) access.
449
## :return: A lock object, which should be passed to Transport.unlock()
451
## # The old RemoteBranch ignore lock for reading, so we will
452
## # continue that tradition and return a bogus lock object.
453
## class BogusLock(object):
454
## def __init__(self, path):
458
## return BogusLock(relpath)
463
def list_dir(self, relpath):
464
resp = self._call2('list_dir', self._remote_path(relpath))
465
if resp[0] == 'names':
466
return [name.encode('ascii') for name in resp[1:]]
467
raise errors.UnexpectedSmartServerResponse(resp)
469
def iter_files_recursive(self):
470
resp = self._call2('iter_files_recursive', self._remote_path(''))
471
if resp[0] == 'names':
473
raise errors.UnexpectedSmartServerResponse(resp)
476
class RemoteTCPTransport(RemoteTransport):
477
"""Connection to smart server over plain tcp.
479
This is essentially just a factory to get 'RemoteTransport(url,
480
SmartTCPClientMedium).
483
def _build_medium(self):
484
client_medium = medium.SmartTCPClientMedium(
485
self._parsed_url.host, self._parsed_url.port, self.base)
486
return client_medium, None
489
class RemoteTCPTransportV2Only(RemoteTransport):
490
"""Connection to smart server over plain tcp with the client hard-coded to
491
assume protocol v2 and remote server version <= 1.6.
493
This should only be used for testing.
496
def _build_medium(self):
497
client_medium = medium.SmartTCPClientMedium(
498
self._parsed_url.host, self._parsed_url.port, self.base)
499
client_medium._protocol_version = 2
500
client_medium._remember_remote_is_before((1, 6))
501
return client_medium, None
504
class RemoteSSHTransport(RemoteTransport):
505
"""Connection to smart server over SSH.
507
This is essentially just a factory to get 'RemoteTransport(url,
508
SmartSSHClientMedium).
511
def _build_medium(self):
512
location_config = config.LocationConfig(self.base)
513
bzr_remote_path = location_config.get_bzr_remote_path()
514
user = self._parsed_url.user
516
auth = config.AuthenticationConfig()
517
user = auth.get_user('ssh', self._parsed_url.host,
518
self._parsed_url.port)
519
ssh_params = medium.SSHParams(self._parsed_url.host,
520
self._parsed_url.port, user, self._parsed_url.password,
522
client_medium = medium.SmartSSHClientMedium(self.base, ssh_params)
523
return client_medium, (user, self._parsed_url.password)
526
class RemoteHTTPTransport(RemoteTransport):
527
"""Just a way to connect between a bzr+http:// url and http://.
529
This connection operates slightly differently than the RemoteSSHTransport.
530
It uses a plain http:// transport underneath, which defines what remote
531
.bzr/smart URL we are connected to. From there, all paths that are sent are
532
sent as relative paths, this way, the remote side can properly
533
de-reference them, since it is likely doing rewrite rules to translate an
534
HTTP path into a local path.
537
def __init__(self, base, _from_transport=None, http_transport=None):
538
if http_transport is None:
539
# FIXME: the password may be lost here because it appears in the
540
# url only for an intial construction (when the url came from the
542
http_url = base[len('bzr+'):]
543
self._http_transport = transport.get_transport_from_url(http_url)
545
self._http_transport = http_transport
546
super(RemoteHTTPTransport, self).__init__(
547
base, _from_transport=_from_transport)
549
def _build_medium(self):
550
# We let http_transport take care of the credentials
551
return self._http_transport.get_smart_medium(), None
553
def _remote_path(self, relpath):
554
"""After connecting, HTTP Transport only deals in relative URLs."""
555
# Adjust the relpath based on which URL this smart transport is
557
http_base = urlutils.normalize_url(self.get_smart_medium().base)
558
url = urlutils.join(self.base[len('bzr+'):], relpath)
559
url = urlutils.normalize_url(url)
560
return urlutils.relative_url(http_base, url)
562
def clone(self, relative_url):
563
"""Make a new RemoteHTTPTransport related to me.
565
This is re-implemented rather than using the default
566
RemoteTransport.clone() because we must be careful about the underlying
569
Also, the cloned smart transport will POST to the same .bzr/smart
570
location as this transport (although obviously the relative paths in the
571
smart requests may be different). This is so that the server doesn't
572
have to handle .bzr/smart requests at arbitrary places inside .bzr
573
directories, just at the initial URL the user uses.
576
abs_url = self.abspath(relative_url)
579
return RemoteHTTPTransport(abs_url,
580
_from_transport=self,
581
http_transport=self._http_transport)
583
def _redirected_to(self, source, target):
584
"""See transport._redirected_to"""
585
redirected = self._http_transport._redirected_to(source, target)
586
if (redirected is not None
587
and isinstance(redirected, type(self._http_transport))):
588
return RemoteHTTPTransport('bzr+' + redirected.external_url(),
589
http_transport=redirected)
591
# Either None or a transport for a different protocol
595
class HintingSSHTransport(transport.Transport):
596
"""Simple transport that handles ssh:// and points out bzr+ssh://."""
598
def __init__(self, url):
599
raise errors.UnsupportedProtocol(url,
600
'bzr supports bzr+ssh to operate over ssh, use "bzr+%s".' % url)
603
def get_test_permutations():
604
"""Return (transport, server) permutations for testing."""
605
### We may need a little more test framework support to construct an
606
### appropriate RemoteTransport in the future.
607
from bzrlib.tests import test_server
608
return [(RemoteTCPTransport, test_server.SmartTCPServer_for_testing)]
added
removed
bzrlib/transport/remote.py
