1
# Copyright (C) 2006 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
"""RemoteTransport client for the smart-server.
19
This module shouldn't be accessed directly. The classes defined here should be
20
imported from bzrlib.smart.
23
__all__ = ['RemoteTransport', 'RemoteTCPTransport', 'RemoteSSHTransport']
25
from cStringIO import StringIO
36
from bzrlib.smart import client, medium
37
from bzrlib.symbol_versioning import (deprecated_method, one_four)
40
class _SmartStat(object):
42
def __init__(self, size, mode):
47
class RemoteTransport(transport.ConnectedTransport):
48
"""Connection to a smart server.
50
The connection holds references to the medium that can be used to send
51
requests to the server.
53
The connection has a notion of the current directory to which it's
54
connected; this is incorporated in filenames passed to the server.
56
This supports some higher-level RPC operations and can also be treated
57
like a Transport to do file-like operations.
59
The connection can be made over a tcp socket, an ssh pipe or a series of
60
http requests. There are concrete subclasses for each type:
61
RemoteTCPTransport, etc.
64
# When making a readv request, cap it at requesting 5MB of data
65
_max_readv_bytes = 5*1024*1024
67
# IMPORTANT FOR IMPLEMENTORS: RemoteTransport MUST NOT be given encoding
68
# responsibilities: Put those on SmartClient or similar. This is vital for
69
# the ability to support multiple versions of the smart protocol over time:
70
# RemoteTransport is an adapter from the Transport object model to the
71
# SmartClient model, not an encoder.
73
# FIXME: the medium parameter should be private, only the tests requires
74
# it. It may be even clearer to define a TestRemoteTransport that handles
75
# the specific cases of providing a _client and/or a _medium, and leave
76
# RemoteTransport as an abstract class.
77
def __init__(self, url, _from_transport=None, medium=None, _client=None):
80
:param _from_transport: Another RemoteTransport instance that this
81
one is being cloned from. Attributes such as the medium will
84
:param medium: The medium to use for this RemoteTransport. If None,
85
the medium from the _from_transport is shared. If both this
86
and _from_transport are None, a new medium will be built.
87
_from_transport and medium cannot both be specified.
89
:param _client: Override the _SmartClient used by this transport. This
90
should only be used for testing purposes; normally this is
91
determined from the medium.
93
super(RemoteTransport, self).__init__(
94
url, _from_transport=_from_transport)
96
# The medium is the connection, except when we need to share it with
97
# other objects (RemoteBzrDir, RemoteRepository etc). In these cases
98
# what we want to share is really the shared connection.
100
if (_from_transport is not None
101
and isinstance(_from_transport, RemoteTransport)):
102
_client = _from_transport._client
103
elif _from_transport is None:
104
# If no _from_transport is specified, we need to intialize the
108
medium, credentials = self._build_medium()
109
if 'hpss' in debug.debug_flags:
110
trace.mutter('hpss: Built a new medium: %s',
111
medium.__class__.__name__)
112
self._shared_connection = transport._SharedConnection(medium,
116
# No medium was specified, so share the medium from the
118
medium = self._shared_connection.connection
120
raise AssertionError(
121
"Both _from_transport (%r) and medium (%r) passed to "
122
"RemoteTransport.__init__, but these parameters are mutally "
123
"exclusive." % (_from_transport, medium))
126
self._client = client._SmartClient(medium)
128
self._client = _client
130
def _build_medium(self):
131
"""Create the medium if _from_transport does not provide one.
133
The medium is analogous to the connection for ConnectedTransport: it
134
allows connection sharing.
139
def is_readonly(self):
140
"""Smart server transport can do read/write file operations."""
142
resp = self._call2('Transport.is_readonly')
143
except errors.UnknownSmartMethod:
144
# XXX: nasty hack: servers before 0.16 don't have a
145
# 'Transport.is_readonly' verb, so we do what clients before 0.16
148
if resp == ('yes', ):
150
elif resp == ('no', ):
153
raise errors.UnexpectedSmartServerResponse(resp)
155
def get_smart_client(self):
156
return self._get_connection()
158
def get_smart_medium(self):
159
return self._get_connection()
161
@deprecated_method(one_four)
162
def get_shared_medium(self):
163
return self._get_shared_connection()
165
def _remote_path(self, relpath):
166
"""Returns the Unicode version of the absolute path for relpath."""
167
return self._combine_paths(self._path, relpath)
169
def _call(self, method, *args):
170
resp = self._call2(method, *args)
171
self._ensure_ok(resp)
173
def _call2(self, method, *args):
174
"""Call a method on the remote server."""
176
return self._client.call(method, *args)
177
except errors.ErrorFromSmartServer, err:
178
# The first argument, if present, is always a path.
180
context = {'relpath': args[0]}
183
self._translate_error(err, **context)
185
def _call_with_body_bytes(self, method, args, body):
186
"""Call a method on the remote server with body bytes."""
188
return self._client.call_with_body_bytes(method, args, body)
189
except errors.ErrorFromSmartServer, err:
190
# The first argument, if present, is always a path.
192
context = {'relpath': args[0]}
195
self._translate_error(err, **context)
197
def has(self, relpath):
198
"""Indicate whether a remote file of the given name exists or not.
200
:see: Transport.has()
202
resp = self._call2('has', self._remote_path(relpath))
203
if resp == ('yes', ):
205
elif resp == ('no', ):
208
raise errors.UnexpectedSmartServerResponse(resp)
210
def get(self, relpath):
211
"""Return file-like object reading the contents of a remote file.
213
:see: Transport.get_bytes()/get_file()
215
return StringIO(self.get_bytes(relpath))
217
def get_bytes(self, relpath):
218
remote = self._remote_path(relpath)
220
resp, response_handler = self._client.call_expecting_body('get', remote)
221
except errors.ErrorFromSmartServer, err:
222
self._translate_error(err, relpath)
224
response_handler.cancel_read_body()
225
raise errors.UnexpectedSmartServerResponse(resp)
226
return response_handler.read_body_bytes()
228
def _serialise_optional_mode(self, mode):
234
def mkdir(self, relpath, mode=None):
235
resp = self._call2('mkdir', self._remote_path(relpath),
236
self._serialise_optional_mode(mode))
238
def open_write_stream(self, relpath, mode=None):
239
"""See Transport.open_write_stream."""
240
self.put_bytes(relpath, "", mode)
241
result = transport.AppendBasedFileStream(self, relpath)
242
transport._file_streams[self.abspath(relpath)] = result
245
def put_bytes(self, relpath, upload_contents, mode=None):
246
# FIXME: upload_file is probably not safe for non-ascii characters -
247
# should probably just pass all parameters as length-delimited
249
if type(upload_contents) is unicode:
250
# Although not strictly correct, we raise UnicodeEncodeError to be
251
# compatible with other transports.
252
raise UnicodeEncodeError(
253
'undefined', upload_contents, 0, 1,
254
'put_bytes must be given bytes, not unicode.')
255
resp = self._call_with_body_bytes('put',
256
(self._remote_path(relpath), self._serialise_optional_mode(mode)),
258
self._ensure_ok(resp)
259
return len(upload_contents)
261
def put_bytes_non_atomic(self, relpath, bytes, mode=None,
262
create_parent_dir=False,
264
"""See Transport.put_bytes_non_atomic."""
265
# FIXME: no encoding in the transport!
266
create_parent_str = 'F'
267
if create_parent_dir:
268
create_parent_str = 'T'
270
resp = self._call_with_body_bytes(
272
(self._remote_path(relpath), self._serialise_optional_mode(mode),
273
create_parent_str, self._serialise_optional_mode(dir_mode)),
275
self._ensure_ok(resp)
277
def put_file(self, relpath, upload_file, mode=None):
278
# its not ideal to seek back, but currently put_non_atomic_file depends
279
# on transports not reading before failing - which is a faulty
280
# assumption I think - RBC 20060915
281
pos = upload_file.tell()
283
return self.put_bytes(relpath, upload_file.read(), mode)
285
upload_file.seek(pos)
288
def put_file_non_atomic(self, relpath, f, mode=None,
289
create_parent_dir=False,
291
return self.put_bytes_non_atomic(relpath, f.read(), mode=mode,
292
create_parent_dir=create_parent_dir,
295
def append_file(self, relpath, from_file, mode=None):
296
return self.append_bytes(relpath, from_file.read(), mode)
298
def append_bytes(self, relpath, bytes, mode=None):
299
resp = self._call_with_body_bytes(
301
(self._remote_path(relpath), self._serialise_optional_mode(mode)),
303
if resp[0] == 'appended':
305
raise errors.UnexpectedSmartServerResponse(resp)
307
def delete(self, relpath):
308
resp = self._call2('delete', self._remote_path(relpath))
309
self._ensure_ok(resp)
311
def external_url(self):
312
"""See bzrlib.transport.Transport.external_url."""
313
# the external path for RemoteTransports is the base
316
def recommended_page_size(self):
317
"""Return the recommended page size for this transport."""
320
def _readv(self, relpath, offsets):
324
offsets = list(offsets)
326
sorted_offsets = sorted(offsets)
327
coalesced = list(self._coalesce_offsets(sorted_offsets,
328
limit=self._max_readv_combine,
329
fudge_factor=self._bytes_to_read_before_seek,
330
max_size=self._max_readv_bytes))
332
# now that we've coallesced things, avoid making enormous requests
337
if c.length + cur_len > self._max_readv_bytes:
338
requests.append(cur_request)
342
cur_request.append(c)
345
requests.append(cur_request)
346
if 'hpss' in debug.debug_flags:
347
trace.mutter('%s.readv %s offsets => %s coalesced'
348
' => %s requests (%s)',
349
self.__class__.__name__, len(offsets), len(coalesced),
350
len(requests), sum(map(len, requests)))
351
# Cache the results, but only until they have been fulfilled
353
# turn the list of offsets into a single stack to iterate
354
offset_stack = iter(offsets)
355
# using a list so it can be modified when passing down and coming back
356
next_offset = [offset_stack.next()]
357
for cur_request in requests:
359
result = self._client.call_with_body_readv_array(
360
('readv', self._remote_path(relpath),),
361
[(c.start, c.length) for c in cur_request])
362
resp, response_handler = result
363
except errors.ErrorFromSmartServer, err:
364
self._translate_error(err, relpath)
366
if resp[0] != 'readv':
367
# This should raise an exception
368
response_handler.cancel_read_body()
369
raise errors.UnexpectedSmartServerResponse(resp)
371
for res in self._handle_response(offset_stack, cur_request,
377
def _handle_response(self, offset_stack, coalesced, response_handler,
378
data_map, next_offset):
379
cur_offset_and_size = next_offset[0]
380
# FIXME: this should know how many bytes are needed, for clarity.
381
data = response_handler.read_body_bytes()
383
for c_offset in coalesced:
384
if len(data) < c_offset.length:
385
raise errors.ShortReadvError(relpath, c_offset.start,
386
c_offset.length, actual=len(data))
387
for suboffset, subsize in c_offset.ranges:
388
key = (c_offset.start+suboffset, subsize)
389
this_data = data[data_offset+suboffset:
390
data_offset+suboffset+subsize]
391
# Special case when the data is in-order, rather than packing
392
# into a map and then back out again. Benchmarking shows that
393
# this has 100% hit rate, but leave in the data_map work just
395
# TODO: Could we get away with using buffer() to avoid the
396
# memory copy? Callers would need to realize they may
397
# not have a real string.
398
if key == cur_offset_and_size:
399
yield cur_offset_and_size[0], this_data
400
cur_offset_and_size = next_offset[0] = offset_stack.next()
402
data_map[key] = this_data
403
data_offset += c_offset.length
405
# Now that we've read some data, see if we can yield anything back
406
while cur_offset_and_size in data_map:
407
this_data = data_map.pop(cur_offset_and_size)
408
yield cur_offset_and_size[0], this_data
409
cur_offset_and_size = next_offset[0] = offset_stack.next()
411
def rename(self, rel_from, rel_to):
413
self._remote_path(rel_from),
414
self._remote_path(rel_to))
416
def move(self, rel_from, rel_to):
418
self._remote_path(rel_from),
419
self._remote_path(rel_to))
421
def rmdir(self, relpath):
422
resp = self._call('rmdir', self._remote_path(relpath))
424
def _ensure_ok(self, resp):
426
raise errors.UnexpectedSmartServerResponse(resp)
428
def _translate_error(self, err, relpath=None):
429
remote._translate_error(err, path=relpath)
431
def disconnect(self):
432
self.get_smart_medium().disconnect()
434
def stat(self, relpath):
435
resp = self._call2('stat', self._remote_path(relpath))
436
if resp[0] == 'stat':
437
return _SmartStat(int(resp[1]), int(resp[2], 8))
438
raise errors.UnexpectedSmartServerResponse(resp)
440
## def lock_read(self, relpath):
441
## """Lock the given file for shared (read) access.
442
## :return: A lock object, which should be passed to Transport.unlock()
444
## # The old RemoteBranch ignore lock for reading, so we will
445
## # continue that tradition and return a bogus lock object.
446
## class BogusLock(object):
447
## def __init__(self, path):
451
## return BogusLock(relpath)
456
def list_dir(self, relpath):
457
resp = self._call2('list_dir', self._remote_path(relpath))
458
if resp[0] == 'names':
459
return [name.encode('ascii') for name in resp[1:]]
460
raise errors.UnexpectedSmartServerResponse(resp)
462
def iter_files_recursive(self):
463
resp = self._call2('iter_files_recursive', self._remote_path(''))
464
if resp[0] == 'names':
466
raise errors.UnexpectedSmartServerResponse(resp)
469
class RemoteTCPTransport(RemoteTransport):
470
"""Connection to smart server over plain tcp.
472
This is essentially just a factory to get 'RemoteTransport(url,
473
SmartTCPClientMedium).
476
def _build_medium(self):
477
client_medium = medium.SmartTCPClientMedium(
478
self._host, self._port, self.base)
479
return client_medium, None
482
class RemoteTCPTransportV2Only(RemoteTransport):
483
"""Connection to smart server over plain tcp with the client hard-coded to
484
assume protocol v2 and remote server version <= 1.6.
486
This should only be used for testing.
489
def _build_medium(self):
490
client_medium = medium.SmartTCPClientMedium(
491
self._host, self._port, self.base)
492
client_medium._protocol_version = 2
493
client_medium._remember_remote_is_before((1, 6))
494
return client_medium, None
497
class RemoteSSHTransport(RemoteTransport):
498
"""Connection to smart server over SSH.
500
This is essentially just a factory to get 'RemoteTransport(url,
501
SmartSSHClientMedium).
504
def _build_medium(self):
505
location_config = config.LocationConfig(self.base)
506
bzr_remote_path = location_config.get_bzr_remote_path()
509
auth = config.AuthenticationConfig()
510
user = auth.get_user('ssh', self._host, self._port)
511
client_medium = medium.SmartSSHClientMedium(self._host, self._port,
512
user, self._password, self.base,
513
bzr_remote_path=bzr_remote_path)
514
return client_medium, (user, self._password)
517
class RemoteHTTPTransport(RemoteTransport):
518
"""Just a way to connect between a bzr+http:// url and http://.
520
This connection operates slightly differently than the RemoteSSHTransport.
521
It uses a plain http:// transport underneath, which defines what remote
522
.bzr/smart URL we are connected to. From there, all paths that are sent are
523
sent as relative paths, this way, the remote side can properly
524
de-reference them, since it is likely doing rewrite rules to translate an
525
HTTP path into a local path.
528
def __init__(self, base, _from_transport=None, http_transport=None):
529
if http_transport is None:
530
# FIXME: the password may be lost here because it appears in the
531
# url only for an intial construction (when the url came from the
533
http_url = base[len('bzr+'):]
534
self._http_transport = transport.get_transport(http_url)
536
self._http_transport = http_transport
537
super(RemoteHTTPTransport, self).__init__(
538
base, _from_transport=_from_transport)
540
def _build_medium(self):
541
# We let http_transport take care of the credentials
542
return self._http_transport.get_smart_medium(), None
544
def _remote_path(self, relpath):
545
"""After connecting, HTTP Transport only deals in relative URLs."""
546
# Adjust the relpath based on which URL this smart transport is
548
http_base = urlutils.normalize_url(self.get_smart_medium().base)
549
url = urlutils.join(self.base[len('bzr+'):], relpath)
550
url = urlutils.normalize_url(url)
551
return urlutils.relative_url(http_base, url)
553
def clone(self, relative_url):
554
"""Make a new RemoteHTTPTransport related to me.
556
This is re-implemented rather than using the default
557
RemoteTransport.clone() because we must be careful about the underlying
560
Also, the cloned smart transport will POST to the same .bzr/smart
561
location as this transport (although obviously the relative paths in the
562
smart requests may be different). This is so that the server doesn't
563
have to handle .bzr/smart requests at arbitrary places inside .bzr
564
directories, just at the initial URL the user uses.
567
abs_url = self.abspath(relative_url)
570
return RemoteHTTPTransport(abs_url,
571
_from_transport=self,
572
http_transport=self._http_transport)
574
def _redirected_to(self, source, target):
575
"""See transport._redirected_to"""
576
redirected = self._http_transport._redirected_to(source, target)
577
if (redirected is not None
578
and isinstance(redirected, type(self._http_transport))):
579
return RemoteHTTPTransport('bzr+' + redirected.external_url(),
580
http_transport=redirected)
582
# Either None or a transport for a different protocol
586
class HintingSSHTransport(transport.Transport):
587
"""Simple transport that handles ssh:// and points out bzr+ssh://."""
589
def __init__(self, url):
590
raise errors.UnsupportedProtocol(url,
591
'bzr supports bzr+ssh to operate over ssh, use "bzr+%s".' % url)
594
def get_test_permutations():
595
"""Return (transport, server) permutations for testing."""
596
### We may need a little more test framework support to construct an
597
### appropriate RemoteTransport in the future.
598
from bzrlib.smart import server
599
return [(RemoteTCPTransport, server.SmartTCPServer_for_testing)]
added
removed
bzrlib/transport/remote.py
