1
# Copyright (C) 2006 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
"""RemoteTransport client for the smart-server.
19
This module shouldn't be accessed directly. The classes defined here should be
20
imported from bzrlib.smart.
23
__all__ = ['RemoteTransport', 'RemoteTCPTransport', 'RemoteSSHTransport']
25
from cStringIO import StringIO
36
from bzrlib.smart import client, medium
37
from bzrlib.symbol_versioning import (deprecated_method, one_four)
40
class _SmartStat(object):
42
def __init__(self, size, mode):
47
class RemoteTransport(transport.ConnectedTransport):
48
"""Connection to a smart server.
50
The connection holds references to the medium that can be used to send
51
requests to the server.
53
The connection has a notion of the current directory to which it's
54
connected; this is incorporated in filenames passed to the server.
56
This supports some higher-level RPC operations and can also be treated
57
like a Transport to do file-like operations.
59
The connection can be made over a tcp socket, an ssh pipe or a series of
60
http requests. There are concrete subclasses for each type:
61
RemoteTCPTransport, etc.
64
# When making a readv request, cap it at requesting 5MB of data
65
_max_readv_bytes = 5*1024*1024
67
# IMPORTANT FOR IMPLEMENTORS: RemoteTransport MUST NOT be given encoding
68
# responsibilities: Put those on SmartClient or similar. This is vital for
69
# the ability to support multiple versions of the smart protocol over time:
70
# RemoteTransport is an adapter from the Transport object model to the
71
# SmartClient model, not an encoder.
73
# FIXME: the medium parameter should be private, only the tests requires
74
# it. It may be even clearer to define a TestRemoteTransport that handles
75
# the specific cases of providing a _client and/or a _medium, and leave
76
# RemoteTransport as an abstract class.
77
def __init__(self, url, _from_transport=None, medium=None, _client=None):
80
:param _from_transport: Another RemoteTransport instance that this
81
one is being cloned from. Attributes such as the medium will
84
:param medium: The medium to use for this RemoteTransport. If None,
85
the medium from the _from_transport is shared. If both this
86
and _from_transport are None, a new medium will be built.
87
_from_transport and medium cannot both be specified.
89
:param _client: Override the _SmartClient used by this transport. This
90
should only be used for testing purposes; normally this is
91
determined from the medium.
93
super(RemoteTransport, self).__init__(url,
94
_from_transport=_from_transport)
96
# The medium is the connection, except when we need to share it with
97
# other objects (RemoteBzrDir, RemoteRepository etc). In these cases
98
# what we want to share is really the shared connection.
100
if _from_transport is None:
101
# If no _from_transport is specified, we need to intialize the
105
medium, credentials = self._build_medium()
106
if 'hpss' in debug.debug_flags:
107
trace.mutter('hpss: Built a new medium: %s',
108
medium.__class__.__name__)
109
self._shared_connection = transport._SharedConnection(medium,
113
# No medium was specified, so share the medium from the
115
medium = self._shared_connection.connection
117
raise AssertionError(
118
"Both _from_transport (%r) and medium (%r) passed to "
119
"RemoteTransport.__init__, but these parameters are mutally "
120
"exclusive." % (_from_transport, medium))
123
self._client = client._SmartClient(medium)
125
self._client = _client
127
def _build_medium(self):
128
"""Create the medium if _from_transport does not provide one.
130
The medium is analogous to the connection for ConnectedTransport: it
131
allows connection sharing.
136
def is_readonly(self):
137
"""Smart server transport can do read/write file operations."""
139
resp = self._call2('Transport.is_readonly')
140
except errors.UnknownSmartMethod:
141
# XXX: nasty hack: servers before 0.16 don't have a
142
# 'Transport.is_readonly' verb, so we do what clients before 0.16
145
if resp == ('yes', ):
147
elif resp == ('no', ):
150
raise errors.UnexpectedSmartServerResponse(resp)
152
def get_smart_client(self):
153
return self._get_connection()
155
def get_smart_medium(self):
156
return self._get_connection()
158
@deprecated_method(one_four)
159
def get_shared_medium(self):
160
return self._get_shared_connection()
162
def _remote_path(self, relpath):
163
"""Returns the Unicode version of the absolute path for relpath."""
164
return self._combine_paths(self._path, relpath)
166
def _call(self, method, *args):
167
resp = self._call2(method, *args)
168
self._ensure_ok(resp)
170
def _call2(self, method, *args):
171
"""Call a method on the remote server."""
173
return self._client.call(method, *args)
174
except errors.ErrorFromSmartServer, err:
175
# The first argument, if present, is always a path.
177
context = {'relpath': args[0]}
180
self._translate_error(err, **context)
182
def _call_with_body_bytes(self, method, args, body):
183
"""Call a method on the remote server with body bytes."""
185
return self._client.call_with_body_bytes(method, args, body)
186
except errors.ErrorFromSmartServer, err:
187
# The first argument, if present, is always a path.
189
context = {'relpath': args[0]}
192
self._translate_error(err, **context)
194
def has(self, relpath):
195
"""Indicate whether a remote file of the given name exists or not.
197
:see: Transport.has()
199
resp = self._call2('has', self._remote_path(relpath))
200
if resp == ('yes', ):
202
elif resp == ('no', ):
205
raise errors.UnexpectedSmartServerResponse(resp)
207
def get(self, relpath):
208
"""Return file-like object reading the contents of a remote file.
210
:see: Transport.get_bytes()/get_file()
212
return StringIO(self.get_bytes(relpath))
214
def get_bytes(self, relpath):
215
remote = self._remote_path(relpath)
217
resp, response_handler = self._client.call_expecting_body('get', remote)
218
except errors.ErrorFromSmartServer, err:
219
self._translate_error(err, relpath)
221
response_handler.cancel_read_body()
222
raise errors.UnexpectedSmartServerResponse(resp)
223
return response_handler.read_body_bytes()
225
def _serialise_optional_mode(self, mode):
231
def mkdir(self, relpath, mode=None):
232
resp = self._call2('mkdir', self._remote_path(relpath),
233
self._serialise_optional_mode(mode))
235
def open_write_stream(self, relpath, mode=None):
236
"""See Transport.open_write_stream."""
237
self.put_bytes(relpath, "", mode)
238
result = transport.AppendBasedFileStream(self, relpath)
239
transport._file_streams[self.abspath(relpath)] = result
242
def put_bytes(self, relpath, upload_contents, mode=None):
243
# FIXME: upload_file is probably not safe for non-ascii characters -
244
# should probably just pass all parameters as length-delimited
246
if type(upload_contents) is unicode:
247
# Although not strictly correct, we raise UnicodeEncodeError to be
248
# compatible with other transports.
249
raise UnicodeEncodeError(
250
'undefined', upload_contents, 0, 1,
251
'put_bytes must be given bytes, not unicode.')
252
resp = self._call_with_body_bytes('put',
253
(self._remote_path(relpath), self._serialise_optional_mode(mode)),
255
self._ensure_ok(resp)
256
return len(upload_contents)
258
def put_bytes_non_atomic(self, relpath, bytes, mode=None,
259
create_parent_dir=False,
261
"""See Transport.put_bytes_non_atomic."""
262
# FIXME: no encoding in the transport!
263
create_parent_str = 'F'
264
if create_parent_dir:
265
create_parent_str = 'T'
267
resp = self._call_with_body_bytes(
269
(self._remote_path(relpath), self._serialise_optional_mode(mode),
270
create_parent_str, self._serialise_optional_mode(dir_mode)),
272
self._ensure_ok(resp)
274
def put_file(self, relpath, upload_file, mode=None):
275
# its not ideal to seek back, but currently put_non_atomic_file depends
276
# on transports not reading before failing - which is a faulty
277
# assumption I think - RBC 20060915
278
pos = upload_file.tell()
280
return self.put_bytes(relpath, upload_file.read(), mode)
282
upload_file.seek(pos)
285
def put_file_non_atomic(self, relpath, f, mode=None,
286
create_parent_dir=False,
288
return self.put_bytes_non_atomic(relpath, f.read(), mode=mode,
289
create_parent_dir=create_parent_dir,
292
def append_file(self, relpath, from_file, mode=None):
293
return self.append_bytes(relpath, from_file.read(), mode)
295
def append_bytes(self, relpath, bytes, mode=None):
296
resp = self._call_with_body_bytes(
298
(self._remote_path(relpath), self._serialise_optional_mode(mode)),
300
if resp[0] == 'appended':
302
raise errors.UnexpectedSmartServerResponse(resp)
304
def delete(self, relpath):
305
resp = self._call2('delete', self._remote_path(relpath))
306
self._ensure_ok(resp)
308
def external_url(self):
309
"""See bzrlib.transport.Transport.external_url."""
310
# the external path for RemoteTransports is the base
313
def recommended_page_size(self):
314
"""Return the recommended page size for this transport."""
317
def _readv(self, relpath, offsets):
321
offsets = list(offsets)
323
sorted_offsets = sorted(offsets)
324
coalesced = list(self._coalesce_offsets(sorted_offsets,
325
limit=self._max_readv_combine,
326
fudge_factor=self._bytes_to_read_before_seek,
327
max_size=self._max_readv_bytes))
329
# now that we've coallesced things, avoid making enormous requests
334
if c.length + cur_len > self._max_readv_bytes:
335
requests.append(cur_request)
339
cur_request.append(c)
342
requests.append(cur_request)
343
if 'hpss' in debug.debug_flags:
344
trace.mutter('%s.readv %s offsets => %s coalesced'
345
' => %s requests (%s)',
346
self.__class__.__name__, len(offsets), len(coalesced),
347
len(requests), sum(map(len, requests)))
348
# Cache the results, but only until they have been fulfilled
350
# turn the list of offsets into a single stack to iterate
351
offset_stack = iter(offsets)
352
# using a list so it can be modified when passing down and coming back
353
next_offset = [offset_stack.next()]
354
for cur_request in requests:
356
result = self._client.call_with_body_readv_array(
357
('readv', self._remote_path(relpath),),
358
[(c.start, c.length) for c in cur_request])
359
resp, response_handler = result
360
except errors.ErrorFromSmartServer, err:
361
self._translate_error(err, relpath)
363
if resp[0] != 'readv':
364
# This should raise an exception
365
response_handler.cancel_read_body()
366
raise errors.UnexpectedSmartServerResponse(resp)
368
for res in self._handle_response(offset_stack, cur_request,
374
def _handle_response(self, offset_stack, coalesced, response_handler,
375
data_map, next_offset):
376
cur_offset_and_size = next_offset[0]
377
# FIXME: this should know how many bytes are needed, for clarity.
378
data = response_handler.read_body_bytes()
380
for c_offset in coalesced:
381
if len(data) < c_offset.length:
382
raise errors.ShortReadvError(relpath, c_offset.start,
383
c_offset.length, actual=len(data))
384
for suboffset, subsize in c_offset.ranges:
385
key = (c_offset.start+suboffset, subsize)
386
this_data = data[data_offset+suboffset:
387
data_offset+suboffset+subsize]
388
# Special case when the data is in-order, rather than packing
389
# into a map and then back out again. Benchmarking shows that
390
# this has 100% hit rate, but leave in the data_map work just
392
# TODO: Could we get away with using buffer() to avoid the
393
# memory copy? Callers would need to realize they may
394
# not have a real string.
395
if key == cur_offset_and_size:
396
yield cur_offset_and_size[0], this_data
397
cur_offset_and_size = next_offset[0] = offset_stack.next()
399
data_map[key] = this_data
400
data_offset += c_offset.length
402
# Now that we've read some data, see if we can yield anything back
403
while cur_offset_and_size in data_map:
404
this_data = data_map.pop(cur_offset_and_size)
405
yield cur_offset_and_size[0], this_data
406
cur_offset_and_size = next_offset[0] = offset_stack.next()
408
def rename(self, rel_from, rel_to):
410
self._remote_path(rel_from),
411
self._remote_path(rel_to))
413
def move(self, rel_from, rel_to):
415
self._remote_path(rel_from),
416
self._remote_path(rel_to))
418
def rmdir(self, relpath):
419
resp = self._call('rmdir', self._remote_path(relpath))
421
def _ensure_ok(self, resp):
423
raise errors.UnexpectedSmartServerResponse(resp)
425
def _translate_error(self, err, relpath=None):
426
remote._translate_error(err, path=relpath)
428
def disconnect(self):
429
self.get_smart_medium().disconnect()
431
def stat(self, relpath):
432
resp = self._call2('stat', self._remote_path(relpath))
433
if resp[0] == 'stat':
434
return _SmartStat(int(resp[1]), int(resp[2], 8))
435
raise errors.UnexpectedSmartServerResponse(resp)
437
## def lock_read(self, relpath):
438
## """Lock the given file for shared (read) access.
439
## :return: A lock object, which should be passed to Transport.unlock()
441
## # The old RemoteBranch ignore lock for reading, so we will
442
## # continue that tradition and return a bogus lock object.
443
## class BogusLock(object):
444
## def __init__(self, path):
448
## return BogusLock(relpath)
453
def list_dir(self, relpath):
454
resp = self._call2('list_dir', self._remote_path(relpath))
455
if resp[0] == 'names':
456
return [name.encode('ascii') for name in resp[1:]]
457
raise errors.UnexpectedSmartServerResponse(resp)
459
def iter_files_recursive(self):
460
resp = self._call2('iter_files_recursive', self._remote_path(''))
461
if resp[0] == 'names':
463
raise errors.UnexpectedSmartServerResponse(resp)
466
class RemoteTCPTransport(RemoteTransport):
467
"""Connection to smart server over plain tcp.
469
This is essentially just a factory to get 'RemoteTransport(url,
470
SmartTCPClientMedium).
473
def _build_medium(self):
474
client_medium = medium.SmartTCPClientMedium(
475
self._host, self._port, self.base)
476
return client_medium, None
479
class RemoteTCPTransportV2Only(RemoteTransport):
480
"""Connection to smart server over plain tcp with the client hard-coded to
481
assume protocol v2 and remote server version <= 1.6.
483
This should only be used for testing.
486
def _build_medium(self):
487
client_medium = medium.SmartTCPClientMedium(
488
self._host, self._port, self.base)
489
client_medium._protocol_version = 2
490
client_medium._remember_remote_is_before((1, 6))
491
return client_medium, None
494
class RemoteSSHTransport(RemoteTransport):
495
"""Connection to smart server over SSH.
497
This is essentially just a factory to get 'RemoteTransport(url,
498
SmartSSHClientMedium).
501
def _build_medium(self):
502
location_config = config.LocationConfig(self.base)
503
bzr_remote_path = location_config.get_bzr_remote_path()
506
auth = config.AuthenticationConfig()
507
user = auth.get_user('ssh', self._host, self._port)
508
client_medium = medium.SmartSSHClientMedium(self._host, self._port,
509
user, self._password, self.base,
510
bzr_remote_path=bzr_remote_path)
511
return client_medium, (user, self._password)
514
class RemoteHTTPTransport(RemoteTransport):
515
"""Just a way to connect between a bzr+http:// url and http://.
517
This connection operates slightly differently than the RemoteSSHTransport.
518
It uses a plain http:// transport underneath, which defines what remote
519
.bzr/smart URL we are connected to. From there, all paths that are sent are
520
sent as relative paths, this way, the remote side can properly
521
de-reference them, since it is likely doing rewrite rules to translate an
522
HTTP path into a local path.
525
def __init__(self, base, _from_transport=None, http_transport=None):
526
if http_transport is None:
527
# FIXME: the password may be lost here because it appears in the
528
# url only for an intial construction (when the url came from the
530
http_url = base[len('bzr+'):]
531
self._http_transport = transport.get_transport(http_url)
533
self._http_transport = http_transport
534
super(RemoteHTTPTransport, self).__init__(
535
base, _from_transport=_from_transport)
537
def _build_medium(self):
538
# We let http_transport take care of the credentials
539
return self._http_transport.get_smart_medium(), None
541
def _remote_path(self, relpath):
542
"""After connecting, HTTP Transport only deals in relative URLs."""
543
# Adjust the relpath based on which URL this smart transport is
545
http_base = urlutils.normalize_url(self.get_smart_medium().base)
546
url = urlutils.join(self.base[len('bzr+'):], relpath)
547
url = urlutils.normalize_url(url)
548
return urlutils.relative_url(http_base, url)
550
def clone(self, relative_url):
551
"""Make a new RemoteHTTPTransport related to me.
553
This is re-implemented rather than using the default
554
RemoteTransport.clone() because we must be careful about the underlying
557
Also, the cloned smart transport will POST to the same .bzr/smart
558
location as this transport (although obviously the relative paths in the
559
smart requests may be different). This is so that the server doesn't
560
have to handle .bzr/smart requests at arbitrary places inside .bzr
561
directories, just at the initial URL the user uses.
564
abs_url = self.abspath(relative_url)
567
return RemoteHTTPTransport(abs_url,
568
_from_transport=self,
569
http_transport=self._http_transport)
571
def _redirected_to(self, source, target):
572
"""See transport._redirected_to"""
573
redirected = self._http_transport._redirected_to(source, target)
574
if (redirected is not None
575
and isinstance(redirected, type(self._http_transport))):
576
return RemoteHTTPTransport('bzr+' + redirected.external_url(),
577
http_transport=redirected)
579
# Either None or a transport for a different protocol
583
def get_test_permutations():
584
"""Return (transport, server) permutations for testing."""
585
### We may need a little more test framework support to construct an
586
### appropriate RemoteTransport in the future.
587
from bzrlib.smart import server
588
return [(RemoteTCPTransport, server.SmartTCPServer_for_testing)]
added
removed
bzrlib/transport/remote.py
