1
# Copyright (C) 2006 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
"""RemoteTransport client for the smart-server.
19
This module shouldn't be accessed directly. The classes defined here should be
20
imported from bzrlib.smart.
23
__all__ = ['RemoteTransport', 'RemoteTCPTransport', 'RemoteSSHTransport']
25
from cStringIO import StringIO
36
from bzrlib.smart import client, medium
37
from bzrlib.symbol_versioning import (deprecated_method, one_four)
40
class _SmartStat(object):
42
def __init__(self, size, mode):
47
class RemoteTransport(transport.ConnectedTransport):
48
"""Connection to a smart server.
50
The connection holds references to the medium that can be used to send
51
requests to the server.
53
The connection has a notion of the current directory to which it's
54
connected; this is incorporated in filenames passed to the server.
56
This supports some higher-level RPC operations and can also be treated
57
like a Transport to do file-like operations.
59
The connection can be made over a tcp socket, an ssh pipe or a series of
60
http requests. There are concrete subclasses for each type:
61
RemoteTCPTransport, etc.
64
# IMPORTANT FOR IMPLEMENTORS: RemoteTransport MUST NOT be given encoding
65
# responsibilities: Put those on SmartClient or similar. This is vital for
66
# the ability to support multiple versions of the smart protocol over time:
67
# RemoteTransport is an adapter from the Transport object model to the
68
# SmartClient model, not an encoder.
70
# FIXME: the medium parameter should be private, only the tests requires
71
# it. It may be even clearer to define a TestRemoteTransport that handles
72
# the specific cases of providing a _client and/or a _medium, and leave
73
# RemoteTransport as an abstract class.
74
def __init__(self, url, _from_transport=None, medium=None, _client=None):
77
:param _from_transport: Another RemoteTransport instance that this
78
one is being cloned from. Attributes such as the medium will
81
:param medium: The medium to use for this RemoteTransport. If None,
82
the medium from the _from_transport is shared. If both this
83
and _from_transport are None, a new medium will be built.
84
_from_transport and medium cannot both be specified.
86
:param _client: Override the _SmartClient used by this transport. This
87
should only be used for testing purposes; normally this is
88
determined from the medium.
90
super(RemoteTransport, self).__init__(url,
91
_from_transport=_from_transport)
93
# The medium is the connection, except when we need to share it with
94
# other objects (RemoteBzrDir, RemoteRepository etc). In these cases
95
# what we want to share is really the shared connection.
97
if _from_transport is None:
98
# If no _from_transport is specified, we need to intialize the
102
medium, credentials = self._build_medium()
103
if 'hpss' in debug.debug_flags:
104
trace.mutter('hpss: Built a new medium: %s',
105
medium.__class__.__name__)
106
self._shared_connection = transport._SharedConnection(medium,
110
# No medium was specified, so share the medium from the
112
medium = self._shared_connection.connection
114
raise AssertionError(
115
"Both _from_transport (%r) and medium (%r) passed to "
116
"RemoteTransport.__init__, but these parameters are mutally "
117
"exclusive." % (_from_transport, medium))
120
self._client = client._SmartClient(medium)
122
self._client = _client
124
def _build_medium(self):
125
"""Create the medium if _from_transport does not provide one.
127
The medium is analogous to the connection for ConnectedTransport: it
128
allows connection sharing.
133
def is_readonly(self):
134
"""Smart server transport can do read/write file operations."""
136
resp = self._call2('Transport.is_readonly')
137
except errors.UnknownSmartMethod:
138
# XXX: nasty hack: servers before 0.16 don't have a
139
# 'Transport.is_readonly' verb, so we do what clients before 0.16
142
if resp == ('yes', ):
144
elif resp == ('no', ):
147
raise errors.UnexpectedSmartServerResponse(resp)
149
def get_smart_client(self):
150
return self._get_connection()
152
def get_smart_medium(self):
153
return self._get_connection()
155
@deprecated_method(one_four)
156
def get_shared_medium(self):
157
return self._get_shared_connection()
159
def _remote_path(self, relpath):
160
"""Returns the Unicode version of the absolute path for relpath."""
161
return self._combine_paths(self._path, relpath)
163
def _call(self, method, *args):
164
resp = self._call2(method, *args)
165
self._ensure_ok(resp)
167
def _call2(self, method, *args):
168
"""Call a method on the remote server."""
170
return self._client.call(method, *args)
171
except errors.ErrorFromSmartServer, err:
172
self._translate_error(err)
174
def _call_with_body_bytes(self, method, args, body):
175
"""Call a method on the remote server with body bytes."""
177
return self._client.call_with_body_bytes(method, args, body)
178
except errors.ErrorFromSmartServer, err:
179
self._translate_error(err)
181
def has(self, relpath):
182
"""Indicate whether a remote file of the given name exists or not.
184
:see: Transport.has()
186
resp = self._call2('has', self._remote_path(relpath))
187
if resp == ('yes', ):
189
elif resp == ('no', ):
192
raise errors.UnexpectedSmartServerResponse(resp)
194
def get(self, relpath):
195
"""Return file-like object reading the contents of a remote file.
197
:see: Transport.get_bytes()/get_file()
199
return StringIO(self.get_bytes(relpath))
201
def get_bytes(self, relpath):
202
remote = self._remote_path(relpath)
204
resp, response_handler = self._client.call_expecting_body('get', remote)
205
except errors.ErrorFromSmartServer, err:
206
self._translate_error(err, relpath)
208
response_handler.cancel_read_body()
209
raise errors.UnexpectedSmartServerResponse(resp)
210
return response_handler.read_body_bytes()
212
def _serialise_optional_mode(self, mode):
218
def mkdir(self, relpath, mode=None):
219
resp = self._call2('mkdir', self._remote_path(relpath),
220
self._serialise_optional_mode(mode))
222
def open_write_stream(self, relpath, mode=None):
223
"""See Transport.open_write_stream."""
224
self.put_bytes(relpath, "", mode)
225
result = transport.AppendBasedFileStream(self, relpath)
226
transport._file_streams[self.abspath(relpath)] = result
229
def put_bytes(self, relpath, upload_contents, mode=None):
230
# FIXME: upload_file is probably not safe for non-ascii characters -
231
# should probably just pass all parameters as length-delimited
233
if type(upload_contents) is unicode:
234
# Although not strictly correct, we raise UnicodeEncodeError to be
235
# compatible with other transports.
236
raise UnicodeEncodeError(
237
'undefined', upload_contents, 0, 1,
238
'put_bytes must be given bytes, not unicode.')
239
resp = self._call_with_body_bytes('put',
240
(self._remote_path(relpath), self._serialise_optional_mode(mode)),
242
self._ensure_ok(resp)
243
return len(upload_contents)
245
def put_bytes_non_atomic(self, relpath, bytes, mode=None,
246
create_parent_dir=False,
248
"""See Transport.put_bytes_non_atomic."""
249
# FIXME: no encoding in the transport!
250
create_parent_str = 'F'
251
if create_parent_dir:
252
create_parent_str = 'T'
254
resp = self._call_with_body_bytes(
256
(self._remote_path(relpath), self._serialise_optional_mode(mode),
257
create_parent_str, self._serialise_optional_mode(dir_mode)),
259
self._ensure_ok(resp)
261
def put_file(self, relpath, upload_file, mode=None):
262
# its not ideal to seek back, but currently put_non_atomic_file depends
263
# on transports not reading before failing - which is a faulty
264
# assumption I think - RBC 20060915
265
pos = upload_file.tell()
267
return self.put_bytes(relpath, upload_file.read(), mode)
269
upload_file.seek(pos)
272
def put_file_non_atomic(self, relpath, f, mode=None,
273
create_parent_dir=False,
275
return self.put_bytes_non_atomic(relpath, f.read(), mode=mode,
276
create_parent_dir=create_parent_dir,
279
def append_file(self, relpath, from_file, mode=None):
280
return self.append_bytes(relpath, from_file.read(), mode)
282
def append_bytes(self, relpath, bytes, mode=None):
283
resp = self._call_with_body_bytes(
285
(self._remote_path(relpath), self._serialise_optional_mode(mode)),
287
if resp[0] == 'appended':
289
raise errors.UnexpectedSmartServerResponse(resp)
291
def delete(self, relpath):
292
resp = self._call2('delete', self._remote_path(relpath))
293
self._ensure_ok(resp)
295
def external_url(self):
296
"""See bzrlib.transport.Transport.external_url."""
297
# the external path for RemoteTransports is the base
300
def recommended_page_size(self):
301
"""Return the recommended page size for this transport."""
304
def _readv(self, relpath, offsets):
308
offsets = list(offsets)
310
sorted_offsets = sorted(offsets)
311
coalesced = list(self._coalesce_offsets(sorted_offsets,
312
limit=self._max_readv_combine,
313
fudge_factor=self._bytes_to_read_before_seek))
316
result = self._client.call_with_body_readv_array(
317
('readv', self._remote_path(relpath),),
318
[(c.start, c.length) for c in coalesced])
319
resp, response_handler = result
320
except errors.ErrorFromSmartServer, err:
321
self._translate_error(err)
323
if resp[0] != 'readv':
324
# This should raise an exception
325
response_handler.cancel_read_body()
326
raise errors.UnexpectedSmartServerResponse(resp)
328
return self._handle_response(offsets, coalesced, response_handler)
330
def _handle_response(self, offsets, coalesced, response_handler):
331
# turn the list of offsets into a stack
332
offset_stack = iter(offsets)
333
cur_offset_and_size = offset_stack.next()
334
# FIXME: this should know how many bytes are needed, for clarity.
335
data = response_handler.read_body_bytes()
336
# Cache the results, but only until they have been fulfilled
339
for c_offset in coalesced:
340
if len(data) < c_offset.length:
341
raise errors.ShortReadvError(relpath, c_offset.start,
342
c_offset.length, actual=len(data))
343
for suboffset, subsize in c_offset.ranges:
344
key = (c_offset.start+suboffset, subsize)
345
this_data = data[data_offset+suboffset:
346
data_offset+suboffset+subsize]
347
# Special case when the data is in-order, rather than packing
348
# into a map and then back out again. Benchmarking shows that
349
# this has 100% hit rate, but leave in the data_map work just
351
# TODO: Could we get away with using buffer() to avoid the
352
# memory copy? Callers would need to realize they may
353
# not have a real string.
354
if key == cur_offset_and_size:
355
yield cur_offset_and_size[0], this_data
356
cur_offset_and_size = offset_stack.next()
358
data_map[key] = this_data
359
data_offset += c_offset.length
361
# Now that we've read some data, see if we can yield anything back
362
while cur_offset_and_size in data_map:
363
this_data = data_map.pop(cur_offset_and_size)
364
yield cur_offset_and_size[0], this_data
365
cur_offset_and_size = offset_stack.next()
367
def rename(self, rel_from, rel_to):
369
self._remote_path(rel_from),
370
self._remote_path(rel_to))
372
def move(self, rel_from, rel_to):
374
self._remote_path(rel_from),
375
self._remote_path(rel_to))
377
def rmdir(self, relpath):
378
resp = self._call('rmdir', self._remote_path(relpath))
380
def _ensure_ok(self, resp):
382
raise errors.UnexpectedSmartServerResponse(resp)
384
def _translate_error(self, err, orig_path=None):
385
remote._translate_error(err, path=orig_path)
387
def disconnect(self):
388
self.get_smart_medium().disconnect()
390
def stat(self, relpath):
391
resp = self._call2('stat', self._remote_path(relpath))
392
if resp[0] == 'stat':
393
return _SmartStat(int(resp[1]), int(resp[2], 8))
394
raise errors.UnexpectedSmartServerResponse(resp)
396
## def lock_read(self, relpath):
397
## """Lock the given file for shared (read) access.
398
## :return: A lock object, which should be passed to Transport.unlock()
400
## # The old RemoteBranch ignore lock for reading, so we will
401
## # continue that tradition and return a bogus lock object.
402
## class BogusLock(object):
403
## def __init__(self, path):
407
## return BogusLock(relpath)
412
def list_dir(self, relpath):
413
resp = self._call2('list_dir', self._remote_path(relpath))
414
if resp[0] == 'names':
415
return [name.encode('ascii') for name in resp[1:]]
416
raise errors.UnexpectedSmartServerResponse(resp)
418
def iter_files_recursive(self):
419
resp = self._call2('iter_files_recursive', self._remote_path(''))
420
if resp[0] == 'names':
422
raise errors.UnexpectedSmartServerResponse(resp)
425
class RemoteTCPTransport(RemoteTransport):
426
"""Connection to smart server over plain tcp.
428
This is essentially just a factory to get 'RemoteTransport(url,
429
SmartTCPClientMedium).
432
def _build_medium(self):
433
client_medium = medium.SmartTCPClientMedium(
434
self._host, self._port, self.base)
435
return client_medium, None
438
class RemoteTCPTransportV2Only(RemoteTransport):
439
"""Connection to smart server over plain tcp with the client hard-coded to
440
assume protocol v2 and remote server version <= 1.6.
442
This should only be used for testing.
445
def _build_medium(self):
446
client_medium = medium.SmartTCPClientMedium(
447
self._host, self._port, self.base)
448
client_medium._protocol_version = 2
449
client_medium._remember_remote_is_before((1, 6))
450
return client_medium, None
453
class RemoteSSHTransport(RemoteTransport):
454
"""Connection to smart server over SSH.
456
This is essentially just a factory to get 'RemoteTransport(url,
457
SmartSSHClientMedium).
460
def _build_medium(self):
461
location_config = config.LocationConfig(self.base)
462
bzr_remote_path = location_config.get_bzr_remote_path()
465
auth = config.AuthenticationConfig()
466
user = auth.get_user('ssh', self._host, self._port)
467
client_medium = medium.SmartSSHClientMedium(self._host, self._port,
468
user, self._password, self.base,
469
bzr_remote_path=bzr_remote_path)
470
return client_medium, (user, self._password)
473
class RemoteHTTPTransport(RemoteTransport):
474
"""Just a way to connect between a bzr+http:// url and http://.
476
This connection operates slightly differently than the RemoteSSHTransport.
477
It uses a plain http:// transport underneath, which defines what remote
478
.bzr/smart URL we are connected to. From there, all paths that are sent are
479
sent as relative paths, this way, the remote side can properly
480
de-reference them, since it is likely doing rewrite rules to translate an
481
HTTP path into a local path.
484
def __init__(self, base, _from_transport=None, http_transport=None):
485
if http_transport is None:
486
# FIXME: the password may be lost here because it appears in the
487
# url only for an intial construction (when the url came from the
489
http_url = base[len('bzr+'):]
490
self._http_transport = transport.get_transport(http_url)
492
self._http_transport = http_transport
493
super(RemoteHTTPTransport, self).__init__(
494
base, _from_transport=_from_transport)
496
def _build_medium(self):
497
# We let http_transport take care of the credentials
498
return self._http_transport.get_smart_medium(), None
500
def _remote_path(self, relpath):
501
"""After connecting, HTTP Transport only deals in relative URLs."""
502
# Adjust the relpath based on which URL this smart transport is
504
http_base = urlutils.normalize_url(self.get_smart_medium().base)
505
url = urlutils.join(self.base[len('bzr+'):], relpath)
506
url = urlutils.normalize_url(url)
507
return urlutils.relative_url(http_base, url)
509
def clone(self, relative_url):
510
"""Make a new RemoteHTTPTransport related to me.
512
This is re-implemented rather than using the default
513
RemoteTransport.clone() because we must be careful about the underlying
516
Also, the cloned smart transport will POST to the same .bzr/smart
517
location as this transport (although obviously the relative paths in the
518
smart requests may be different). This is so that the server doesn't
519
have to handle .bzr/smart requests at arbitrary places inside .bzr
520
directories, just at the initial URL the user uses.
523
abs_url = self.abspath(relative_url)
526
return RemoteHTTPTransport(abs_url,
527
_from_transport=self,
528
http_transport=self._http_transport)
531
def get_test_permutations():
532
"""Return (transport, server) permutations for testing."""
533
### We may need a little more test framework support to construct an
534
### appropriate RemoteTransport in the future.
535
from bzrlib.smart import server
536
return [(RemoteTCPTransport, server.SmartTCPServer_for_testing)]
added
removed
bzrlib/transport/remote.py
