1
# Copyright (C) 2006 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
"""RemoteTransport client for the smart-server.
19
This module shouldn't be accessed directly. The classes defined here should be
20
imported from bzrlib.smart.
23
__all__ = ['RemoteTransport', 'RemoteTCPTransport', 'RemoteSSHTransport']
25
from cStringIO import StringIO
36
from bzrlib.smart import client, medium, protocol
38
# must do this otherwise urllib can't parse the urls properly :(
39
for scheme in ['ssh', 'bzr', 'bzr+loopback', 'bzr+ssh', 'bzr+http']:
40
transport.register_urlparse_netloc_protocol(scheme)
44
# Port 4155 is the default port for bzr://, registered with IANA.
45
BZR_DEFAULT_INTERFACE = '0.0.0.0'
46
BZR_DEFAULT_PORT = 4155
49
class _SmartStat(object):
51
def __init__(self, size, mode):
56
class RemoteTransport(transport.ConnectedTransport):
57
"""Connection to a smart server.
59
The connection holds references to the medium that can be used to send
60
requests to the server.
62
The connection has a notion of the current directory to which it's
63
connected; this is incorporated in filenames passed to the server.
65
This supports some higher-level RPC operations and can also be treated
66
like a Transport to do file-like operations.
68
The connection can be made over a tcp socket, an ssh pipe or a series of
69
http requests. There are concrete subclasses for each type:
70
RemoteTCPTransport, etc.
73
# IMPORTANT FOR IMPLEMENTORS: RemoteTransport MUST NOT be given encoding
74
# responsibilities: Put those on SmartClient or similar. This is vital for
75
# the ability to support multiple versions of the smart protocol over time:
76
# RemoteTransport is an adapter from the Transport object model to the
77
# SmartClient model, not an encoder.
79
# FIXME: the medium parameter should be private, only the tests requires
80
# it. It may be even clearer to define a TestRemoteTransport that handles
81
# the specific cases of providing a _client and/or a _medium, and leave
82
# RemoteTransport as an abstract class.
83
def __init__(self, url, _from_transport=None, medium=None, _client=None):
86
:param _from_transport: Another RemoteTransport instance that this
87
one is being cloned from. Attributes such as the medium will
90
:param medium: The medium to use for this RemoteTransport. This must be
91
supplied if _from_transport is None.
93
:param _client: Override the _SmartClient used by this transport. This
94
should only be used for testing purposes; normally this is
95
determined from the medium.
97
super(RemoteTransport, self).__init__(url,
98
_from_transport=_from_transport)
100
# The medium is the connection, except when we need to share it with
101
# other objects (RemoteBzrDir, RemoteRepository etc). In these cases
102
# what we want to share is really the shared connection.
104
if _from_transport is None:
105
# If no _from_transport is specified, we need to intialize the
109
medium, credentials = self._build_medium()
110
if 'hpss' in debug.debug_flags:
111
trace.mutter('hpss: Built a new medium: %s',
112
medium.__class__.__name__)
113
self._shared_connection = transport._SharedConnection(medium,
117
self._client = client._SmartClient(self.get_shared_medium())
119
self._client = _client
121
def _build_medium(self):
122
"""Create the medium if _from_transport does not provide one.
124
The medium is analogous to the connection for ConnectedTransport: it
125
allows connection sharing.
130
def is_readonly(self):
131
"""Smart server transport can do read/write file operations."""
132
resp = self._call2('Transport.is_readonly')
133
if resp == ('yes', ):
135
elif resp == ('no', ):
137
elif (resp == ('error', "Generic bzr smart protocol error: "
138
"bad request 'Transport.is_readonly'") or
139
resp == ('error', "Generic bzr smart protocol error: "
140
"bad request u'Transport.is_readonly'")):
141
# XXX: nasty hack: servers before 0.16 don't have a
142
# 'Transport.is_readonly' verb, so we do what clients before 0.16
146
self._translate_error(resp)
147
raise errors.UnexpectedSmartServerResponse(resp)
149
def get_smart_client(self):
150
return self._get_connection()
152
def get_smart_medium(self):
153
return self._get_connection()
155
def get_shared_medium(self):
156
return self._get_shared_connection()
158
def _remote_path(self, relpath):
159
"""Returns the Unicode version of the absolute path for relpath."""
160
return self._combine_paths(self._path, relpath)
162
def _call(self, method, *args):
163
resp = self._call2(method, *args)
164
self._translate_error(resp)
166
def _call2(self, method, *args):
167
"""Call a method on the remote server."""
168
return self._client.call(method, *args)
170
def _call_with_body_bytes(self, method, args, body):
171
"""Call a method on the remote server with body bytes."""
172
return self._client.call_with_body_bytes(method, args, body)
174
def has(self, relpath):
175
"""Indicate whether a remote file of the given name exists or not.
177
:see: Transport.has()
179
resp = self._call2('has', self._remote_path(relpath))
180
if resp == ('yes', ):
182
elif resp == ('no', ):
185
self._translate_error(resp)
187
def get(self, relpath):
188
"""Return file-like object reading the contents of a remote file.
190
:see: Transport.get_bytes()/get_file()
192
return StringIO(self.get_bytes(relpath))
194
def get_bytes(self, relpath):
195
remote = self._remote_path(relpath)
196
request = self.get_smart_medium().get_request()
197
smart_protocol = protocol.SmartClientRequestProtocolOne(request)
198
smart_protocol.call('get', remote)
199
resp = smart_protocol.read_response_tuple(True)
201
smart_protocol.cancel_read_body()
202
self._translate_error(resp, relpath)
203
return smart_protocol.read_body_bytes()
205
def _serialise_optional_mode(self, mode):
211
def mkdir(self, relpath, mode=None):
212
resp = self._call2('mkdir', self._remote_path(relpath),
213
self._serialise_optional_mode(mode))
214
self._translate_error(resp)
216
def open_write_stream(self, relpath, mode=None):
217
"""See Transport.open_write_stream."""
218
self.put_bytes(relpath, "", mode)
219
result = transport.AppendBasedFileStream(self, relpath)
220
transport._file_streams[self.abspath(relpath)] = result
223
def put_bytes(self, relpath, upload_contents, mode=None):
224
# FIXME: upload_file is probably not safe for non-ascii characters -
225
# should probably just pass all parameters as length-delimited
227
if type(upload_contents) is unicode:
228
# Although not strictly correct, we raise UnicodeEncodeError to be
229
# compatible with other transports.
230
raise UnicodeEncodeError(
231
'undefined', upload_contents, 0, 1,
232
'put_bytes must be given bytes, not unicode.')
233
resp = self._call_with_body_bytes('put',
234
(self._remote_path(relpath), self._serialise_optional_mode(mode)),
236
self._translate_error(resp)
237
return len(upload_contents)
239
def put_bytes_non_atomic(self, relpath, bytes, mode=None,
240
create_parent_dir=False,
242
"""See Transport.put_bytes_non_atomic."""
243
# FIXME: no encoding in the transport!
244
create_parent_str = 'F'
245
if create_parent_dir:
246
create_parent_str = 'T'
248
resp = self._call_with_body_bytes(
250
(self._remote_path(relpath), self._serialise_optional_mode(mode),
251
create_parent_str, self._serialise_optional_mode(dir_mode)),
253
self._translate_error(resp)
255
def put_file(self, relpath, upload_file, mode=None):
256
# its not ideal to seek back, but currently put_non_atomic_file depends
257
# on transports not reading before failing - which is a faulty
258
# assumption I think - RBC 20060915
259
pos = upload_file.tell()
261
return self.put_bytes(relpath, upload_file.read(), mode)
263
upload_file.seek(pos)
266
def put_file_non_atomic(self, relpath, f, mode=None,
267
create_parent_dir=False,
269
return self.put_bytes_non_atomic(relpath, f.read(), mode=mode,
270
create_parent_dir=create_parent_dir,
273
def append_file(self, relpath, from_file, mode=None):
274
return self.append_bytes(relpath, from_file.read(), mode)
276
def append_bytes(self, relpath, bytes, mode=None):
277
resp = self._call_with_body_bytes(
279
(self._remote_path(relpath), self._serialise_optional_mode(mode)),
281
if resp[0] == 'appended':
283
self._translate_error(resp)
285
def delete(self, relpath):
286
resp = self._call2('delete', self._remote_path(relpath))
287
self._translate_error(resp)
289
def external_url(self):
290
"""See bzrlib.transport.Transport.external_url."""
291
# the external path for RemoteTransports is the base
294
def _readv(self, relpath, offsets):
298
offsets = list(offsets)
300
sorted_offsets = sorted(offsets)
301
# turn the list of offsets into a stack
302
offset_stack = iter(offsets)
303
cur_offset_and_size = offset_stack.next()
304
coalesced = list(self._coalesce_offsets(sorted_offsets,
305
limit=self._max_readv_combine,
306
fudge_factor=self._bytes_to_read_before_seek))
308
request = self.get_smart_medium().get_request()
309
smart_protocol = protocol.SmartClientRequestProtocolOne(request)
310
smart_protocol.call_with_body_readv_array(
311
('readv', self._remote_path(relpath)),
312
[(c.start, c.length) for c in coalesced])
313
resp = smart_protocol.read_response_tuple(True)
315
if resp[0] != 'readv':
316
# This should raise an exception
317
smart_protocol.cancel_read_body()
318
self._translate_error(resp)
321
# FIXME: this should know how many bytes are needed, for clarity.
322
data = smart_protocol.read_body_bytes()
323
# Cache the results, but only until they have been fulfilled
325
for c_offset in coalesced:
326
if len(data) < c_offset.length:
327
raise errors.ShortReadvError(relpath, c_offset.start,
328
c_offset.length, actual=len(data))
329
for suboffset, subsize in c_offset.ranges:
330
key = (c_offset.start+suboffset, subsize)
331
data_map[key] = data[suboffset:suboffset+subsize]
332
data = data[c_offset.length:]
334
# Now that we've read some data, see if we can yield anything back
335
while cur_offset_and_size in data_map:
336
this_data = data_map.pop(cur_offset_and_size)
337
yield cur_offset_and_size[0], this_data
338
cur_offset_and_size = offset_stack.next()
340
def rename(self, rel_from, rel_to):
342
self._remote_path(rel_from),
343
self._remote_path(rel_to))
345
def move(self, rel_from, rel_to):
347
self._remote_path(rel_from),
348
self._remote_path(rel_to))
350
def rmdir(self, relpath):
351
resp = self._call('rmdir', self._remote_path(relpath))
353
def _translate_error(self, resp, orig_path=None):
354
"""Raise an exception from a response"""
361
elif what == 'NoSuchFile':
362
if orig_path is not None:
363
error_path = orig_path
366
raise errors.NoSuchFile(error_path)
367
elif what == 'error':
368
raise errors.SmartProtocolError(unicode(resp[1]))
369
elif what == 'FileExists':
370
raise errors.FileExists(resp[1])
371
elif what == 'DirectoryNotEmpty':
372
raise errors.DirectoryNotEmpty(resp[1])
373
elif what == 'ShortReadvError':
374
raise errors.ShortReadvError(resp[1], int(resp[2]),
375
int(resp[3]), int(resp[4]))
376
elif what in ('UnicodeEncodeError', 'UnicodeDecodeError'):
377
encoding = str(resp[1]) # encoding must always be a string
381
reason = str(resp[5]) # reason must always be a string
382
if val.startswith('u:'):
383
val = val[2:].decode('utf-8')
384
elif val.startswith('s:'):
385
val = val[2:].decode('base64')
386
if what == 'UnicodeDecodeError':
387
raise UnicodeDecodeError(encoding, val, start, end, reason)
388
elif what == 'UnicodeEncodeError':
389
raise UnicodeEncodeError(encoding, val, start, end, reason)
390
elif what == "ReadOnlyError":
391
raise errors.TransportNotPossible('readonly transport')
392
elif what == "ReadError":
393
if orig_path is not None:
394
error_path = orig_path
397
raise errors.ReadError(error_path)
399
raise errors.SmartProtocolError('unexpected smart server error: %r' % (resp,))
401
def disconnect(self):
402
self.get_smart_medium().disconnect()
404
def delete_tree(self, relpath):
405
raise errors.TransportNotPossible('readonly transport')
407
def stat(self, relpath):
408
resp = self._call2('stat', self._remote_path(relpath))
409
if resp[0] == 'stat':
410
return _SmartStat(int(resp[1]), int(resp[2], 8))
412
self._translate_error(resp)
414
## def lock_read(self, relpath):
415
## """Lock the given file for shared (read) access.
416
## :return: A lock object, which should be passed to Transport.unlock()
418
## # The old RemoteBranch ignore lock for reading, so we will
419
## # continue that tradition and return a bogus lock object.
420
## class BogusLock(object):
421
## def __init__(self, path):
425
## return BogusLock(relpath)
430
def list_dir(self, relpath):
431
resp = self._call2('list_dir', self._remote_path(relpath))
432
if resp[0] == 'names':
433
return [name.encode('ascii') for name in resp[1:]]
435
self._translate_error(resp)
437
def iter_files_recursive(self):
438
resp = self._call2('iter_files_recursive', self._remote_path(''))
439
if resp[0] == 'names':
442
self._translate_error(resp)
445
class RemoteTCPTransport(RemoteTransport):
446
"""Connection to smart server over plain tcp.
448
This is essentially just a factory to get 'RemoteTransport(url,
449
SmartTCPClientMedium).
452
def _build_medium(self):
453
assert self.base.startswith('bzr://')
454
if self._port is None:
455
self._port = BZR_DEFAULT_PORT
456
return medium.SmartTCPClientMedium(self._host, self._port), None
459
class RemoteSSHTransport(RemoteTransport):
460
"""Connection to smart server over SSH.
462
This is essentially just a factory to get 'RemoteTransport(url,
463
SmartSSHClientMedium).
466
def _build_medium(self):
467
assert self.base.startswith('bzr+ssh://')
468
# ssh will prompt the user for a password if needed and if none is
469
# provided but it will not give it back, so no credentials can be
471
return medium.SmartSSHClientMedium(self._host, self._port,
472
self._user, self._password), None
475
class RemoteHTTPTransport(RemoteTransport):
476
"""Just a way to connect between a bzr+http:// url and http://.
478
This connection operates slightly differently than the RemoteSSHTransport.
479
It uses a plain http:// transport underneath, which defines what remote
480
.bzr/smart URL we are connected to. From there, all paths that are sent are
481
sent as relative paths, this way, the remote side can properly
482
de-reference them, since it is likely doing rewrite rules to translate an
483
HTTP path into a local path.
486
def __init__(self, base, _from_transport=None, http_transport=None):
487
assert base.startswith('bzr+http://')
489
if http_transport is None:
490
# FIXME: the password may be lost here because it appears in the
491
# url only for an intial construction (when the url came from the
493
http_url = base[len('bzr+'):]
494
self._http_transport = transport.get_transport(http_url)
496
self._http_transport = http_transport
497
super(RemoteHTTPTransport, self).__init__(
498
base, _from_transport=_from_transport)
500
def _build_medium(self):
501
# We let http_transport take care of the credentials
502
return self._http_transport.get_smart_medium(), None
504
def _remote_path(self, relpath):
505
"""After connecting, HTTP Transport only deals in relative URLs."""
506
# Adjust the relpath based on which URL this smart transport is
508
http_base = urlutils.normalize_url(self._http_transport.base)
509
url = urlutils.join(self.base[len('bzr+'):], relpath)
510
url = urlutils.normalize_url(url)
511
return urlutils.relative_url(http_base, url)
513
def clone(self, relative_url):
514
"""Make a new RemoteHTTPTransport related to me.
516
This is re-implemented rather than using the default
517
RemoteTransport.clone() because we must be careful about the underlying
520
Also, the cloned smart transport will POST to the same .bzr/smart
521
location as this transport (although obviously the relative paths in the
522
smart requests may be different). This is so that the server doesn't
523
have to handle .bzr/smart requests at arbitrary places inside .bzr
524
directories, just at the initial URL the user uses.
526
The exception is parent paths (i.e. relative_url of "..").
529
abs_url = self.abspath(relative_url)
532
# We either use the exact same http_transport (for child locations), or
533
# a clone of the underlying http_transport (for parent locations). This
534
# means we share the connection.
535
norm_base = urlutils.normalize_url(self.base)
536
norm_abs_url = urlutils.normalize_url(abs_url)
537
normalized_rel_url = urlutils.relative_url(norm_base, norm_abs_url)
538
if normalized_rel_url == ".." or normalized_rel_url.startswith("../"):
539
http_transport = self._http_transport.clone(normalized_rel_url)
541
http_transport = self._http_transport
542
return RemoteHTTPTransport(abs_url,
543
_from_transport=self,
544
http_transport=http_transport)
547
def get_test_permutations():
548
"""Return (transport, server) permutations for testing."""
549
### We may need a little more test framework support to construct an
550
### appropriate RemoteTransport in the future.
551
from bzrlib.smart import server
552
return [(RemoteTCPTransport, server.SmartTCPServer_for_testing)]
added
removed
bzrlib/transport/remote.py
