1
# Copyright (C) 2006 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
"""RemoteTransport client for the smart-server.
19
This module shouldn't be accessed directly. The classes defined here should be
20
imported from bzrlib.smart.
23
__all__ = ['RemoteTransport', 'RemoteTCPTransport', 'RemoteSSHTransport']
25
from cStringIO import StringIO
34
from bzrlib.smart import client, medium, protocol
36
# must do this otherwise urllib can't parse the urls properly :(
37
for scheme in ['ssh', 'bzr', 'bzr+loopback', 'bzr+ssh', 'bzr+http']:
38
transport.register_urlparse_netloc_protocol(scheme)
42
# Port 4155 is the default port for bzr://, registered with IANA.
43
BZR_DEFAULT_INTERFACE = '0.0.0.0'
44
BZR_DEFAULT_PORT = 4155
47
class _SmartStat(object):
49
def __init__(self, size, mode):
54
class RemoteTransport(transport.Transport):
55
"""Connection to a smart server.
57
The connection holds references to the medium that can be used to send
58
requests to the server.
60
The connection has a notion of the current directory to which it's
61
connected; this is incorporated in filenames passed to the server.
63
This supports some higher-level RPC operations and can also be treated
64
like a Transport to do file-like operations.
66
The connection can be made over a tcp socket, an ssh pipe or a series of
67
http requests. There are concrete subclasses for each type:
68
RemoteTCPTransport, etc.
71
# IMPORTANT FOR IMPLEMENTORS: RemoteTransport MUST NOT be given encoding
72
# responsibilities: Put those on SmartClient or similar. This is vital for
73
# the ability to support multiple versions of the smart protocol over time:
74
# RemoteTransport is an adapter from the Transport object model to the
75
# SmartClient model, not an encoder.
77
def __init__(self, url, clone_from=None, medium=None, _client=None):
80
:param clone_from: Another RemoteTransport instance that this one is
81
being cloned from. Attributes such as credentials and the medium
83
:param medium: The medium to use for this RemoteTransport. This must be
84
supplied if clone_from is None.
85
:param _client: Override the _SmartClient used by this transport. This
86
should only be used for testing purposes; normally this is
87
determined from the medium.
89
### Technically super() here is faulty because Transport's __init__
90
### fails to take 2 parameters, and if super were to choose a silly
91
### initialisation order things would blow up.
92
if not url.endswith('/'):
94
super(RemoteTransport, self).__init__(url)
95
self._scheme, self._username, self._password, self._host, self._port, self._path = \
96
transport.split_url(url)
97
if clone_from is None:
100
# credentials may be stripped from the base in some circumstances
101
# as yet to be clearly defined or documented, so copy them.
102
self._username = clone_from._username
103
# reuse same connection
104
self._medium = clone_from._medium
105
assert self._medium is not None
107
self._client = client._SmartClient(self._medium)
109
self._client = _client
111
def abspath(self, relpath):
112
"""Return the full url to the given relative path.
114
@param relpath: the relative path or path components
115
@type relpath: str or list
117
return self._unparse_url(self._remote_path(relpath))
119
def clone(self, relative_url):
120
"""Make a new RemoteTransport related to me, sharing the same connection.
122
This essentially opens a handle on a different remote directory.
124
if relative_url is None:
125
return RemoteTransport(self.base, self)
127
return RemoteTransport(self.abspath(relative_url), self)
129
def is_readonly(self):
130
"""Smart server transport can do read/write file operations."""
131
resp = self._call2('Transport.is_readonly')
132
if resp == ('yes', ):
134
elif resp == ('no', ):
136
elif (resp == ('error', "Generic bzr smart protocol error: "
137
"bad request 'Transport.is_readonly'") or
138
resp == ('error', "Generic bzr smart protocol error: "
139
"bad request u'Transport.is_readonly'")):
140
# XXX: nasty hack: servers before 0.16 don't have a
141
# 'Transport.is_readonly' verb, so we do what clients before 0.16
145
self._translate_error(resp)
146
raise errors.UnexpectedSmartServerResponse(resp)
148
def get_smart_client(self):
151
def get_smart_medium(self):
154
def _unparse_url(self, path):
155
"""Return URL for a path.
157
:see: SFTPUrlHandling._unparse_url
159
# TODO: Eventually it should be possible to unify this with
160
# SFTPUrlHandling._unparse_url?
163
path = urllib.quote(path)
164
netloc = urllib.quote(self._host)
165
if self._username is not None:
166
netloc = '%s@%s' % (urllib.quote(self._username), netloc)
167
if self._port is not None:
168
netloc = '%s:%d' % (netloc, self._port)
169
return urlparse.urlunparse((self._scheme, netloc, path, '', '', ''))
171
def _remote_path(self, relpath):
172
"""Returns the Unicode version of the absolute path for relpath."""
173
return self._combine_paths(self._path, relpath)
175
def _call(self, method, *args):
176
resp = self._call2(method, *args)
177
self._translate_error(resp)
179
def _call2(self, method, *args):
180
"""Call a method on the remote server."""
181
return self._client.call(method, *args)
183
def _call_with_body_bytes(self, method, args, body):
184
"""Call a method on the remote server with body bytes."""
185
return self._client.call_with_body_bytes(method, args, body)
187
def has(self, relpath):
188
"""Indicate whether a remote file of the given name exists or not.
190
:see: Transport.has()
192
resp = self._call2('has', self._remote_path(relpath))
193
if resp == ('yes', ):
195
elif resp == ('no', ):
198
self._translate_error(resp)
200
def get(self, relpath):
201
"""Return file-like object reading the contents of a remote file.
203
:see: Transport.get_bytes()/get_file()
205
return StringIO(self.get_bytes(relpath))
207
def get_bytes(self, relpath):
208
remote = self._remote_path(relpath)
209
request = self._medium.get_request()
210
smart_protocol = protocol.SmartClientRequestProtocolOne(request)
211
smart_protocol.call('get', remote)
212
resp = smart_protocol.read_response_tuple(True)
214
smart_protocol.cancel_read_body()
215
self._translate_error(resp, relpath)
216
return smart_protocol.read_body_bytes()
218
def _serialise_optional_mode(self, mode):
224
def mkdir(self, relpath, mode=None):
225
resp = self._call2('mkdir', self._remote_path(relpath),
226
self._serialise_optional_mode(mode))
227
self._translate_error(resp)
229
def put_bytes(self, relpath, upload_contents, mode=None):
230
# FIXME: upload_file is probably not safe for non-ascii characters -
231
# should probably just pass all parameters as length-delimited
233
if type(upload_contents) is unicode:
234
# Although not strictly correct, we raise UnicodeEncodeError to be
235
# compatible with other transports.
236
raise UnicodeEncodeError(
237
'undefined', upload_contents, 0, 1,
238
'put_bytes must be given bytes, not unicode.')
239
resp = self._call_with_body_bytes('put',
240
(self._remote_path(relpath), self._serialise_optional_mode(mode)),
242
self._translate_error(resp)
244
def put_bytes_non_atomic(self, relpath, bytes, mode=None,
245
create_parent_dir=False,
247
"""See Transport.put_bytes_non_atomic."""
248
# FIXME: no encoding in the transport!
249
create_parent_str = 'F'
250
if create_parent_dir:
251
create_parent_str = 'T'
253
resp = self._call_with_body_bytes(
255
(self._remote_path(relpath), self._serialise_optional_mode(mode),
256
create_parent_str, self._serialise_optional_mode(dir_mode)),
258
self._translate_error(resp)
260
def put_file(self, relpath, upload_file, mode=None):
261
# its not ideal to seek back, but currently put_non_atomic_file depends
262
# on transports not reading before failing - which is a faulty
263
# assumption I think - RBC 20060915
264
pos = upload_file.tell()
266
return self.put_bytes(relpath, upload_file.read(), mode)
268
upload_file.seek(pos)
271
def put_file_non_atomic(self, relpath, f, mode=None,
272
create_parent_dir=False,
274
return self.put_bytes_non_atomic(relpath, f.read(), mode=mode,
275
create_parent_dir=create_parent_dir,
278
def append_file(self, relpath, from_file, mode=None):
279
return self.append_bytes(relpath, from_file.read(), mode)
281
def append_bytes(self, relpath, bytes, mode=None):
282
resp = self._call_with_body_bytes(
284
(self._remote_path(relpath), self._serialise_optional_mode(mode)),
286
if resp[0] == 'appended':
288
self._translate_error(resp)
290
def delete(self, relpath):
291
resp = self._call2('delete', self._remote_path(relpath))
292
self._translate_error(resp)
294
def external_url(self):
295
"""See bzrlib.transport.Transport.external_url."""
296
# the external path for RemoteTransports is the base
299
def readv(self, relpath, offsets):
303
offsets = list(offsets)
305
sorted_offsets = sorted(offsets)
306
# turn the list of offsets into a stack
307
offset_stack = iter(offsets)
308
cur_offset_and_size = offset_stack.next()
309
coalesced = list(self._coalesce_offsets(sorted_offsets,
310
limit=self._max_readv_combine,
311
fudge_factor=self._bytes_to_read_before_seek))
313
request = self._medium.get_request()
314
smart_protocol = protocol.SmartClientRequestProtocolOne(request)
315
smart_protocol.call_with_body_readv_array(
316
('readv', self._remote_path(relpath)),
317
[(c.start, c.length) for c in coalesced])
318
resp = smart_protocol.read_response_tuple(True)
320
if resp[0] != 'readv':
321
# This should raise an exception
322
smart_protocol.cancel_read_body()
323
self._translate_error(resp)
326
# FIXME: this should know how many bytes are needed, for clarity.
327
data = smart_protocol.read_body_bytes()
328
# Cache the results, but only until they have been fulfilled
330
for c_offset in coalesced:
331
if len(data) < c_offset.length:
332
raise errors.ShortReadvError(relpath, c_offset.start,
333
c_offset.length, actual=len(data))
334
for suboffset, subsize in c_offset.ranges:
335
key = (c_offset.start+suboffset, subsize)
336
data_map[key] = data[suboffset:suboffset+subsize]
337
data = data[c_offset.length:]
339
# Now that we've read some data, see if we can yield anything back
340
while cur_offset_and_size in data_map:
341
this_data = data_map.pop(cur_offset_and_size)
342
yield cur_offset_and_size[0], this_data
343
cur_offset_and_size = offset_stack.next()
345
def rename(self, rel_from, rel_to):
347
self._remote_path(rel_from),
348
self._remote_path(rel_to))
350
def move(self, rel_from, rel_to):
352
self._remote_path(rel_from),
353
self._remote_path(rel_to))
355
def rmdir(self, relpath):
356
resp = self._call('rmdir', self._remote_path(relpath))
358
def _translate_error(self, resp, orig_path=None):
359
"""Raise an exception from a response"""
366
elif what == 'NoSuchFile':
367
if orig_path is not None:
368
error_path = orig_path
371
raise errors.NoSuchFile(error_path)
372
elif what == 'error':
373
raise errors.SmartProtocolError(unicode(resp[1]))
374
elif what == 'FileExists':
375
raise errors.FileExists(resp[1])
376
elif what == 'DirectoryNotEmpty':
377
raise errors.DirectoryNotEmpty(resp[1])
378
elif what == 'ShortReadvError':
379
raise errors.ShortReadvError(resp[1], int(resp[2]),
380
int(resp[3]), int(resp[4]))
381
elif what in ('UnicodeEncodeError', 'UnicodeDecodeError'):
382
encoding = str(resp[1]) # encoding must always be a string
386
reason = str(resp[5]) # reason must always be a string
387
if val.startswith('u:'):
388
val = val[2:].decode('utf-8')
389
elif val.startswith('s:'):
390
val = val[2:].decode('base64')
391
if what == 'UnicodeDecodeError':
392
raise UnicodeDecodeError(encoding, val, start, end, reason)
393
elif what == 'UnicodeEncodeError':
394
raise UnicodeEncodeError(encoding, val, start, end, reason)
395
elif what == "ReadOnlyError":
396
raise errors.TransportNotPossible('readonly transport')
397
elif what == "ReadError":
398
if orig_path is not None:
399
error_path = orig_path
402
raise errors.ReadError(error_path)
404
raise errors.SmartProtocolError('unexpected smart server error: %r' % (resp,))
406
def disconnect(self):
407
self._medium.disconnect()
409
def delete_tree(self, relpath):
410
raise errors.TransportNotPossible('readonly transport')
412
def stat(self, relpath):
413
resp = self._call2('stat', self._remote_path(relpath))
414
if resp[0] == 'stat':
415
return _SmartStat(int(resp[1]), int(resp[2], 8))
417
self._translate_error(resp)
419
## def lock_read(self, relpath):
420
## """Lock the given file for shared (read) access.
421
## :return: A lock object, which should be passed to Transport.unlock()
423
## # The old RemoteBranch ignore lock for reading, so we will
424
## # continue that tradition and return a bogus lock object.
425
## class BogusLock(object):
426
## def __init__(self, path):
430
## return BogusLock(relpath)
435
def list_dir(self, relpath):
436
resp = self._call2('list_dir', self._remote_path(relpath))
437
if resp[0] == 'names':
438
return [name.encode('ascii') for name in resp[1:]]
440
self._translate_error(resp)
442
def iter_files_recursive(self):
443
resp = self._call2('iter_files_recursive', self._remote_path(''))
444
if resp[0] == 'names':
447
self._translate_error(resp)
450
class RemoteTCPTransport(RemoteTransport):
451
"""Connection to smart server over plain tcp.
453
This is essentially just a factory to get 'RemoteTransport(url,
454
SmartTCPClientMedium).
457
def __init__(self, url):
458
_scheme, _username, _password, _host, _port, _path = \
459
transport.split_url(url)
461
_port = BZR_DEFAULT_PORT
465
except (ValueError, TypeError), e:
466
raise errors.InvalidURL(
467
path=url, extra="invalid port %s" % _port)
468
client_medium = medium.SmartTCPClientMedium(_host, _port)
469
super(RemoteTCPTransport, self).__init__(url, medium=client_medium)
472
class RemoteSSHTransport(RemoteTransport):
473
"""Connection to smart server over SSH.
475
This is essentially just a factory to get 'RemoteTransport(url,
476
SmartSSHClientMedium).
479
def __init__(self, url):
480
_scheme, _username, _password, _host, _port, _path = \
481
transport.split_url(url)
483
if _port is not None:
485
except (ValueError, TypeError), e:
486
raise errors.InvalidURL(path=url, extra="invalid port %s" %
488
client_medium = medium.SmartSSHClientMedium(_host, _port,
489
_username, _password)
490
super(RemoteSSHTransport, self).__init__(url, medium=client_medium)
493
class RemoteHTTPTransport(RemoteTransport):
494
"""Just a way to connect between a bzr+http:// url and http://.
496
This connection operates slightly differently than the RemoteSSHTransport.
497
It uses a plain http:// transport underneath, which defines what remote
498
.bzr/smart URL we are connected to. From there, all paths that are sent are
499
sent as relative paths, this way, the remote side can properly
500
de-reference them, since it is likely doing rewrite rules to translate an
501
HTTP path into a local path.
504
def __init__(self, url, http_transport=None):
505
assert url.startswith('bzr+http://')
507
if http_transport is None:
508
http_url = url[len('bzr+'):]
509
self._http_transport = transport.get_transport(http_url)
511
self._http_transport = http_transport
512
http_medium = self._http_transport.get_smart_medium()
513
super(RemoteHTTPTransport, self).__init__(url, medium=http_medium)
515
def _remote_path(self, relpath):
516
"""After connecting HTTP Transport only deals in relative URLs."""
517
# Adjust the relpath based on which URL this smart transport is
519
base = urlutils.normalize_url(self._http_transport.base)
520
url = urlutils.join(self.base[len('bzr+'):], relpath)
521
url = urlutils.normalize_url(url)
522
return urlutils.relative_url(base, url)
524
def abspath(self, relpath):
525
"""Return the full url to the given relative path.
527
:param relpath: the relative path or path components
528
:type relpath: str or list
530
return self._unparse_url(self._combine_paths(self._path, relpath))
532
def clone(self, relative_url):
533
"""Make a new RemoteHTTPTransport related to me.
535
This is re-implemented rather than using the default
536
RemoteTransport.clone() because we must be careful about the underlying
539
Also, the cloned smart transport will POST to the same .bzr/smart
540
location as this transport (although obviously the relative paths in the
541
smart requests may be different). This is so that the server doesn't
542
have to handle .bzr/smart requests at arbitrary places inside .bzr
543
directories, just at the initial URL the user uses.
545
The exception is parent paths (i.e. relative_url of "..").
548
abs_url = self.abspath(relative_url)
551
# We either use the exact same http_transport (for child locations), or
552
# a clone of the underlying http_transport (for parent locations). This
553
# means we share the connection.
554
norm_base = urlutils.normalize_url(self.base)
555
norm_abs_url = urlutils.normalize_url(abs_url)
556
normalized_rel_url = urlutils.relative_url(norm_base, norm_abs_url)
557
if normalized_rel_url == ".." or normalized_rel_url.startswith("../"):
558
http_transport = self._http_transport.clone(normalized_rel_url)
560
http_transport = self._http_transport
561
return RemoteHTTPTransport(abs_url, http_transport=http_transport)
564
def get_test_permutations():
565
"""Return (transport, server) permutations for testing."""
566
### We may need a little more test framework support to construct an
567
### appropriate RemoteTransport in the future.
568
from bzrlib.smart import server
569
return [(RemoteTCPTransport, server.SmartTCPServer_for_testing)]
added
removed
bzrlib/transport/remote.py
