63
68
RemoteTCPTransport, etc.
66
# When making a readv request, cap it at requesting 5MB of data
67
_max_readv_bytes = 5*1024*1024
69
71
# IMPORTANT FOR IMPLEMENTORS: RemoteTransport MUST NOT be given encoding
70
72
# responsibilities: Put those on SmartClient or similar. This is vital for
71
73
# the ability to support multiple versions of the smart protocol over time:
72
# RemoteTransport is an adapter from the Transport object model to the
74
# RemoteTransport is an adapter from the Transport object model to the
73
75
# SmartClient model, not an encoder.
75
# FIXME: the medium parameter should be private, only the tests requires
76
# it. It may be even clearer to define a TestRemoteTransport that handles
77
# the specific cases of providing a _client and/or a _medium, and leave
78
# RemoteTransport as an abstract class.
79
def __init__(self, url, _from_transport=None, medium=None, _client=None):
77
def __init__(self, url, clone_from=None, medium=None, _client=None):
82
:param _from_transport: Another RemoteTransport instance that this
83
one is being cloned from. Attributes such as the medium will
86
:param medium: The medium to use for this RemoteTransport. If None,
87
the medium from the _from_transport is shared. If both this
88
and _from_transport are None, a new medium will be built.
89
_from_transport and medium cannot both be specified.
80
:param clone_from: Another RemoteTransport instance that this one is
81
being cloned from. Attributes such as credentials and the medium
83
:param medium: The medium to use for this RemoteTransport. This must be
84
supplied if clone_from is None.
91
85
:param _client: Override the _SmartClient used by this transport. This
92
86
should only be used for testing purposes; normally this is
93
87
determined from the medium.
95
super(RemoteTransport, self).__init__(
96
url, _from_transport=_from_transport)
98
# The medium is the connection, except when we need to share it with
99
# other objects (RemoteBzrDir, RemoteRepository etc). In these cases
100
# what we want to share is really the shared connection.
102
if (_from_transport is not None
103
and isinstance(_from_transport, RemoteTransport)):
104
_client = _from_transport._client
105
elif _from_transport is None:
106
# If no _from_transport is specified, we need to intialize the
110
medium, credentials = self._build_medium()
111
if 'hpss' in debug.debug_flags:
112
trace.mutter('hpss: Built a new medium: %s',
113
medium.__class__.__name__)
114
self._shared_connection = transport._SharedConnection(medium,
118
# No medium was specified, so share the medium from the
120
medium = self._shared_connection.connection
89
### Technically super() here is faulty because Transport's __init__
90
### fails to take 2 parameters, and if super were to choose a silly
91
### initialisation order things would blow up.
92
if not url.endswith('/'):
94
super(RemoteTransport, self).__init__(url)
95
self._scheme, self._username, self._password, self._host, self._port, self._path = \
96
transport.split_url(url)
97
if clone_from is None:
122
raise AssertionError(
123
"Both _from_transport (%r) and medium (%r) passed to "
124
"RemoteTransport.__init__, but these parameters are mutally "
125
"exclusive." % (_from_transport, medium))
100
# credentials may be stripped from the base in some circumstances
101
# as yet to be clearly defined or documented, so copy them.
102
self._username = clone_from._username
103
# reuse same connection
104
self._medium = clone_from._medium
105
assert self._medium is not None
127
106
if _client is None:
128
self._client = client._SmartClient(medium)
107
self._client = client._SmartClient(self._medium)
130
109
self._client = _client
132
def _build_medium(self):
133
"""Create the medium if _from_transport does not provide one.
135
The medium is analogous to the connection for ConnectedTransport: it
136
allows connection sharing.
141
def _report_activity(self, bytes, direction):
142
"""See Transport._report_activity.
144
Does nothing; the smart medium will report activity triggered by a
111
def abspath(self, relpath):
112
"""Return the full url to the given relative path.
114
@param relpath: the relative path or path components
115
@type relpath: str or list
117
return self._unparse_url(self._remote_path(relpath))
119
def clone(self, relative_url):
120
"""Make a new RemoteTransport related to me, sharing the same connection.
122
This essentially opens a handle on a different remote directory.
124
if relative_url is None:
125
return RemoteTransport(self.base, self)
127
return RemoteTransport(self.abspath(relative_url), self)
149
129
def is_readonly(self):
150
130
"""Smart server transport can do read/write file operations."""
152
resp = self._call2('Transport.is_readonly')
153
except errors.UnknownSmartMethod:
131
resp = self._call2('Transport.is_readonly')
132
if resp == ('yes', ):
134
elif resp == ('no', ):
136
elif (resp == ('error', "Generic bzr smart protocol error: "
137
"bad request 'Transport.is_readonly'") or
138
resp == ('error', "Generic bzr smart protocol error: "
139
"bad request u'Transport.is_readonly'")):
154
140
# XXX: nasty hack: servers before 0.16 don't have a
155
141
# 'Transport.is_readonly' verb, so we do what clients before 0.16
156
142
# did: assume False.
158
if resp == ('yes', ):
160
elif resp == ('no', ):
163
raise errors.UnexpectedSmartServerResponse(resp)
145
self._translate_error(resp)
146
raise errors.UnexpectedSmartServerResponse(resp)
165
148
def get_smart_client(self):
166
return self._get_connection()
168
151
def get_smart_medium(self):
169
return self._get_connection()
154
def _unparse_url(self, path):
155
"""Return URL for a path.
157
:see: SFTPUrlHandling._unparse_url
159
# TODO: Eventually it should be possible to unify this with
160
# SFTPUrlHandling._unparse_url?
163
path = urllib.quote(path)
164
netloc = urllib.quote(self._host)
165
if self._username is not None:
166
netloc = '%s@%s' % (urllib.quote(self._username), netloc)
167
if self._port is not None:
168
netloc = '%s:%d' % (netloc, self._port)
169
return urlparse.urlunparse((self._scheme, netloc, path, '', '', ''))
171
171
def _remote_path(self, relpath):
172
172
"""Returns the Unicode version of the absolute path for relpath."""
175
175
def _call(self, method, *args):
176
176
resp = self._call2(method, *args)
177
self._ensure_ok(resp)
177
self._translate_error(resp)
179
179
def _call2(self, method, *args):
180
180
"""Call a method on the remote server."""
182
return self._client.call(method, *args)
183
except errors.ErrorFromSmartServer, err:
184
# The first argument, if present, is always a path.
186
context = {'relpath': args[0]}
189
self._translate_error(err, **context)
181
return self._client.call(method, *args)
191
183
def _call_with_body_bytes(self, method, args, body):
192
184
"""Call a method on the remote server with body bytes."""
194
return self._client.call_with_body_bytes(method, args, body)
195
except errors.ErrorFromSmartServer, err:
196
# The first argument, if present, is always a path.
198
context = {'relpath': args[0]}
201
self._translate_error(err, **context)
185
return self._client.call_with_body_bytes(method, args, body)
203
187
def has(self, relpath):
204
188
"""Indicate whether a remote file of the given name exists or not.
211
195
elif resp == ('no', ):
214
raise errors.UnexpectedSmartServerResponse(resp)
198
self._translate_error(resp)
216
200
def get(self, relpath):
217
201
"""Return file-like object reading the contents of a remote file.
219
203
:see: Transport.get_bytes()/get_file()
221
205
return StringIO(self.get_bytes(relpath))
223
207
def get_bytes(self, relpath):
224
208
remote = self._remote_path(relpath)
226
resp, response_handler = self._client.call_expecting_body('get', remote)
227
except errors.ErrorFromSmartServer, err:
228
self._translate_error(err, relpath)
209
request = self._medium.get_request()
210
smart_protocol = protocol.SmartClientRequestProtocolOne(request)
211
smart_protocol.call('get', remote)
212
resp = smart_protocol.read_response_tuple(True)
229
213
if resp != ('ok', ):
230
response_handler.cancel_read_body()
231
raise errors.UnexpectedSmartServerResponse(resp)
232
return response_handler.read_body_bytes()
214
smart_protocol.cancel_read_body()
215
self._translate_error(resp, relpath)
216
return smart_protocol.read_body_bytes()
234
218
def _serialise_optional_mode(self, mode):
309
286
if resp[0] == 'appended':
310
287
return int(resp[1])
311
raise errors.UnexpectedSmartServerResponse(resp)
288
self._translate_error(resp)
313
290
def delete(self, relpath):
314
291
resp = self._call2('delete', self._remote_path(relpath))
315
self._ensure_ok(resp)
317
def external_url(self):
318
"""See bzrlib.transport.Transport.external_url."""
319
# the external path for RemoteTransports is the base
322
def recommended_page_size(self):
323
"""Return the recommended page size for this transport."""
326
def _readv(self, relpath, offsets):
292
self._translate_error(resp)
294
def readv(self, relpath, offsets):
330
298
offsets = list(offsets)
332
300
sorted_offsets = sorted(offsets)
301
# turn the list of offsets into a stack
302
offset_stack = iter(offsets)
303
cur_offset_and_size = offset_stack.next()
333
304
coalesced = list(self._coalesce_offsets(sorted_offsets,
334
305
limit=self._max_readv_combine,
335
fudge_factor=self._bytes_to_read_before_seek,
336
max_size=self._max_readv_bytes))
338
# now that we've coallesced things, avoid making enormous requests
343
if c.length + cur_len > self._max_readv_bytes:
344
requests.append(cur_request)
348
cur_request.append(c)
351
requests.append(cur_request)
352
if 'hpss' in debug.debug_flags:
353
trace.mutter('%s.readv %s offsets => %s coalesced'
354
' => %s requests (%s)',
355
self.__class__.__name__, len(offsets), len(coalesced),
356
len(requests), sum(map(len, requests)))
306
fudge_factor=self._bytes_to_read_before_seek))
308
request = self._medium.get_request()
309
smart_protocol = protocol.SmartClientRequestProtocolOne(request)
310
smart_protocol.call_with_body_readv_array(
311
('readv', self._remote_path(relpath)),
312
[(c.start, c.length) for c in coalesced])
313
resp = smart_protocol.read_response_tuple(True)
315
if resp[0] != 'readv':
316
# This should raise an exception
317
smart_protocol.cancel_read_body()
318
self._translate_error(resp)
321
# FIXME: this should know how many bytes are needed, for clarity.
322
data = smart_protocol.read_body_bytes()
357
323
# Cache the results, but only until they have been fulfilled
359
# turn the list of offsets into a single stack to iterate
360
offset_stack = iter(offsets)
361
# using a list so it can be modified when passing down and coming back
362
next_offset = [offset_stack.next()]
363
for cur_request in requests:
365
result = self._client.call_with_body_readv_array(
366
('readv', self._remote_path(relpath),),
367
[(c.start, c.length) for c in cur_request])
368
resp, response_handler = result
369
except errors.ErrorFromSmartServer, err:
370
self._translate_error(err, relpath)
372
if resp[0] != 'readv':
373
# This should raise an exception
374
response_handler.cancel_read_body()
375
raise errors.UnexpectedSmartServerResponse(resp)
377
for res in self._handle_response(offset_stack, cur_request,
383
def _handle_response(self, offset_stack, coalesced, response_handler,
384
data_map, next_offset):
385
cur_offset_and_size = next_offset[0]
386
# FIXME: this should know how many bytes are needed, for clarity.
387
data = response_handler.read_body_bytes()
389
325
for c_offset in coalesced:
390
326
if len(data) < c_offset.length:
391
327
raise errors.ShortReadvError(relpath, c_offset.start,
392
328
c_offset.length, actual=len(data))
393
329
for suboffset, subsize in c_offset.ranges:
394
330
key = (c_offset.start+suboffset, subsize)
395
this_data = data[data_offset+suboffset:
396
data_offset+suboffset+subsize]
397
# Special case when the data is in-order, rather than packing
398
# into a map and then back out again. Benchmarking shows that
399
# this has 100% hit rate, but leave in the data_map work just
401
# TODO: Could we get away with using buffer() to avoid the
402
# memory copy? Callers would need to realize they may
403
# not have a real string.
404
if key == cur_offset_and_size:
405
yield cur_offset_and_size[0], this_data
406
cur_offset_and_size = next_offset[0] = offset_stack.next()
408
data_map[key] = this_data
409
data_offset += c_offset.length
331
data_map[key] = data[suboffset:suboffset+subsize]
332
data = data[c_offset.length:]
411
334
# Now that we've read some data, see if we can yield anything back
412
335
while cur_offset_and_size in data_map:
413
336
this_data = data_map.pop(cur_offset_and_size)
414
337
yield cur_offset_and_size[0], this_data
415
cur_offset_and_size = next_offset[0] = offset_stack.next()
338
cur_offset_and_size = offset_stack.next()
417
340
def rename(self, rel_from, rel_to):
418
341
self._call('rename',
427
350
def rmdir(self, relpath):
428
351
resp = self._call('rmdir', self._remote_path(relpath))
430
def _ensure_ok(self, resp):
432
raise errors.UnexpectedSmartServerResponse(resp)
434
def _translate_error(self, err, relpath=None):
435
remote._translate_error(err, path=relpath)
353
def _translate_error(self, resp, orig_path=None):
354
"""Raise an exception from a response"""
361
elif what == 'NoSuchFile':
362
if orig_path is not None:
363
error_path = orig_path
366
raise errors.NoSuchFile(error_path)
367
elif what == 'error':
368
raise errors.SmartProtocolError(unicode(resp[1]))
369
elif what == 'FileExists':
370
raise errors.FileExists(resp[1])
371
elif what == 'DirectoryNotEmpty':
372
raise errors.DirectoryNotEmpty(resp[1])
373
elif what == 'ShortReadvError':
374
raise errors.ShortReadvError(resp[1], int(resp[2]),
375
int(resp[3]), int(resp[4]))
376
elif what in ('UnicodeEncodeError', 'UnicodeDecodeError'):
377
encoding = str(resp[1]) # encoding must always be a string
381
reason = str(resp[5]) # reason must always be a string
382
if val.startswith('u:'):
383
val = val[2:].decode('utf-8')
384
elif val.startswith('s:'):
385
val = val[2:].decode('base64')
386
if what == 'UnicodeDecodeError':
387
raise UnicodeDecodeError(encoding, val, start, end, reason)
388
elif what == 'UnicodeEncodeError':
389
raise UnicodeEncodeError(encoding, val, start, end, reason)
390
elif what == "ReadOnlyError":
391
raise errors.TransportNotPossible('readonly transport')
392
elif what == "ReadError":
393
if orig_path is not None:
394
error_path = orig_path
397
raise errors.ReadError(error_path)
399
raise errors.SmartProtocolError('unexpected smart server error: %r' % (resp,))
437
401
def disconnect(self):
438
self.get_smart_medium().disconnect()
402
self._medium.disconnect()
404
def delete_tree(self, relpath):
405
raise errors.TransportNotPossible('readonly transport')
440
407
def stat(self, relpath):
441
408
resp = self._call2('stat', self._remote_path(relpath))
442
409
if resp[0] == 'stat':
443
410
return _SmartStat(int(resp[1]), int(resp[2], 8))
444
raise errors.UnexpectedSmartServerResponse(resp)
412
self._translate_error(resp)
446
414
## def lock_read(self, relpath):
447
415
## """Lock the given file for shared (read) access.
463
431
resp = self._call2('list_dir', self._remote_path(relpath))
464
432
if resp[0] == 'names':
465
433
return [name.encode('ascii') for name in resp[1:]]
466
raise errors.UnexpectedSmartServerResponse(resp)
435
self._translate_error(resp)
468
437
def iter_files_recursive(self):
469
438
resp = self._call2('iter_files_recursive', self._remote_path(''))
470
439
if resp[0] == 'names':
472
raise errors.UnexpectedSmartServerResponse(resp)
442
self._translate_error(resp)
475
445
class RemoteTCPTransport(RemoteTransport):
476
446
"""Connection to smart server over plain tcp.
478
448
This is essentially just a factory to get 'RemoteTransport(url,
479
449
SmartTCPClientMedium).
482
def _build_medium(self):
483
client_medium = medium.SmartTCPClientMedium(
484
self._host, self._port, self.base)
485
return client_medium, None
488
class RemoteTCPTransportV2Only(RemoteTransport):
489
"""Connection to smart server over plain tcp with the client hard-coded to
490
assume protocol v2 and remote server version <= 1.6.
492
This should only be used for testing.
495
def _build_medium(self):
496
client_medium = medium.SmartTCPClientMedium(
497
self._host, self._port, self.base)
498
client_medium._protocol_version = 2
499
client_medium._remember_remote_is_before((1, 6))
500
return client_medium, None
452
def __init__(self, url):
453
_scheme, _username, _password, _host, _port, _path = \
454
transport.split_url(url)
456
_port = BZR_DEFAULT_PORT
460
except (ValueError, TypeError), e:
461
raise errors.InvalidURL(
462
path=url, extra="invalid port %s" % _port)
463
client_medium = medium.SmartTCPClientMedium(_host, _port)
464
super(RemoteTCPTransport, self).__init__(url, medium=client_medium)
503
467
class RemoteSSHTransport(RemoteTransport):
507
471
SmartSSHClientMedium).
510
def _build_medium(self):
511
location_config = config.LocationConfig(self.base)
512
bzr_remote_path = location_config.get_bzr_remote_path()
515
auth = config.AuthenticationConfig()
516
user = auth.get_user('ssh', self._host, self._port)
517
client_medium = medium.SmartSSHClientMedium(self._host, self._port,
518
user, self._password, self.base,
519
bzr_remote_path=bzr_remote_path)
520
return client_medium, (user, self._password)
474
def __init__(self, url):
475
_scheme, _username, _password, _host, _port, _path = \
476
transport.split_url(url)
478
if _port is not None:
480
except (ValueError, TypeError), e:
481
raise errors.InvalidURL(path=url, extra="invalid port %s" %
483
client_medium = medium.SmartSSHClientMedium(_host, _port,
484
_username, _password)
485
super(RemoteSSHTransport, self).__init__(url, medium=client_medium)
523
488
class RemoteHTTPTransport(RemoteTransport):
524
489
"""Just a way to connect between a bzr+http:// url and http://.
526
491
This connection operates slightly differently than the RemoteSSHTransport.
527
492
It uses a plain http:// transport underneath, which defines what remote
528
493
.bzr/smart URL we are connected to. From there, all paths that are sent are
531
496
HTTP path into a local path.
534
def __init__(self, base, _from_transport=None, http_transport=None):
499
def __init__(self, url, http_transport=None):
500
assert url.startswith('bzr+http://')
535
502
if http_transport is None:
536
# FIXME: the password may be lost here because it appears in the
537
# url only for an intial construction (when the url came from the
539
http_url = base[len('bzr+'):]
503
http_url = url[len('bzr+'):]
540
504
self._http_transport = transport.get_transport(http_url)
542
506
self._http_transport = http_transport
543
super(RemoteHTTPTransport, self).__init__(
544
base, _from_transport=_from_transport)
546
def _build_medium(self):
547
# We let http_transport take care of the credentials
548
return self._http_transport.get_smart_medium(), None
507
http_medium = self._http_transport.get_smart_medium()
508
super(RemoteHTTPTransport, self).__init__(url, medium=http_medium)
550
510
def _remote_path(self, relpath):
551
"""After connecting, HTTP Transport only deals in relative URLs."""
511
"""After connecting HTTP Transport only deals in relative URLs."""
552
512
# Adjust the relpath based on which URL this smart transport is
554
http_base = urlutils.normalize_url(self.get_smart_medium().base)
514
base = urlutils.normalize_url(self._http_transport.base)
555
515
url = urlutils.join(self.base[len('bzr+'):], relpath)
556
516
url = urlutils.normalize_url(url)
557
return urlutils.relative_url(http_base, url)
517
return urlutils.relative_url(base, url)
519
def abspath(self, relpath):
520
"""Return the full url to the given relative path.
522
:param relpath: the relative path or path components
523
:type relpath: str or list
525
return self._unparse_url(self._combine_paths(self._path, relpath))
559
527
def clone(self, relative_url):
560
528
"""Make a new RemoteHTTPTransport related to me.
568
536
smart requests may be different). This is so that the server doesn't
569
537
have to handle .bzr/smart requests at arbitrary places inside .bzr
570
538
directories, just at the initial URL the user uses.
540
The exception is parent paths (i.e. relative_url of "..").
573
543
abs_url = self.abspath(relative_url)
575
545
abs_url = self.base
576
return RemoteHTTPTransport(abs_url,
577
_from_transport=self,
578
http_transport=self._http_transport)
580
def _redirected_to(self, source, target):
581
"""See transport._redirected_to"""
582
redirected = self._http_transport._redirected_to(source, target)
583
if (redirected is not None
584
and isinstance(redirected, type(self._http_transport))):
585
return RemoteHTTPTransport('bzr+' + redirected.external_url(),
586
http_transport=redirected)
546
# We either use the exact same http_transport (for child locations), or
547
# a clone of the underlying http_transport (for parent locations). This
548
# means we share the connection.
549
norm_base = urlutils.normalize_url(self.base)
550
norm_abs_url = urlutils.normalize_url(abs_url)
551
normalized_rel_url = urlutils.relative_url(norm_base, norm_abs_url)
552
if normalized_rel_url == ".." or normalized_rel_url.startswith("../"):
553
http_transport = self._http_transport.clone(normalized_rel_url)
588
# Either None or a transport for a different protocol
592
class HintingSSHTransport(transport.Transport):
593
"""Simple transport that handles ssh:// and points out bzr+ssh://."""
595
def __init__(self, url):
596
raise errors.UnsupportedProtocol(url,
597
'bzr supports bzr+ssh to operate over ssh, use "bzr+%s".' % url)
555
http_transport = self._http_transport
556
return RemoteHTTPTransport(abs_url, http_transport=http_transport)
600
559
def get_test_permutations():
added
removed
bzrlib/transport/remote.py
