83
87
one is being cloned from. Attributes such as the medium will
86
:param medium: The medium to use for this RemoteTransport. If None,
87
the medium from the _from_transport is shared. If both this
88
and _from_transport are None, a new medium will be built.
89
_from_transport and medium cannot both be specified.
90
:param medium: The medium to use for this RemoteTransport. This must be
91
supplied if _from_transport is None.
91
93
:param _client: Override the _SmartClient used by this transport. This
92
94
should only be used for testing purposes; normally this is
93
95
determined from the medium.
95
super(RemoteTransport, self).__init__(
96
url, _from_transport=_from_transport)
97
super(RemoteTransport, self).__init__(url,
98
_from_transport=_from_transport)
98
100
# The medium is the connection, except when we need to share it with
99
101
# other objects (RemoteBzrDir, RemoteRepository etc). In these cases
100
102
# what we want to share is really the shared connection.
102
if (_from_transport is not None
103
and isinstance(_from_transport, RemoteTransport)):
104
_client = _from_transport._client
105
elif _from_transport is None:
104
if _from_transport is None:
106
105
# If no _from_transport is specified, we need to intialize the
108
107
credentials = None
139
128
return None, None
141
def _report_activity(self, bytes, direction):
142
"""See Transport._report_activity.
144
Does nothing; the smart medium will report activity triggered by a
149
130
def is_readonly(self):
150
131
"""Smart server transport can do read/write file operations."""
152
resp = self._call2('Transport.is_readonly')
153
except errors.UnknownSmartMethod:
132
resp = self._call2('Transport.is_readonly')
133
if resp == ('yes', ):
135
elif resp == ('no', ):
137
elif (resp == ('error', "Generic bzr smart protocol error: "
138
"bad request 'Transport.is_readonly'") or
139
resp == ('error', "Generic bzr smart protocol error: "
140
"bad request u'Transport.is_readonly'")):
154
141
# XXX: nasty hack: servers before 0.16 don't have a
155
142
# 'Transport.is_readonly' verb, so we do what clients before 0.16
156
143
# did: assume False.
158
if resp == ('yes', ):
160
elif resp == ('no', ):
163
raise errors.UnexpectedSmartServerResponse(resp)
146
self._translate_error(resp)
147
raise errors.UnexpectedSmartServerResponse(resp)
165
149
def get_smart_client(self):
166
150
return self._get_connection()
168
152
def get_smart_medium(self):
169
153
return self._get_connection()
155
def get_shared_medium(self):
156
return self._get_shared_connection()
171
158
def _remote_path(self, relpath):
172
159
"""Returns the Unicode version of the absolute path for relpath."""
173
return urlutils.URL._combine_paths(self._parsed_url.path, relpath)
160
return self._combine_paths(self._path, relpath)
175
162
def _call(self, method, *args):
176
163
resp = self._call2(method, *args)
177
self._ensure_ok(resp)
164
self._translate_error(resp)
179
166
def _call2(self, method, *args):
180
167
"""Call a method on the remote server."""
182
return self._client.call(method, *args)
183
except errors.ErrorFromSmartServer, err:
184
# The first argument, if present, is always a path.
186
context = {'relpath': args[0]}
189
self._translate_error(err, **context)
168
return self._client.call(method, *args)
191
170
def _call_with_body_bytes(self, method, args, body):
192
171
"""Call a method on the remote server with body bytes."""
194
return self._client.call_with_body_bytes(method, args, body)
195
except errors.ErrorFromSmartServer, err:
196
# The first argument, if present, is always a path.
198
context = {'relpath': args[0]}
201
self._translate_error(err, **context)
172
return self._client.call_with_body_bytes(method, args, body)
203
174
def has(self, relpath):
204
175
"""Indicate whether a remote file of the given name exists or not.
211
182
elif resp == ('no', ):
214
raise errors.UnexpectedSmartServerResponse(resp)
185
self._translate_error(resp)
216
187
def get(self, relpath):
217
188
"""Return file-like object reading the contents of a remote file.
219
190
:see: Transport.get_bytes()/get_file()
221
192
return StringIO(self.get_bytes(relpath))
223
194
def get_bytes(self, relpath):
224
195
remote = self._remote_path(relpath)
226
resp, response_handler = self._client.call_expecting_body('get', remote)
227
except errors.ErrorFromSmartServer, err:
228
self._translate_error(err, relpath)
196
request = self.get_smart_medium().get_request()
197
smart_protocol = protocol.SmartClientRequestProtocolOne(request)
198
smart_protocol.call('get', remote)
199
resp = smart_protocol.read_response_tuple(True)
229
200
if resp != ('ok', ):
230
response_handler.cancel_read_body()
231
raise errors.UnexpectedSmartServerResponse(resp)
232
return response_handler.read_body_bytes()
201
smart_protocol.cancel_read_body()
202
self._translate_error(resp, relpath)
203
return smart_protocol.read_body_bytes()
234
205
def _serialise_optional_mode(self, mode):
309
273
if resp[0] == 'appended':
310
274
return int(resp[1])
311
raise errors.UnexpectedSmartServerResponse(resp)
275
self._translate_error(resp)
313
277
def delete(self, relpath):
314
278
resp = self._call2('delete', self._remote_path(relpath))
315
self._ensure_ok(resp)
279
self._translate_error(resp)
317
281
def external_url(self):
318
282
"""See bzrlib.transport.Transport.external_url."""
319
283
# the external path for RemoteTransports is the base
322
def recommended_page_size(self):
323
"""Return the recommended page size for this transport."""
326
def _readv(self, relpath, offsets):
286
def readv(self, relpath, offsets):
330
290
offsets = list(offsets)
332
292
sorted_offsets = sorted(offsets)
293
# turn the list of offsets into a stack
294
offset_stack = iter(offsets)
295
cur_offset_and_size = offset_stack.next()
333
296
coalesced = list(self._coalesce_offsets(sorted_offsets,
334
297
limit=self._max_readv_combine,
335
fudge_factor=self._bytes_to_read_before_seek,
336
max_size=self._max_readv_bytes))
338
# now that we've coallesced things, avoid making enormous requests
343
if c.length + cur_len > self._max_readv_bytes:
344
requests.append(cur_request)
348
cur_request.append(c)
351
requests.append(cur_request)
352
if 'hpss' in debug.debug_flags:
353
trace.mutter('%s.readv %s offsets => %s coalesced'
354
' => %s requests (%s)',
355
self.__class__.__name__, len(offsets), len(coalesced),
356
len(requests), sum(map(len, requests)))
298
fudge_factor=self._bytes_to_read_before_seek))
300
request = self.get_smart_medium().get_request()
301
smart_protocol = protocol.SmartClientRequestProtocolOne(request)
302
smart_protocol.call_with_body_readv_array(
303
('readv', self._remote_path(relpath)),
304
[(c.start, c.length) for c in coalesced])
305
resp = smart_protocol.read_response_tuple(True)
307
if resp[0] != 'readv':
308
# This should raise an exception
309
smart_protocol.cancel_read_body()
310
self._translate_error(resp)
313
# FIXME: this should know how many bytes are needed, for clarity.
314
data = smart_protocol.read_body_bytes()
357
315
# Cache the results, but only until they have been fulfilled
359
# turn the list of offsets into a single stack to iterate
360
offset_stack = iter(offsets)
361
# using a list so it can be modified when passing down and coming back
362
next_offset = [offset_stack.next()]
363
for cur_request in requests:
365
result = self._client.call_with_body_readv_array(
366
('readv', self._remote_path(relpath),),
367
[(c.start, c.length) for c in cur_request])
368
resp, response_handler = result
369
except errors.ErrorFromSmartServer, err:
370
self._translate_error(err, relpath)
372
if resp[0] != 'readv':
373
# This should raise an exception
374
response_handler.cancel_read_body()
375
raise errors.UnexpectedSmartServerResponse(resp)
377
for res in self._handle_response(offset_stack, cur_request,
383
def _handle_response(self, offset_stack, coalesced, response_handler,
384
data_map, next_offset):
385
cur_offset_and_size = next_offset[0]
386
# FIXME: this should know how many bytes are needed, for clarity.
387
data = response_handler.read_body_bytes()
389
317
for c_offset in coalesced:
390
318
if len(data) < c_offset.length:
391
319
raise errors.ShortReadvError(relpath, c_offset.start,
392
320
c_offset.length, actual=len(data))
393
321
for suboffset, subsize in c_offset.ranges:
394
322
key = (c_offset.start+suboffset, subsize)
395
this_data = data[data_offset+suboffset:
396
data_offset+suboffset+subsize]
397
# Special case when the data is in-order, rather than packing
398
# into a map and then back out again. Benchmarking shows that
399
# this has 100% hit rate, but leave in the data_map work just
401
# TODO: Could we get away with using buffer() to avoid the
402
# memory copy? Callers would need to realize they may
403
# not have a real string.
404
if key == cur_offset_and_size:
405
yield cur_offset_and_size[0], this_data
406
cur_offset_and_size = next_offset[0] = offset_stack.next()
408
data_map[key] = this_data
409
data_offset += c_offset.length
323
data_map[key] = data[suboffset:suboffset+subsize]
324
data = data[c_offset.length:]
411
326
# Now that we've read some data, see if we can yield anything back
412
327
while cur_offset_and_size in data_map:
413
328
this_data = data_map.pop(cur_offset_and_size)
414
329
yield cur_offset_and_size[0], this_data
415
cur_offset_and_size = next_offset[0] = offset_stack.next()
330
cur_offset_and_size = offset_stack.next()
417
332
def rename(self, rel_from, rel_to):
418
333
self._call('rename',
427
342
def rmdir(self, relpath):
428
343
resp = self._call('rmdir', self._remote_path(relpath))
430
def _ensure_ok(self, resp):
432
raise errors.UnexpectedSmartServerResponse(resp)
434
def _translate_error(self, err, relpath=None):
435
remote._translate_error(err, path=relpath)
345
def _translate_error(self, resp, orig_path=None):
346
"""Raise an exception from a response"""
353
elif what == 'NoSuchFile':
354
if orig_path is not None:
355
error_path = orig_path
358
raise errors.NoSuchFile(error_path)
359
elif what == 'error':
360
raise errors.SmartProtocolError(unicode(resp[1]))
361
elif what == 'FileExists':
362
raise errors.FileExists(resp[1])
363
elif what == 'DirectoryNotEmpty':
364
raise errors.DirectoryNotEmpty(resp[1])
365
elif what == 'ShortReadvError':
366
raise errors.ShortReadvError(resp[1], int(resp[2]),
367
int(resp[3]), int(resp[4]))
368
elif what in ('UnicodeEncodeError', 'UnicodeDecodeError'):
369
encoding = str(resp[1]) # encoding must always be a string
373
reason = str(resp[5]) # reason must always be a string
374
if val.startswith('u:'):
375
val = val[2:].decode('utf-8')
376
elif val.startswith('s:'):
377
val = val[2:].decode('base64')
378
if what == 'UnicodeDecodeError':
379
raise UnicodeDecodeError(encoding, val, start, end, reason)
380
elif what == 'UnicodeEncodeError':
381
raise UnicodeEncodeError(encoding, val, start, end, reason)
382
elif what == "ReadOnlyError":
383
raise errors.TransportNotPossible('readonly transport')
384
elif what == "ReadError":
385
if orig_path is not None:
386
error_path = orig_path
389
raise errors.ReadError(error_path)
391
raise errors.SmartProtocolError('unexpected smart server error: %r' % (resp,))
437
393
def disconnect(self):
438
m = self.get_smart_medium()
394
self.get_smart_medium().disconnect()
396
def delete_tree(self, relpath):
397
raise errors.TransportNotPossible('readonly transport')
442
399
def stat(self, relpath):
443
400
resp = self._call2('stat', self._remote_path(relpath))
444
401
if resp[0] == 'stat':
445
402
return _SmartStat(int(resp[1]), int(resp[2], 8))
446
raise errors.UnexpectedSmartServerResponse(resp)
404
self._translate_error(resp)
448
406
## def lock_read(self, relpath):
449
407
## """Lock the given file for shared (read) access.
465
423
resp = self._call2('list_dir', self._remote_path(relpath))
466
424
if resp[0] == 'names':
467
425
return [name.encode('ascii') for name in resp[1:]]
468
raise errors.UnexpectedSmartServerResponse(resp)
427
self._translate_error(resp)
470
429
def iter_files_recursive(self):
471
430
resp = self._call2('iter_files_recursive', self._remote_path(''))
472
431
if resp[0] == 'names':
474
raise errors.UnexpectedSmartServerResponse(resp)
434
self._translate_error(resp)
477
437
class RemoteTCPTransport(RemoteTransport):
478
438
"""Connection to smart server over plain tcp.
480
440
This is essentially just a factory to get 'RemoteTransport(url,
481
441
SmartTCPClientMedium).
484
444
def _build_medium(self):
485
client_medium = medium.SmartTCPClientMedium(
486
self._parsed_url.host, self._parsed_url.port, self.base)
487
return client_medium, None
490
class RemoteTCPTransportV2Only(RemoteTransport):
491
"""Connection to smart server over plain tcp with the client hard-coded to
492
assume protocol v2 and remote server version <= 1.6.
494
This should only be used for testing.
497
def _build_medium(self):
498
client_medium = medium.SmartTCPClientMedium(
499
self._parsed_url.host, self._parsed_url.port, self.base)
500
client_medium._protocol_version = 2
501
client_medium._remember_remote_is_before((1, 6))
502
return client_medium, None
445
assert self.base.startswith('bzr://')
446
if self._port is None:
447
self._port = BZR_DEFAULT_PORT
448
return medium.SmartTCPClientMedium(self._host, self._port), None
505
451
class RemoteSSHTransport(RemoteTransport):
512
458
def _build_medium(self):
513
location_config = config.LocationConfig(self.base)
514
bzr_remote_path = location_config.get_bzr_remote_path()
515
user = self._parsed_url.user
517
auth = config.AuthenticationConfig()
518
user = auth.get_user('ssh', self._parsed_url.host,
519
self._parsed_url.port)
520
ssh_params = medium.SSHParams(self._parsed_url.host,
521
self._parsed_url.port, user, self._parsed_url.password,
523
client_medium = medium.SmartSSHClientMedium(self.base, ssh_params)
524
return client_medium, (user, self._parsed_url.password)
459
assert self.base.startswith('bzr+ssh://')
460
# ssh will prompt the user for a password if needed and if none is
461
# provided but it will not give it back, so no credentials can be
463
return medium.SmartSSHClientMedium(self._host, self._port,
464
self._user, self._password), None
527
467
class RemoteHTTPTransport(RemoteTransport):
528
468
"""Just a way to connect between a bzr+http:// url and http://.
530
470
This connection operates slightly differently than the RemoteSSHTransport.
531
471
It uses a plain http:// transport underneath, which defines what remote
532
472
.bzr/smart URL we are connected to. From there, all paths that are sent are
572
514
smart requests may be different). This is so that the server doesn't
573
515
have to handle .bzr/smart requests at arbitrary places inside .bzr
574
516
directories, just at the initial URL the user uses.
518
The exception is parent paths (i.e. relative_url of "..").
577
521
abs_url = self.abspath(relative_url)
579
523
abs_url = self.base
524
# We either use the exact same http_transport (for child locations), or
525
# a clone of the underlying http_transport (for parent locations). This
526
# means we share the connection.
527
norm_base = urlutils.normalize_url(self.base)
528
norm_abs_url = urlutils.normalize_url(abs_url)
529
normalized_rel_url = urlutils.relative_url(norm_base, norm_abs_url)
530
if normalized_rel_url == ".." or normalized_rel_url.startswith("../"):
531
http_transport = self._http_transport.clone(normalized_rel_url)
533
http_transport = self._http_transport
580
534
return RemoteHTTPTransport(abs_url,
581
535
_from_transport=self,
582
http_transport=self._http_transport)
584
def _redirected_to(self, source, target):
585
"""See transport._redirected_to"""
586
redirected = self._http_transport._redirected_to(source, target)
587
if (redirected is not None
588
and isinstance(redirected, type(self._http_transport))):
589
return RemoteHTTPTransport('bzr+' + redirected.external_url(),
590
http_transport=redirected)
592
# Either None or a transport for a different protocol
596
class HintingSSHTransport(transport.Transport):
597
"""Simple transport that handles ssh:// and points out bzr+ssh://."""
599
def __init__(self, url):
600
raise errors.UnsupportedProtocol(url,
601
'bzr supports bzr+ssh to operate over ssh, use "bzr+%s".' % url)
536
http_transport=http_transport)
604
539
def get_test_permutations():
605
540
"""Return (transport, server) permutations for testing."""
606
541
### We may need a little more test framework support to construct an
607
542
### appropriate RemoteTransport in the future.
608
from bzrlib.tests import test_server
609
return [(RemoteTCPTransport, test_server.SmartTCPServer_for_testing)]
543
from bzrlib.smart import server
544
return [(RemoteTCPTransport, server.SmartTCPServer_for_testing)]
added
removed
bzrlib/transport/remote.py
