78
82
one is being cloned from. Attributes such as the medium will
81
:param medium: The medium to use for this RemoteTransport. This must be
82
supplied if _from_transport is None.
85
:param medium: The medium to use for this RemoteTransport. If None,
86
the medium from the _from_transport is shared. If both this
87
and _from_transport are None, a new medium will be built.
88
_from_transport and medium cannot both be specified.
84
90
:param _client: Override the _SmartClient used by this transport. This
85
91
should only be used for testing purposes; normally this is
86
92
determined from the medium.
88
super(RemoteTransport, self).__init__(url,
89
_from_transport=_from_transport)
94
super(RemoteTransport, self).__init__(
95
url, _from_transport=_from_transport)
91
97
# The medium is the connection, except when we need to share it with
92
98
# other objects (RemoteBzrDir, RemoteRepository etc). In these cases
93
99
# what we want to share is really the shared connection.
95
if _from_transport is None:
101
if (_from_transport is not None
102
and isinstance(_from_transport, RemoteTransport)):
103
_client = _from_transport._client
104
elif _from_transport is None:
96
105
# If no _from_transport is specified, we need to intialize the
98
107
credentials = None
119
138
return None, None
140
def _report_activity(self, bytes, direction):
141
"""See Transport._report_activity.
143
Does nothing; the smart medium will report activity triggered by a
121
148
def is_readonly(self):
122
149
"""Smart server transport can do read/write file operations."""
123
resp = self._call2('Transport.is_readonly')
124
if resp == ('yes', ):
126
elif resp == ('no', ):
128
elif (resp == ('error', "Generic bzr smart protocol error: "
129
"bad request 'Transport.is_readonly'") or
130
resp == ('error', "Generic bzr smart protocol error: "
131
"bad request u'Transport.is_readonly'")):
151
resp = self._call2('Transport.is_readonly')
152
except errors.UnknownSmartMethod:
132
153
# XXX: nasty hack: servers before 0.16 don't have a
133
154
# 'Transport.is_readonly' verb, so we do what clients before 0.16
134
155
# did: assume False.
157
if resp == ('yes', ):
159
elif resp == ('no', ):
137
self._translate_error(resp)
138
raise errors.UnexpectedSmartServerResponse(resp)
162
raise errors.UnexpectedSmartServerResponse(resp)
140
164
def get_smart_client(self):
141
165
return self._get_connection()
143
167
def get_smart_medium(self):
144
168
return self._get_connection()
146
def get_shared_medium(self):
147
return self._get_shared_connection()
149
170
def _remote_path(self, relpath):
150
171
"""Returns the Unicode version of the absolute path for relpath."""
151
return self._combine_paths(self._path, relpath)
172
return urlutils.URL._combine_paths(self._parsed_url.path, relpath)
153
174
def _call(self, method, *args):
154
175
resp = self._call2(method, *args)
155
self._translate_error(resp)
176
self._ensure_ok(resp)
157
178
def _call2(self, method, *args):
158
179
"""Call a method on the remote server."""
159
return self._client.call(method, *args)
181
return self._client.call(method, *args)
182
except errors.ErrorFromSmartServer, err:
183
# The first argument, if present, is always a path.
185
context = {'relpath': args[0]}
188
self._translate_error(err, **context)
161
190
def _call_with_body_bytes(self, method, args, body):
162
191
"""Call a method on the remote server with body bytes."""
163
return self._client.call_with_body_bytes(method, args, body)
193
return self._client.call_with_body_bytes(method, args, body)
194
except errors.ErrorFromSmartServer, err:
195
# The first argument, if present, is always a path.
197
context = {'relpath': args[0]}
200
self._translate_error(err, **context)
165
202
def has(self, relpath):
166
203
"""Indicate whether a remote file of the given name exists or not.
173
210
elif resp == ('no', ):
176
self._translate_error(resp)
213
raise errors.UnexpectedSmartServerResponse(resp)
178
215
def get(self, relpath):
179
216
"""Return file-like object reading the contents of a remote file.
181
218
:see: Transport.get_bytes()/get_file()
183
220
return StringIO(self.get_bytes(relpath))
185
222
def get_bytes(self, relpath):
186
223
remote = self._remote_path(relpath)
187
request = self.get_smart_medium().get_request()
188
smart_protocol = protocol.SmartClientRequestProtocolOne(request)
189
smart_protocol.call('get', remote)
190
resp = smart_protocol.read_response_tuple(True)
225
resp, response_handler = self._client.call_expecting_body('get', remote)
226
except errors.ErrorFromSmartServer, err:
227
self._translate_error(err, relpath)
191
228
if resp != ('ok', ):
192
smart_protocol.cancel_read_body()
193
self._translate_error(resp, relpath)
194
return smart_protocol.read_body_bytes()
229
response_handler.cancel_read_body()
230
raise errors.UnexpectedSmartServerResponse(resp)
231
return response_handler.read_body_bytes()
196
233
def _serialise_optional_mode(self, mode):
211
247
transport._file_streams[self.abspath(relpath)] = result
214
def put_bytes(self, relpath, upload_contents, mode=None):
215
# FIXME: upload_file is probably not safe for non-ascii characters -
216
# should probably just pass all parameters as length-delimited
218
if type(upload_contents) is unicode:
219
# Although not strictly correct, we raise UnicodeEncodeError to be
220
# compatible with other transports.
221
raise UnicodeEncodeError(
222
'undefined', upload_contents, 0, 1,
223
'put_bytes must be given bytes, not unicode.')
224
resp = self._call_with_body_bytes('put',
250
def put_bytes(self, relpath, raw_bytes, mode=None):
251
if not isinstance(raw_bytes, str):
253
'raw_bytes must be a plain string, not %s' % type(raw_bytes))
254
resp = self._call_with_body_bytes(
225
256
(self._remote_path(relpath), self._serialise_optional_mode(mode)),
227
self._translate_error(resp)
228
return len(upload_contents)
258
self._ensure_ok(resp)
259
return len(raw_bytes)
230
def put_bytes_non_atomic(self, relpath, bytes, mode=None,
261
def put_bytes_non_atomic(self, relpath, raw_bytes, mode=None,
231
262
create_parent_dir=False,
233
264
"""See Transport.put_bytes_non_atomic."""
289
324
offsets = list(offsets)
291
326
sorted_offsets = sorted(offsets)
292
# turn the list of offsets into a stack
293
offset_stack = iter(offsets)
294
cur_offset_and_size = offset_stack.next()
295
327
coalesced = list(self._coalesce_offsets(sorted_offsets,
296
328
limit=self._max_readv_combine,
297
fudge_factor=self._bytes_to_read_before_seek))
299
request = self.get_smart_medium().get_request()
300
smart_protocol = protocol.SmartClientRequestProtocolOne(request)
301
smart_protocol.call_with_body_readv_array(
302
('readv', self._remote_path(relpath)),
303
[(c.start, c.length) for c in coalesced])
304
resp = smart_protocol.read_response_tuple(True)
306
if resp[0] != 'readv':
307
# This should raise an exception
308
smart_protocol.cancel_read_body()
309
self._translate_error(resp)
312
# FIXME: this should know how many bytes are needed, for clarity.
313
data = smart_protocol.read_body_bytes()
329
fudge_factor=self._bytes_to_read_before_seek,
330
max_size=self._max_readv_bytes))
332
# now that we've coallesced things, avoid making enormous requests
337
if c.length + cur_len > self._max_readv_bytes:
338
requests.append(cur_request)
342
cur_request.append(c)
345
requests.append(cur_request)
346
if 'hpss' in debug.debug_flags:
347
trace.mutter('%s.readv %s offsets => %s coalesced'
348
' => %s requests (%s)',
349
self.__class__.__name__, len(offsets), len(coalesced),
350
len(requests), sum(map(len, requests)))
314
351
# Cache the results, but only until they have been fulfilled
353
# turn the list of offsets into a single stack to iterate
354
offset_stack = iter(offsets)
355
# using a list so it can be modified when passing down and coming back
356
next_offset = [offset_stack.next()]
357
for cur_request in requests:
359
result = self._client.call_with_body_readv_array(
360
('readv', self._remote_path(relpath),),
361
[(c.start, c.length) for c in cur_request])
362
resp, response_handler = result
363
except errors.ErrorFromSmartServer, err:
364
self._translate_error(err, relpath)
366
if resp[0] != 'readv':
367
# This should raise an exception
368
response_handler.cancel_read_body()
369
raise errors.UnexpectedSmartServerResponse(resp)
371
for res in self._handle_response(offset_stack, cur_request,
377
def _handle_response(self, offset_stack, coalesced, response_handler,
378
data_map, next_offset):
379
cur_offset_and_size = next_offset[0]
380
# FIXME: this should know how many bytes are needed, for clarity.
381
data = response_handler.read_body_bytes()
316
383
for c_offset in coalesced:
317
384
if len(data) < c_offset.length:
318
385
raise errors.ShortReadvError(relpath, c_offset.start,
319
386
c_offset.length, actual=len(data))
320
387
for suboffset, subsize in c_offset.ranges:
321
388
key = (c_offset.start+suboffset, subsize)
322
data_map[key] = data[suboffset:suboffset+subsize]
323
data = data[c_offset.length:]
389
this_data = data[data_offset+suboffset:
390
data_offset+suboffset+subsize]
391
# Special case when the data is in-order, rather than packing
392
# into a map and then back out again. Benchmarking shows that
393
# this has 100% hit rate, but leave in the data_map work just
395
# TODO: Could we get away with using buffer() to avoid the
396
# memory copy? Callers would need to realize they may
397
# not have a real string.
398
if key == cur_offset_and_size:
399
yield cur_offset_and_size[0], this_data
400
cur_offset_and_size = next_offset[0] = offset_stack.next()
402
data_map[key] = this_data
403
data_offset += c_offset.length
325
405
# Now that we've read some data, see if we can yield anything back
326
406
while cur_offset_and_size in data_map:
327
407
this_data = data_map.pop(cur_offset_and_size)
328
408
yield cur_offset_and_size[0], this_data
329
cur_offset_and_size = offset_stack.next()
409
cur_offset_and_size = next_offset[0] = offset_stack.next()
331
411
def rename(self, rel_from, rel_to):
332
412
self._call('rename',
341
421
def rmdir(self, relpath):
342
422
resp = self._call('rmdir', self._remote_path(relpath))
344
def _translate_error(self, resp, orig_path=None):
345
"""Raise an exception from a response"""
352
elif what == 'NoSuchFile':
353
if orig_path is not None:
354
error_path = orig_path
357
raise errors.NoSuchFile(error_path)
358
elif what == 'error':
359
raise errors.SmartProtocolError(unicode(resp[1]))
360
elif what == 'FileExists':
361
raise errors.FileExists(resp[1])
362
elif what == 'DirectoryNotEmpty':
363
raise errors.DirectoryNotEmpty(resp[1])
364
elif what == 'ShortReadvError':
365
raise errors.ShortReadvError(resp[1], int(resp[2]),
366
int(resp[3]), int(resp[4]))
367
elif what in ('UnicodeEncodeError', 'UnicodeDecodeError'):
368
encoding = str(resp[1]) # encoding must always be a string
372
reason = str(resp[5]) # reason must always be a string
373
if val.startswith('u:'):
374
val = val[2:].decode('utf-8')
375
elif val.startswith('s:'):
376
val = val[2:].decode('base64')
377
if what == 'UnicodeDecodeError':
378
raise UnicodeDecodeError(encoding, val, start, end, reason)
379
elif what == 'UnicodeEncodeError':
380
raise UnicodeEncodeError(encoding, val, start, end, reason)
381
elif what == "ReadOnlyError":
382
raise errors.TransportNotPossible('readonly transport')
383
elif what == "ReadError":
384
if orig_path is not None:
385
error_path = orig_path
388
raise errors.ReadError(error_path)
390
raise errors.SmartProtocolError('unexpected smart server error: %r' % (resp,))
424
def _ensure_ok(self, resp):
426
raise errors.UnexpectedSmartServerResponse(resp)
428
def _translate_error(self, err, relpath=None):
429
remote._translate_error(err, path=relpath)
392
431
def disconnect(self):
393
self.get_smart_medium().disconnect()
395
def delete_tree(self, relpath):
396
raise errors.TransportNotPossible('readonly transport')
432
m = self.get_smart_medium()
398
436
def stat(self, relpath):
399
437
resp = self._call2('stat', self._remote_path(relpath))
400
438
if resp[0] == 'stat':
401
439
return _SmartStat(int(resp[1]), int(resp[2], 8))
403
self._translate_error(resp)
440
raise errors.UnexpectedSmartServerResponse(resp)
405
442
## def lock_read(self, relpath):
406
443
## """Lock the given file for shared (read) access.
422
459
resp = self._call2('list_dir', self._remote_path(relpath))
423
460
if resp[0] == 'names':
424
461
return [name.encode('ascii') for name in resp[1:]]
426
self._translate_error(resp)
462
raise errors.UnexpectedSmartServerResponse(resp)
428
464
def iter_files_recursive(self):
429
465
resp = self._call2('iter_files_recursive', self._remote_path(''))
430
466
if resp[0] == 'names':
433
self._translate_error(resp)
468
raise errors.UnexpectedSmartServerResponse(resp)
436
471
class RemoteTCPTransport(RemoteTransport):
437
472
"""Connection to smart server over plain tcp.
439
474
This is essentially just a factory to get 'RemoteTransport(url,
440
475
SmartTCPClientMedium).
443
478
def _build_medium(self):
444
assert self.base.startswith('bzr://')
445
return medium.SmartTCPClientMedium(self._host, self._port), None
479
client_medium = medium.SmartTCPClientMedium(
480
self._parsed_url.host, self._parsed_url.port, self.base)
481
return client_medium, None
484
class RemoteTCPTransportV2Only(RemoteTransport):
485
"""Connection to smart server over plain tcp with the client hard-coded to
486
assume protocol v2 and remote server version <= 1.6.
488
This should only be used for testing.
491
def _build_medium(self):
492
client_medium = medium.SmartTCPClientMedium(
493
self._parsed_url.host, self._parsed_url.port, self.base)
494
client_medium._protocol_version = 2
495
client_medium._remember_remote_is_before((1, 6))
496
return client_medium, None
448
499
class RemoteSSHTransport(RemoteTransport):
455
506
def _build_medium(self):
456
assert self.base.startswith('bzr+ssh://')
457
# ssh will prompt the user for a password if needed and if none is
458
# provided but it will not give it back, so no credentials can be
460
507
location_config = config.LocationConfig(self.base)
461
508
bzr_remote_path = location_config.get_bzr_remote_path()
462
return medium.SmartSSHClientMedium(self._host, self._port,
463
self._user, self._password, bzr_remote_path=bzr_remote_path), None
509
user = self._parsed_url.user
511
auth = config.AuthenticationConfig()
512
user = auth.get_user('ssh', self._parsed_url.host,
513
self._parsed_url.port)
514
ssh_params = medium.SSHParams(self._parsed_url.host,
515
self._parsed_url.port, user, self._parsed_url.password,
517
client_medium = medium.SmartSSHClientMedium(self.base, ssh_params)
518
return client_medium, (user, self._parsed_url.password)
466
521
class RemoteHTTPTransport(RemoteTransport):
467
522
"""Just a way to connect between a bzr+http:// url and http://.
469
524
This connection operates slightly differently than the RemoteSSHTransport.
470
525
It uses a plain http:// transport underneath, which defines what remote
471
526
.bzr/smart URL we are connected to. From there, all paths that are sent are
513
566
smart requests may be different). This is so that the server doesn't
514
567
have to handle .bzr/smart requests at arbitrary places inside .bzr
515
568
directories, just at the initial URL the user uses.
517
The exception is parent paths (i.e. relative_url of "..").
520
571
abs_url = self.abspath(relative_url)
522
573
abs_url = self.base
523
# We either use the exact same http_transport (for child locations), or
524
# a clone of the underlying http_transport (for parent locations). This
525
# means we share the connection.
526
norm_base = urlutils.normalize_url(self.base)
527
norm_abs_url = urlutils.normalize_url(abs_url)
528
normalized_rel_url = urlutils.relative_url(norm_base, norm_abs_url)
529
if normalized_rel_url == ".." or normalized_rel_url.startswith("../"):
530
http_transport = self._http_transport.clone(normalized_rel_url)
532
http_transport = self._http_transport
533
574
return RemoteHTTPTransport(abs_url,
534
575
_from_transport=self,
535
http_transport=http_transport)
576
http_transport=self._http_transport)
578
def _redirected_to(self, source, target):
579
"""See transport._redirected_to"""
580
redirected = self._http_transport._redirected_to(source, target)
581
if (redirected is not None
582
and isinstance(redirected, type(self._http_transport))):
583
return RemoteHTTPTransport('bzr+' + redirected.external_url(),
584
http_transport=redirected)
586
# Either None or a transport for a different protocol
590
class HintingSSHTransport(transport.Transport):
591
"""Simple transport that handles ssh:// and points out bzr+ssh://."""
593
def __init__(self, url):
594
raise errors.UnsupportedProtocol(url,
595
'bzr supports bzr+ssh to operate over ssh, use "bzr+%s".' % url)
538
598
def get_test_permutations():
539
599
"""Return (transport, server) permutations for testing."""
540
600
### We may need a little more test framework support to construct an
541
601
### appropriate RemoteTransport in the future.
542
from bzrlib.smart import server
543
return [(RemoteTCPTransport, server.SmartTCPServer_for_testing)]
602
from bzrlib.tests import test_server
603
return [(RemoteTCPTransport, test_server.SmartTCPServer_for_testing)]
added
removed
bzrlib/transport/remote.py
