82
83
one is being cloned from. Attributes such as the medium will
85
:param medium: The medium to use for this RemoteTransport. If None,
86
the medium from the _from_transport is shared. If both this
87
and _from_transport are None, a new medium will be built.
88
_from_transport and medium cannot both be specified.
86
:param medium: The medium to use for this RemoteTransport. This must be
87
supplied if _from_transport is None.
90
89
:param _client: Override the _SmartClient used by this transport. This
91
90
should only be used for testing purposes; normally this is
92
91
determined from the medium.
94
super(RemoteTransport, self).__init__(
95
url, _from_transport=_from_transport)
93
super(RemoteTransport, self).__init__(url,
94
_from_transport=_from_transport)
97
96
# The medium is the connection, except when we need to share it with
98
97
# other objects (RemoteBzrDir, RemoteRepository etc). In these cases
99
98
# what we want to share is really the shared connection.
101
if (_from_transport is not None
102
and isinstance(_from_transport, RemoteTransport)):
103
_client = _from_transport._client
104
elif _from_transport is None:
100
if _from_transport is None:
105
101
# If no _from_transport is specified, we need to intialize the
107
103
credentials = None
138
124
return None, None
140
def _report_activity(self, bytes, direction):
141
"""See Transport._report_activity.
143
Does nothing; the smart medium will report activity triggered by a
148
126
def is_readonly(self):
149
127
"""Smart server transport can do read/write file operations."""
151
resp = self._call2('Transport.is_readonly')
152
except errors.UnknownSmartMethod:
128
resp = self._call2('Transport.is_readonly')
129
if resp == ('yes', ):
131
elif resp == ('no', ):
133
elif (resp == ('error', "Generic bzr smart protocol error: "
134
"bad request 'Transport.is_readonly'") or
135
resp == ('error', "Generic bzr smart protocol error: "
136
"bad request u'Transport.is_readonly'")):
153
137
# XXX: nasty hack: servers before 0.16 don't have a
154
138
# 'Transport.is_readonly' verb, so we do what clients before 0.16
155
139
# did: assume False.
157
if resp == ('yes', ):
159
elif resp == ('no', ):
162
raise errors.UnexpectedSmartServerResponse(resp)
142
self._translate_error(resp)
143
raise errors.UnexpectedSmartServerResponse(resp)
164
145
def get_smart_client(self):
165
146
return self._get_connection()
167
148
def get_smart_medium(self):
168
149
return self._get_connection()
151
def get_shared_medium(self):
152
return self._get_shared_connection()
170
154
def _remote_path(self, relpath):
171
155
"""Returns the Unicode version of the absolute path for relpath."""
172
return urlutils.URL._combine_paths(self._parsed_url.path, relpath)
156
return self._combine_paths(self._path, relpath)
174
158
def _call(self, method, *args):
175
159
resp = self._call2(method, *args)
176
self._ensure_ok(resp)
160
self._translate_error(resp)
178
162
def _call2(self, method, *args):
179
163
"""Call a method on the remote server."""
181
return self._client.call(method, *args)
182
except errors.ErrorFromSmartServer, err:
183
# The first argument, if present, is always a path.
185
context = {'relpath': args[0]}
188
self._translate_error(err, **context)
164
return self._client.call(method, *args)
190
166
def _call_with_body_bytes(self, method, args, body):
191
167
"""Call a method on the remote server with body bytes."""
193
return self._client.call_with_body_bytes(method, args, body)
194
except errors.ErrorFromSmartServer, err:
195
# The first argument, if present, is always a path.
197
context = {'relpath': args[0]}
200
self._translate_error(err, **context)
168
return self._client.call_with_body_bytes(method, args, body)
202
170
def has(self, relpath):
203
171
"""Indicate whether a remote file of the given name exists or not.
210
178
elif resp == ('no', ):
213
raise errors.UnexpectedSmartServerResponse(resp)
181
self._translate_error(resp)
215
183
def get(self, relpath):
216
184
"""Return file-like object reading the contents of a remote file.
218
186
:see: Transport.get_bytes()/get_file()
220
188
return StringIO(self.get_bytes(relpath))
222
190
def get_bytes(self, relpath):
223
191
remote = self._remote_path(relpath)
225
resp, response_handler = self._client.call_expecting_body('get', remote)
226
except errors.ErrorFromSmartServer, err:
227
self._translate_error(err, relpath)
192
request = self.get_smart_medium().get_request()
193
smart_protocol = protocol.SmartClientRequestProtocolOne(request)
194
smart_protocol.call('get', remote)
195
resp = smart_protocol.read_response_tuple(True)
228
196
if resp != ('ok', ):
229
response_handler.cancel_read_body()
230
raise errors.UnexpectedSmartServerResponse(resp)
231
return response_handler.read_body_bytes()
197
smart_protocol.cancel_read_body()
198
self._translate_error(resp, relpath)
199
return smart_protocol.read_body_bytes()
233
201
def _serialise_optional_mode(self, mode):
247
216
transport._file_streams[self.abspath(relpath)] = result
250
def put_bytes(self, relpath, raw_bytes, mode=None):
251
if not isinstance(raw_bytes, str):
253
'raw_bytes must be a plain string, not %s' % type(raw_bytes))
254
resp = self._call_with_body_bytes(
219
def put_bytes(self, relpath, upload_contents, mode=None):
220
# FIXME: upload_file is probably not safe for non-ascii characters -
221
# should probably just pass all parameters as length-delimited
223
if type(upload_contents) is unicode:
224
# Although not strictly correct, we raise UnicodeEncodeError to be
225
# compatible with other transports.
226
raise UnicodeEncodeError(
227
'undefined', upload_contents, 0, 1,
228
'put_bytes must be given bytes, not unicode.')
229
resp = self._call_with_body_bytes('put',
256
230
(self._remote_path(relpath), self._serialise_optional_mode(mode)),
258
self._ensure_ok(resp)
259
return len(raw_bytes)
232
self._translate_error(resp)
233
return len(upload_contents)
261
def put_bytes_non_atomic(self, relpath, raw_bytes, mode=None,
235
def put_bytes_non_atomic(self, relpath, bytes, mode=None,
262
236
create_parent_dir=False,
264
238
"""See Transport.put_bytes_non_atomic."""
324
294
offsets = list(offsets)
326
296
sorted_offsets = sorted(offsets)
297
# turn the list of offsets into a stack
298
offset_stack = iter(offsets)
299
cur_offset_and_size = offset_stack.next()
327
300
coalesced = list(self._coalesce_offsets(sorted_offsets,
328
301
limit=self._max_readv_combine,
329
fudge_factor=self._bytes_to_read_before_seek,
330
max_size=self._max_readv_bytes))
332
# now that we've coallesced things, avoid making enormous requests
337
if c.length + cur_len > self._max_readv_bytes:
338
requests.append(cur_request)
342
cur_request.append(c)
345
requests.append(cur_request)
346
if 'hpss' in debug.debug_flags:
347
trace.mutter('%s.readv %s offsets => %s coalesced'
348
' => %s requests (%s)',
349
self.__class__.__name__, len(offsets), len(coalesced),
350
len(requests), sum(map(len, requests)))
302
fudge_factor=self._bytes_to_read_before_seek))
304
request = self.get_smart_medium().get_request()
305
smart_protocol = protocol.SmartClientRequestProtocolOne(request)
306
smart_protocol.call_with_body_readv_array(
307
('readv', self._remote_path(relpath)),
308
[(c.start, c.length) for c in coalesced])
309
resp = smart_protocol.read_response_tuple(True)
311
if resp[0] != 'readv':
312
# This should raise an exception
313
smart_protocol.cancel_read_body()
314
self._translate_error(resp)
317
# FIXME: this should know how many bytes are needed, for clarity.
318
data = smart_protocol.read_body_bytes()
351
319
# Cache the results, but only until they have been fulfilled
353
# turn the list of offsets into a single stack to iterate
354
offset_stack = iter(offsets)
355
# using a list so it can be modified when passing down and coming back
356
next_offset = [offset_stack.next()]
357
for cur_request in requests:
359
result = self._client.call_with_body_readv_array(
360
('readv', self._remote_path(relpath),),
361
[(c.start, c.length) for c in cur_request])
362
resp, response_handler = result
363
except errors.ErrorFromSmartServer, err:
364
self._translate_error(err, relpath)
366
if resp[0] != 'readv':
367
# This should raise an exception
368
response_handler.cancel_read_body()
369
raise errors.UnexpectedSmartServerResponse(resp)
371
for res in self._handle_response(offset_stack, cur_request,
377
def _handle_response(self, offset_stack, coalesced, response_handler,
378
data_map, next_offset):
379
cur_offset_and_size = next_offset[0]
380
# FIXME: this should know how many bytes are needed, for clarity.
381
data = response_handler.read_body_bytes()
383
321
for c_offset in coalesced:
384
322
if len(data) < c_offset.length:
385
323
raise errors.ShortReadvError(relpath, c_offset.start,
386
324
c_offset.length, actual=len(data))
387
325
for suboffset, subsize in c_offset.ranges:
388
326
key = (c_offset.start+suboffset, subsize)
389
this_data = data[data_offset+suboffset:
390
data_offset+suboffset+subsize]
391
# Special case when the data is in-order, rather than packing
392
# into a map and then back out again. Benchmarking shows that
393
# this has 100% hit rate, but leave in the data_map work just
395
# TODO: Could we get away with using buffer() to avoid the
396
# memory copy? Callers would need to realize they may
397
# not have a real string.
398
if key == cur_offset_and_size:
399
yield cur_offset_and_size[0], this_data
400
cur_offset_and_size = next_offset[0] = offset_stack.next()
402
data_map[key] = this_data
403
data_offset += c_offset.length
327
data_map[key] = data[suboffset:suboffset+subsize]
328
data = data[c_offset.length:]
405
330
# Now that we've read some data, see if we can yield anything back
406
331
while cur_offset_and_size in data_map:
407
332
this_data = data_map.pop(cur_offset_and_size)
408
333
yield cur_offset_and_size[0], this_data
409
cur_offset_and_size = next_offset[0] = offset_stack.next()
334
cur_offset_and_size = offset_stack.next()
411
336
def rename(self, rel_from, rel_to):
412
337
self._call('rename',
421
346
def rmdir(self, relpath):
422
347
resp = self._call('rmdir', self._remote_path(relpath))
424
def _ensure_ok(self, resp):
426
raise errors.UnexpectedSmartServerResponse(resp)
428
def _translate_error(self, err, relpath=None):
429
remote._translate_error(err, path=relpath)
349
def _translate_error(self, resp, orig_path=None):
350
"""Raise an exception from a response"""
357
elif what == 'NoSuchFile':
358
if orig_path is not None:
359
error_path = orig_path
362
raise errors.NoSuchFile(error_path)
363
elif what == 'error':
364
raise errors.SmartProtocolError(unicode(resp[1]))
365
elif what == 'FileExists':
366
raise errors.FileExists(resp[1])
367
elif what == 'DirectoryNotEmpty':
368
raise errors.DirectoryNotEmpty(resp[1])
369
elif what == 'ShortReadvError':
370
raise errors.ShortReadvError(resp[1], int(resp[2]),
371
int(resp[3]), int(resp[4]))
372
elif what in ('UnicodeEncodeError', 'UnicodeDecodeError'):
373
encoding = str(resp[1]) # encoding must always be a string
377
reason = str(resp[5]) # reason must always be a string
378
if val.startswith('u:'):
379
val = val[2:].decode('utf-8')
380
elif val.startswith('s:'):
381
val = val[2:].decode('base64')
382
if what == 'UnicodeDecodeError':
383
raise UnicodeDecodeError(encoding, val, start, end, reason)
384
elif what == 'UnicodeEncodeError':
385
raise UnicodeEncodeError(encoding, val, start, end, reason)
386
elif what == "ReadOnlyError":
387
raise errors.TransportNotPossible('readonly transport')
388
elif what == "ReadError":
389
if orig_path is not None:
390
error_path = orig_path
393
raise errors.ReadError(error_path)
395
raise errors.SmartProtocolError('unexpected smart server error: %r' % (resp,))
431
397
def disconnect(self):
432
m = self.get_smart_medium()
398
self.get_smart_medium().disconnect()
400
def delete_tree(self, relpath):
401
raise errors.TransportNotPossible('readonly transport')
436
403
def stat(self, relpath):
437
404
resp = self._call2('stat', self._remote_path(relpath))
438
405
if resp[0] == 'stat':
439
406
return _SmartStat(int(resp[1]), int(resp[2], 8))
440
raise errors.UnexpectedSmartServerResponse(resp)
408
self._translate_error(resp)
442
410
## def lock_read(self, relpath):
443
411
## """Lock the given file for shared (read) access.
459
427
resp = self._call2('list_dir', self._remote_path(relpath))
460
428
if resp[0] == 'names':
461
429
return [name.encode('ascii') for name in resp[1:]]
462
raise errors.UnexpectedSmartServerResponse(resp)
431
self._translate_error(resp)
464
433
def iter_files_recursive(self):
465
434
resp = self._call2('iter_files_recursive', self._remote_path(''))
466
435
if resp[0] == 'names':
468
raise errors.UnexpectedSmartServerResponse(resp)
438
self._translate_error(resp)
471
441
class RemoteTCPTransport(RemoteTransport):
472
442
"""Connection to smart server over plain tcp.
474
444
This is essentially just a factory to get 'RemoteTransport(url,
475
445
SmartTCPClientMedium).
478
448
def _build_medium(self):
479
client_medium = medium.SmartTCPClientMedium(
480
self._parsed_url.host, self._parsed_url.port, self.base)
481
return client_medium, None
484
class RemoteTCPTransportV2Only(RemoteTransport):
485
"""Connection to smart server over plain tcp with the client hard-coded to
486
assume protocol v2 and remote server version <= 1.6.
488
This should only be used for testing.
491
def _build_medium(self):
492
client_medium = medium.SmartTCPClientMedium(
493
self._parsed_url.host, self._parsed_url.port, self.base)
494
client_medium._protocol_version = 2
495
client_medium._remember_remote_is_before((1, 6))
496
return client_medium, None
449
assert self.base.startswith('bzr://')
450
return medium.SmartTCPClientMedium(self._host, self._port), None
499
453
class RemoteSSHTransport(RemoteTransport):
506
460
def _build_medium(self):
461
assert self.base.startswith('bzr+ssh://')
462
# ssh will prompt the user for a password if needed and if none is
463
# provided but it will not give it back, so no credentials can be
507
465
location_config = config.LocationConfig(self.base)
508
466
bzr_remote_path = location_config.get_bzr_remote_path()
509
user = self._parsed_url.user
511
auth = config.AuthenticationConfig()
512
user = auth.get_user('ssh', self._parsed_url.host,
513
self._parsed_url.port)
514
ssh_params = medium.SSHParams(self._parsed_url.host,
515
self._parsed_url.port, user, self._parsed_url.password,
517
client_medium = medium.SmartSSHClientMedium(self.base, ssh_params)
518
return client_medium, (user, self._parsed_url.password)
467
return medium.SmartSSHClientMedium(self._host, self._port,
468
self._user, self._password, bzr_remote_path=bzr_remote_path), None
521
471
class RemoteHTTPTransport(RemoteTransport):
522
472
"""Just a way to connect between a bzr+http:// url and http://.
524
474
This connection operates slightly differently than the RemoteSSHTransport.
525
475
It uses a plain http:// transport underneath, which defines what remote
526
476
.bzr/smart URL we are connected to. From there, all paths that are sent are
566
518
smart requests may be different). This is so that the server doesn't
567
519
have to handle .bzr/smart requests at arbitrary places inside .bzr
568
520
directories, just at the initial URL the user uses.
522
The exception is parent paths (i.e. relative_url of "..").
571
525
abs_url = self.abspath(relative_url)
573
527
abs_url = self.base
528
# We either use the exact same http_transport (for child locations), or
529
# a clone of the underlying http_transport (for parent locations). This
530
# means we share the connection.
531
norm_base = urlutils.normalize_url(self.base)
532
norm_abs_url = urlutils.normalize_url(abs_url)
533
normalized_rel_url = urlutils.relative_url(norm_base, norm_abs_url)
534
if normalized_rel_url == ".." or normalized_rel_url.startswith("../"):
535
http_transport = self._http_transport.clone(normalized_rel_url)
537
http_transport = self._http_transport
574
538
return RemoteHTTPTransport(abs_url,
575
539
_from_transport=self,
576
http_transport=self._http_transport)
578
def _redirected_to(self, source, target):
579
"""See transport._redirected_to"""
580
redirected = self._http_transport._redirected_to(source, target)
581
if (redirected is not None
582
and isinstance(redirected, type(self._http_transport))):
583
return RemoteHTTPTransport('bzr+' + redirected.external_url(),
584
http_transport=redirected)
586
# Either None or a transport for a different protocol
590
class HintingSSHTransport(transport.Transport):
591
"""Simple transport that handles ssh:// and points out bzr+ssh://."""
593
def __init__(self, url):
594
raise errors.UnsupportedProtocol(url,
595
'bzr supports bzr+ssh to operate over ssh, use "bzr+%s".' % url)
540
http_transport=http_transport)
598
543
def get_test_permutations():
599
544
"""Return (transport, server) permutations for testing."""
600
545
### We may need a little more test framework support to construct an
601
546
### appropriate RemoteTransport in the future.
602
from bzrlib.tests import test_server
603
return [(RemoteTCPTransport, test_server.SmartTCPServer_for_testing)]
547
from bzrlib.smart import server
548
return [(RemoteTCPTransport, server.SmartTCPServer_for_testing)]
added
removed
bzrlib/transport/remote.py
