« back to all changes in this revision
Viewing changes to bzrlib/transport/ftp.py
- browse files at revision 2646
- compare with another revision
- compare with revision 2485.8.2
- download diff
- download tarball
- view history from revision 2646
- Committer: Canonical.com Patch Queue Manager
- Date: 2007-07-22 18:09:04 UTC
- mfrom: (2485.8.63 bzr.connection.sharing)
- Revision ID: pqm@pqm.ubuntu.com-20070722180904-wy7y7oyi32wbghgf
Transport connection sharing
- files added:
- bzrlib/tests/commands
- files modified:
- NEWS
added
removed
bzrlib/transport/ftp.py
47
47
from bzrlib.trace import mutter, warning
48
48
from bzrlib.transport import (
49
49
Server,
50
split_url,
51
Transport,
50
ConnectedTransport,
52
51
)
53
52
from bzrlib.transport.local import LocalURLServer
54
53
import bzrlib.ui
60
59
"""FTP failed for path: %(path)s%(extra)s"""
61
60
62
61
63
_FTP_cache = {}
64
def _find_FTP(hostname, port, username, password, is_active):
65
"""Find an ftplib.FTP instance attached to this triplet."""
66
key = (hostname, port, username, password, is_active)
67
alt_key = (hostname, port, username, '********', is_active)
68
if key not in _FTP_cache:
69
mutter("Constructing FTP instance against %r" % (alt_key,))
70
conn = ftplib.FTP()
71
72
conn.connect(host=hostname, port=port)
73
if username and username != 'anonymous' and not password:
74
password = bzrlib.ui.ui_factory.get_password(
75
prompt='FTP %(user)s@%(host)s password',
76
user=username, host=hostname)
77
conn.login(user=username, passwd=password)
78
conn.set_pasv(not is_active)
79
80
_FTP_cache[key] = conn
81
82
return _FTP_cache[key]
83
84
85
62
class FtpStatResult(object):
86
63
def __init__(self, f, relpath):
87
64
try:
99
76
_number_of_retries = 2
100
77
_sleep_between_retries = 5
101
78
102
class FtpTransport(Transport):
79
# FIXME: there are inconsistencies in the way temporary errors are
80
# handled. Sometimes we reconnect, sometimes we raise an exception. Care should
81
# be taken to analyze the implications for write operations (read operations
82
# are safe to retry). Overall even some read operations are never
83
# retried. --vila 20070720 (Bug #127164)
84
class FtpTransport(ConnectedTransport):
103
85
"""This is the transport agent for ftp:// access."""
104
86
105
def __init__(self, base, _provided_instance=None):
87
def __init__(self, base, _from_transport=None):
106
88
"""Set the base path where files will be stored."""
107
89
assert base.startswith('ftp://') or base.startswith('aftp://')
108
109
self.is_active = base.startswith('aftp://')
110
if self.is_active:
111
# urlparse won't handle aftp://
112
base = base[1:]
113
if not base.endswith('/'):
114
base += '/'
115
(self._proto, self._username,
116
self._password, self._host,
117
self._port, self._path) = split_url(base)
118
base = self._unparse_url()
119
120
super(FtpTransport, self).__init__(base)
121
self._FTP_instance = _provided_instance
122
123
def _unparse_url(self, path=None):
124
if path is None:
125
path = self._path
126
path = urllib.quote(path)
127
netloc = urllib.quote(self._host)
128
if self._username is not None:
129
netloc = '%s@%s' % (urllib.quote(self._username), netloc)
130
if self._port is not None:
131
netloc = '%s:%d' % (netloc, self._port)
132
proto = 'ftp'
133
if self.is_active:
134
proto = 'aftp'
135
return urlparse.urlunparse((proto, netloc, path, '', '', ''))
90
super(FtpTransport, self).__init__(base,
91
_from_transport=_from_transport)
92
self._unqualified_scheme = 'ftp'
93
if self._scheme == 'aftp':
94
self.is_active = True
95
else:
96
self.is_active = False
136
97
137
98
def _get_FTP(self):
138
99
"""Return the ftplib.FTP instance for this object."""
139
if self._FTP_instance is not None:
140
return self._FTP_instance
141
100
# Ensures that a connection is established
101
connection = self._get_connection()
102
if connection is None:
103
# First connection ever
104
connection, credentials = self._create_connection()
105
self._set_connection(connection, credentials)
106
return connection
107
108
def _create_connection(self, credentials=None):
109
"""Create a new connection with the provided credentials.
110
111
:param credentials: The credentials needed to establish the connection.
112
113
:return: The created connection and its associated credentials.
114
115
The credentials are only the password as it may have been entered
116
interactively by the user and may be different from the one provided
117
in base url at transport creation time.
118
"""
119
if credentials is None:
120
password = self._password
121
else:
122
password = credentials
123
124
mutter("Constructing FTP instance against %r" %
125
((self._host, self._port, self._user, '********',
126
self.is_active),))
142
127
try:
143
self._FTP_instance = _find_FTP(self._host, self._port,
144
self._username, self._password,
145
self.is_active)
146
return self._FTP_instance
128
connection = ftplib.FTP()
129
connection.connect(host=self._host, port=self._port)
130
if self._user and self._user != 'anonymous' and \
131
password is not None: # '' is a valid password
132
get_password = bzrlib.ui.ui_factory.get_password
133
password = get_password(prompt='FTP %(user)s@%(host)s password',
134
user=self._user, host=self._host)
135
connection.login(user=self._user, passwd=password)
136
connection.set_pasv(not self.is_active)
147
137
except ftplib.error_perm, e:
148
raise errors.TransportError(msg="Error setting up connection: %s"
149
% str(e), orig_error=e)
150
151
def _translate_perm_error(self, err, path, extra=None, unknown_exc=FtpPathError):
138
raise errors.TransportError(msg="Error setting up connection:"
139
" %s" % str(e), orig_error=e)
140
return connection, password
141
142
def _reconnect(self):
143
"""Create a new connection with the previously used credentials"""
144
credentials = self.get_credentials()
145
connection, credentials = self._create_connection(credentials)
146
self._set_connection(connection, credentials)
147
148
def _translate_perm_error(self, err, path, extra=None,
149
unknown_exc=FtpPathError):
152
150
"""Try to translate an ftplib.error_perm exception.
153
151
154
152
:param err: The error to translate into a bzr error
190
188
"""
191
189
return True
192
190
193
def clone(self, offset=None):
194
"""Return a new FtpTransport with root at self.base + offset.
195
"""
196
mutter("FTP clone")
197
if offset is None:
198
return FtpTransport(self.base, self._FTP_instance)
199
else:
200
return FtpTransport(self.abspath(offset), self._FTP_instance)
201
202
def _abspath(self, relpath):
203
assert isinstance(relpath, basestring)
204
relpath = urlutils.unescape(relpath)
205
if relpath.startswith('/'):
206
basepath = []
207
else:
208
basepath = self._path.split('/')
209
if len(basepath) > 0 and basepath[-1] == '':
210
basepath = basepath[:-1]
211
for p in relpath.split('/'):
212
if p == '..':
213
if len(basepath) == 0:
214
# In most filesystems, a request for the parent
215
# of root, just returns root.
216
continue
217
basepath.pop()
218
elif p == '.' or p == '':
219
continue # No-op
220
else:
221
basepath.append(p)
222
# Possibly, we could use urlparse.urljoin() here, but
223
# I'm concerned about when it chooses to strip the last
224
# portion of the path, and when it doesn't.
225
191
def _remote_path(self, relpath):
226
192
# XXX: It seems that ftplib does not handle Unicode paths
227
# at the same time, medusa won't handle utf8 paths
228
# So if we .encode(utf8) here, then we get a Server failure.
229
# while if we use str(), we get a UnicodeError, and the test suite
230
# just skips testing UnicodePaths.
231
return str('/'.join(basepath) or '/')
232
233
def abspath(self, relpath):
234
"""Return the full url to the given relative path.
235
This can be supplied with a string or a list
236
"""
237
path = self._abspath(relpath)
238
return self._unparse_url(path)
193
# at the same time, medusa won't handle utf8 paths So if
194
# we .encode(utf8) here (see ConnectedTransport
195
# implementation), then we get a Server failure. while
196
# if we use str(), we get a UnicodeError, and the test
197
# suite just skips testing UnicodePaths.
198
relative = str(urlutils.unescape(relpath))
199
remote_path = self._combine_paths(self._path, relative)
200
return remote_path
239
201
240
202
def has(self, relpath):
241
203
"""Does the target location exist?"""
244
206
# XXX: I assume we're never asked has(dirname) and thus I use
245
207
# the FTP size command and assume that if it doesn't raise,
246
208
# all is good.
247
abspath = self._abspath(relpath)
209
abspath = self._remote_path(relpath)
248
210
try:
249
211
f = self._get_FTP()
250
212
mutter('FTP has check: %s => %s', relpath, abspath)
270
232
"""
271
233
# TODO: decode should be deprecated
272
234
try:
273
mutter("FTP get: %s", self._abspath(relpath))
235
mutter("FTP get: %s", self._remote_path(relpath))
274
236
f = self._get_FTP()
275
237
ret = StringIO()
276
f.retrbinary('RETR '+self._abspath(relpath), ret.write, 8192)
238
f.retrbinary('RETR '+self._remote_path(relpath), ret.write, 8192)
277
239
ret.seek(0)
278
240
return ret
279
241
except ftplib.error_perm, e:
285
247
orig_error=e)
286
248
else:
287
249
warning("FTP temporary error: %s. Retrying.", str(e))
288
self._FTP_instance = None
250
self._reconnect()
289
251
return self.get(relpath, decode, retries+1)
290
252
except EOFError, e:
291
253
if retries > _number_of_retries:
295
257
else:
296
258
warning("FTP control connection closed. Trying to reopen.")
297
259
time.sleep(_sleep_between_retries)
298
self._FTP_instance = None
260
self._reconnect()
299
261
return self.get(relpath, decode, retries+1)
300
262
301
263
def put_file(self, relpath, fp, mode=None, retries=0):
309
271
TODO: jam 20051215 ftp as a protocol seems to support chmod, but
310
272
ftplib does not
311
273
"""
312
abspath = self._abspath(relpath)
274
abspath = self._remote_path(relpath)
313
275
tmp_abspath = '%s.tmp.%.9f.%d.%d' % (abspath, time.time(),
314
276
os.getpid(), random.randint(0,0x7FFFFFFF))
315
277
if getattr(fp, 'read', None) is None:
338
300
% self.abspath(relpath), orig_error=e)
339
301
else:
340
302
warning("FTP temporary error: %s. Retrying.", str(e))
341
self._FTP_instance = None
303
self._reconnect()
342
304
self.put_file(relpath, fp, mode, retries+1)
343
305
except EOFError:
344
306
if retries > _number_of_retries:
347
309
else:
348
310
warning("FTP control connection closed. Trying to reopen.")
349
311
time.sleep(_sleep_between_retries)
350
self._FTP_instance = None
312
self._reconnect()
351
313
self.put_file(relpath, fp, mode, retries+1)
352
314
353
315
def mkdir(self, relpath, mode=None):
354
316
"""Create a directory at the given path."""
355
abspath = self._abspath(relpath)
317
abspath = self._remote_path(relpath)
356
318
try:
357
319
mutter("FTP mkd: %s", abspath)
358
320
f = self._get_FTP()
363
325
364
326
def rmdir(self, rel_path):
365
327
"""Delete the directory at rel_path"""
366
abspath = self._abspath(rel_path)
328
abspath = self._remote_path(rel_path)
367
329
try:
368
330
mutter("FTP rmd: %s", abspath)
369
331
f = self._get_FTP()
375
337
"""Append the text in the file-like object into the final
376
338
location.
377
339
"""
378
abspath = self._abspath(relpath)
340
abspath = self._remote_path(relpath)
379
341
if self.has(relpath):
380
342
ftp = self._get_FTP()
381
343
result = ftp.size(abspath)
394
356
number of retries is exceeded.
395
357
"""
396
358
try:
397
abspath = self._abspath(relpath)
359
abspath = self._remote_path(relpath)
398
360
mutter("FTP appe (try %d) to %s", retries, abspath)
399
361
ftp = self._get_FTP()
400
362
ftp.voidcmd("TYPE I")
414
376
"Aborting." % abspath, orig_error=e)
415
377
else:
416
378
warning("FTP temporary error: %s. Retrying.", str(e))
417
self._FTP_instance = None
379
self._reconnect()
418
380
self._try_append(relpath, text, mode, retries+1)
419
381
420
382
def _setmode(self, relpath, mode):
425
387
"""
426
388
try:
427
389
mutter("FTP site chmod: setting permissions to %s on %s",
428
str(mode), self._abspath(relpath))
390
str(mode), self._remote_path(relpath))
429
391
ftp = self._get_FTP()
430
cmd = "SITE CHMOD %s %s" % (self._abspath(relpath), str(mode))
392
cmd = "SITE CHMOD %s %s" % (self._remote_path(relpath), str(mode))
431
393
ftp.sendcmd(cmd)
432
394
except ftplib.error_perm, e:
433
395
# Command probably not available on this server
434
396
warning("FTP Could not set permissions to %s on %s. %s",
435
str(mode), self._abspath(relpath), str(e))
397
str(mode), self._remote_path(relpath), str(e))
436
398
437
399
# TODO: jam 20060516 I believe ftp allows you to tell an ftp server
438
400
# to copy something to another machine. And you may be able
440
402
# So implement a fancier 'copy()'
441
403
442
404
def rename(self, rel_from, rel_to):
443
abs_from = self._abspath(rel_from)
444
abs_to = self._abspath(rel_to)
405
abs_from = self._remote_path(rel_from)
406
abs_to = self._remote_path(rel_to)
445
407
mutter("FTP rename: %s => %s", abs_from, abs_to)
446
408
f = self._get_FTP()
447
409
return self._rename(abs_from, abs_to, f)
455
417
456
418
def move(self, rel_from, rel_to):
457
419
"""Move the item at rel_from to the location at rel_to"""
458
abs_from = self._abspath(rel_from)
459
abs_to = self._abspath(rel_to)
420
abs_from = self._remote_path(rel_from)
421
abs_to = self._remote_path(rel_to)
460
422
try:
461
423
mutter("FTP mv: %s => %s", abs_from, abs_to)
462
424
f = self._get_FTP()
477
439
478
440
def delete(self, relpath):
479
441
"""Delete the item at relpath"""
480
abspath = self._abspath(relpath)
442
abspath = self._remote_path(relpath)
481
443
f = self._get_FTP()
482
444
self._delete(abspath, f)
483
445
500
462
501
463
def list_dir(self, relpath):
502
464
"""See Transport.list_dir."""
503
basepath = self._abspath(relpath)
465
basepath = self._remote_path(relpath)
504
466
mutter("FTP nlst: %s", basepath)
505
467
f = self._get_FTP()
506
468
try:
533
495
534
496
def stat(self, relpath):
535
497
"""Return the stat information for a file."""
536
abspath = self._abspath(relpath)
498
abspath = self._remote_path(relpath)
537
499
try:
538
500
mutter("FTP stat: %s", abspath)
539
501
f = self._get_FTP()
603
565
)
604
566
self._port = self._ftp_server.getsockname()[1]
605
567
# Don't let it loop forever, or handle an infinite number of requests.
606
# In this case it will run for 100s, or 1000 requests
568
# In this case it will run for 1000s, or 10000 requests
607
569
self._async_thread = threading.Thread(
608
570
target=FtpServer._asyncore_loop_ignore_EBADF,
609
kwargs={'timeout':0.1, 'count':1000})
571
kwargs={'timeout':0.1, 'count':10000})
610
572
self._async_thread.setDaemon(True)
611
573
self._async_thread.start()
612
574
626
588
"""
627
589
try:
628
590
asyncore.loop(*args, **kwargs)
591
# FIXME: If we reach that point, we should raise an exception
592
# explaining that the 'count' parameter in setUp is too low or
593
# testers may wonder why their test just sits there waiting for a
594
# server that is already dead. Note that if the tester waits too
595
# long under pdb the server will also die.
629
596
except select.error, e:
630
597
if e.args[0] != errno.EBADF:
631
598
raise
677
644
def log(self, message):
678
645
"""Redirect logging requests."""
679
646
mutter('_ftp_channel: %s', message)
680
647
681
648
def log_info(self, message, type='info'):
682
649
"""Redirect logging requests."""
683
650
mutter('_ftp_channel %s: %s', type, message)
684
651
685
652
def cmd_rnfr(self, line):
686
653
"""Prepare for renaming a file."""
687
654
self._renaming = line[1]
Loggerhead is a web-based interface for Breezy
Version: 2.0.1