2338.1.1
by John Arbash Meinel
(Arnaud Fontaine) Compatibility with vsftpd error messages. |
1 |
# Copyright (C) 2005, 2006, 2007 Canonical Ltd
|
1887.1.1
by Adeodato Simó
Do not separate paragraphs in the copyright statement with blank lines, |
2 |
#
|
1185.36.4
by Daniel Silverstone
Add FTP transport |
3 |
# This program is free software; you can redistribute it and/or modify
|
4 |
# it under the terms of the GNU General Public License as published by
|
|
5 |
# the Free Software Foundation; either version 2 of the License, or
|
|
6 |
# (at your option) any later version.
|
|
1887.1.1
by Adeodato Simó
Do not separate paragraphs in the copyright statement with blank lines, |
7 |
#
|
1185.36.4
by Daniel Silverstone
Add FTP transport |
8 |
# This program is distributed in the hope that it will be useful,
|
9 |
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
10 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
11 |
# GNU General Public License for more details.
|
|
1887.1.1
by Adeodato Simó
Do not separate paragraphs in the copyright statement with blank lines, |
12 |
#
|
1185.36.4
by Daniel Silverstone
Add FTP transport |
13 |
# You should have received a copy of the GNU General Public License
|
14 |
# along with this program; if not, write to the Free Software
|
|
15 |
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
16 |
"""Implementation of Transport over ftp.
|
|
17 |
||
18 |
Written by Daniel Silverstone <dsilvers@digital-scurf.org> with serious
|
|
19 |
cargo-culting from the sftp transport and the http transport.
|
|
20 |
||
21 |
It provides the ftp:// and aftp:// protocols where ftp:// is passive ftp
|
|
22 |
and aftp:// is active ftp. Most people will want passive ftp for traversing
|
|
23 |
NAT and other firewalls, so it's best to use it unless you explicitly want
|
|
24 |
active, in which case aftp:// will be your friend.
|
|
25 |
"""
|
|
26 |
||
27 |
from cStringIO import StringIO |
|
1707.3.1
by John Arbash Meinel
Have a FtpServer, but the tests fail |
28 |
import asyncore |
1530.1.11
by Robert Collins
Push the transport permutations list into each transport module allowing for automatic testing of new modules that are registered as transports. |
29 |
import errno |
1185.36.4
by Daniel Silverstone
Add FTP transport |
30 |
import ftplib |
1530.1.11
by Robert Collins
Push the transport permutations list into each transport module allowing for automatic testing of new modules that are registered as transports. |
31 |
import os |
1551.15.2
by Aaron Bentley
FTP does not require atomic rename. Fixes #89436 |
32 |
import os.path |
1530.1.11
by Robert Collins
Push the transport permutations list into each transport module allowing for automatic testing of new modules that are registered as transports. |
33 |
import urllib |
1185.36.4
by Daniel Silverstone
Add FTP transport |
34 |
import urlparse |
2413.3.1
by John Arbash Meinel
Trap EBADF during shutdown of asyncore.loop() |
35 |
import select |
1185.36.4
by Daniel Silverstone
Add FTP transport |
36 |
import stat |
1707.3.1
by John Arbash Meinel
Have a FtpServer, but the tests fail |
37 |
import threading |
1185.72.14
by Matthieu Moy
Sleep between retries when the connection closes |
38 |
import time |
1185.72.13
by Matthieu Moy
Make ftp put atomic |
39 |
import random |
1530.1.11
by Robert Collins
Push the transport permutations list into each transport module allowing for automatic testing of new modules that are registered as transports. |
40 |
from warnings import warn |
41 |
||
1948.1.7
by John Arbash Meinel
Fix FtpTransport so that it raises UnicodeError and skips the unicode path tests |
42 |
from bzrlib import ( |
43 |
errors, |
|
1551.15.2
by Aaron Bentley
FTP does not require atomic rename. Fixes #89436 |
44 |
osutils, |
1948.1.7
by John Arbash Meinel
Fix FtpTransport so that it raises UnicodeError and skips the unicode path tests |
45 |
urlutils, |
46 |
)
|
|
47 |
from bzrlib.trace import mutter, warning |
|
1707.3.4
by John Arbash Meinel
Moved most of sftp.split_url into a Transport function. |
48 |
from bzrlib.transport import ( |
2671.3.6
by Robert Collins
Review feedback. |
49 |
AppendBasedFileStream, |
2671.3.2
by Robert Collins
Start open_file_stream logic. |
50 |
_file_streams, |
1707.3.4
by John Arbash Meinel
Moved most of sftp.split_url into a Transport function. |
51 |
Server, |
2485.8.16
by Vincent Ladeuil
Create a new, empty, ConnectedTransport class. |
52 |
ConnectedTransport, |
1707.3.4
by John Arbash Meinel
Moved most of sftp.split_url into a Transport function. |
53 |
)
|
2413.1.1
by John Arbash Meinel
Implement TestCaseWithFTPServer using the new shiny Feature mechanism. |
54 |
from bzrlib.transport.local import LocalURLServer |
1707.3.24
by John Arbash Meinel
Add prompt for password for ftp. |
55 |
import bzrlib.ui |
1551.3.11
by Aaron Bentley
Merge from Robert |
56 |
|
1707.3.30
by John Arbash Meinel
Only load medusa if you are running the test suite. |
57 |
_have_medusa = False |
58 |
||
1551.3.11
by Aaron Bentley
Merge from Robert |
59 |
|
1707.3.25
by John Arbash Meinel
Setting some default exception classes, which can help a lot. |
60 |
class FtpPathError(errors.PathError): |
61 |
"""FTP failed for path: %(path)s%(extra)s"""
|
|
62 |
||
63 |
||
1185.36.4
by Daniel Silverstone
Add FTP transport |
64 |
class FtpStatResult(object): |
65 |
def __init__(self, f, relpath): |
|
66 |
try: |
|
67 |
self.st_size = f.size(relpath) |
|
68 |
self.st_mode = stat.S_IFREG |
|
69 |
except ftplib.error_perm: |
|
70 |
pwd = f.pwd() |
|
71 |
try: |
|
72 |
f.cwd(relpath) |
|
73 |
self.st_mode = stat.S_IFDIR |
|
74 |
finally: |
|
75 |
f.cwd(pwd) |
|
76 |
||
77 |
||
1185.72.15
by Matthieu Moy
better error messages on ftp failures |
78 |
_number_of_retries = 2 |
1185.72.14
by Matthieu Moy
Sleep between retries when the connection closes |
79 |
_sleep_between_retries = 5 |
1185.72.12
by Matthieu Moy
made __number_of_retries global |
80 |
|
2485.8.46
by Vincent Ladeuil
Add some remarks about current limitations in connection sharing. |
81 |
# FIXME: there are inconsistencies in the way temporary errors are
|
82 |
# handled. Sometimes we reconnect, sometimes we raise an exception. Care should
|
|
83 |
# be taken to analyze the implications for write operations (read operations
|
|
2485.8.59
by Vincent Ladeuil
Update from review comments. |
84 |
# are safe to retry). Overall even some read operations are never
|
85 |
# retried. --vila 20070720 (Bug #127164)
|
|
2485.8.16
by Vincent Ladeuil
Create a new, empty, ConnectedTransport class. |
86 |
class FtpTransport(ConnectedTransport): |
1185.36.4
by Daniel Silverstone
Add FTP transport |
87 |
"""This is the transport agent for ftp:// access."""
|
88 |
||
2485.8.59
by Vincent Ladeuil
Update from review comments. |
89 |
def __init__(self, base, _from_transport=None): |
1185.36.4
by Daniel Silverstone
Add FTP transport |
90 |
"""Set the base path where files will be stored."""
|
2485.8.23
by Vincent Ladeuil
Assert the accepted schemes for sftp and ftp. |
91 |
assert base.startswith('ftp://') or base.startswith('aftp://') |
2485.8.59
by Vincent Ladeuil
Update from review comments. |
92 |
super(FtpTransport, self).__init__(base, |
93 |
_from_transport=_from_transport) |
|
2485.8.25
by Vincent Ladeuil
Separate abspath from _remote_path, the intents are different. |
94 |
self._unqualified_scheme = 'ftp' |
2485.8.22
by Vincent Ladeuil
Finish ftp refactoring. Test suite passing. |
95 |
if self._scheme == 'aftp': |
96 |
self.is_active = True |
|
97 |
else: |
|
98 |
self.is_active = False |
|
1707.3.4
by John Arbash Meinel
Moved most of sftp.split_url into a Transport function. |
99 |
|
1185.36.4
by Daniel Silverstone
Add FTP transport |
100 |
def _get_FTP(self): |
101 |
"""Return the ftplib.FTP instance for this object."""
|
|
2485.8.33
by Vincent Ladeuil
ftp connection sharing refactored. Tests passing. |
102 |
# Ensures that a connection is established
|
103 |
connection = self._get_connection() |
|
104 |
if connection is None: |
|
105 |
# First connection ever
|
|
106 |
connection, credentials = self._create_connection() |
|
107 |
self._set_connection(connection, credentials) |
|
108 |
return connection |
|
109 |
||
110 |
def _create_connection(self, credentials=None): |
|
111 |
"""Create a new connection with the provided credentials.
|
|
112 |
||
113 |
:param credentials: The credentials needed to establish the connection.
|
|
114 |
||
115 |
:return: The created connection and its associated credentials.
|
|
116 |
||
117 |
The credentials are only the password as it may have been entered
|
|
118 |
interactively by the user and may be different from the one provided
|
|
119 |
in base url at transport creation time.
|
|
120 |
"""
|
|
121 |
if credentials is None: |
|
122 |
password = self._password |
|
123 |
else: |
|
124 |
password = credentials |
|
125 |
||
126 |
mutter("Constructing FTP instance against %r" % |
|
127 |
((self._host, self._port, self._user, '********', |
|
128 |
self.is_active),)) |
|
129 |
try: |
|
130 |
connection = ftplib.FTP() |
|
131 |
connection.connect(host=self._host, port=self._port) |
|
132 |
if self._user and self._user != 'anonymous' and \ |
|
2790.1.1
by Vincent Ladeuil
Fix #137044 by prompting for a password if *none* is provided for ftp. |
133 |
password is None: # '' is a valid password |
2485.8.33
by Vincent Ladeuil
ftp connection sharing refactored. Tests passing. |
134 |
get_password = bzrlib.ui.ui_factory.get_password |
135 |
password = get_password(prompt='FTP %(user)s@%(host)s password', |
|
136 |
user=self._user, host=self._host) |
|
137 |
connection.login(user=self._user, passwd=password) |
|
138 |
connection.set_pasv(not self.is_active) |
|
139 |
except ftplib.error_perm, e: |
|
140 |
raise errors.TransportError(msg="Error setting up connection:" |
|
141 |
" %s" % str(e), orig_error=e) |
|
142 |
return connection, password |
|
143 |
||
144 |
def _reconnect(self): |
|
145 |
"""Create a new connection with the previously used credentials"""
|
|
2917.1.2
by Vincent Ladeuil
Fix typo in ftp.py making the reconnection fail on temporary errors (154259). |
146 |
credentials = self._get_credentials() |
2485.8.33
by Vincent Ladeuil
ftp connection sharing refactored. Tests passing. |
147 |
connection, credentials = self._create_connection(credentials) |
148 |
self._set_connection(connection, credentials) |
|
149 |
||
150 |
def _translate_perm_error(self, err, path, extra=None, |
|
151 |
unknown_exc=FtpPathError): |
|
1707.3.19
by John Arbash Meinel
For some reason after RNTO we get a failure for has() |
152 |
"""Try to translate an ftplib.error_perm exception.
|
153 |
||
154 |
:param err: The error to translate into a bzr error
|
|
155 |
:param path: The path which had problems
|
|
156 |
:param extra: Extra information which can be included
|
|
157 |
:param unknown_exc: If None, we will just raise the original exception
|
|
158 |
otherwise we raise unknown_exc(path, extra=extra)
|
|
159 |
"""
|
|
1707.3.10
by John Arbash Meinel
Getting the append() tests to pass, and adding better error translation. These are probably dependent on the ftp test server, but for now it works. |
160 |
s = str(err).lower() |
161 |
if not extra: |
|
162 |
extra = str(err) |
|
1707.3.25
by John Arbash Meinel
Setting some default exception classes, which can help a lot. |
163 |
else: |
164 |
extra += ': ' + str(err) |
|
1707.3.10
by John Arbash Meinel
Getting the append() tests to pass, and adding better error translation. These are probably dependent on the ftp test server, but for now it works. |
165 |
if ('no such file' in s |
1707.3.13
by John Arbash Meinel
Changing raise Error to raise errors.Error |
166 |
or 'could not open' in s |
167 |
or 'no such dir' in s |
|
2338.1.1
by John Arbash Meinel
(Arnaud Fontaine) Compatibility with vsftpd error messages. |
168 |
or 'could not create file' in s # vsftpd |
2570.1.1
by John Arbash Meinel
(John Arbash Meinel) A couple small updates for pushing over ftp. |
169 |
or 'file doesn\'t exist' in s |
2885.3.1
by Gary van der Merwe
Correctly detect a NoSuchFile when using a filezilla server. |
170 |
or 'file/directory not found' in s # filezilla server |
1707.3.13
by John Arbash Meinel
Changing raise Error to raise errors.Error |
171 |
):
|
172 |
raise errors.NoSuchFile(path, extra=extra) |
|
1707.3.10
by John Arbash Meinel
Getting the append() tests to pass, and adding better error translation. These are probably dependent on the ftp test server, but for now it works. |
173 |
if ('file exists' in s): |
1707.3.15
by John Arbash Meinel
Handle trying to list a file instead of a dir |
174 |
raise errors.FileExists(path, extra=extra) |
175 |
if ('not a directory' in s): |
|
176 |
raise errors.PathError(path, extra=extra) |
|
177 |
||
1707.3.18
by John Arbash Meinel
Fix error handling for ftp.put() |
178 |
mutter('unable to understand error for path: %s: %s', path, err) |
1707.3.19
by John Arbash Meinel
For some reason after RNTO we get a failure for has() |
179 |
|
180 |
if unknown_exc: |
|
181 |
raise unknown_exc(path, extra=extra) |
|
1707.3.10
by John Arbash Meinel
Getting the append() tests to pass, and adding better error translation. These are probably dependent on the ftp test server, but for now it works. |
182 |
# TODO: jam 20060516 Consider re-raising the error wrapped in
|
183 |
# something like TransportError, but this loses the traceback
|
|
1707.3.15
by John Arbash Meinel
Handle trying to list a file instead of a dir |
184 |
# Also, 'sftp' has a generic 'Failure' mode, which we use failure_exc
|
185 |
# to handle. Consider doing something like that here.
|
|
1707.3.10
by John Arbash Meinel
Getting the append() tests to pass, and adding better error translation. These are probably dependent on the ftp test server, but for now it works. |
186 |
#raise TransportError(msg='Error for path: %s' % (path,), orig_error=e)
|
1707.3.15
by John Arbash Meinel
Handle trying to list a file instead of a dir |
187 |
raise
|
1707.3.10
by John Arbash Meinel
Getting the append() tests to pass, and adding better error translation. These are probably dependent on the ftp test server, but for now it works. |
188 |
|
2485.8.22
by Vincent Ladeuil
Finish ftp refactoring. Test suite passing. |
189 |
def _remote_path(self, relpath): |
1948.1.7
by John Arbash Meinel
Fix FtpTransport so that it raises UnicodeError and skips the unicode path tests |
190 |
# XXX: It seems that ftplib does not handle Unicode paths
|
2485.8.22
by Vincent Ladeuil
Finish ftp refactoring. Test suite passing. |
191 |
# at the same time, medusa won't handle utf8 paths So if
|
192 |
# we .encode(utf8) here (see ConnectedTransport
|
|
193 |
# implementation), then we get a Server failure. while
|
|
194 |
# if we use str(), we get a UnicodeError, and the test
|
|
195 |
# suite just skips testing UnicodePaths.
|
|
196 |
relative = str(urlutils.unescape(relpath)) |
|
197 |
remote_path = self._combine_paths(self._path, relative) |
|
198 |
return remote_path |
|
1185.36.4
by Daniel Silverstone
Add FTP transport |
199 |
|
200 |
def has(self, relpath): |
|
1707.3.16
by John Arbash Meinel
Overloading cmd_size and cmd_mkd so that the test suite can handle directories. |
201 |
"""Does the target location exist?"""
|
202 |
# FIXME jam 20060516 We *do* ask about directories in the test suite
|
|
203 |
# We don't seem to in the actual codebase
|
|
204 |
# XXX: I assume we're never asked has(dirname) and thus I use
|
|
205 |
# the FTP size command and assume that if it doesn't raise,
|
|
206 |
# all is good.
|
|
2485.8.22
by Vincent Ladeuil
Finish ftp refactoring. Test suite passing. |
207 |
abspath = self._remote_path(relpath) |
1185.36.4
by Daniel Silverstone
Add FTP transport |
208 |
try: |
209 |
f = self._get_FTP() |
|
1707.3.19
by John Arbash Meinel
For some reason after RNTO we get a failure for has() |
210 |
mutter('FTP has check: %s => %s', relpath, abspath) |
1707.3.16
by John Arbash Meinel
Overloading cmd_size and cmd_mkd so that the test suite can handle directories. |
211 |
s = f.size(abspath) |
212 |
mutter("FTP has: %s", abspath) |
|
1185.36.4
by Daniel Silverstone
Add FTP transport |
213 |
return True |
1707.3.16
by John Arbash Meinel
Overloading cmd_size and cmd_mkd so that the test suite can handle directories. |
214 |
except ftplib.error_perm, e: |
215 |
if ('is a directory' in str(e).lower()): |
|
216 |
mutter("FTP has dir: %s: %s", abspath, e) |
|
217 |
return True |
|
218 |
mutter("FTP has not: %s: %s", abspath, e) |
|
1185.36.4
by Daniel Silverstone
Add FTP transport |
219 |
return False |
220 |
||
2164.2.15
by Vincent Ladeuil
Http redirections are not followed by default. Do not use hints |
221 |
def get(self, relpath, decode=False, retries=0): |
1185.36.4
by Daniel Silverstone
Add FTP transport |
222 |
"""Get the file at the given relative path.
|
223 |
||
224 |
:param relpath: The relative path to the file
|
|
1185.72.9
by Matthieu Moy
suggestions by jam and robertc |
225 |
:param retries: Number of retries after temporary failures so far
|
226 |
for this operation.
|
|
1185.36.4
by Daniel Silverstone
Add FTP transport |
227 |
|
228 |
We're meant to return a file-like object which bzr will
|
|
229 |
then read from. For now we do this via the magic of StringIO
|
|
230 |
"""
|
|
1540.3.6
by Martin Pool
[merge] update from bzr.dev |
231 |
# TODO: decode should be deprecated
|
1185.36.4
by Daniel Silverstone
Add FTP transport |
232 |
try: |
2485.8.22
by Vincent Ladeuil
Finish ftp refactoring. Test suite passing. |
233 |
mutter("FTP get: %s", self._remote_path(relpath)) |
1185.36.4
by Daniel Silverstone
Add FTP transport |
234 |
f = self._get_FTP() |
235 |
ret = StringIO() |
|
2485.8.22
by Vincent Ladeuil
Finish ftp refactoring. Test suite passing. |
236 |
f.retrbinary('RETR '+self._remote_path(relpath), ret.write, 8192) |
1185.36.4
by Daniel Silverstone
Add FTP transport |
237 |
ret.seek(0) |
238 |
return ret |
|
239 |
except ftplib.error_perm, e: |
|
1707.3.13
by John Arbash Meinel
Changing raise Error to raise errors.Error |
240 |
raise errors.NoSuchFile(self.abspath(relpath), extra=str(e)) |
1185.72.8
by Matthieu Moy
try to solve the problem of ftp servers closing the connection |
241 |
except ftplib.error_temp, e: |
1185.72.15
by Matthieu Moy
better error messages on ftp failures |
242 |
if retries > _number_of_retries: |
1707.3.13
by John Arbash Meinel
Changing raise Error to raise errors.Error |
243 |
raise errors.TransportError(msg="FTP temporary error during GET %s. Aborting." |
1185.72.15
by Matthieu Moy
better error messages on ftp failures |
244 |
% self.abspath(relpath), |
245 |
orig_error=e) |
|
1185.72.8
by Matthieu Moy
try to solve the problem of ftp servers closing the connection |
246 |
else: |
1707.3.8
by John Arbash Meinel
[patch] Alexandre Saint: enable append support for ftp |
247 |
warning("FTP temporary error: %s. Retrying.", str(e)) |
2485.8.33
by Vincent Ladeuil
ftp connection sharing refactored. Tests passing. |
248 |
self._reconnect() |
1185.72.8
by Matthieu Moy
try to solve the problem of ftp servers closing the connection |
249 |
return self.get(relpath, decode, retries+1) |
1185.72.15
by Matthieu Moy
better error messages on ftp failures |
250 |
except EOFError, e: |
1185.72.9
by Matthieu Moy
suggestions by jam and robertc |
251 |
if retries > _number_of_retries: |
1707.3.13
by John Arbash Meinel
Changing raise Error to raise errors.Error |
252 |
raise errors.TransportError("FTP control connection closed during GET %s." |
1185.72.15
by Matthieu Moy
better error messages on ftp failures |
253 |
% self.abspath(relpath), |
254 |
orig_error=e) |
|
1185.72.8
by Matthieu Moy
try to solve the problem of ftp servers closing the connection |
255 |
else: |
1185.72.9
by Matthieu Moy
suggestions by jam and robertc |
256 |
warning("FTP control connection closed. Trying to reopen.") |
1185.72.14
by Matthieu Moy
Sleep between retries when the connection closes |
257 |
time.sleep(_sleep_between_retries) |
2485.8.33
by Vincent Ladeuil
ftp connection sharing refactored. Tests passing. |
258 |
self._reconnect() |
1185.72.8
by Matthieu Moy
try to solve the problem of ftp servers closing the connection |
259 |
return self.get(relpath, decode, retries+1) |
1185.36.4
by Daniel Silverstone
Add FTP transport |
260 |
|
1955.3.6
by John Arbash Meinel
Lots of deprecation warnings, but no errors |
261 |
def put_file(self, relpath, fp, mode=None, retries=0): |
1185.36.4
by Daniel Silverstone
Add FTP transport |
262 |
"""Copy the file-like or string object into the location.
|
263 |
||
264 |
:param relpath: Location to put the contents, relative to base.
|
|
1185.72.13
by Matthieu Moy
Make ftp put atomic |
265 |
:param fp: File-like or string object.
|
1185.72.9
by Matthieu Moy
suggestions by jam and robertc |
266 |
:param retries: Number of retries after temporary failures so far
|
267 |
for this operation.
|
|
268 |
||
1551.15.2
by Aaron Bentley
FTP does not require atomic rename. Fixes #89436 |
269 |
TODO: jam 20051215 ftp as a protocol seems to support chmod, but
|
270 |
ftplib does not
|
|
1185.36.4
by Daniel Silverstone
Add FTP transport |
271 |
"""
|
2485.8.22
by Vincent Ladeuil
Finish ftp refactoring. Test suite passing. |
272 |
abspath = self._remote_path(relpath) |
1707.3.18
by John Arbash Meinel
Fix error handling for ftp.put() |
273 |
tmp_abspath = '%s.tmp.%.9f.%d.%d' % (abspath, time.time(), |
1185.72.13
by Matthieu Moy
Make ftp put atomic |
274 |
os.getpid(), random.randint(0,0x7FFFFFFF)) |
2745.5.2
by Robert Collins
* ``bzrlib.transport.Transport.put_file`` now returns the number of bytes |
275 |
bytes = None |
1963.2.6
by Robey Pointer
pychecker is on crack; go back to using 'is None'. |
276 |
if getattr(fp, 'read', None) is None: |
2745.5.2
by Robert Collins
* ``bzrlib.transport.Transport.put_file`` now returns the number of bytes |
277 |
# hand in a string IO
|
278 |
bytes = fp |
|
279 |
fp = StringIO(bytes) |
|
280 |
else: |
|
281 |
# capture the byte count; .read() may be read only so
|
|
282 |
# decorate it.
|
|
283 |
class byte_counter(object): |
|
284 |
def __init__(self, fp): |
|
285 |
self.fp = fp |
|
286 |
self.counted_bytes = 0 |
|
287 |
def read(self, count): |
|
288 |
result = self.fp.read(count) |
|
289 |
self.counted_bytes += len(result) |
|
290 |
return result |
|
291 |
fp = byte_counter(fp) |
|
1185.36.4
by Daniel Silverstone
Add FTP transport |
292 |
try: |
1707.3.18
by John Arbash Meinel
Fix error handling for ftp.put() |
293 |
mutter("FTP put: %s", abspath) |
1185.36.4
by Daniel Silverstone
Add FTP transport |
294 |
f = self._get_FTP() |
1185.72.13
by Matthieu Moy
Make ftp put atomic |
295 |
try: |
296 |
f.storbinary('STOR '+tmp_abspath, fp) |
|
1551.15.2
by Aaron Bentley
FTP does not require atomic rename. Fixes #89436 |
297 |
self._rename_and_overwrite(tmp_abspath, abspath, f) |
2745.5.2
by Robert Collins
* ``bzrlib.transport.Transport.put_file`` now returns the number of bytes |
298 |
if bytes is not None: |
299 |
return len(bytes) |
|
300 |
else: |
|
301 |
return fp.counted_bytes |
|
1185.72.13
by Matthieu Moy
Make ftp put atomic |
302 |
except (ftplib.error_temp,EOFError), e: |
1185.72.15
by Matthieu Moy
better error messages on ftp failures |
303 |
warning("Failure during ftp PUT. Deleting temporary file.") |
1185.72.13
by Matthieu Moy
Make ftp put atomic |
304 |
try: |
305 |
f.delete(tmp_abspath) |
|
306 |
except: |
|
1707.3.8
by John Arbash Meinel
[patch] Alexandre Saint: enable append support for ftp |
307 |
warning("Failed to delete temporary file on the" |
308 |
" server.\nFile: %s", tmp_abspath) |
|
1185.72.13
by Matthieu Moy
Make ftp put atomic |
309 |
raise e |
310 |
raise
|
|
1185.36.4
by Daniel Silverstone
Add FTP transport |
311 |
except ftplib.error_perm, e: |
2570.1.1
by John Arbash Meinel
(John Arbash Meinel) A couple small updates for pushing over ftp. |
312 |
self._translate_perm_error(e, abspath, extra='could not store', |
313 |
unknown_exc=errors.NoSuchFile) |
|
1185.72.8
by Matthieu Moy
try to solve the problem of ftp servers closing the connection |
314 |
except ftplib.error_temp, e: |
1185.72.9
by Matthieu Moy
suggestions by jam and robertc |
315 |
if retries > _number_of_retries: |
1707.3.13
by John Arbash Meinel
Changing raise Error to raise errors.Error |
316 |
raise errors.TransportError("FTP temporary error during PUT %s. Aborting." |
1185.72.15
by Matthieu Moy
better error messages on ftp failures |
317 |
% self.abspath(relpath), orig_error=e) |
1185.72.8
by Matthieu Moy
try to solve the problem of ftp servers closing the connection |
318 |
else: |
1707.3.8
by John Arbash Meinel
[patch] Alexandre Saint: enable append support for ftp |
319 |
warning("FTP temporary error: %s. Retrying.", str(e)) |
2485.8.33
by Vincent Ladeuil
ftp connection sharing refactored. Tests passing. |
320 |
self._reconnect() |
1955.3.6
by John Arbash Meinel
Lots of deprecation warnings, but no errors |
321 |
self.put_file(relpath, fp, mode, retries+1) |
1185.72.9
by Matthieu Moy
suggestions by jam and robertc |
322 |
except EOFError: |
1185.72.10
by Matthieu Moy
One 1 -> _number_of_retries was missing |
323 |
if retries > _number_of_retries: |
1707.3.13
by John Arbash Meinel
Changing raise Error to raise errors.Error |
324 |
raise errors.TransportError("FTP control connection closed during PUT %s." |
1185.72.15
by Matthieu Moy
better error messages on ftp failures |
325 |
% self.abspath(relpath), orig_error=e) |
1185.72.8
by Matthieu Moy
try to solve the problem of ftp servers closing the connection |
326 |
else: |
1185.72.14
by Matthieu Moy
Sleep between retries when the connection closes |
327 |
warning("FTP control connection closed. Trying to reopen.") |
328 |
time.sleep(_sleep_between_retries) |
|
2485.8.33
by Vincent Ladeuil
ftp connection sharing refactored. Tests passing. |
329 |
self._reconnect() |
1955.3.6
by John Arbash Meinel
Lots of deprecation warnings, but no errors |
330 |
self.put_file(relpath, fp, mode, retries+1) |
1185.72.8
by Matthieu Moy
try to solve the problem of ftp servers closing the connection |
331 |
|
1185.58.2
by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work. |
332 |
def mkdir(self, relpath, mode=None): |
1185.36.4
by Daniel Silverstone
Add FTP transport |
333 |
"""Create a directory at the given path."""
|
2485.8.22
by Vincent Ladeuil
Finish ftp refactoring. Test suite passing. |
334 |
abspath = self._remote_path(relpath) |
1185.36.4
by Daniel Silverstone
Add FTP transport |
335 |
try: |
1707.3.14
by John Arbash Meinel
Cleanup mkdir and rmdir error handling. |
336 |
mutter("FTP mkd: %s", abspath) |
1185.36.4
by Daniel Silverstone
Add FTP transport |
337 |
f = self._get_FTP() |
1707.3.14
by John Arbash Meinel
Cleanup mkdir and rmdir error handling. |
338 |
f.mkd(abspath) |
1185.36.4
by Daniel Silverstone
Add FTP transport |
339 |
except ftplib.error_perm, e: |
1707.3.25
by John Arbash Meinel
Setting some default exception classes, which can help a lot. |
340 |
self._translate_perm_error(e, abspath, |
341 |
unknown_exc=errors.FileExists) |
|
1185.36.4
by Daniel Silverstone
Add FTP transport |
342 |
|
2671.3.9
by Robert Collins
Review feedback and fix VFat emulated transports to not claim to have unix permissions. |
343 |
def open_write_stream(self, relpath, mode=None): |
344 |
"""See Transport.open_write_stream."""
|
|
2671.3.6
by Robert Collins
Review feedback. |
345 |
self.put_bytes(relpath, "", mode) |
346 |
result = AppendBasedFileStream(self, relpath) |
|
347 |
_file_streams[self.abspath(relpath)] = result |
|
348 |
return result |
|
2671.3.2
by Robert Collins
Start open_file_stream logic. |
349 |
|
2671.3.1
by Robert Collins
* New method ``bzrlib.transport.Transport.get_recommended_page_size``. |
350 |
def recommended_page_size(self): |
351 |
"""See Transport.recommended_page_size().
|
|
352 |
||
353 |
For FTP we suggest a large page size to reduce the overhead
|
|
354 |
introduced by latency.
|
|
355 |
"""
|
|
356 |
return 64 * 1024 |
|
357 |
||
1658.1.1
by Martin Pool
Fix FTP push with metadir format (Alexandre Saint) |
358 |
def rmdir(self, rel_path): |
359 |
"""Delete the directory at rel_path"""
|
|
2485.8.22
by Vincent Ladeuil
Finish ftp refactoring. Test suite passing. |
360 |
abspath = self._remote_path(rel_path) |
1658.1.1
by Martin Pool
Fix FTP push with metadir format (Alexandre Saint) |
361 |
try: |
1707.3.14
by John Arbash Meinel
Cleanup mkdir and rmdir error handling. |
362 |
mutter("FTP rmd: %s", abspath) |
1551.15.4
by Aaron Bentley
Revert now-unnecessary changes |
363 |
f = self._get_FTP() |
1707.3.14
by John Arbash Meinel
Cleanup mkdir and rmdir error handling. |
364 |
f.rmd(abspath) |
1658.1.1
by Martin Pool
Fix FTP push with metadir format (Alexandre Saint) |
365 |
except ftplib.error_perm, e: |
1551.15.4
by Aaron Bentley
Revert now-unnecessary changes |
366 |
self._translate_perm_error(e, abspath, unknown_exc=errors.PathError) |
1658.1.1
by Martin Pool
Fix FTP push with metadir format (Alexandre Saint) |
367 |
|
1955.3.15
by John Arbash Meinel
Deprecate 'Transport.append' in favor of Transport.append_file or Transport.append_bytes |
368 |
def append_file(self, relpath, f, mode=None): |
1185.36.4
by Daniel Silverstone
Add FTP transport |
369 |
"""Append the text in the file-like object into the final
|
370 |
location.
|
|
371 |
"""
|
|
2485.8.22
by Vincent Ladeuil
Finish ftp refactoring. Test suite passing. |
372 |
abspath = self._remote_path(relpath) |
1707.3.8
by John Arbash Meinel
[patch] Alexandre Saint: enable append support for ftp |
373 |
if self.has(relpath): |
374 |
ftp = self._get_FTP() |
|
1707.3.19
by John Arbash Meinel
For some reason after RNTO we get a failure for has() |
375 |
result = ftp.size(abspath) |
1707.3.8
by John Arbash Meinel
[patch] Alexandre Saint: enable append support for ftp |
376 |
else: |
377 |
result = 0 |
|
378 |
||
1707.3.19
by John Arbash Meinel
For some reason after RNTO we get a failure for has() |
379 |
mutter("FTP appe to %s", abspath) |
1707.3.8
by John Arbash Meinel
[patch] Alexandre Saint: enable append support for ftp |
380 |
self._try_append(relpath, f.read(), mode) |
381 |
||
382 |
return result |
|
383 |
||
384 |
def _try_append(self, relpath, text, mode=None, retries=0): |
|
385 |
"""Try repeatedly to append the given text to the file at relpath.
|
|
386 |
|
|
387 |
This is a recursive function. On errors, it will be called until the
|
|
388 |
number of retries is exceeded.
|
|
389 |
"""
|
|
390 |
try: |
|
2485.8.22
by Vincent Ladeuil
Finish ftp refactoring. Test suite passing. |
391 |
abspath = self._remote_path(relpath) |
1707.3.8
by John Arbash Meinel
[patch] Alexandre Saint: enable append support for ftp |
392 |
mutter("FTP appe (try %d) to %s", retries, abspath) |
393 |
ftp = self._get_FTP() |
|
394 |
ftp.voidcmd("TYPE I") |
|
395 |
cmd = "APPE %s" % abspath |
|
396 |
conn = ftp.transfercmd(cmd) |
|
397 |
conn.sendall(text) |
|
398 |
conn.close() |
|
1707.3.25
by John Arbash Meinel
Setting some default exception classes, which can help a lot. |
399 |
if mode: |
1707.3.8
by John Arbash Meinel
[patch] Alexandre Saint: enable append support for ftp |
400 |
self._setmode(relpath, mode) |
401 |
ftp.getresp() |
|
402 |
except ftplib.error_perm, e: |
|
1707.3.26
by John Arbash Meinel
Added a default exception for append |
403 |
self._translate_perm_error(e, abspath, extra='error appending', |
404 |
unknown_exc=errors.NoSuchFile) |
|
1707.3.8
by John Arbash Meinel
[patch] Alexandre Saint: enable append support for ftp |
405 |
except ftplib.error_temp, e: |
406 |
if retries > _number_of_retries: |
|
1707.3.13
by John Arbash Meinel
Changing raise Error to raise errors.Error |
407 |
raise errors.TransportError("FTP temporary error during APPEND %s." \ |
1707.3.8
by John Arbash Meinel
[patch] Alexandre Saint: enable append support for ftp |
408 |
"Aborting." % abspath, orig_error=e) |
409 |
else: |
|
410 |
warning("FTP temporary error: %s. Retrying.", str(e)) |
|
2485.8.33
by Vincent Ladeuil
ftp connection sharing refactored. Tests passing. |
411 |
self._reconnect() |
1707.3.8
by John Arbash Meinel
[patch] Alexandre Saint: enable append support for ftp |
412 |
self._try_append(relpath, text, mode, retries+1) |
413 |
||
414 |
def _setmode(self, relpath, mode): |
|
415 |
"""Set permissions on a path.
|
|
416 |
||
417 |
Only set permissions if the FTP server supports the 'SITE CHMOD'
|
|
418 |
extension.
|
|
419 |
"""
|
|
420 |
try: |
|
421 |
mutter("FTP site chmod: setting permissions to %s on %s", |
|
2485.8.22
by Vincent Ladeuil
Finish ftp refactoring. Test suite passing. |
422 |
str(mode), self._remote_path(relpath)) |
1707.3.8
by John Arbash Meinel
[patch] Alexandre Saint: enable append support for ftp |
423 |
ftp = self._get_FTP() |
2485.8.22
by Vincent Ladeuil
Finish ftp refactoring. Test suite passing. |
424 |
cmd = "SITE CHMOD %s %s" % (self._remote_path(relpath), str(mode)) |
1707.3.8
by John Arbash Meinel
[patch] Alexandre Saint: enable append support for ftp |
425 |
ftp.sendcmd(cmd) |
426 |
except ftplib.error_perm, e: |
|
427 |
# Command probably not available on this server
|
|
428 |
warning("FTP Could not set permissions to %s on %s. %s", |
|
2485.8.22
by Vincent Ladeuil
Finish ftp refactoring. Test suite passing. |
429 |
str(mode), self._remote_path(relpath), str(e)) |
1185.36.4
by Daniel Silverstone
Add FTP transport |
430 |
|
1707.3.11
by John Arbash Meinel
fixing more tests. |
431 |
# TODO: jam 20060516 I believe ftp allows you to tell an ftp server
|
432 |
# to copy something to another machine. And you may be able
|
|
433 |
# to give it its own address as the 'to' location.
|
|
434 |
# So implement a fancier 'copy()'
|
|
1185.36.4
by Daniel Silverstone
Add FTP transport |
435 |
|
1551.15.3
by Aaron Bentley
Separate FtpTransport.rename from FtpTransport.move |
436 |
def rename(self, rel_from, rel_to): |
2485.8.22
by Vincent Ladeuil
Finish ftp refactoring. Test suite passing. |
437 |
abs_from = self._remote_path(rel_from) |
438 |
abs_to = self._remote_path(rel_to) |
|
1551.15.3
by Aaron Bentley
Separate FtpTransport.rename from FtpTransport.move |
439 |
mutter("FTP rename: %s => %s", abs_from, abs_to) |
440 |
f = self._get_FTP() |
|
441 |
return self._rename(abs_from, abs_to, f) |
|
442 |
||
443 |
def _rename(self, abs_from, abs_to, f): |
|
444 |
try: |
|
445 |
f.rename(abs_from, abs_to) |
|
446 |
except ftplib.error_perm, e: |
|
447 |
self._translate_perm_error(e, abs_from, |
|
448 |
': unable to rename to %r' % (abs_to)) |
|
449 |
||
1185.36.4
by Daniel Silverstone
Add FTP transport |
450 |
def move(self, rel_from, rel_to): |
451 |
"""Move the item at rel_from to the location at rel_to"""
|
|
2485.8.22
by Vincent Ladeuil
Finish ftp refactoring. Test suite passing. |
452 |
abs_from = self._remote_path(rel_from) |
453 |
abs_to = self._remote_path(rel_to) |
|
1185.36.4
by Daniel Silverstone
Add FTP transport |
454 |
try: |
1707.3.19
by John Arbash Meinel
For some reason after RNTO we get a failure for has() |
455 |
mutter("FTP mv: %s => %s", abs_from, abs_to) |
1185.36.4
by Daniel Silverstone
Add FTP transport |
456 |
f = self._get_FTP() |
1551.15.2
by Aaron Bentley
FTP does not require atomic rename. Fixes #89436 |
457 |
self._rename_and_overwrite(abs_from, abs_to, f) |
1185.36.4
by Daniel Silverstone
Add FTP transport |
458 |
except ftplib.error_perm, e: |
1707.3.19
by John Arbash Meinel
For some reason after RNTO we get a failure for has() |
459 |
self._translate_perm_error(e, abs_from, |
460 |
extra='unable to rename to %r' % (rel_to,), |
|
461 |
unknown_exc=errors.PathError) |
|
1185.36.4
by Daniel Silverstone
Add FTP transport |
462 |
|
1551.15.2
by Aaron Bentley
FTP does not require atomic rename. Fixes #89436 |
463 |
def _rename_and_overwrite(self, abs_from, abs_to, f): |
464 |
"""Do a fancy rename on the remote server.
|
|
465 |
||
466 |
Using the implementation provided by osutils.
|
|
467 |
"""
|
|
1551.15.3
by Aaron Bentley
Separate FtpTransport.rename from FtpTransport.move |
468 |
osutils.fancy_rename(abs_from, abs_to, |
1551.15.4
by Aaron Bentley
Revert now-unnecessary changes |
469 |
rename_func=lambda p1, p2: self._rename(p1, p2, f), |
470 |
unlink_func=lambda p: self._delete(p, f)) |
|
1551.15.2
by Aaron Bentley
FTP does not require atomic rename. Fixes #89436 |
471 |
|
1185.36.4
by Daniel Silverstone
Add FTP transport |
472 |
def delete(self, relpath): |
473 |
"""Delete the item at relpath"""
|
|
2485.8.22
by Vincent Ladeuil
Finish ftp refactoring. Test suite passing. |
474 |
abspath = self._remote_path(relpath) |
1551.15.2
by Aaron Bentley
FTP does not require atomic rename. Fixes #89436 |
475 |
f = self._get_FTP() |
476 |
self._delete(abspath, f) |
|
477 |
||
478 |
def _delete(self, abspath, f): |
|
1185.36.4
by Daniel Silverstone
Add FTP transport |
479 |
try: |
1707.3.11
by John Arbash Meinel
fixing more tests. |
480 |
mutter("FTP rm: %s", abspath) |
481 |
f.delete(abspath) |
|
1185.36.4
by Daniel Silverstone
Add FTP transport |
482 |
except ftplib.error_perm, e: |
1707.3.25
by John Arbash Meinel
Setting some default exception classes, which can help a lot. |
483 |
self._translate_perm_error(e, abspath, 'error deleting', |
484 |
unknown_exc=errors.NoSuchFile) |
|
1185.36.4
by Daniel Silverstone
Add FTP transport |
485 |
|
2634.1.1
by Robert Collins
(robertc) Reinstate the accidentally backed out external_url patch. |
486 |
def external_url(self): |
487 |
"""See bzrlib.transport.Transport.external_url."""
|
|
488 |
# FTP URL's are externally usable.
|
|
489 |
return self.base |
|
490 |
||
1185.36.4
by Daniel Silverstone
Add FTP transport |
491 |
def listable(self): |
492 |
"""See Transport.listable."""
|
|
493 |
return True |
|
494 |
||
495 |
def list_dir(self, relpath): |
|
496 |
"""See Transport.list_dir."""
|
|
2485.8.22
by Vincent Ladeuil
Finish ftp refactoring. Test suite passing. |
497 |
basepath = self._remote_path(relpath) |
1959.2.5
by John Arbash Meinel
Small fixes in ftp.list_dir() |
498 |
mutter("FTP nlst: %s", basepath) |
1959.2.1
by John Arbash Meinel
David Allouche: Make transports return escaped paths |
499 |
f = self._get_FTP() |
1185.36.4
by Daniel Silverstone
Add FTP transport |
500 |
try: |
1707.3.8
by John Arbash Meinel
[patch] Alexandre Saint: enable append support for ftp |
501 |
paths = f.nlst(basepath) |
1185.36.4
by Daniel Silverstone
Add FTP transport |
502 |
except ftplib.error_perm, e: |
1707.3.13
by John Arbash Meinel
Changing raise Error to raise errors.Error |
503 |
self._translate_perm_error(e, relpath, extra='error with list_dir') |
1959.2.1
by John Arbash Meinel
David Allouche: Make transports return escaped paths |
504 |
# If FTP.nlst returns paths prefixed by relpath, strip 'em
|
505 |
if paths and paths[0].startswith(basepath): |
|
506 |
entries = [path[len(basepath)+1:] for path in paths] |
|
507 |
else: |
|
508 |
entries = paths |
|
509 |
# Remove . and .. if present
|
|
1959.2.5
by John Arbash Meinel
Small fixes in ftp.list_dir() |
510 |
return [urlutils.escape(entry) for entry in entries |
511 |
if entry not in ('.', '..')] |
|
1185.36.4
by Daniel Silverstone
Add FTP transport |
512 |
|
513 |
def iter_files_recursive(self): |
|
514 |
"""See Transport.iter_files_recursive.
|
|
515 |
||
516 |
This is cargo-culted from the SFTP transport"""
|
|
517 |
mutter("FTP iter_files_recursive") |
|
518 |
queue = list(self.list_dir(".")) |
|
519 |
while queue: |
|
1959.2.1
by John Arbash Meinel
David Allouche: Make transports return escaped paths |
520 |
relpath = queue.pop(0) |
1185.36.4
by Daniel Silverstone
Add FTP transport |
521 |
st = self.stat(relpath) |
522 |
if stat.S_ISDIR(st.st_mode): |
|
523 |
for i, basename in enumerate(self.list_dir(relpath)): |
|
524 |
queue.insert(i, relpath+"/"+basename) |
|
525 |
else: |
|
526 |
yield relpath |
|
527 |
||
528 |
def stat(self, relpath): |
|
1707.3.13
by John Arbash Meinel
Changing raise Error to raise errors.Error |
529 |
"""Return the stat information for a file."""
|
2485.8.22
by Vincent Ladeuil
Finish ftp refactoring. Test suite passing. |
530 |
abspath = self._remote_path(relpath) |
1185.36.4
by Daniel Silverstone
Add FTP transport |
531 |
try: |
1707.3.13
by John Arbash Meinel
Changing raise Error to raise errors.Error |
532 |
mutter("FTP stat: %s", abspath) |
1185.36.4
by Daniel Silverstone
Add FTP transport |
533 |
f = self._get_FTP() |
1707.3.13
by John Arbash Meinel
Changing raise Error to raise errors.Error |
534 |
return FtpStatResult(f, abspath) |
1185.36.4
by Daniel Silverstone
Add FTP transport |
535 |
except ftplib.error_perm, e: |
1707.3.13
by John Arbash Meinel
Changing raise Error to raise errors.Error |
536 |
self._translate_perm_error(e, abspath, extra='error w/ stat') |
1185.36.4
by Daniel Silverstone
Add FTP transport |
537 |
|
538 |
def lock_read(self, relpath): |
|
539 |
"""Lock the given file for shared (read) access.
|
|
540 |
:return: A lock object, which should be passed to Transport.unlock()
|
|
541 |
"""
|
|
542 |
# The old RemoteBranch ignore lock for reading, so we will
|
|
543 |
# continue that tradition and return a bogus lock object.
|
|
544 |
class BogusLock(object): |
|
545 |
def __init__(self, path): |
|
546 |
self.path = path |
|
547 |
def unlock(self): |
|
548 |
pass
|
|
549 |
return BogusLock(relpath) |
|
550 |
||
551 |
def lock_write(self, relpath): |
|
552 |
"""Lock the given file for exclusive (write) access.
|
|
553 |
WARNING: many transports do not support this, so trying avoid using it
|
|
554 |
||
555 |
:return: A lock object, which should be passed to Transport.unlock()
|
|
556 |
"""
|
|
557 |
return self.lock_read(relpath) |
|
1530.1.11
by Robert Collins
Push the transport permutations list into each transport module allowing for automatic testing of new modules that are registered as transports. |
558 |
|
559 |
||
1707.3.30
by John Arbash Meinel
Only load medusa if you are running the test suite. |
560 |
class FtpServer(Server): |
2052.6.1
by Robert Collins
``Transport.get`` has had its interface made more clear for ease of use. |
561 |
"""Common code for FTP server facilities."""
|
1707.3.30
by John Arbash Meinel
Only load medusa if you are running the test suite. |
562 |
|
563 |
def __init__(self): |
|
564 |
self._root = None |
|
565 |
self._ftp_server = None |
|
566 |
self._port = None |
|
567 |
self._async_thread = None |
|
568 |
# ftp server logs
|
|
569 |
self.logs = [] |
|
570 |
||
571 |
def get_url(self): |
|
572 |
"""Calculate an ftp url to this server."""
|
|
573 |
return 'ftp://foo:bar@localhost:%d/' % (self._port) |
|
574 |
||
575 |
# def get_bogus_url(self):
|
|
576 |
# """Return a URL which cannot be connected to."""
|
|
577 |
# return 'ftp://127.0.0.1:1'
|
|
578 |
||
579 |
def log(self, message): |
|
580 |
"""This is used by medusa.ftp_server to log connections, etc."""
|
|
581 |
self.logs.append(message) |
|
582 |
||
2413.1.1
by John Arbash Meinel
Implement TestCaseWithFTPServer using the new shiny Feature mechanism. |
583 |
def setUp(self, vfs_server=None): |
1707.3.1
by John Arbash Meinel
Have a FtpServer, but the tests fail |
584 |
if not _have_medusa: |
1707.3.30
by John Arbash Meinel
Only load medusa if you are running the test suite. |
585 |
raise RuntimeError('Must have medusa to run the FtpServer') |
586 |
||
2413.1.1
by John Arbash Meinel
Implement TestCaseWithFTPServer using the new shiny Feature mechanism. |
587 |
assert vfs_server is None or isinstance(vfs_server, LocalURLServer), \ |
588 |
"FtpServer currently assumes local transport, got %s" % vfs_server |
|
589 |
||
1707.3.30
by John Arbash Meinel
Only load medusa if you are running the test suite. |
590 |
self._root = os.getcwdu() |
591 |
self._ftp_server = _ftp_server( |
|
592 |
authorizer=_test_authorizer(root=self._root), |
|
593 |
ip='localhost', |
|
594 |
port=0, # bind to a random port |
|
595 |
resolver=None, |
|
596 |
logger_object=self # Use FtpServer.log() for messages |
|
597 |
)
|
|
598 |
self._port = self._ftp_server.getsockname()[1] |
|
599 |
# Don't let it loop forever, or handle an infinite number of requests.
|
|
2485.8.12
by Vincent Ladeuil
Cosmetic change. |
600 |
# In this case it will run for 1000s, or 10000 requests
|
2413.3.1
by John Arbash Meinel
Trap EBADF during shutdown of asyncore.loop() |
601 |
self._async_thread = threading.Thread( |
602 |
target=FtpServer._asyncore_loop_ignore_EBADF, |
|
2485.8.6
by Vincent Ladeuil
Fix the 'FtpServer' so that it can handle full bzr commands. |
603 |
kwargs={'timeout':0.1, 'count':10000}) |
1707.3.30
by John Arbash Meinel
Only load medusa if you are running the test suite. |
604 |
self._async_thread.setDaemon(True) |
605 |
self._async_thread.start() |
|
606 |
||
607 |
def tearDown(self): |
|
608 |
"""See bzrlib.transport.Server.tearDown."""
|
|
2825.1.1
by Vincent Ladeuil
Fix #140055 by properly closing the http and ftp test servers. |
609 |
self._ftp_server.close() |
1707.3.30
by John Arbash Meinel
Only load medusa if you are running the test suite. |
610 |
asyncore.close_all() |
611 |
self._async_thread.join() |
|
612 |
||
2413.3.1
by John Arbash Meinel
Trap EBADF during shutdown of asyncore.loop() |
613 |
@staticmethod
|
614 |
def _asyncore_loop_ignore_EBADF(*args, **kwargs): |
|
2413.3.2
by John Arbash Meinel
cleanup comment (from Martin) |
615 |
"""Ignore EBADF during server shutdown.
|
616 |
||
617 |
We close the socket to get the server to shutdown, but this causes
|
|
618 |
select.select() to raise EBADF.
|
|
619 |
"""
|
|
2413.3.1
by John Arbash Meinel
Trap EBADF during shutdown of asyncore.loop() |
620 |
try: |
621 |
asyncore.loop(*args, **kwargs) |
|
2485.8.37
by Vincent Ladeuil
Fix merge multiple connections. Test suite *not* passing (sftp |
622 |
# FIXME: If we reach that point, we should raise an exception
|
623 |
# explaining that the 'count' parameter in setUp is too low or
|
|
624 |
# testers may wonder why their test just sits there waiting for a
|
|
625 |
# server that is already dead. Note that if the tester waits too
|
|
626 |
# long under pdb the server will also die.
|
|
2413.3.1
by John Arbash Meinel
Trap EBADF during shutdown of asyncore.loop() |
627 |
except select.error, e: |
628 |
if e.args[0] != errno.EBADF: |
|
629 |
raise
|
|
630 |
||
1707.3.30
by John Arbash Meinel
Only load medusa if you are running the test suite. |
631 |
|
632 |
_ftp_channel = None |
|
633 |
_ftp_server = None |
|
634 |
_test_authorizer = None |
|
635 |
||
636 |
||
637 |
def _setup_medusa(): |
|
638 |
global _have_medusa, _ftp_channel, _ftp_server, _test_authorizer |
|
639 |
try: |
|
640 |
import medusa |
|
641 |
import medusa.filesys |
|
642 |
import medusa.ftp_server |
|
643 |
except ImportError: |
|
644 |
return False |
|
645 |
||
646 |
_have_medusa = True |
|
647 |
||
648 |
class test_authorizer(object): |
|
649 |
"""A custom Authorizer object for running the test suite.
|
|
650 |
||
651 |
The reason we cannot use dummy_authorizer, is because it sets the
|
|
652 |
channel to readonly, which we don't always want to do.
|
|
653 |
"""
|
|
654 |
||
655 |
def __init__(self, root): |
|
656 |
self.root = root |
|
2790.1.1
by Vincent Ladeuil
Fix #137044 by prompting for a password if *none* is provided for ftp. |
657 |
# If secured_user is set secured_password will be checked
|
658 |
self.secured_user = None |
|
659 |
self.secured_password = None |
|
1707.3.30
by John Arbash Meinel
Only load medusa if you are running the test suite. |
660 |
|
661 |
def authorize(self, channel, username, password): |
|
662 |
"""Return (success, reply_string, filesystem)"""
|
|
663 |
if not _have_medusa: |
|
664 |
return 0, 'No Medusa.', None |
|
665 |
||
666 |
channel.persona = -1, -1 |
|
667 |
if username == 'anonymous': |
|
668 |
channel.read_only = 1 |
|
669 |
else: |
|
670 |
channel.read_only = 0 |
|
671 |
||
2790.1.3
by Vincent Ladeuil
Cosmetic change. |
672 |
# Check secured_user if set
|
2790.1.2
by Vincent Ladeuil
Review feedback. |
673 |
if (self.secured_user is not None |
674 |
and username == self.secured_user |
|
675 |
and password != self.secured_password): |
|
2790.1.1
by Vincent Ladeuil
Fix #137044 by prompting for a password if *none* is provided for ftp. |
676 |
return 0, 'Password invalid.', None |
677 |
else: |
|
678 |
return 1, 'OK.', medusa.filesys.os_filesystem(self.root) |
|
1707.3.30
by John Arbash Meinel
Only load medusa if you are running the test suite. |
679 |
|
680 |
||
681 |
class ftp_channel(medusa.ftp_server.ftp_channel): |
|
1707.3.7
by John Arbash Meinel
Creating child of ftp_channel to support renaming, need to add appending. |
682 |
"""Customized ftp channel"""
|
683 |
||
1707.3.19
by John Arbash Meinel
For some reason after RNTO we get a failure for has() |
684 |
def log(self, message): |
685 |
"""Redirect logging requests."""
|
|
686 |
mutter('_ftp_channel: %s', message) |
|
2485.8.6
by Vincent Ladeuil
Fix the 'FtpServer' so that it can handle full bzr commands. |
687 |
|
1707.3.7
by John Arbash Meinel
Creating child of ftp_channel to support renaming, need to add appending. |
688 |
def log_info(self, message, type='info'): |
689 |
"""Redirect logging requests."""
|
|
690 |
mutter('_ftp_channel %s: %s', type, message) |
|
2485.8.6
by Vincent Ladeuil
Fix the 'FtpServer' so that it can handle full bzr commands. |
691 |
|
1707.3.7
by John Arbash Meinel
Creating child of ftp_channel to support renaming, need to add appending. |
692 |
def cmd_rnfr(self, line): |
693 |
"""Prepare for renaming a file."""
|
|
694 |
self._renaming = line[1] |
|
695 |
self.respond('350 Ready for RNTO') |
|
696 |
# TODO: jam 20060516 in testing, the ftp server seems to
|
|
697 |
# check that the file already exists, or it sends
|
|
698 |
# 550 RNFR command failed
|
|
699 |
||
700 |
def cmd_rnto(self, line): |
|
701 |
"""Rename a file based on the target given.
|
|
702 |
||
703 |
rnto must be called after calling rnfr.
|
|
704 |
"""
|
|
705 |
if not self._renaming: |
|
706 |
self.respond('503 RNFR required first.') |
|
707 |
pfrom = self.filesystem.translate(self._renaming) |
|
1707.3.19
by John Arbash Meinel
For some reason after RNTO we get a failure for has() |
708 |
self._renaming = None |
1707.3.7
by John Arbash Meinel
Creating child of ftp_channel to support renaming, need to add appending. |
709 |
pto = self.filesystem.translate(line[1]) |
1551.15.2
by Aaron Bentley
FTP does not require atomic rename. Fixes #89436 |
710 |
if os.path.exists(pto): |
711 |
self.respond('550 RNTO failed: file exists') |
|
712 |
return
|
|
1707.3.7
by John Arbash Meinel
Creating child of ftp_channel to support renaming, need to add appending. |
713 |
try: |
714 |
os.rename(pfrom, pto) |
|
1707.3.19
by John Arbash Meinel
For some reason after RNTO we get a failure for has() |
715 |
except (IOError, OSError), e: |
1707.3.7
by John Arbash Meinel
Creating child of ftp_channel to support renaming, need to add appending. |
716 |
# TODO: jam 20060516 return custom responses based on
|
717 |
# why the command failed
|
|
2423.2.2
by Alexander Belchenko
Explicit error messages for test FTP server (Python 2.5 @ win32 compatibility) |
718 |
# (bialix 20070418) str(e) on Python 2.5 @ Windows
|
719 |
# sometimes don't provide expected error message;
|
|
720 |
# so we obtain such message via os.strerror()
|
|
721 |
self.respond('550 RNTO failed: %s' % os.strerror(e.errno)) |
|
1707.3.7
by John Arbash Meinel
Creating child of ftp_channel to support renaming, need to add appending. |
722 |
except: |
723 |
self.respond('550 RNTO failed') |
|
724 |
# For a test server, we will go ahead and just die
|
|
725 |
raise
|
|
1707.3.20
by John Arbash Meinel
Found the problem, was sending 2 responses |
726 |
else: |
727 |
self.respond('250 Rename successful.') |
|
1707.3.7
by John Arbash Meinel
Creating child of ftp_channel to support renaming, need to add appending. |
728 |
|
1707.3.16
by John Arbash Meinel
Overloading cmd_size and cmd_mkd so that the test suite can handle directories. |
729 |
def cmd_size(self, line): |
730 |
"""Return the size of a file
|
|
731 |
||
732 |
This is overloaded to help the test suite determine if the
|
|
733 |
target is a directory.
|
|
734 |
"""
|
|
735 |
filename = line[1] |
|
736 |
if not self.filesystem.isfile(filename): |
|
737 |
if self.filesystem.isdir(filename): |
|
738 |
self.respond('550 "%s" is a directory' % (filename,)) |
|
739 |
else: |
|
740 |
self.respond('550 "%s" is not a file' % (filename,)) |
|
741 |
else: |
|
742 |
self.respond('213 %d' |
|
1707.3.17
by John Arbash Meinel
Fix errors in cmd_size |
743 |
% (self.filesystem.stat(filename)[stat.ST_SIZE]),) |
1707.3.16
by John Arbash Meinel
Overloading cmd_size and cmd_mkd so that the test suite can handle directories. |
744 |
|
745 |
def cmd_mkd(self, line): |
|
746 |
"""Create a directory.
|
|
747 |
||
748 |
Overloaded because default implementation does not distinguish
|
|
749 |
*why* it cannot make a directory.
|
|
750 |
"""
|
|
751 |
if len (line) != 2: |
|
1773.4.1
by Martin Pool
Add pyflakes makefile target; fix many warnings |
752 |
self.command_not_understood(''.join(line)) |
1707.3.16
by John Arbash Meinel
Overloading cmd_size and cmd_mkd so that the test suite can handle directories. |
753 |
else: |
754 |
path = line[1] |
|
755 |
try: |
|
756 |
self.filesystem.mkdir (path) |
|
757 |
self.respond ('257 MKD command successful.') |
|
758 |
except (IOError, OSError), e: |
|
2423.2.2
by Alexander Belchenko
Explicit error messages for test FTP server (Python 2.5 @ win32 compatibility) |
759 |
# (bialix 20070418) str(e) on Python 2.5 @ Windows
|
760 |
# sometimes don't provide expected error message;
|
|
761 |
# so we obtain such message via os.strerror()
|
|
762 |
self.respond ('550 error creating directory: %s' % |
|
763 |
os.strerror(e.errno)) |
|
1707.3.16
by John Arbash Meinel
Overloading cmd_size and cmd_mkd so that the test suite can handle directories. |
764 |
except: |
765 |
self.respond ('550 error creating directory.') |
|
766 |
||
1707.3.7
by John Arbash Meinel
Creating child of ftp_channel to support renaming, need to add appending. |
767 |
|
1707.3.30
by John Arbash Meinel
Only load medusa if you are running the test suite. |
768 |
class ftp_server(medusa.ftp_server.ftp_server): |
1707.3.2
by John Arbash Meinel
Removing stipple from ftp tests. |
769 |
"""Customize the behavior of the Medusa ftp_server.
|
770 |
||
771 |
There are a few warts on the ftp_server, based on how it expects
|
|
772 |
to be used.
|
|
773 |
"""
|
|
1707.3.7
by John Arbash Meinel
Creating child of ftp_channel to support renaming, need to add appending. |
774 |
_renaming = None |
1707.3.30
by John Arbash Meinel
Only load medusa if you are running the test suite. |
775 |
ftp_channel_class = ftp_channel |
1707.3.7
by John Arbash Meinel
Creating child of ftp_channel to support renaming, need to add appending. |
776 |
|
777 |
def __init__(self, *args, **kwargs): |
|
778 |
mutter('Initializing _ftp_server: %r, %r', args, kwargs) |
|
779 |
medusa.ftp_server.ftp_server.__init__(self, *args, **kwargs) |
|
1707.3.2
by John Arbash Meinel
Removing stipple from ftp tests. |
780 |
|
1707.3.19
by John Arbash Meinel
For some reason after RNTO we get a failure for has() |
781 |
def log(self, message): |
782 |
"""Redirect logging requests."""
|
|
1707.3.30
by John Arbash Meinel
Only load medusa if you are running the test suite. |
783 |
mutter('_ftp_server: %s', message) |
1707.3.19
by John Arbash Meinel
For some reason after RNTO we get a failure for has() |
784 |
|
1707.3.2
by John Arbash Meinel
Removing stipple from ftp tests. |
785 |
def log_info(self, message, type='info'): |
786 |
"""Override the asyncore.log_info so we don't stipple the screen."""
|
|
1707.3.7
by John Arbash Meinel
Creating child of ftp_channel to support renaming, need to add appending. |
787 |
mutter('_ftp_server %s: %s', type, message) |
1707.3.2
by John Arbash Meinel
Removing stipple from ftp tests. |
788 |
|
1707.3.30
by John Arbash Meinel
Only load medusa if you are running the test suite. |
789 |
_test_authorizer = test_authorizer |
790 |
_ftp_channel = ftp_channel |
|
791 |
_ftp_server = ftp_server |
|
792 |
||
793 |
return True |
|
1707.3.1
by John Arbash Meinel
Have a FtpServer, but the tests fail |
794 |
|
795 |
||
1530.1.11
by Robert Collins
Push the transport permutations list into each transport module allowing for automatic testing of new modules that are registered as transports. |
796 |
def get_test_permutations(): |
797 |
"""Return the permutations to be used in testing."""
|
|
1707.3.30
by John Arbash Meinel
Only load medusa if you are running the test suite. |
798 |
if not _setup_medusa(): |
1707.3.1
by John Arbash Meinel
Have a FtpServer, but the tests fail |
799 |
warn("You must install medusa (http://www.amk.ca/python/code/medusa.html) for FTP tests") |
800 |
return [] |
|
801 |
else: |
|
802 |
return [(FtpTransport, FtpServer)] |