2338.1.1
by John Arbash Meinel
(Arnaud Fontaine) Compatibility with vsftpd error messages. |
1 |
# Copyright (C) 2005, 2006, 2007 Canonical Ltd
|
1887.1.1
by Adeodato Simó
Do not separate paragraphs in the copyright statement with blank lines, |
2 |
#
|
1185.36.4
by Daniel Silverstone
Add FTP transport |
3 |
# This program is free software; you can redistribute it and/or modify
|
4 |
# it under the terms of the GNU General Public License as published by
|
|
5 |
# the Free Software Foundation; either version 2 of the License, or
|
|
6 |
# (at your option) any later version.
|
|
1887.1.1
by Adeodato Simó
Do not separate paragraphs in the copyright statement with blank lines, |
7 |
#
|
1185.36.4
by Daniel Silverstone
Add FTP transport |
8 |
# This program is distributed in the hope that it will be useful,
|
9 |
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
10 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
11 |
# GNU General Public License for more details.
|
|
1887.1.1
by Adeodato Simó
Do not separate paragraphs in the copyright statement with blank lines, |
12 |
#
|
1185.36.4
by Daniel Silverstone
Add FTP transport |
13 |
# You should have received a copy of the GNU General Public License
|
14 |
# along with this program; if not, write to the Free Software
|
|
15 |
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
16 |
"""Implementation of Transport over ftp.
|
|
17 |
||
18 |
Written by Daniel Silverstone <dsilvers@digital-scurf.org> with serious
|
|
19 |
cargo-culting from the sftp transport and the http transport.
|
|
20 |
||
21 |
It provides the ftp:// and aftp:// protocols where ftp:// is passive ftp
|
|
22 |
and aftp:// is active ftp. Most people will want passive ftp for traversing
|
|
23 |
NAT and other firewalls, so it's best to use it unless you explicitly want
|
|
24 |
active, in which case aftp:// will be your friend.
|
|
25 |
"""
|
|
26 |
||
27 |
from cStringIO import StringIO |
|
1707.3.1
by John Arbash Meinel
Have a FtpServer, but the tests fail |
28 |
import asyncore |
1530.1.11
by Robert Collins
Push the transport permutations list into each transport module allowing for automatic testing of new modules that are registered as transports. |
29 |
import errno |
1185.36.4
by Daniel Silverstone
Add FTP transport |
30 |
import ftplib |
1530.1.11
by Robert Collins
Push the transport permutations list into each transport module allowing for automatic testing of new modules that are registered as transports. |
31 |
import os |
1551.15.2
by Aaron Bentley
FTP does not require atomic rename. Fixes #89436 |
32 |
import os.path |
1530.1.11
by Robert Collins
Push the transport permutations list into each transport module allowing for automatic testing of new modules that are registered as transports. |
33 |
import urllib |
1185.36.4
by Daniel Silverstone
Add FTP transport |
34 |
import urlparse |
2413.3.1
by John Arbash Meinel
Trap EBADF during shutdown of asyncore.loop() |
35 |
import select |
1185.36.4
by Daniel Silverstone
Add FTP transport |
36 |
import stat |
1707.3.1
by John Arbash Meinel
Have a FtpServer, but the tests fail |
37 |
import threading |
1185.72.14
by Matthieu Moy
Sleep between retries when the connection closes |
38 |
import time |
1185.72.13
by Matthieu Moy
Make ftp put atomic |
39 |
import random |
1530.1.11
by Robert Collins
Push the transport permutations list into each transport module allowing for automatic testing of new modules that are registered as transports. |
40 |
from warnings import warn |
41 |
||
1948.1.7
by John Arbash Meinel
Fix FtpTransport so that it raises UnicodeError and skips the unicode path tests |
42 |
from bzrlib import ( |
2900.2.5
by Vincent Ladeuil
ake ftp aware of authentication config. |
43 |
config, |
1948.1.7
by John Arbash Meinel
Fix FtpTransport so that it raises UnicodeError and skips the unicode path tests |
44 |
errors, |
1551.15.2
by Aaron Bentley
FTP does not require atomic rename. Fixes #89436 |
45 |
osutils, |
1948.1.7
by John Arbash Meinel
Fix FtpTransport so that it raises UnicodeError and skips the unicode path tests |
46 |
urlutils, |
47 |
)
|
|
48 |
from bzrlib.trace import mutter, warning |
|
1707.3.4
by John Arbash Meinel
Moved most of sftp.split_url into a Transport function. |
49 |
from bzrlib.transport import ( |
2671.3.6
by Robert Collins
Review feedback. |
50 |
AppendBasedFileStream, |
2900.2.16
by Vincent Ladeuil
Make hhtp proxy aware of AuthenticationConfig (for password). |
51 |
ConnectedTransport, |
2671.3.2
by Robert Collins
Start open_file_stream logic. |
52 |
_file_streams, |
2900.2.16
by Vincent Ladeuil
Make hhtp proxy aware of AuthenticationConfig (for password). |
53 |
register_urlparse_netloc_protocol, |
1707.3.4
by John Arbash Meinel
Moved most of sftp.split_url into a Transport function. |
54 |
Server, |
55 |
)
|
|
2413.1.1
by John Arbash Meinel
Implement TestCaseWithFTPServer using the new shiny Feature mechanism. |
56 |
from bzrlib.transport.local import LocalURLServer |
1707.3.24
by John Arbash Meinel
Add prompt for password for ftp. |
57 |
import bzrlib.ui |
1551.3.11
by Aaron Bentley
Merge from Robert |
58 |
|
2900.2.16
by Vincent Ladeuil
Make hhtp proxy aware of AuthenticationConfig (for password). |
59 |
|
60 |
register_urlparse_netloc_protocol('aftp') |
|
1707.3.30
by John Arbash Meinel
Only load medusa if you are running the test suite. |
61 |
|
1551.3.11
by Aaron Bentley
Merge from Robert |
62 |
|
1707.3.25
by John Arbash Meinel
Setting some default exception classes, which can help a lot. |
63 |
class FtpPathError(errors.PathError): |
64 |
"""FTP failed for path: %(path)s%(extra)s"""
|
|
65 |
||
66 |
||
1185.36.4
by Daniel Silverstone
Add FTP transport |
67 |
class FtpStatResult(object): |
68 |
def __init__(self, f, relpath): |
|
69 |
try: |
|
70 |
self.st_size = f.size(relpath) |
|
71 |
self.st_mode = stat.S_IFREG |
|
72 |
except ftplib.error_perm: |
|
73 |
pwd = f.pwd() |
|
74 |
try: |
|
75 |
f.cwd(relpath) |
|
76 |
self.st_mode = stat.S_IFDIR |
|
77 |
finally: |
|
78 |
f.cwd(pwd) |
|
79 |
||
80 |
||
1185.72.15
by Matthieu Moy
better error messages on ftp failures |
81 |
_number_of_retries = 2 |
1185.72.14
by Matthieu Moy
Sleep between retries when the connection closes |
82 |
_sleep_between_retries = 5 |
1185.72.12
by Matthieu Moy
made __number_of_retries global |
83 |
|
2485.8.46
by Vincent Ladeuil
Add some remarks about current limitations in connection sharing. |
84 |
# FIXME: there are inconsistencies in the way temporary errors are
|
85 |
# handled. Sometimes we reconnect, sometimes we raise an exception. Care should
|
|
86 |
# be taken to analyze the implications for write operations (read operations
|
|
2485.8.59
by Vincent Ladeuil
Update from review comments. |
87 |
# are safe to retry). Overall even some read operations are never
|
88 |
# retried. --vila 20070720 (Bug #127164)
|
|
2485.8.16
by Vincent Ladeuil
Create a new, empty, ConnectedTransport class. |
89 |
class FtpTransport(ConnectedTransport): |
1185.36.4
by Daniel Silverstone
Add FTP transport |
90 |
"""This is the transport agent for ftp:// access."""
|
91 |
||
2485.8.59
by Vincent Ladeuil
Update from review comments. |
92 |
def __init__(self, base, _from_transport=None): |
1185.36.4
by Daniel Silverstone
Add FTP transport |
93 |
"""Set the base path where files will be stored."""
|
2485.8.23
by Vincent Ladeuil
Assert the accepted schemes for sftp and ftp. |
94 |
assert base.startswith('ftp://') or base.startswith('aftp://') |
2485.8.59
by Vincent Ladeuil
Update from review comments. |
95 |
super(FtpTransport, self).__init__(base, |
96 |
_from_transport=_from_transport) |
|
2485.8.25
by Vincent Ladeuil
Separate abspath from _remote_path, the intents are different. |
97 |
self._unqualified_scheme = 'ftp' |
2485.8.22
by Vincent Ladeuil
Finish ftp refactoring. Test suite passing. |
98 |
if self._scheme == 'aftp': |
99 |
self.is_active = True |
|
100 |
else: |
|
101 |
self.is_active = False |
|
1707.3.4
by John Arbash Meinel
Moved most of sftp.split_url into a Transport function. |
102 |
|
1185.36.4
by Daniel Silverstone
Add FTP transport |
103 |
def _get_FTP(self): |
104 |
"""Return the ftplib.FTP instance for this object."""
|
|
2485.8.33
by Vincent Ladeuil
ftp connection sharing refactored. Tests passing. |
105 |
# Ensures that a connection is established
|
106 |
connection = self._get_connection() |
|
107 |
if connection is None: |
|
108 |
# First connection ever
|
|
109 |
connection, credentials = self._create_connection() |
|
110 |
self._set_connection(connection, credentials) |
|
111 |
return connection |
|
112 |
||
113 |
def _create_connection(self, credentials=None): |
|
114 |
"""Create a new connection with the provided credentials.
|
|
115 |
||
116 |
:param credentials: The credentials needed to establish the connection.
|
|
117 |
||
118 |
:return: The created connection and its associated credentials.
|
|
119 |
||
120 |
The credentials are only the password as it may have been entered
|
|
121 |
interactively by the user and may be different from the one provided
|
|
122 |
in base url at transport creation time.
|
|
123 |
"""
|
|
124 |
if credentials is None: |
|
2900.2.15
by Vincent Ladeuil
AuthenticationConfig can be queried for logins too (first step). |
125 |
user, password = self._user, self._password |
2485.8.33
by Vincent Ladeuil
ftp connection sharing refactored. Tests passing. |
126 |
else: |
2900.2.15
by Vincent Ladeuil
AuthenticationConfig can be queried for logins too (first step). |
127 |
user, password = credentials |
128 |
||
129 |
auth = config.AuthenticationConfig() |
|
130 |
if user is None: |
|
131 |
user = auth.get_user('ftp', self._host, port=self._port) |
|
132 |
if user is None: |
|
133 |
# Default to local user
|
|
134 |
user = getpass.getuser() |
|
2485.8.33
by Vincent Ladeuil
ftp connection sharing refactored. Tests passing. |
135 |
|
136 |
mutter("Constructing FTP instance against %r" % |
|
2900.2.15
by Vincent Ladeuil
AuthenticationConfig can be queried for logins too (first step). |
137 |
((self._host, self._port, user, '********', |
2485.8.33
by Vincent Ladeuil
ftp connection sharing refactored. Tests passing. |
138 |
self.is_active),)) |
139 |
try: |
|
140 |
connection = ftplib.FTP() |
|
141 |
connection.connect(host=self._host, port=self._port) |
|
2900.2.15
by Vincent Ladeuil
AuthenticationConfig can be queried for logins too (first step). |
142 |
if user and user != 'anonymous' and \ |
2790.1.1
by Vincent Ladeuil
Fix #137044 by prompting for a password if *none* is provided for ftp. |
143 |
password is None: # '' is a valid password |
2900.2.15
by Vincent Ladeuil
AuthenticationConfig can be queried for logins too (first step). |
144 |
password = auth.get_password('ftp', self._host, user, |
2900.2.12
by Vincent Ladeuil
Since all schemes query AuthenticationConfig then prompt user, make that |
145 |
port=self._port) |
2900.2.15
by Vincent Ladeuil
AuthenticationConfig can be queried for logins too (first step). |
146 |
connection.login(user=user, passwd=password) |
2485.8.33
by Vincent Ladeuil
ftp connection sharing refactored. Tests passing. |
147 |
connection.set_pasv(not self.is_active) |
148 |
except ftplib.error_perm, e: |
|
149 |
raise errors.TransportError(msg="Error setting up connection:" |
|
150 |
" %s" % str(e), orig_error=e) |
|
2900.2.15
by Vincent Ladeuil
AuthenticationConfig can be queried for logins too (first step). |
151 |
return connection, (user, password) |
2485.8.33
by Vincent Ladeuil
ftp connection sharing refactored. Tests passing. |
152 |
|
153 |
def _reconnect(self): |
|
154 |
"""Create a new connection with the previously used credentials"""
|
|
155 |
credentials = self.get_credentials() |
|
156 |
connection, credentials = self._create_connection(credentials) |
|
157 |
self._set_connection(connection, credentials) |
|
158 |
||
159 |
def _translate_perm_error(self, err, path, extra=None, |
|
160 |
unknown_exc=FtpPathError): |
|
1707.3.19
by John Arbash Meinel
For some reason after RNTO we get a failure for has() |
161 |
"""Try to translate an ftplib.error_perm exception.
|
162 |
||
163 |
:param err: The error to translate into a bzr error
|
|
164 |
:param path: The path which had problems
|
|
165 |
:param extra: Extra information which can be included
|
|
166 |
:param unknown_exc: If None, we will just raise the original exception
|
|
167 |
otherwise we raise unknown_exc(path, extra=extra)
|
|
168 |
"""
|
|
1707.3.10
by John Arbash Meinel
Getting the append() tests to pass, and adding better error translation. These are probably dependent on the ftp test server, but for now it works. |
169 |
s = str(err).lower() |
170 |
if not extra: |
|
171 |
extra = str(err) |
|
1707.3.25
by John Arbash Meinel
Setting some default exception classes, which can help a lot. |
172 |
else: |
173 |
extra += ': ' + str(err) |
|
1707.3.10
by John Arbash Meinel
Getting the append() tests to pass, and adding better error translation. These are probably dependent on the ftp test server, but for now it works. |
174 |
if ('no such file' in s |
1707.3.13
by John Arbash Meinel
Changing raise Error to raise errors.Error |
175 |
or 'could not open' in s |
176 |
or 'no such dir' in s |
|
2338.1.1
by John Arbash Meinel
(Arnaud Fontaine) Compatibility with vsftpd error messages. |
177 |
or 'could not create file' in s # vsftpd |
2570.1.1
by John Arbash Meinel
(John Arbash Meinel) A couple small updates for pushing over ftp. |
178 |
or 'file doesn\'t exist' in s |
2885.3.1
by Gary van der Merwe
Correctly detect a NoSuchFile when using a filezilla server. |
179 |
or 'file/directory not found' in s # filezilla server |
1707.3.13
by John Arbash Meinel
Changing raise Error to raise errors.Error |
180 |
):
|
181 |
raise errors.NoSuchFile(path, extra=extra) |
|
1707.3.10
by John Arbash Meinel
Getting the append() tests to pass, and adding better error translation. These are probably dependent on the ftp test server, but for now it works. |
182 |
if ('file exists' in s): |
1707.3.15
by John Arbash Meinel
Handle trying to list a file instead of a dir |
183 |
raise errors.FileExists(path, extra=extra) |
184 |
if ('not a directory' in s): |
|
185 |
raise errors.PathError(path, extra=extra) |
|
186 |
||
1707.3.18
by John Arbash Meinel
Fix error handling for ftp.put() |
187 |
mutter('unable to understand error for path: %s: %s', path, err) |
1707.3.19
by John Arbash Meinel
For some reason after RNTO we get a failure for has() |
188 |
|
189 |
if unknown_exc: |
|
190 |
raise unknown_exc(path, extra=extra) |
|
1707.3.10
by John Arbash Meinel
Getting the append() tests to pass, and adding better error translation. These are probably dependent on the ftp test server, but for now it works. |
191 |
# TODO: jam 20060516 Consider re-raising the error wrapped in
|
192 |
# something like TransportError, but this loses the traceback
|
|
1707.3.15
by John Arbash Meinel
Handle trying to list a file instead of a dir |
193 |
# Also, 'sftp' has a generic 'Failure' mode, which we use failure_exc
|
194 |
# to handle. Consider doing something like that here.
|
|
1707.3.10
by John Arbash Meinel
Getting the append() tests to pass, and adding better error translation. These are probably dependent on the ftp test server, but for now it works. |
195 |
#raise TransportError(msg='Error for path: %s' % (path,), orig_error=e)
|
1707.3.15
by John Arbash Meinel
Handle trying to list a file instead of a dir |
196 |
raise
|
1707.3.10
by John Arbash Meinel
Getting the append() tests to pass, and adding better error translation. These are probably dependent on the ftp test server, but for now it works. |
197 |
|
2485.8.22
by Vincent Ladeuil
Finish ftp refactoring. Test suite passing. |
198 |
def _remote_path(self, relpath): |
1948.1.7
by John Arbash Meinel
Fix FtpTransport so that it raises UnicodeError and skips the unicode path tests |
199 |
# XXX: It seems that ftplib does not handle Unicode paths
|
2485.8.22
by Vincent Ladeuil
Finish ftp refactoring. Test suite passing. |
200 |
# at the same time, medusa won't handle utf8 paths So if
|
201 |
# we .encode(utf8) here (see ConnectedTransport
|
|
202 |
# implementation), then we get a Server failure. while
|
|
203 |
# if we use str(), we get a UnicodeError, and the test
|
|
204 |
# suite just skips testing UnicodePaths.
|
|
205 |
relative = str(urlutils.unescape(relpath)) |
|
206 |
remote_path = self._combine_paths(self._path, relative) |
|
207 |
return remote_path |
|
1185.36.4
by Daniel Silverstone
Add FTP transport |
208 |
|
209 |
def has(self, relpath): |
|
1707.3.16
by John Arbash Meinel
Overloading cmd_size and cmd_mkd so that the test suite can handle directories. |
210 |
"""Does the target location exist?"""
|
211 |
# FIXME jam 20060516 We *do* ask about directories in the test suite
|
|
212 |
# We don't seem to in the actual codebase
|
|
213 |
# XXX: I assume we're never asked has(dirname) and thus I use
|
|
214 |
# the FTP size command and assume that if it doesn't raise,
|
|
215 |
# all is good.
|
|
2485.8.22
by Vincent Ladeuil
Finish ftp refactoring. Test suite passing. |
216 |
abspath = self._remote_path(relpath) |
1185.36.4
by Daniel Silverstone
Add FTP transport |
217 |
try: |
218 |
f = self._get_FTP() |
|
1707.3.19
by John Arbash Meinel
For some reason after RNTO we get a failure for has() |
219 |
mutter('FTP has check: %s => %s', relpath, abspath) |
1707.3.16
by John Arbash Meinel
Overloading cmd_size and cmd_mkd so that the test suite can handle directories. |
220 |
s = f.size(abspath) |
221 |
mutter("FTP has: %s", abspath) |
|
1185.36.4
by Daniel Silverstone
Add FTP transport |
222 |
return True |
1707.3.16
by John Arbash Meinel
Overloading cmd_size and cmd_mkd so that the test suite can handle directories. |
223 |
except ftplib.error_perm, e: |
224 |
if ('is a directory' in str(e).lower()): |
|
225 |
mutter("FTP has dir: %s: %s", abspath, e) |
|
226 |
return True |
|
227 |
mutter("FTP has not: %s: %s", abspath, e) |
|
1185.36.4
by Daniel Silverstone
Add FTP transport |
228 |
return False |
229 |
||
2164.2.15
by Vincent Ladeuil
Http redirections are not followed by default. Do not use hints |
230 |
def get(self, relpath, decode=False, retries=0): |
1185.36.4
by Daniel Silverstone
Add FTP transport |
231 |
"""Get the file at the given relative path.
|
232 |
||
233 |
:param relpath: The relative path to the file
|
|
1185.72.9
by Matthieu Moy
suggestions by jam and robertc |
234 |
:param retries: Number of retries after temporary failures so far
|
235 |
for this operation.
|
|
1185.36.4
by Daniel Silverstone
Add FTP transport |
236 |
|
237 |
We're meant to return a file-like object which bzr will
|
|
238 |
then read from. For now we do this via the magic of StringIO
|
|
239 |
"""
|
|
1540.3.6
by Martin Pool
[merge] update from bzr.dev |
240 |
# TODO: decode should be deprecated
|
1185.36.4
by Daniel Silverstone
Add FTP transport |
241 |
try: |
2485.8.22
by Vincent Ladeuil
Finish ftp refactoring. Test suite passing. |
242 |
mutter("FTP get: %s", self._remote_path(relpath)) |
1185.36.4
by Daniel Silverstone
Add FTP transport |
243 |
f = self._get_FTP() |
244 |
ret = StringIO() |
|
2485.8.22
by Vincent Ladeuil
Finish ftp refactoring. Test suite passing. |
245 |
f.retrbinary('RETR '+self._remote_path(relpath), ret.write, 8192) |
1185.36.4
by Daniel Silverstone
Add FTP transport |
246 |
ret.seek(0) |
247 |
return ret |
|
248 |
except ftplib.error_perm, e: |
|
1707.3.13
by John Arbash Meinel
Changing raise Error to raise errors.Error |
249 |
raise errors.NoSuchFile(self.abspath(relpath), extra=str(e)) |
1185.72.8
by Matthieu Moy
try to solve the problem of ftp servers closing the connection |
250 |
except ftplib.error_temp, e: |
1185.72.15
by Matthieu Moy
better error messages on ftp failures |
251 |
if retries > _number_of_retries: |
1707.3.13
by John Arbash Meinel
Changing raise Error to raise errors.Error |
252 |
raise errors.TransportError(msg="FTP temporary error during GET %s. Aborting." |
1185.72.15
by Matthieu Moy
better error messages on ftp failures |
253 |
% self.abspath(relpath), |
254 |
orig_error=e) |
|
1185.72.8
by Matthieu Moy
try to solve the problem of ftp servers closing the connection |
255 |
else: |
1707.3.8
by John Arbash Meinel
[patch] Alexandre Saint: enable append support for ftp |
256 |
warning("FTP temporary error: %s. Retrying.", str(e)) |
2485.8.33
by Vincent Ladeuil
ftp connection sharing refactored. Tests passing. |
257 |
self._reconnect() |
1185.72.8
by Matthieu Moy
try to solve the problem of ftp servers closing the connection |
258 |
return self.get(relpath, decode, retries+1) |
1185.72.15
by Matthieu Moy
better error messages on ftp failures |
259 |
except EOFError, e: |
1185.72.9
by Matthieu Moy
suggestions by jam and robertc |
260 |
if retries > _number_of_retries: |
1707.3.13
by John Arbash Meinel
Changing raise Error to raise errors.Error |
261 |
raise errors.TransportError("FTP control connection closed during GET %s." |
1185.72.15
by Matthieu Moy
better error messages on ftp failures |
262 |
% self.abspath(relpath), |
263 |
orig_error=e) |
|
1185.72.8
by Matthieu Moy
try to solve the problem of ftp servers closing the connection |
264 |
else: |
1185.72.9
by Matthieu Moy
suggestions by jam and robertc |
265 |
warning("FTP control connection closed. Trying to reopen.") |
1185.72.14
by Matthieu Moy
Sleep between retries when the connection closes |
266 |
time.sleep(_sleep_between_retries) |
2485.8.33
by Vincent Ladeuil
ftp connection sharing refactored. Tests passing. |
267 |
self._reconnect() |
1185.72.8
by Matthieu Moy
try to solve the problem of ftp servers closing the connection |
268 |
return self.get(relpath, decode, retries+1) |
1185.36.4
by Daniel Silverstone
Add FTP transport |
269 |
|
1955.3.6
by John Arbash Meinel
Lots of deprecation warnings, but no errors |
270 |
def put_file(self, relpath, fp, mode=None, retries=0): |
1185.36.4
by Daniel Silverstone
Add FTP transport |
271 |
"""Copy the file-like or string object into the location.
|
272 |
||
273 |
:param relpath: Location to put the contents, relative to base.
|
|
1185.72.13
by Matthieu Moy
Make ftp put atomic |
274 |
:param fp: File-like or string object.
|
1185.72.9
by Matthieu Moy
suggestions by jam and robertc |
275 |
:param retries: Number of retries after temporary failures so far
|
276 |
for this operation.
|
|
277 |
||
1551.15.2
by Aaron Bentley
FTP does not require atomic rename. Fixes #89436 |
278 |
TODO: jam 20051215 ftp as a protocol seems to support chmod, but
|
279 |
ftplib does not
|
|
1185.36.4
by Daniel Silverstone
Add FTP transport |
280 |
"""
|
2485.8.22
by Vincent Ladeuil
Finish ftp refactoring. Test suite passing. |
281 |
abspath = self._remote_path(relpath) |
1707.3.18
by John Arbash Meinel
Fix error handling for ftp.put() |
282 |
tmp_abspath = '%s.tmp.%.9f.%d.%d' % (abspath, time.time(), |
1185.72.13
by Matthieu Moy
Make ftp put atomic |
283 |
os.getpid(), random.randint(0,0x7FFFFFFF)) |
2745.5.2
by Robert Collins
* ``bzrlib.transport.Transport.put_file`` now returns the number of bytes |
284 |
bytes = None |
1963.2.6
by Robey Pointer
pychecker is on crack; go back to using 'is None'. |
285 |
if getattr(fp, 'read', None) is None: |
2745.5.2
by Robert Collins
* ``bzrlib.transport.Transport.put_file`` now returns the number of bytes |
286 |
# hand in a string IO
|
287 |
bytes = fp |
|
288 |
fp = StringIO(bytes) |
|
289 |
else: |
|
290 |
# capture the byte count; .read() may be read only so
|
|
291 |
# decorate it.
|
|
292 |
class byte_counter(object): |
|
293 |
def __init__(self, fp): |
|
294 |
self.fp = fp |
|
295 |
self.counted_bytes = 0 |
|
296 |
def read(self, count): |
|
297 |
result = self.fp.read(count) |
|
298 |
self.counted_bytes += len(result) |
|
299 |
return result |
|
300 |
fp = byte_counter(fp) |
|
1185.36.4
by Daniel Silverstone
Add FTP transport |
301 |
try: |
1707.3.18
by John Arbash Meinel
Fix error handling for ftp.put() |
302 |
mutter("FTP put: %s", abspath) |
1185.36.4
by Daniel Silverstone
Add FTP transport |
303 |
f = self._get_FTP() |
1185.72.13
by Matthieu Moy
Make ftp put atomic |
304 |
try: |
305 |
f.storbinary('STOR '+tmp_abspath, fp) |
|
1551.15.2
by Aaron Bentley
FTP does not require atomic rename. Fixes #89436 |
306 |
self._rename_and_overwrite(tmp_abspath, abspath, f) |
2745.5.2
by Robert Collins
* ``bzrlib.transport.Transport.put_file`` now returns the number of bytes |
307 |
if bytes is not None: |
308 |
return len(bytes) |
|
309 |
else: |
|
310 |
return fp.counted_bytes |
|
1185.72.13
by Matthieu Moy
Make ftp put atomic |
311 |
except (ftplib.error_temp,EOFError), e: |
1185.72.15
by Matthieu Moy
better error messages on ftp failures |
312 |
warning("Failure during ftp PUT. Deleting temporary file.") |
1185.72.13
by Matthieu Moy
Make ftp put atomic |
313 |
try: |
314 |
f.delete(tmp_abspath) |
|
315 |
except: |
|
1707.3.8
by John Arbash Meinel
[patch] Alexandre Saint: enable append support for ftp |
316 |
warning("Failed to delete temporary file on the" |
317 |
" server.\nFile: %s", tmp_abspath) |
|
1185.72.13
by Matthieu Moy
Make ftp put atomic |
318 |
raise e |
319 |
raise
|
|
1185.36.4
by Daniel Silverstone
Add FTP transport |
320 |
except ftplib.error_perm, e: |
2570.1.1
by John Arbash Meinel
(John Arbash Meinel) A couple small updates for pushing over ftp. |
321 |
self._translate_perm_error(e, abspath, extra='could not store', |
322 |
unknown_exc=errors.NoSuchFile) |
|
1185.72.8
by Matthieu Moy
try to solve the problem of ftp servers closing the connection |
323 |
except ftplib.error_temp, e: |
1185.72.9
by Matthieu Moy
suggestions by jam and robertc |
324 |
if retries > _number_of_retries: |
1707.3.13
by John Arbash Meinel
Changing raise Error to raise errors.Error |
325 |
raise errors.TransportError("FTP temporary error during PUT %s. Aborting." |
1185.72.15
by Matthieu Moy
better error messages on ftp failures |
326 |
% self.abspath(relpath), orig_error=e) |
1185.72.8
by Matthieu Moy
try to solve the problem of ftp servers closing the connection |
327 |
else: |
1707.3.8
by John Arbash Meinel
[patch] Alexandre Saint: enable append support for ftp |
328 |
warning("FTP temporary error: %s. Retrying.", str(e)) |
2485.8.33
by Vincent Ladeuil
ftp connection sharing refactored. Tests passing. |
329 |
self._reconnect() |
1955.3.6
by John Arbash Meinel
Lots of deprecation warnings, but no errors |
330 |
self.put_file(relpath, fp, mode, retries+1) |
1185.72.9
by Matthieu Moy
suggestions by jam and robertc |
331 |
except EOFError: |
1185.72.10
by Matthieu Moy
One 1 -> _number_of_retries was missing |
332 |
if retries > _number_of_retries: |
1707.3.13
by John Arbash Meinel
Changing raise Error to raise errors.Error |
333 |
raise errors.TransportError("FTP control connection closed during PUT %s." |
1185.72.15
by Matthieu Moy
better error messages on ftp failures |
334 |
% self.abspath(relpath), orig_error=e) |
1185.72.8
by Matthieu Moy
try to solve the problem of ftp servers closing the connection |
335 |
else: |
1185.72.14
by Matthieu Moy
Sleep between retries when the connection closes |
336 |
warning("FTP control connection closed. Trying to reopen.") |
337 |
time.sleep(_sleep_between_retries) |
|
2485.8.33
by Vincent Ladeuil
ftp connection sharing refactored. Tests passing. |
338 |
self._reconnect() |
1955.3.6
by John Arbash Meinel
Lots of deprecation warnings, but no errors |
339 |
self.put_file(relpath, fp, mode, retries+1) |
1185.72.8
by Matthieu Moy
try to solve the problem of ftp servers closing the connection |
340 |
|
1185.58.2
by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work. |
341 |
def mkdir(self, relpath, mode=None): |
1185.36.4
by Daniel Silverstone
Add FTP transport |
342 |
"""Create a directory at the given path."""
|
2485.8.22
by Vincent Ladeuil
Finish ftp refactoring. Test suite passing. |
343 |
abspath = self._remote_path(relpath) |
1185.36.4
by Daniel Silverstone
Add FTP transport |
344 |
try: |
1707.3.14
by John Arbash Meinel
Cleanup mkdir and rmdir error handling. |
345 |
mutter("FTP mkd: %s", abspath) |
1185.36.4
by Daniel Silverstone
Add FTP transport |
346 |
f = self._get_FTP() |
1707.3.14
by John Arbash Meinel
Cleanup mkdir and rmdir error handling. |
347 |
f.mkd(abspath) |
1185.36.4
by Daniel Silverstone
Add FTP transport |
348 |
except ftplib.error_perm, e: |
1707.3.25
by John Arbash Meinel
Setting some default exception classes, which can help a lot. |
349 |
self._translate_perm_error(e, abspath, |
350 |
unknown_exc=errors.FileExists) |
|
1185.36.4
by Daniel Silverstone
Add FTP transport |
351 |
|
2671.3.9
by Robert Collins
Review feedback and fix VFat emulated transports to not claim to have unix permissions. |
352 |
def open_write_stream(self, relpath, mode=None): |
353 |
"""See Transport.open_write_stream."""
|
|
2671.3.6
by Robert Collins
Review feedback. |
354 |
self.put_bytes(relpath, "", mode) |
355 |
result = AppendBasedFileStream(self, relpath) |
|
356 |
_file_streams[self.abspath(relpath)] = result |
|
357 |
return result |
|
2671.3.2
by Robert Collins
Start open_file_stream logic. |
358 |
|
2671.3.1
by Robert Collins
* New method ``bzrlib.transport.Transport.get_recommended_page_size``. |
359 |
def recommended_page_size(self): |
360 |
"""See Transport.recommended_page_size().
|
|
361 |
||
362 |
For FTP we suggest a large page size to reduce the overhead
|
|
363 |
introduced by latency.
|
|
364 |
"""
|
|
365 |
return 64 * 1024 |
|
366 |
||
1658.1.1
by Martin Pool
Fix FTP push with metadir format (Alexandre Saint) |
367 |
def rmdir(self, rel_path): |
368 |
"""Delete the directory at rel_path"""
|
|
2485.8.22
by Vincent Ladeuil
Finish ftp refactoring. Test suite passing. |
369 |
abspath = self._remote_path(rel_path) |
1658.1.1
by Martin Pool
Fix FTP push with metadir format (Alexandre Saint) |
370 |
try: |
1707.3.14
by John Arbash Meinel
Cleanup mkdir and rmdir error handling. |
371 |
mutter("FTP rmd: %s", abspath) |
1551.15.4
by Aaron Bentley
Revert now-unnecessary changes |
372 |
f = self._get_FTP() |
1707.3.14
by John Arbash Meinel
Cleanup mkdir and rmdir error handling. |
373 |
f.rmd(abspath) |
1658.1.1
by Martin Pool
Fix FTP push with metadir format (Alexandre Saint) |
374 |
except ftplib.error_perm, e: |
1551.15.4
by Aaron Bentley
Revert now-unnecessary changes |
375 |
self._translate_perm_error(e, abspath, unknown_exc=errors.PathError) |
1658.1.1
by Martin Pool
Fix FTP push with metadir format (Alexandre Saint) |
376 |
|
1955.3.15
by John Arbash Meinel
Deprecate 'Transport.append' in favor of Transport.append_file or Transport.append_bytes |
377 |
def append_file(self, relpath, f, mode=None): |
1185.36.4
by Daniel Silverstone
Add FTP transport |
378 |
"""Append the text in the file-like object into the final
|
379 |
location.
|
|
380 |
"""
|
|
2485.8.22
by Vincent Ladeuil
Finish ftp refactoring. Test suite passing. |
381 |
abspath = self._remote_path(relpath) |
1707.3.8
by John Arbash Meinel
[patch] Alexandre Saint: enable append support for ftp |
382 |
if self.has(relpath): |
383 |
ftp = self._get_FTP() |
|
1707.3.19
by John Arbash Meinel
For some reason after RNTO we get a failure for has() |
384 |
result = ftp.size(abspath) |
1707.3.8
by John Arbash Meinel
[patch] Alexandre Saint: enable append support for ftp |
385 |
else: |
386 |
result = 0 |
|
387 |
||
1707.3.19
by John Arbash Meinel
For some reason after RNTO we get a failure for has() |
388 |
mutter("FTP appe to %s", abspath) |
1707.3.8
by John Arbash Meinel
[patch] Alexandre Saint: enable append support for ftp |
389 |
self._try_append(relpath, f.read(), mode) |
390 |
||
391 |
return result |
|
392 |
||
393 |
def _try_append(self, relpath, text, mode=None, retries=0): |
|
394 |
"""Try repeatedly to append the given text to the file at relpath.
|
|
395 |
|
|
396 |
This is a recursive function. On errors, it will be called until the
|
|
397 |
number of retries is exceeded.
|
|
398 |
"""
|
|
399 |
try: |
|
2485.8.22
by Vincent Ladeuil
Finish ftp refactoring. Test suite passing. |
400 |
abspath = self._remote_path(relpath) |
1707.3.8
by John Arbash Meinel
[patch] Alexandre Saint: enable append support for ftp |
401 |
mutter("FTP appe (try %d) to %s", retries, abspath) |
402 |
ftp = self._get_FTP() |
|
403 |
ftp.voidcmd("TYPE I") |
|
404 |
cmd = "APPE %s" % abspath |
|
405 |
conn = ftp.transfercmd(cmd) |
|
406 |
conn.sendall(text) |
|
407 |
conn.close() |
|
1707.3.25
by John Arbash Meinel
Setting some default exception classes, which can help a lot. |
408 |
if mode: |
1707.3.8
by John Arbash Meinel
[patch] Alexandre Saint: enable append support for ftp |
409 |
self._setmode(relpath, mode) |
410 |
ftp.getresp() |
|
411 |
except ftplib.error_perm, e: |
|
1707.3.26
by John Arbash Meinel
Added a default exception for append |
412 |
self._translate_perm_error(e, abspath, extra='error appending', |
413 |
unknown_exc=errors.NoSuchFile) |
|
1707.3.8
by John Arbash Meinel
[patch] Alexandre Saint: enable append support for ftp |
414 |
except ftplib.error_temp, e: |
415 |
if retries > _number_of_retries: |
|
1707.3.13
by John Arbash Meinel
Changing raise Error to raise errors.Error |
416 |
raise errors.TransportError("FTP temporary error during APPEND %s." \ |
1707.3.8
by John Arbash Meinel
[patch] Alexandre Saint: enable append support for ftp |
417 |
"Aborting." % abspath, orig_error=e) |
418 |
else: |
|
419 |
warning("FTP temporary error: %s. Retrying.", str(e)) |
|
2485.8.33
by Vincent Ladeuil
ftp connection sharing refactored. Tests passing. |
420 |
self._reconnect() |
1707.3.8
by John Arbash Meinel
[patch] Alexandre Saint: enable append support for ftp |
421 |
self._try_append(relpath, text, mode, retries+1) |
422 |
||
423 |
def _setmode(self, relpath, mode): |
|
424 |
"""Set permissions on a path.
|
|
425 |
||
426 |
Only set permissions if the FTP server supports the 'SITE CHMOD'
|
|
427 |
extension.
|
|
428 |
"""
|
|
429 |
try: |
|
430 |
mutter("FTP site chmod: setting permissions to %s on %s", |
|
2485.8.22
by Vincent Ladeuil
Finish ftp refactoring. Test suite passing. |
431 |
str(mode), self._remote_path(relpath)) |
1707.3.8
by John Arbash Meinel
[patch] Alexandre Saint: enable append support for ftp |
432 |
ftp = self._get_FTP() |
2485.8.22
by Vincent Ladeuil
Finish ftp refactoring. Test suite passing. |
433 |
cmd = "SITE CHMOD %s %s" % (self._remote_path(relpath), str(mode)) |
1707.3.8
by John Arbash Meinel
[patch] Alexandre Saint: enable append support for ftp |
434 |
ftp.sendcmd(cmd) |
435 |
except ftplib.error_perm, e: |
|
436 |
# Command probably not available on this server
|
|
437 |
warning("FTP Could not set permissions to %s on %s. %s", |
|
2485.8.22
by Vincent Ladeuil
Finish ftp refactoring. Test suite passing. |
438 |
str(mode), self._remote_path(relpath), str(e)) |
1185.36.4
by Daniel Silverstone
Add FTP transport |
439 |
|
1707.3.11
by John Arbash Meinel
fixing more tests. |
440 |
# TODO: jam 20060516 I believe ftp allows you to tell an ftp server
|
441 |
# to copy something to another machine. And you may be able
|
|
442 |
# to give it its own address as the 'to' location.
|
|
443 |
# So implement a fancier 'copy()'
|
|
1185.36.4
by Daniel Silverstone
Add FTP transport |
444 |
|
1551.15.3
by Aaron Bentley
Separate FtpTransport.rename from FtpTransport.move |
445 |
def rename(self, rel_from, rel_to): |
2485.8.22
by Vincent Ladeuil
Finish ftp refactoring. Test suite passing. |
446 |
abs_from = self._remote_path(rel_from) |
447 |
abs_to = self._remote_path(rel_to) |
|
1551.15.3
by Aaron Bentley
Separate FtpTransport.rename from FtpTransport.move |
448 |
mutter("FTP rename: %s => %s", abs_from, abs_to) |
449 |
f = self._get_FTP() |
|
450 |
return self._rename(abs_from, abs_to, f) |
|
451 |
||
452 |
def _rename(self, abs_from, abs_to, f): |
|
453 |
try: |
|
454 |
f.rename(abs_from, abs_to) |
|
455 |
except ftplib.error_perm, e: |
|
456 |
self._translate_perm_error(e, abs_from, |
|
457 |
': unable to rename to %r' % (abs_to)) |
|
458 |
||
1185.36.4
by Daniel Silverstone
Add FTP transport |
459 |
def move(self, rel_from, rel_to): |
460 |
"""Move the item at rel_from to the location at rel_to"""
|
|
2485.8.22
by Vincent Ladeuil
Finish ftp refactoring. Test suite passing. |
461 |
abs_from = self._remote_path(rel_from) |
462 |
abs_to = self._remote_path(rel_to) |
|
1185.36.4
by Daniel Silverstone
Add FTP transport |
463 |
try: |
1707.3.19
by John Arbash Meinel
For some reason after RNTO we get a failure for has() |
464 |
mutter("FTP mv: %s => %s", abs_from, abs_to) |
1185.36.4
by Daniel Silverstone
Add FTP transport |
465 |
f = self._get_FTP() |
1551.15.2
by Aaron Bentley
FTP does not require atomic rename. Fixes #89436 |
466 |
self._rename_and_overwrite(abs_from, abs_to, f) |
1185.36.4
by Daniel Silverstone
Add FTP transport |
467 |
except ftplib.error_perm, e: |
1707.3.19
by John Arbash Meinel
For some reason after RNTO we get a failure for has() |
468 |
self._translate_perm_error(e, abs_from, |
469 |
extra='unable to rename to %r' % (rel_to,), |
|
470 |
unknown_exc=errors.PathError) |
|
1185.36.4
by Daniel Silverstone
Add FTP transport |
471 |
|
1551.15.2
by Aaron Bentley
FTP does not require atomic rename. Fixes #89436 |
472 |
def _rename_and_overwrite(self, abs_from, abs_to, f): |
473 |
"""Do a fancy rename on the remote server.
|
|
474 |
||
475 |
Using the implementation provided by osutils.
|
|
476 |
"""
|
|
1551.15.3
by Aaron Bentley
Separate FtpTransport.rename from FtpTransport.move |
477 |
osutils.fancy_rename(abs_from, abs_to, |
1551.15.4
by Aaron Bentley
Revert now-unnecessary changes |
478 |
rename_func=lambda p1, p2: self._rename(p1, p2, f), |
479 |
unlink_func=lambda p: self._delete(p, f)) |
|
1551.15.2
by Aaron Bentley
FTP does not require atomic rename. Fixes #89436 |
480 |
|
1185.36.4
by Daniel Silverstone
Add FTP transport |
481 |
def delete(self, relpath): |
482 |
"""Delete the item at relpath"""
|
|
2485.8.22
by Vincent Ladeuil
Finish ftp refactoring. Test suite passing. |
483 |
abspath = self._remote_path(relpath) |
1551.15.2
by Aaron Bentley
FTP does not require atomic rename. Fixes #89436 |
484 |
f = self._get_FTP() |
485 |
self._delete(abspath, f) |
|
486 |
||
487 |
def _delete(self, abspath, f): |
|
1185.36.4
by Daniel Silverstone
Add FTP transport |
488 |
try: |
1707.3.11
by John Arbash Meinel
fixing more tests. |
489 |
mutter("FTP rm: %s", abspath) |
490 |
f.delete(abspath) |
|
1185.36.4
by Daniel Silverstone
Add FTP transport |
491 |
except ftplib.error_perm, e: |
1707.3.25
by John Arbash Meinel
Setting some default exception classes, which can help a lot. |
492 |
self._translate_perm_error(e, abspath, 'error deleting', |
493 |
unknown_exc=errors.NoSuchFile) |
|
1185.36.4
by Daniel Silverstone
Add FTP transport |
494 |
|
2634.1.1
by Robert Collins
(robertc) Reinstate the accidentally backed out external_url patch. |
495 |
def external_url(self): |
496 |
"""See bzrlib.transport.Transport.external_url."""
|
|
497 |
# FTP URL's are externally usable.
|
|
498 |
return self.base |
|
499 |
||
1185.36.4
by Daniel Silverstone
Add FTP transport |
500 |
def listable(self): |
501 |
"""See Transport.listable."""
|
|
502 |
return True |
|
503 |
||
504 |
def list_dir(self, relpath): |
|
505 |
"""See Transport.list_dir."""
|
|
2485.8.22
by Vincent Ladeuil
Finish ftp refactoring. Test suite passing. |
506 |
basepath = self._remote_path(relpath) |
1959.2.5
by John Arbash Meinel
Small fixes in ftp.list_dir() |
507 |
mutter("FTP nlst: %s", basepath) |
1959.2.1
by John Arbash Meinel
David Allouche: Make transports return escaped paths |
508 |
f = self._get_FTP() |
1185.36.4
by Daniel Silverstone
Add FTP transport |
509 |
try: |
1707.3.8
by John Arbash Meinel
[patch] Alexandre Saint: enable append support for ftp |
510 |
paths = f.nlst(basepath) |
1185.36.4
by Daniel Silverstone
Add FTP transport |
511 |
except ftplib.error_perm, e: |
1707.3.13
by John Arbash Meinel
Changing raise Error to raise errors.Error |
512 |
self._translate_perm_error(e, relpath, extra='error with list_dir') |
1959.2.1
by John Arbash Meinel
David Allouche: Make transports return escaped paths |
513 |
# If FTP.nlst returns paths prefixed by relpath, strip 'em
|
514 |
if paths and paths[0].startswith(basepath): |
|
515 |
entries = [path[len(basepath)+1:] for path in paths] |
|
516 |
else: |
|
517 |
entries = paths |
|
518 |
# Remove . and .. if present
|
|
1959.2.5
by John Arbash Meinel
Small fixes in ftp.list_dir() |
519 |
return [urlutils.escape(entry) for entry in entries |
520 |
if entry not in ('.', '..')] |
|
1185.36.4
by Daniel Silverstone
Add FTP transport |
521 |
|
522 |
def iter_files_recursive(self): |
|
523 |
"""See Transport.iter_files_recursive.
|
|
524 |
||
525 |
This is cargo-culted from the SFTP transport"""
|
|
526 |
mutter("FTP iter_files_recursive") |
|
527 |
queue = list(self.list_dir(".")) |
|
528 |
while queue: |
|
1959.2.1
by John Arbash Meinel
David Allouche: Make transports return escaped paths |
529 |
relpath = queue.pop(0) |
1185.36.4
by Daniel Silverstone
Add FTP transport |
530 |
st = self.stat(relpath) |
531 |
if stat.S_ISDIR(st.st_mode): |
|
532 |
for i, basename in enumerate(self.list_dir(relpath)): |
|
533 |
queue.insert(i, relpath+"/"+basename) |
|
534 |
else: |
|
535 |
yield relpath |
|
536 |
||
537 |
def stat(self, relpath): |
|
1707.3.13
by John Arbash Meinel
Changing raise Error to raise errors.Error |
538 |
"""Return the stat information for a file."""
|
2485.8.22
by Vincent Ladeuil
Finish ftp refactoring. Test suite passing. |
539 |
abspath = self._remote_path(relpath) |
1185.36.4
by Daniel Silverstone
Add FTP transport |
540 |
try: |
1707.3.13
by John Arbash Meinel
Changing raise Error to raise errors.Error |
541 |
mutter("FTP stat: %s", abspath) |
1185.36.4
by Daniel Silverstone
Add FTP transport |
542 |
f = self._get_FTP() |
1707.3.13
by John Arbash Meinel
Changing raise Error to raise errors.Error |
543 |
return FtpStatResult(f, abspath) |
1185.36.4
by Daniel Silverstone
Add FTP transport |
544 |
except ftplib.error_perm, e: |
1707.3.13
by John Arbash Meinel
Changing raise Error to raise errors.Error |
545 |
self._translate_perm_error(e, abspath, extra='error w/ stat') |
1185.36.4
by Daniel Silverstone
Add FTP transport |
546 |
|
547 |
def lock_read(self, relpath): |
|
548 |
"""Lock the given file for shared (read) access.
|
|
549 |
:return: A lock object, which should be passed to Transport.unlock()
|
|
550 |
"""
|
|
551 |
# The old RemoteBranch ignore lock for reading, so we will
|
|
552 |
# continue that tradition and return a bogus lock object.
|
|
553 |
class BogusLock(object): |
|
554 |
def __init__(self, path): |
|
555 |
self.path = path |
|
556 |
def unlock(self): |
|
557 |
pass
|
|
558 |
return BogusLock(relpath) |
|
559 |
||
560 |
def lock_write(self, relpath): |
|
561 |
"""Lock the given file for exclusive (write) access.
|
|
562 |
WARNING: many transports do not support this, so trying avoid using it
|
|
563 |
||
564 |
:return: A lock object, which should be passed to Transport.unlock()
|
|
565 |
"""
|
|
566 |
return self.lock_read(relpath) |
|
1530.1.11
by Robert Collins
Push the transport permutations list into each transport module allowing for automatic testing of new modules that are registered as transports. |
567 |
|
568 |
||
1707.3.30
by John Arbash Meinel
Only load medusa if you are running the test suite. |
569 |
class FtpServer(Server): |
2052.6.1
by Robert Collins
``Transport.get`` has had its interface made more clear for ease of use. |
570 |
"""Common code for FTP server facilities."""
|
1707.3.30
by John Arbash Meinel
Only load medusa if you are running the test suite. |
571 |
|
572 |
def __init__(self): |
|
573 |
self._root = None |
|
574 |
self._ftp_server = None |
|
575 |
self._port = None |
|
576 |
self._async_thread = None |
|
577 |
# ftp server logs
|
|
578 |
self.logs = [] |
|
579 |
||
580 |
def get_url(self): |
|
581 |
"""Calculate an ftp url to this server."""
|
|
582 |
return 'ftp://foo:bar@localhost:%d/' % (self._port) |
|
583 |
||
584 |
# def get_bogus_url(self):
|
|
585 |
# """Return a URL which cannot be connected to."""
|
|
586 |
# return 'ftp://127.0.0.1:1'
|
|
587 |
||
588 |
def log(self, message): |
|
589 |
"""This is used by medusa.ftp_server to log connections, etc."""
|
|
590 |
self.logs.append(message) |
|
591 |
||
2413.1.1
by John Arbash Meinel
Implement TestCaseWithFTPServer using the new shiny Feature mechanism. |
592 |
def setUp(self, vfs_server=None): |
1707.3.1
by John Arbash Meinel
Have a FtpServer, but the tests fail |
593 |
if not _have_medusa: |
1707.3.30
by John Arbash Meinel
Only load medusa if you are running the test suite. |
594 |
raise RuntimeError('Must have medusa to run the FtpServer') |
595 |
||
2413.1.1
by John Arbash Meinel
Implement TestCaseWithFTPServer using the new shiny Feature mechanism. |
596 |
assert vfs_server is None or isinstance(vfs_server, LocalURLServer), \ |
597 |
"FtpServer currently assumes local transport, got %s" % vfs_server |
|
598 |
||
1707.3.30
by John Arbash Meinel
Only load medusa if you are running the test suite. |
599 |
self._root = os.getcwdu() |
600 |
self._ftp_server = _ftp_server( |
|
601 |
authorizer=_test_authorizer(root=self._root), |
|
602 |
ip='localhost', |
|
603 |
port=0, # bind to a random port |
|
604 |
resolver=None, |
|
605 |
logger_object=self # Use FtpServer.log() for messages |
|
606 |
)
|
|
607 |
self._port = self._ftp_server.getsockname()[1] |
|
608 |
# Don't let it loop forever, or handle an infinite number of requests.
|
|
2485.8.12
by Vincent Ladeuil
Cosmetic change. |
609 |
# In this case it will run for 1000s, or 10000 requests
|
2413.3.1
by John Arbash Meinel
Trap EBADF during shutdown of asyncore.loop() |
610 |
self._async_thread = threading.Thread( |
611 |
target=FtpServer._asyncore_loop_ignore_EBADF, |
|
2485.8.6
by Vincent Ladeuil
Fix the 'FtpServer' so that it can handle full bzr commands. |
612 |
kwargs={'timeout':0.1, 'count':10000}) |
1707.3.30
by John Arbash Meinel
Only load medusa if you are running the test suite. |
613 |
self._async_thread.setDaemon(True) |
614 |
self._async_thread.start() |
|
615 |
||
616 |
def tearDown(self): |
|
617 |
"""See bzrlib.transport.Server.tearDown."""
|
|
2825.1.1
by Vincent Ladeuil
Fix #140055 by properly closing the http and ftp test servers. |
618 |
self._ftp_server.close() |
1707.3.30
by John Arbash Meinel
Only load medusa if you are running the test suite. |
619 |
asyncore.close_all() |
620 |
self._async_thread.join() |
|
621 |
||
2413.3.1
by John Arbash Meinel
Trap EBADF during shutdown of asyncore.loop() |
622 |
@staticmethod
|
623 |
def _asyncore_loop_ignore_EBADF(*args, **kwargs): |
|
2413.3.2
by John Arbash Meinel
cleanup comment (from Martin) |
624 |
"""Ignore EBADF during server shutdown.
|
625 |
||
626 |
We close the socket to get the server to shutdown, but this causes
|
|
627 |
select.select() to raise EBADF.
|
|
628 |
"""
|
|
2413.3.1
by John Arbash Meinel
Trap EBADF during shutdown of asyncore.loop() |
629 |
try: |
630 |
asyncore.loop(*args, **kwargs) |
|
2485.8.37
by Vincent Ladeuil
Fix merge multiple connections. Test suite *not* passing (sftp |
631 |
# FIXME: If we reach that point, we should raise an exception
|
632 |
# explaining that the 'count' parameter in setUp is too low or
|
|
633 |
# testers may wonder why their test just sits there waiting for a
|
|
634 |
# server that is already dead. Note that if the tester waits too
|
|
635 |
# long under pdb the server will also die.
|
|
2413.3.1
by John Arbash Meinel
Trap EBADF during shutdown of asyncore.loop() |
636 |
except select.error, e: |
637 |
if e.args[0] != errno.EBADF: |
|
638 |
raise
|
|
639 |
||
1707.3.30
by John Arbash Meinel
Only load medusa if you are running the test suite. |
640 |
|
641 |
_ftp_channel = None |
|
642 |
_ftp_server = None |
|
643 |
_test_authorizer = None |
|
644 |
||
645 |
||
646 |
def _setup_medusa(): |
|
647 |
global _have_medusa, _ftp_channel, _ftp_server, _test_authorizer |
|
648 |
try: |
|
649 |
import medusa |
|
650 |
import medusa.filesys |
|
651 |
import medusa.ftp_server |
|
652 |
except ImportError: |
|
653 |
return False |
|
654 |
||
655 |
_have_medusa = True |
|
656 |
||
657 |
class test_authorizer(object): |
|
658 |
"""A custom Authorizer object for running the test suite.
|
|
659 |
||
660 |
The reason we cannot use dummy_authorizer, is because it sets the
|
|
661 |
channel to readonly, which we don't always want to do.
|
|
662 |
"""
|
|
663 |
||
664 |
def __init__(self, root): |
|
665 |
self.root = root |
|
2790.1.1
by Vincent Ladeuil
Fix #137044 by prompting for a password if *none* is provided for ftp. |
666 |
# If secured_user is set secured_password will be checked
|
667 |
self.secured_user = None |
|
668 |
self.secured_password = None |
|
1707.3.30
by John Arbash Meinel
Only load medusa if you are running the test suite. |
669 |
|
670 |
def authorize(self, channel, username, password): |
|
671 |
"""Return (success, reply_string, filesystem)"""
|
|
672 |
if not _have_medusa: |
|
673 |
return 0, 'No Medusa.', None |
|
674 |
||
675 |
channel.persona = -1, -1 |
|
676 |
if username == 'anonymous': |
|
677 |
channel.read_only = 1 |
|
678 |
else: |
|
679 |
channel.read_only = 0 |
|
680 |
||
2790.1.3
by Vincent Ladeuil
Cosmetic change. |
681 |
# Check secured_user if set
|
2790.1.2
by Vincent Ladeuil
Review feedback. |
682 |
if (self.secured_user is not None |
683 |
and username == self.secured_user |
|
684 |
and password != self.secured_password): |
|
2790.1.1
by Vincent Ladeuil
Fix #137044 by prompting for a password if *none* is provided for ftp. |
685 |
return 0, 'Password invalid.', None |
686 |
else: |
|
687 |
return 1, 'OK.', medusa.filesys.os_filesystem(self.root) |
|
1707.3.30
by John Arbash Meinel
Only load medusa if you are running the test suite. |
688 |
|
689 |
||
690 |
class ftp_channel(medusa.ftp_server.ftp_channel): |
|
1707.3.7
by John Arbash Meinel
Creating child of ftp_channel to support renaming, need to add appending. |
691 |
"""Customized ftp channel"""
|
692 |
||
1707.3.19
by John Arbash Meinel
For some reason after RNTO we get a failure for has() |
693 |
def log(self, message): |
694 |
"""Redirect logging requests."""
|
|
695 |
mutter('_ftp_channel: %s', message) |
|
2485.8.6
by Vincent Ladeuil
Fix the 'FtpServer' so that it can handle full bzr commands. |
696 |
|
1707.3.7
by John Arbash Meinel
Creating child of ftp_channel to support renaming, need to add appending. |
697 |
def log_info(self, message, type='info'): |
698 |
"""Redirect logging requests."""
|
|
699 |
mutter('_ftp_channel %s: %s', type, message) |
|
2485.8.6
by Vincent Ladeuil
Fix the 'FtpServer' so that it can handle full bzr commands. |
700 |
|
1707.3.7
by John Arbash Meinel
Creating child of ftp_channel to support renaming, need to add appending. |
701 |
def cmd_rnfr(self, line): |
702 |
"""Prepare for renaming a file."""
|
|
703 |
self._renaming = line[1] |
|
704 |
self.respond('350 Ready for RNTO') |
|
705 |
# TODO: jam 20060516 in testing, the ftp server seems to
|
|
706 |
# check that the file already exists, or it sends
|
|
707 |
# 550 RNFR command failed
|
|
708 |
||
709 |
def cmd_rnto(self, line): |
|
710 |
"""Rename a file based on the target given.
|
|
711 |
||
712 |
rnto must be called after calling rnfr.
|
|
713 |
"""
|
|
714 |
if not self._renaming: |
|
715 |
self.respond('503 RNFR required first.') |
|
716 |
pfrom = self.filesystem.translate(self._renaming) |
|
1707.3.19
by John Arbash Meinel
For some reason after RNTO we get a failure for has() |
717 |
self._renaming = None |
1707.3.7
by John Arbash Meinel
Creating child of ftp_channel to support renaming, need to add appending. |
718 |
pto = self.filesystem.translate(line[1]) |
1551.15.2
by Aaron Bentley
FTP does not require atomic rename. Fixes #89436 |
719 |
if os.path.exists(pto): |
720 |
self.respond('550 RNTO failed: file exists') |
|
721 |
return
|
|
1707.3.7
by John Arbash Meinel
Creating child of ftp_channel to support renaming, need to add appending. |
722 |
try: |
723 |
os.rename(pfrom, pto) |
|
1707.3.19
by John Arbash Meinel
For some reason after RNTO we get a failure for has() |
724 |
except (IOError, OSError), e: |
1707.3.7
by John Arbash Meinel
Creating child of ftp_channel to support renaming, need to add appending. |
725 |
# TODO: jam 20060516 return custom responses based on
|
726 |
# why the command failed
|
|
2423.2.2
by Alexander Belchenko
Explicit error messages for test FTP server (Python 2.5 @ win32 compatibility) |
727 |
# (bialix 20070418) str(e) on Python 2.5 @ Windows
|
728 |
# sometimes don't provide expected error message;
|
|
729 |
# so we obtain such message via os.strerror()
|
|
730 |
self.respond('550 RNTO failed: %s' % os.strerror(e.errno)) |
|
1707.3.7
by John Arbash Meinel
Creating child of ftp_channel to support renaming, need to add appending. |
731 |
except: |
732 |
self.respond('550 RNTO failed') |
|
733 |
# For a test server, we will go ahead and just die
|
|
734 |
raise
|
|
1707.3.20
by John Arbash Meinel
Found the problem, was sending 2 responses |
735 |
else: |
736 |
self.respond('250 Rename successful.') |
|
1707.3.7
by John Arbash Meinel
Creating child of ftp_channel to support renaming, need to add appending. |
737 |
|
1707.3.16
by John Arbash Meinel
Overloading cmd_size and cmd_mkd so that the test suite can handle directories. |
738 |
def cmd_size(self, line): |
739 |
"""Return the size of a file
|
|
740 |
||
741 |
This is overloaded to help the test suite determine if the
|
|
742 |
target is a directory.
|
|
743 |
"""
|
|
744 |
filename = line[1] |
|
745 |
if not self.filesystem.isfile(filename): |
|
746 |
if self.filesystem.isdir(filename): |
|
747 |
self.respond('550 "%s" is a directory' % (filename,)) |
|
748 |
else: |
|
749 |
self.respond('550 "%s" is not a file' % (filename,)) |
|
750 |
else: |
|
751 |
self.respond('213 %d' |
|
1707.3.17
by John Arbash Meinel
Fix errors in cmd_size |
752 |
% (self.filesystem.stat(filename)[stat.ST_SIZE]),) |
1707.3.16
by John Arbash Meinel
Overloading cmd_size and cmd_mkd so that the test suite can handle directories. |
753 |
|
754 |
def cmd_mkd(self, line): |
|
755 |
"""Create a directory.
|
|
756 |
||
757 |
Overloaded because default implementation does not distinguish
|
|
758 |
*why* it cannot make a directory.
|
|
759 |
"""
|
|
760 |
if len (line) != 2: |
|
1773.4.1
by Martin Pool
Add pyflakes makefile target; fix many warnings |
761 |
self.command_not_understood(''.join(line)) |
1707.3.16
by John Arbash Meinel
Overloading cmd_size and cmd_mkd so that the test suite can handle directories. |
762 |
else: |
763 |
path = line[1] |
|
764 |
try: |
|
765 |
self.filesystem.mkdir (path) |
|
766 |
self.respond ('257 MKD command successful.') |
|
767 |
except (IOError, OSError), e: |
|
2423.2.2
by Alexander Belchenko
Explicit error messages for test FTP server (Python 2.5 @ win32 compatibility) |
768 |
# (bialix 20070418) str(e) on Python 2.5 @ Windows
|
769 |
# sometimes don't provide expected error message;
|
|
770 |
# so we obtain such message via os.strerror()
|
|
771 |
self.respond ('550 error creating directory: %s' % |
|
772 |
os.strerror(e.errno)) |
|
1707.3.16
by John Arbash Meinel
Overloading cmd_size and cmd_mkd so that the test suite can handle directories. |
773 |
except: |
774 |
self.respond ('550 error creating directory.') |
|
775 |
||
1707.3.7
by John Arbash Meinel
Creating child of ftp_channel to support renaming, need to add appending. |
776 |
|
1707.3.30
by John Arbash Meinel
Only load medusa if you are running the test suite. |
777 |
class ftp_server(medusa.ftp_server.ftp_server): |
1707.3.2
by John Arbash Meinel
Removing stipple from ftp tests. |
778 |
"""Customize the behavior of the Medusa ftp_server.
|
779 |
||
780 |
There are a few warts on the ftp_server, based on how it expects
|
|
781 |
to be used.
|
|
782 |
"""
|
|
1707.3.7
by John Arbash Meinel
Creating child of ftp_channel to support renaming, need to add appending. |
783 |
_renaming = None |
1707.3.30
by John Arbash Meinel
Only load medusa if you are running the test suite. |
784 |
ftp_channel_class = ftp_channel |
1707.3.7
by John Arbash Meinel
Creating child of ftp_channel to support renaming, need to add appending. |
785 |
|
786 |
def __init__(self, *args, **kwargs): |
|
787 |
mutter('Initializing _ftp_server: %r, %r', args, kwargs) |
|
788 |
medusa.ftp_server.ftp_server.__init__(self, *args, **kwargs) |
|
1707.3.2
by John Arbash Meinel
Removing stipple from ftp tests. |
789 |
|
1707.3.19
by John Arbash Meinel
For some reason after RNTO we get a failure for has() |
790 |
def log(self, message): |
791 |
"""Redirect logging requests."""
|
|
1707.3.30
by John Arbash Meinel
Only load medusa if you are running the test suite. |
792 |
mutter('_ftp_server: %s', message) |
1707.3.19
by John Arbash Meinel
For some reason after RNTO we get a failure for has() |
793 |
|
1707.3.2
by John Arbash Meinel
Removing stipple from ftp tests. |
794 |
def log_info(self, message, type='info'): |
795 |
"""Override the asyncore.log_info so we don't stipple the screen."""
|
|
1707.3.7
by John Arbash Meinel
Creating child of ftp_channel to support renaming, need to add appending. |
796 |
mutter('_ftp_server %s: %s', type, message) |
1707.3.2
by John Arbash Meinel
Removing stipple from ftp tests. |
797 |
|
1707.3.30
by John Arbash Meinel
Only load medusa if you are running the test suite. |
798 |
_test_authorizer = test_authorizer |
799 |
_ftp_channel = ftp_channel |
|
800 |
_ftp_server = ftp_server |
|
801 |
||
802 |
return True |
|
1707.3.1
by John Arbash Meinel
Have a FtpServer, but the tests fail |
803 |
|
804 |
||
1530.1.11
by Robert Collins
Push the transport permutations list into each transport module allowing for automatic testing of new modules that are registered as transports. |
805 |
def get_test_permutations(): |
806 |
"""Return the permutations to be used in testing."""
|
|
1707.3.30
by John Arbash Meinel
Only load medusa if you are running the test suite. |
807 |
if not _setup_medusa(): |
1707.3.1
by John Arbash Meinel
Have a FtpServer, but the tests fail |
808 |
warn("You must install medusa (http://www.amk.ca/python/code/medusa.html) for FTP tests") |
809 |
return [] |
|
810 |
else: |
|
811 |
return [(FtpTransport, FtpServer)] |