← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/tests/test_sftp_transport.py

  • Committer: John Arbash Meinel
  • Author(s): Mark Hammond
  • Date: 2008-09-09 17:02:21 UTC
  • mto: This revision was merged to the branch mainline in revision 3697.
  • Revision ID: john@arbash-meinel.com-20080909170221-svim3jw2mrz0amp3
An updated transparent icon for bzr.

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/tests/test_sftp_transport.py
1
 
# Copyright (C) 2005-2012, 2016 Robey Pointer <robey@lag.net>
 
1
# Copyright (C) 2005 Robey Pointer <robey@lag.net>
2
2
# Copyright (C) 2005, 2006, 2007 Canonical Ltd
3
3
#
4
4
# This program is free software; you can redistribute it and/or modify
13
13
#
14
14
# You should have received a copy of the GNU General Public License
15
15
# along with this program; if not, write to the Free Software
16
 
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 
16
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
17
17
 
18
18
import os
19
19
import socket
20
20
import sys
 
21
import threading
21
22
import time
22
23
 
 
24
try:
 
25
    import paramiko
 
26
    paramiko_loaded = True
 
27
except ImportError:
 
28
    paramiko_loaded = False
 
29
 
23
30
from bzrlib import (
24
 
    config,
25
 
    controldir,
 
31
    bzrdir,
26
32
    errors,
27
 
    tests,
28
 
    transport as _mod_transport,
29
 
    ui,
30
33
    )
31
34
from bzrlib.osutils import (
 
35
    pathjoin,
32
36
    lexists,
 
37
    set_or_unset_env,
33
38
    )
34
39
from bzrlib.tests import (
35
 
    features,
36
40
    TestCaseWithTransport,
37
41
    TestCase,
38
42
    TestSkipped,
39
43
    )
40
44
from bzrlib.tests.http_server import HttpServer
 
45
from bzrlib.transport import get_transport
41
46
import bzrlib.transport.http
42
47
 
43
 
if features.paramiko.available():
44
 
    from bzrlib.transport import sftp as _mod_sftp
45
 
    from bzrlib.tests import stub_sftp
 
48
if paramiko_loaded:
 
49
    from bzrlib.transport.sftp import (
 
50
        SFTPAbsoluteServer,
 
51
        SFTPHomeDirServer,
 
52
        SFTPTransport,
 
53
        )
 
54
 
 
55
from bzrlib.workingtree import WorkingTree
46
56
 
47
57
 
48
58
def set_test_transport_to_sftp(testcase):
50
60
    if getattr(testcase, '_get_remote_is_absolute', None) is None:
51
61
        testcase._get_remote_is_absolute = True
52
62
    if testcase._get_remote_is_absolute:
53
 
        testcase.transport_server = stub_sftp.SFTPAbsoluteServer
 
63
        testcase.transport_server = SFTPAbsoluteServer
54
64
    else:
55
 
        testcase.transport_server = stub_sftp.SFTPHomeDirServer
 
65
        testcase.transport_server = SFTPHomeDirServer
56
66
    testcase.transport_readonly_server = HttpServer
57
67
 
58
68
 
61
71
 
62
72
    def setUp(self):
63
73
        super(TestCaseWithSFTPServer, self).setUp()
64
 
        self.requireFeature(features.paramiko)
 
74
        if not paramiko_loaded:
 
75
            raise TestSkipped('you must have paramiko to run this test')
65
76
        set_test_transport_to_sftp(self)
66
77
 
67
78
 
68
 
class SFTPLockTests(TestCaseWithSFTPServer):
 
79
class SFTPLockTests (TestCaseWithSFTPServer):
69
80
 
70
81
    def test_sftp_locks(self):
71
82
        from bzrlib.errors import LockError
72
83
        t = self.get_transport()
73
84
 
74
85
        l = t.lock_write('bogus')
75
 
        self.assertPathExists('bogus.write-lock')
 
86
        self.failUnlessExists('bogus.write-lock')
76
87
 
77
88
        # Don't wait for the lock, locking an already locked
78
89
        # file should raise an assert
79
90
        self.assertRaises(LockError, t.lock_write, 'bogus')
80
91
 
81
92
        l.unlock()
82
 
        self.assertFalse(lexists('bogus.write-lock'))
 
93
        self.failIf(lexists('bogus.write-lock'))
83
94
 
84
 
        with open('something.write-lock', 'wb') as f: f.write('fake lock\n')
 
95
        open('something.write-lock', 'wb').write('fake lock\n')
85
96
        self.assertRaises(LockError, t.lock_write, 'something')
86
97
        os.remove('something.write-lock')
87
98
 
140
151
    def test__remote_path_relative_root(self):
141
152
        # relative paths are preserved
142
153
        t = self.get_transport('')
143
 
        self.assertEqual('/~/', t._parsed_url.path)
 
154
        self.assertEqual('/~/', t._path)
144
155
        # the remote path should be relative to home dir
145
156
        # (i.e. not begining with a '/')
146
157
        self.assertEqual('a', t._remote_path('a'))
147
158
 
148
159
 
149
160
class SFTPNonServerTest(TestCase):
150
 
 
151
161
    def setUp(self):
152
 
        super(SFTPNonServerTest, self).setUp()
153
 
        self.requireFeature(features.paramiko)
 
162
        TestCase.setUp(self)
 
163
        if not paramiko_loaded:
 
164
            raise TestSkipped('you must have paramiko to run this test')
154
165
 
155
166
    def test_parse_url_with_home_dir(self):
156
 
        s = _mod_sftp.SFTPTransport(
157
 
            'sftp://ro%62ey:h%40t@example.com:2222/~/relative')
158
 
        self.assertEqual(s._parsed_url.host, 'example.com')
159
 
        self.assertEqual(s._parsed_url.port, 2222)
160
 
        self.assertEqual(s._parsed_url.user, 'robey')
161
 
        self.assertEqual(s._parsed_url.password, 'h@t')
162
 
        self.assertEqual(s._parsed_url.path, '/~/relative/')
 
167
        s = SFTPTransport('sftp://ro%62ey:h%40t@example.com:2222/~/relative')
 
168
        self.assertEquals(s._host, 'example.com')
 
169
        self.assertEquals(s._port, 2222)
 
170
        self.assertEquals(s._user, 'robey')
 
171
        self.assertEquals(s._password, 'h@t')
 
172
        self.assertEquals(s._path, '/~/relative/')
163
173
 
164
174
    def test_relpath(self):
165
 
        s = _mod_sftp.SFTPTransport('sftp://user@host.com/abs/path')
 
175
        s = SFTPTransport('sftp://user@host.com/abs/path')
166
176
        self.assertRaises(errors.PathNotChild, s.relpath,
167
177
                          'sftp://user@host.com/~/rel/path/sub')
168
178
 
170
180
        """Test that if no 'ssh' is available we get builtin paramiko"""
171
181
        from bzrlib.transport import ssh
172
182
        # set '.' as the only location in the path, forcing no 'ssh' to exist
173
 
        self.overrideAttr(ssh, '_ssh_vendor_manager')
174
 
        self.overrideEnv('PATH', '.')
175
 
        ssh._ssh_vendor_manager.clear_cache()
176
 
        vendor = ssh._get_ssh_vendor()
177
 
        self.assertIsInstance(vendor, ssh.ParamikoVendor)
 
183
        orig_vendor = ssh._ssh_vendor_manager._cached_ssh_vendor
 
184
        orig_path = set_or_unset_env('PATH', '.')
 
185
        try:
 
186
            # No vendor defined yet, query for one
 
187
            ssh._ssh_vendor_manager.clear_cache()
 
188
            vendor = ssh._get_ssh_vendor()
 
189
            self.assertIsInstance(vendor, ssh.ParamikoVendor)
 
190
        finally:
 
191
            set_or_unset_env('PATH', orig_path)
 
192
            ssh._ssh_vendor_manager._cached_ssh_vendor = orig_vendor
178
193
 
179
194
    def test_abspath_root_sibling_server(self):
180
 
        server = stub_sftp.SFTPSiblingAbsoluteServer()
181
 
        server.start_server()
182
 
        self.addCleanup(server.stop_server)
183
 
 
184
 
        transport = _mod_transport.get_transport_from_url(server.get_url())
185
 
        self.assertFalse(transport.abspath('/').endswith('/~/'))
186
 
        self.assertTrue(transport.abspath('/').endswith('/'))
187
 
        del transport
 
195
        from bzrlib.transport.sftp import SFTPSiblingAbsoluteServer
 
196
        server = SFTPSiblingAbsoluteServer()
 
197
        server.setUp()
 
198
        try:
 
199
            transport = get_transport(server.get_url())
 
200
            self.assertFalse(transport.abspath('/').endswith('/~/'))
 
201
            self.assertTrue(transport.abspath('/').endswith('/'))
 
202
            del transport
 
203
        finally:
 
204
            server.tearDown()
188
205
 
189
206
 
190
207
class SFTPBranchTest(TestCaseWithSFTPServer):
191
208
    """Test some stuff when accessing a bzr Branch over sftp"""
192
209
 
 
210
    def test_lock_file(self):
 
211
        # old format branches use a special lock file on sftp.
 
212
        b = self.make_branch('', format=bzrdir.BzrDirFormat6())
 
213
        b = bzrlib.branch.Branch.open(self.get_url())
 
214
        self.failUnlessExists('.bzr/')
 
215
        self.failUnlessExists('.bzr/branch-format')
 
216
        self.failUnlessExists('.bzr/branch-lock')
 
217
 
 
218
        self.failIf(lexists('.bzr/branch-lock.write-lock'))
 
219
        b.lock_write()
 
220
        self.failUnlessExists('.bzr/branch-lock.write-lock')
 
221
        b.unlock()
 
222
        self.failIf(lexists('.bzr/branch-lock.write-lock'))
 
223
 
193
224
    def test_push_support(self):
194
225
        self.build_tree(['a/', 'a/foo'])
195
 
        t = controldir.ControlDir.create_standalone_workingtree('a')
 
226
        t = bzrdir.BzrDir.create_standalone_workingtree('a')
196
227
        b = t.branch
197
228
        t.add('foo')
198
229
        t.commit('foo', rev_id='a1')
199
230
 
200
 
        b2 = controldir.ControlDir.create_branch_and_repo(self.get_url('/b'))
 
231
        b2 = bzrdir.BzrDir.create_branch_and_repo(self.get_url('/b'))
201
232
        b2.pull(b)
202
233
 
203
 
        self.assertEqual(b2.last_revision(), 'a1')
 
234
        self.assertEquals(b2.revision_history(), ['a1'])
204
235
 
205
 
        with open('a/foo', 'wt') as f: f.write('something new in foo\n')
 
236
        open('a/foo', 'wt').write('something new in foo\n')
206
237
        t.commit('new', rev_id='a2')
207
238
        b2.pull(b)
208
239
 
209
 
        self.assertEqual(b2.last_revision(), 'a2')
 
240
        self.assertEquals(b2.revision_history(), ['a1', 'a2'])
210
241
 
211
242
 
212
243
class SSHVendorConnection(TestCaseWithSFTPServer):
223
254
      None:       If 'ssh' exists on the machine, then it will be spawned as a
224
255
                  child process.
225
256
    """
226
 
 
 
257
    
227
258
    def setUp(self):
228
259
        super(SSHVendorConnection, self).setUp()
 
260
        from bzrlib.transport.sftp import SFTPFullAbsoluteServer
229
261
 
230
262
        def create_server():
231
263
            """Just a wrapper so that when created, it will set _vendor"""
232
264
            # SFTPFullAbsoluteServer can handle any vendor,
233
265
            # it just needs to be set between the time it is instantiated
234
266
            # and the time .setUp() is called
235
 
            server = stub_sftp.SFTPFullAbsoluteServer()
 
267
            server = SFTPFullAbsoluteServer()
236
268
            server._vendor = self._test_vendor
237
269
            return server
238
270
        self._test_vendor = 'loopback'
270
302
    """
271
303
 
272
304
    def setUp(self):
273
 
        self.requireFeature(features.paramiko)
 
305
        if not paramiko_loaded:
 
306
            raise TestSkipped('you must have paramiko to run this test')
274
307
        super(SSHVendorBadConnection, self).setUp()
 
308
        import bzrlib.transport.ssh
275
309
 
276
310
        # open a random port, so we know nobody else is using it
277
311
        # but don't actually listen on the port.
278
312
        s = socket.socket()
279
313
        s.bind(('localhost', 0))
280
 
        self.addCleanup(s.close)
281
314
        self.bogus_url = 'sftp://%s:%s/' % s.getsockname()
282
315
 
283
 
    def set_vendor(self, vendor, subprocess_stderr=None):
284
 
        from bzrlib.transport import ssh
285
 
        self.overrideAttr(ssh._ssh_vendor_manager, '_cached_ssh_vendor', vendor)
286
 
        if subprocess_stderr is not None:
287
 
            self.overrideAttr(ssh.SubprocessVendor, "_stderr_target",
288
 
                subprocess_stderr)
 
316
        orig_vendor = bzrlib.transport.ssh._ssh_vendor_manager._cached_ssh_vendor
 
317
        def reset():
 
318
            bzrlib.transport.ssh._ssh_vendor_manager._cached_ssh_vendor = orig_vendor
 
319
            s.close()
 
320
        self.addCleanup(reset)
 
321
 
 
322
    def set_vendor(self, vendor):
 
323
        import bzrlib.transport.ssh
 
324
        bzrlib.transport.ssh._ssh_vendor_manager._cached_ssh_vendor = vendor
289
325
 
290
326
    def test_bad_connection_paramiko(self):
291
327
        """Test that a real connection attempt raises the right error"""
292
328
        from bzrlib.transport import ssh
293
329
        self.set_vendor(ssh.ParamikoVendor())
294
 
        t = _mod_transport.get_transport_from_url(self.bogus_url)
 
330
        t = bzrlib.transport.get_transport(self.bogus_url)
295
331
        self.assertRaises(errors.ConnectionError, t.get, 'foobar')
296
332
 
297
333
    def test_bad_connection_ssh(self):
298
334
        """None => auto-detect vendor"""
299
 
        f = file(os.devnull, "wb")
300
 
        self.addCleanup(f.close)
301
 
        self.set_vendor(None, f)
302
 
        t = _mod_transport.get_transport_from_url(self.bogus_url)
303
 
        try:
304
 
            self.assertRaises(errors.ConnectionError, t.get, 'foobar')
305
 
        except NameError, e:
306
 
            if "global name 'SSHException'" in str(e):
307
 
                self.knownFailure('Known NameError bug in paramiko 1.6.1')
308
 
            raise
 
335
        self.set_vendor(None)
 
336
        # This is how I would normally test the connection code
 
337
        # it makes it very clear what we are testing.
 
338
        # However, 'ssh' will create stipple on the output, so instead
 
339
        # I'm using run_bzr_subprocess, and parsing the output
 
340
        # try:
 
341
        #     t = bzrlib.transport.get_transport(self.bogus_url)
 
342
        # except errors.ConnectionError:
 
343
        #     # Correct error
 
344
        #     pass
 
345
        # except errors.NameError, e:
 
346
        #     if 'SSHException' in str(e):
 
347
        #         raise TestSkipped('Known NameError bug in paramiko 1.6.1')
 
348
        #     raise
 
349
        # else:
 
350
        #     self.fail('Excepted ConnectionError to be raised')
 
351
 
 
352
        out, err = self.run_bzr_subprocess(['log', self.bogus_url], retcode=3)
 
353
        self.assertEqual('', out)
 
354
        if "NameError: global name 'SSHException'" in err:
 
355
            # We aren't fixing this bug, because it is a bug in
 
356
            # paramiko, but we know about it, so we don't have to
 
357
            # fail the test
 
358
            raise TestSkipped('Known NameError bug with paramiko-1.6.1')
 
359
        self.assertContainsRe(err, r'bzr: ERROR: Unable to connect to SSH host'
 
360
                                   r' 127\.0\.0\.1:\d+; ')
309
361
 
310
362
 
311
363
class SFTPLatencyKnob(TestCaseWithSFTPServer):
312
364
    """Test that the testing SFTPServer's latency knob works."""
313
365
 
314
366
    def test_latency_knob_slows_transport(self):
315
 
        # change the latency knob to 500ms. We take about 40ms for a
 
367
        # change the latency knob to 500ms. We take about 40ms for a 
316
368
        # loopback connection ordinarily.
317
369
        start_time = time.time()
318
370
        self.get_server().add_latency = 0.5
362
414
class TestSocketDelay(TestCase):
363
415
 
364
416
    def setUp(self):
365
 
        super(TestSocketDelay, self).setUp()
366
 
        self.requireFeature(features.paramiko)
 
417
        TestCase.setUp(self)
 
418
        if not paramiko_loaded:
 
419
            raise TestSkipped('you must have paramiko to run this test')
367
420
 
368
421
    def test_delay(self):
 
422
        from bzrlib.transport.sftp import SocketDelay
369
423
        sending = FakeSocket()
370
 
        receiving = stub_sftp.SocketDelay(sending, 0.1, bandwidth=1000000,
371
 
                                          really_sleep=False)
 
424
        receiving = SocketDelay(sending, 0.1, bandwidth=1000000,
 
425
                                really_sleep=False)
372
426
        # check that simulated time is charged only per round-trip:
373
 
        t1 = stub_sftp.SocketDelay.simulated_time
 
427
        t1 = SocketDelay.simulated_time
374
428
        receiving.send("connect1")
375
429
        self.assertEqual(sending.recv(1024), "connect1")
376
 
        t2 = stub_sftp.SocketDelay.simulated_time
 
430
        t2 = SocketDelay.simulated_time
377
431
        self.assertAlmostEqual(t2 - t1, 0.1)
378
432
        receiving.send("connect2")
379
433
        self.assertEqual(sending.recv(1024), "connect2")
380
434
        sending.send("hello")
381
435
        self.assertEqual(receiving.recv(1024), "hello")
382
 
        t3 = stub_sftp.SocketDelay.simulated_time
 
436
        t3 = SocketDelay.simulated_time
383
437
        self.assertAlmostEqual(t3 - t2, 0.1)
384
438
        sending.send("hello")
385
439
        self.assertEqual(receiving.recv(1024), "hello")
387
441
        self.assertEqual(receiving.recv(1024), "hello")
388
442
        sending.send("hello")
389
443
        self.assertEqual(receiving.recv(1024), "hello")
390
 
        t4 = stub_sftp.SocketDelay.simulated_time
 
444
        t4 = SocketDelay.simulated_time
391
445
        self.assertAlmostEqual(t4, t3)
392
446
 
393
447
    def test_bandwidth(self):
 
448
        from bzrlib.transport.sftp import SocketDelay
394
449
        sending = FakeSocket()
395
 
        receiving = stub_sftp.SocketDelay(sending, 0, bandwidth=8.0/(1024*1024),
396
 
                                          really_sleep=False)
 
450
        receiving = SocketDelay(sending, 0, bandwidth=8.0/(1024*1024),
 
451
                                really_sleep=False)
397
452
        # check that simulated time is charged only per round-trip:
398
 
        t1 = stub_sftp.SocketDelay.simulated_time
 
453
        t1 = SocketDelay.simulated_time
399
454
        receiving.send("connect")
400
455
        self.assertEqual(sending.recv(1024), "connect")
401
456
        sending.send("a" * 100)
402
457
        self.assertEqual(receiving.recv(1024), "a" * 100)
403
 
        t2 = stub_sftp.SocketDelay.simulated_time
 
458
        t2 = SocketDelay.simulated_time
404
459
        self.assertAlmostEqual(t2 - t1, 100 + 7)
405
460
 
406
461
 
407
 
class ReadvFile(object):
408
 
    """An object that acts like Paramiko's SFTPFile when readv() is used"""
409
 
 
410
 
    def __init__(self, data):
411
 
        self._data = data
412
 
 
413
 
    def readv(self, requests):
414
 
        for start, length in requests:
415
 
            yield self._data[start:start+length]
416
 
 
417
 
    def close(self):
418
 
        pass
419
 
 
420
 
 
421
 
def _null_report_activity(*a, **k):
422
 
    pass
423
 
 
424
 
 
425
 
class Test_SFTPReadvHelper(tests.TestCase):
426
 
 
427
 
    def checkGetRequests(self, expected_requests, offsets):
428
 
        self.requireFeature(features.paramiko)
429
 
        helper = _mod_sftp._SFTPReadvHelper(offsets, 'artificial_test',
430
 
            _null_report_activity)
431
 
        self.assertEqual(expected_requests, helper._get_requests())
432
 
 
433
 
    def test__get_requests(self):
434
 
        # Small single requests become a single readv request
435
 
        self.checkGetRequests([(0, 100)],
436
 
                              [(0, 20), (30, 50), (20, 10), (80, 20)])
437
 
        # Non-contiguous ranges are given as multiple requests
438
 
        self.checkGetRequests([(0, 20), (30, 50)],
439
 
                              [(10, 10), (30, 20), (0, 10), (50, 30)])
440
 
        # Ranges larger than _max_request_size (32kB) are broken up into
441
 
        # multiple requests, even if it actually spans multiple logical
442
 
        # requests
443
 
        self.checkGetRequests([(0, 32768), (32768, 32768), (65536, 464)],
444
 
                              [(0, 40000), (40000, 100), (40100, 1900),
445
 
                               (42000, 24000)])
446
 
 
447
 
    def checkRequestAndYield(self, expected, data, offsets):
448
 
        self.requireFeature(features.paramiko)
449
 
        helper = _mod_sftp._SFTPReadvHelper(offsets, 'artificial_test',
450
 
            _null_report_activity)
451
 
        data_f = ReadvFile(data)
452
 
        result = list(helper.request_and_yield_offsets(data_f))
453
 
        self.assertEqual(expected, result)
454
 
 
455
 
    def test_request_and_yield_offsets(self):
456
 
        data = 'abcdefghijklmnopqrstuvwxyz'
457
 
        self.checkRequestAndYield([(0, 'a'), (5, 'f'), (10, 'klm')], data,
458
 
                                  [(0, 1), (5, 1), (10, 3)])
459
 
        # Should combine requests, and split them again
460
 
        self.checkRequestAndYield([(0, 'a'), (1, 'b'), (10, 'klm')], data,
461
 
                                  [(0, 1), (1, 1), (10, 3)])
462
 
        # Out of order requests. The requests should get combined, but then be
463
 
        # yielded out-of-order. We also need one that is at the end of a
464
 
        # previous range. See bug #293746
465
 
        self.checkRequestAndYield([(0, 'a'), (10, 'k'), (4, 'efg'), (1, 'bcd')],
466
 
                                  data, [(0, 1), (10, 1), (4, 3), (1, 3)])
467
 
 
468
 
 
469
 
class TestUsesAuthConfig(TestCaseWithSFTPServer):
470
 
    """Test that AuthenticationConfig can supply default usernames."""
471
 
 
472
 
    def get_transport_for_connection(self, set_config):
473
 
        port = self.get_server().port
474
 
        if set_config:
475
 
            conf = config.AuthenticationConfig()
476
 
            conf._get_config().update(
477
 
                {'sftptest': {'scheme': 'ssh', 'port': port, 'user': 'bar'}})
478
 
            conf._save()
479
 
        t = _mod_transport.get_transport_from_url(
480
 
            'sftp://localhost:%d' % port)
481
 
        # force a connection to be performed.
482
 
        t.has('foo')
483
 
        return t
484
 
 
485
 
    def test_sftp_uses_config(self):
486
 
        t = self.get_transport_for_connection(set_config=True)
487
 
        self.assertEqual('bar', t._get_credentials()[0])
488
 
 
489
 
    def test_sftp_is_none_if_no_config(self):
490
 
        t = self.get_transport_for_connection(set_config=False)
491
 
        self.assertIs(None, t._get_credentials()[0])
492
 
 
493
 
    def test_sftp_doesnt_prompt_username(self):
494
 
        stdout = tests.StringIOWrapper()
495
 
        ui.ui_factory = tests.TestUIFactory(stdin='joe\nfoo\n', stdout=stdout)
496
 
        t = self.get_transport_for_connection(set_config=False)
497
 
        self.assertIs(None, t._get_credentials()[0])
498
 
        # No prompts should've been printed, stdin shouldn't have been read
499
 
        self.assertEqual("", stdout.getvalue())
500
 
        self.assertEqual(0, ui.ui_factory.stdin.tell())