← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/tests/test_sftp_transport.py

  • Committer: Patch Queue Manager
  • Date: 2016-04-21 04:10:52 UTC
  • mfrom: (6616.1.1 fix-en-user-guide)
  • Revision ID: pqm@pqm.ubuntu.com-20160421041052-clcye7ns1qcl2n7w
(richard-wilbur) Ensure build of English use guide always uses English text
 even when user's locale specifies a different language. (Jelmer Vernooij)

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/tests/test_sftp_transport.py
1
 
# Copyright (C) 2005 Robey Pointer <robey@lag.net>
 
1
# Copyright (C) 2005-2012, 2016 Robey Pointer <robey@lag.net>
2
2
# Copyright (C) 2005, 2006, 2007 Canonical Ltd
3
3
#
4
4
# This program is free software; you can redistribute it and/or modify
13
13
#
14
14
# You should have received a copy of the GNU General Public License
15
15
# along with this program; if not, write to the Free Software
16
 
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
16
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
 
18
18
import os
19
19
import socket
20
20
import sys
21
 
import threading
22
21
import time
23
22
 
24
 
try:
25
 
    import paramiko
26
 
    paramiko_loaded = True
27
 
except ImportError:
28
 
    paramiko_loaded = False
29
 
 
30
23
from bzrlib import (
31
 
    bzrdir,
 
24
    config,
 
25
    controldir,
32
26
    errors,
 
27
    tests,
 
28
    transport as _mod_transport,
 
29
    ui,
33
30
    )
34
31
from bzrlib.osutils import (
35
 
    pathjoin,
36
32
    lexists,
37
 
    set_or_unset_env,
38
33
    )
39
34
from bzrlib.tests import (
 
35
    features,
40
36
    TestCaseWithTransport,
41
37
    TestCase,
42
38
    TestSkipped,
43
39
    )
44
40
from bzrlib.tests.http_server import HttpServer
45
 
from bzrlib.transport import get_transport
46
41
import bzrlib.transport.http
47
42
 
48
 
if paramiko_loaded:
49
 
    from bzrlib.transport.sftp import (
50
 
        SFTPAbsoluteServer,
51
 
        SFTPHomeDirServer,
52
 
        SFTPTransport,
53
 
        )
54
 
 
55
 
from bzrlib.workingtree import WorkingTree
 
43
if features.paramiko.available():
 
44
    from bzrlib.transport import sftp as _mod_sftp
 
45
    from bzrlib.tests import stub_sftp
56
46
 
57
47
 
58
48
def set_test_transport_to_sftp(testcase):
60
50
    if getattr(testcase, '_get_remote_is_absolute', None) is None:
61
51
        testcase._get_remote_is_absolute = True
62
52
    if testcase._get_remote_is_absolute:
63
 
        testcase.transport_server = SFTPAbsoluteServer
 
53
        testcase.transport_server = stub_sftp.SFTPAbsoluteServer
64
54
    else:
65
 
        testcase.transport_server = SFTPHomeDirServer
 
55
        testcase.transport_server = stub_sftp.SFTPHomeDirServer
66
56
    testcase.transport_readonly_server = HttpServer
67
57
 
68
58
 
71
61
 
72
62
    def setUp(self):
73
63
        super(TestCaseWithSFTPServer, self).setUp()
74
 
        if not paramiko_loaded:
75
 
            raise TestSkipped('you must have paramiko to run this test')
 
64
        self.requireFeature(features.paramiko)
76
65
        set_test_transport_to_sftp(self)
77
66
 
78
67
 
79
 
class SFTPLockTests (TestCaseWithSFTPServer):
 
68
class SFTPLockTests(TestCaseWithSFTPServer):
80
69
 
81
70
    def test_sftp_locks(self):
82
71
        from bzrlib.errors import LockError
83
72
        t = self.get_transport()
84
73
 
85
74
        l = t.lock_write('bogus')
86
 
        self.failUnlessExists('bogus.write-lock')
 
75
        self.assertPathExists('bogus.write-lock')
87
76
 
88
77
        # Don't wait for the lock, locking an already locked
89
78
        # file should raise an assert
90
79
        self.assertRaises(LockError, t.lock_write, 'bogus')
91
80
 
92
81
        l.unlock()
93
 
        self.failIf(lexists('bogus.write-lock'))
 
82
        self.assertFalse(lexists('bogus.write-lock'))
94
83
 
95
 
        open('something.write-lock', 'wb').write('fake lock\n')
 
84
        with open('something.write-lock', 'wb') as f: f.write('fake lock\n')
96
85
        self.assertRaises(LockError, t.lock_write, 'something')
97
86
        os.remove('something.write-lock')
98
87
 
151
140
    def test__remote_path_relative_root(self):
152
141
        # relative paths are preserved
153
142
        t = self.get_transport('')
154
 
        self.assertEqual('/~/', t._path)
 
143
        self.assertEqual('/~/', t._parsed_url.path)
155
144
        # the remote path should be relative to home dir
156
145
        # (i.e. not begining with a '/')
157
146
        self.assertEqual('a', t._remote_path('a'))
158
147
 
159
148
 
160
149
class SFTPNonServerTest(TestCase):
 
150
 
161
151
    def setUp(self):
162
 
        TestCase.setUp(self)
163
 
        if not paramiko_loaded:
164
 
            raise TestSkipped('you must have paramiko to run this test')
 
152
        super(SFTPNonServerTest, self).setUp()
 
153
        self.requireFeature(features.paramiko)
165
154
 
166
155
    def test_parse_url_with_home_dir(self):
167
 
        s = SFTPTransport('sftp://ro%62ey:h%40t@example.com:2222/~/relative')
168
 
        self.assertEquals(s._host, 'example.com')
169
 
        self.assertEquals(s._port, 2222)
170
 
        self.assertEquals(s._user, 'robey')
171
 
        self.assertEquals(s._password, 'h@t')
172
 
        self.assertEquals(s._path, '/~/relative/')
 
156
        s = _mod_sftp.SFTPTransport(
 
157
            'sftp://ro%62ey:h%40t@example.com:2222/~/relative')
 
158
        self.assertEqual(s._parsed_url.host, 'example.com')
 
159
        self.assertEqual(s._parsed_url.port, 2222)
 
160
        self.assertEqual(s._parsed_url.user, 'robey')
 
161
        self.assertEqual(s._parsed_url.password, 'h@t')
 
162
        self.assertEqual(s._parsed_url.path, '/~/relative/')
173
163
 
174
164
    def test_relpath(self):
175
 
        s = SFTPTransport('sftp://user@host.com/abs/path')
 
165
        s = _mod_sftp.SFTPTransport('sftp://user@host.com/abs/path')
176
166
        self.assertRaises(errors.PathNotChild, s.relpath,
177
167
                          'sftp://user@host.com/~/rel/path/sub')
178
168
 
180
170
        """Test that if no 'ssh' is available we get builtin paramiko"""
181
171
        from bzrlib.transport import ssh
182
172
        # set '.' as the only location in the path, forcing no 'ssh' to exist
183
 
        orig_vendor = ssh._ssh_vendor_manager._cached_ssh_vendor
184
 
        orig_path = set_or_unset_env('PATH', '.')
185
 
        try:
186
 
            # No vendor defined yet, query for one
187
 
            ssh._ssh_vendor_manager.clear_cache()
188
 
            vendor = ssh._get_ssh_vendor()
189
 
            self.assertIsInstance(vendor, ssh.ParamikoVendor)
190
 
        finally:
191
 
            set_or_unset_env('PATH', orig_path)
192
 
            ssh._ssh_vendor_manager._cached_ssh_vendor = orig_vendor
 
173
        self.overrideAttr(ssh, '_ssh_vendor_manager')
 
174
        self.overrideEnv('PATH', '.')
 
175
        ssh._ssh_vendor_manager.clear_cache()
 
176
        vendor = ssh._get_ssh_vendor()
 
177
        self.assertIsInstance(vendor, ssh.ParamikoVendor)
193
178
 
194
179
    def test_abspath_root_sibling_server(self):
195
 
        from bzrlib.transport.sftp import SFTPSiblingAbsoluteServer
196
 
        server = SFTPSiblingAbsoluteServer()
197
 
        server.setUp()
198
 
        try:
199
 
            transport = get_transport(server.get_url())
200
 
            self.assertFalse(transport.abspath('/').endswith('/~/'))
201
 
            self.assertTrue(transport.abspath('/').endswith('/'))
202
 
            del transport
203
 
        finally:
204
 
            server.tearDown()
 
180
        server = stub_sftp.SFTPSiblingAbsoluteServer()
 
181
        server.start_server()
 
182
        self.addCleanup(server.stop_server)
 
183
 
 
184
        transport = _mod_transport.get_transport_from_url(server.get_url())
 
185
        self.assertFalse(transport.abspath('/').endswith('/~/'))
 
186
        self.assertTrue(transport.abspath('/').endswith('/'))
 
187
        del transport
205
188
 
206
189
 
207
190
class SFTPBranchTest(TestCaseWithSFTPServer):
208
191
    """Test some stuff when accessing a bzr Branch over sftp"""
209
192
 
210
 
    def test_lock_file(self):
211
 
        # old format branches use a special lock file on sftp.
212
 
        b = self.make_branch('', format=bzrdir.BzrDirFormat6())
213
 
        b = bzrlib.branch.Branch.open(self.get_url())
214
 
        self.failUnlessExists('.bzr/')
215
 
        self.failUnlessExists('.bzr/branch-format')
216
 
        self.failUnlessExists('.bzr/branch-lock')
217
 
 
218
 
        self.failIf(lexists('.bzr/branch-lock.write-lock'))
219
 
        b.lock_write()
220
 
        self.failUnlessExists('.bzr/branch-lock.write-lock')
221
 
        b.unlock()
222
 
        self.failIf(lexists('.bzr/branch-lock.write-lock'))
223
 
 
224
193
    def test_push_support(self):
225
194
        self.build_tree(['a/', 'a/foo'])
226
 
        t = bzrdir.BzrDir.create_standalone_workingtree('a')
 
195
        t = controldir.ControlDir.create_standalone_workingtree('a')
227
196
        b = t.branch
228
197
        t.add('foo')
229
198
        t.commit('foo', rev_id='a1')
230
199
 
231
 
        b2 = bzrdir.BzrDir.create_branch_and_repo(self.get_url('/b'))
 
200
        b2 = controldir.ControlDir.create_branch_and_repo(self.get_url('/b'))
232
201
        b2.pull(b)
233
202
 
234
 
        self.assertEquals(b2.revision_history(), ['a1'])
 
203
        self.assertEqual(b2.last_revision(), 'a1')
235
204
 
236
 
        open('a/foo', 'wt').write('something new in foo\n')
 
205
        with open('a/foo', 'wt') as f: f.write('something new in foo\n')
237
206
        t.commit('new', rev_id='a2')
238
207
        b2.pull(b)
239
208
 
240
 
        self.assertEquals(b2.revision_history(), ['a1', 'a2'])
 
209
        self.assertEqual(b2.last_revision(), 'a2')
241
210
 
242
211
 
243
212
class SSHVendorConnection(TestCaseWithSFTPServer):
254
223
      None:       If 'ssh' exists on the machine, then it will be spawned as a
255
224
                  child process.
256
225
    """
257
 
    
 
226
 
258
227
    def setUp(self):
259
228
        super(SSHVendorConnection, self).setUp()
260
 
        from bzrlib.transport.sftp import SFTPFullAbsoluteServer
261
229
 
262
230
        def create_server():
263
231
            """Just a wrapper so that when created, it will set _vendor"""
264
232
            # SFTPFullAbsoluteServer can handle any vendor,
265
233
            # it just needs to be set between the time it is instantiated
266
234
            # and the time .setUp() is called
267
 
            server = SFTPFullAbsoluteServer()
 
235
            server = stub_sftp.SFTPFullAbsoluteServer()
268
236
            server._vendor = self._test_vendor
269
237
            return server
270
238
        self._test_vendor = 'loopback'
302
270
    """
303
271
 
304
272
    def setUp(self):
305
 
        if not paramiko_loaded:
306
 
            raise TestSkipped('you must have paramiko to run this test')
 
273
        self.requireFeature(features.paramiko)
307
274
        super(SSHVendorBadConnection, self).setUp()
308
 
        import bzrlib.transport.ssh
309
275
 
310
276
        # open a random port, so we know nobody else is using it
311
277
        # but don't actually listen on the port.
312
278
        s = socket.socket()
313
279
        s.bind(('localhost', 0))
 
280
        self.addCleanup(s.close)
314
281
        self.bogus_url = 'sftp://%s:%s/' % s.getsockname()
315
282
 
316
 
        orig_vendor = bzrlib.transport.ssh._ssh_vendor_manager._cached_ssh_vendor
317
 
        def reset():
318
 
            bzrlib.transport.ssh._ssh_vendor_manager._cached_ssh_vendor = orig_vendor
319
 
            s.close()
320
 
        self.addCleanup(reset)
321
 
 
322
 
    def set_vendor(self, vendor):
323
 
        import bzrlib.transport.ssh
324
 
        bzrlib.transport.ssh._ssh_vendor_manager._cached_ssh_vendor = vendor
 
283
    def set_vendor(self, vendor, subprocess_stderr=None):
 
284
        from bzrlib.transport import ssh
 
285
        self.overrideAttr(ssh._ssh_vendor_manager, '_cached_ssh_vendor', vendor)
 
286
        if subprocess_stderr is not None:
 
287
            self.overrideAttr(ssh.SubprocessVendor, "_stderr_target",
 
288
                subprocess_stderr)
325
289
 
326
290
    def test_bad_connection_paramiko(self):
327
291
        """Test that a real connection attempt raises the right error"""
328
292
        from bzrlib.transport import ssh
329
293
        self.set_vendor(ssh.ParamikoVendor())
330
 
        t = bzrlib.transport.get_transport(self.bogus_url)
 
294
        t = _mod_transport.get_transport_from_url(self.bogus_url)
331
295
        self.assertRaises(errors.ConnectionError, t.get, 'foobar')
332
296
 
333
297
    def test_bad_connection_ssh(self):
334
298
        """None => auto-detect vendor"""
335
 
        self.set_vendor(None)
336
 
        # This is how I would normally test the connection code
337
 
        # it makes it very clear what we are testing.
338
 
        # However, 'ssh' will create stipple on the output, so instead
339
 
        # I'm using run_bzr_subprocess, and parsing the output
340
 
        # try:
341
 
        #     t = bzrlib.transport.get_transport(self.bogus_url)
342
 
        # except errors.ConnectionError:
343
 
        #     # Correct error
344
 
        #     pass
345
 
        # except errors.NameError, e:
346
 
        #     if 'SSHException' in str(e):
347
 
        #         raise TestSkipped('Known NameError bug in paramiko 1.6.1')
348
 
        #     raise
349
 
        # else:
350
 
        #     self.fail('Excepted ConnectionError to be raised')
351
 
 
352
 
        out, err = self.run_bzr_subprocess(['log', self.bogus_url], retcode=3)
353
 
        self.assertEqual('', out)
354
 
        if "NameError: global name 'SSHException'" in err:
355
 
            # We aren't fixing this bug, because it is a bug in
356
 
            # paramiko, but we know about it, so we don't have to
357
 
            # fail the test
358
 
            raise TestSkipped('Known NameError bug with paramiko-1.6.1')
359
 
        self.assertContainsRe(err, r'bzr: ERROR: Unable to connect to SSH host'
360
 
                                   r' 127\.0\.0\.1:\d+; ')
 
299
        f = file(os.devnull, "wb")
 
300
        self.addCleanup(f.close)
 
301
        self.set_vendor(None, f)
 
302
        t = _mod_transport.get_transport_from_url(self.bogus_url)
 
303
        try:
 
304
            self.assertRaises(errors.ConnectionError, t.get, 'foobar')
 
305
        except NameError, e:
 
306
            if "global name 'SSHException'" in str(e):
 
307
                self.knownFailure('Known NameError bug in paramiko 1.6.1')
 
308
            raise
361
309
 
362
310
 
363
311
class SFTPLatencyKnob(TestCaseWithSFTPServer):
364
312
    """Test that the testing SFTPServer's latency knob works."""
365
313
 
366
314
    def test_latency_knob_slows_transport(self):
367
 
        # change the latency knob to 500ms. We take about 40ms for a 
 
315
        # change the latency knob to 500ms. We take about 40ms for a
368
316
        # loopback connection ordinarily.
369
317
        start_time = time.time()
370
318
        self.get_server().add_latency = 0.5
414
362
class TestSocketDelay(TestCase):
415
363
 
416
364
    def setUp(self):
417
 
        TestCase.setUp(self)
418
 
        if not paramiko_loaded:
419
 
            raise TestSkipped('you must have paramiko to run this test')
 
365
        super(TestSocketDelay, self).setUp()
 
366
        self.requireFeature(features.paramiko)
420
367
 
421
368
    def test_delay(self):
422
 
        from bzrlib.transport.sftp import SocketDelay
423
369
        sending = FakeSocket()
424
 
        receiving = SocketDelay(sending, 0.1, bandwidth=1000000,
425
 
                                really_sleep=False)
 
370
        receiving = stub_sftp.SocketDelay(sending, 0.1, bandwidth=1000000,
 
371
                                          really_sleep=False)
426
372
        # check that simulated time is charged only per round-trip:
427
 
        t1 = SocketDelay.simulated_time
 
373
        t1 = stub_sftp.SocketDelay.simulated_time
428
374
        receiving.send("connect1")
429
375
        self.assertEqual(sending.recv(1024), "connect1")
430
 
        t2 = SocketDelay.simulated_time
 
376
        t2 = stub_sftp.SocketDelay.simulated_time
431
377
        self.assertAlmostEqual(t2 - t1, 0.1)
432
378
        receiving.send("connect2")
433
379
        self.assertEqual(sending.recv(1024), "connect2")
434
380
        sending.send("hello")
435
381
        self.assertEqual(receiving.recv(1024), "hello")
436
 
        t3 = SocketDelay.simulated_time
 
382
        t3 = stub_sftp.SocketDelay.simulated_time
437
383
        self.assertAlmostEqual(t3 - t2, 0.1)
438
384
        sending.send("hello")
439
385
        self.assertEqual(receiving.recv(1024), "hello")
441
387
        self.assertEqual(receiving.recv(1024), "hello")
442
388
        sending.send("hello")
443
389
        self.assertEqual(receiving.recv(1024), "hello")
444
 
        t4 = SocketDelay.simulated_time
 
390
        t4 = stub_sftp.SocketDelay.simulated_time
445
391
        self.assertAlmostEqual(t4, t3)
446
392
 
447
393
    def test_bandwidth(self):
448
 
        from bzrlib.transport.sftp import SocketDelay
449
394
        sending = FakeSocket()
450
 
        receiving = SocketDelay(sending, 0, bandwidth=8.0/(1024*1024),
451
 
                                really_sleep=False)
 
395
        receiving = stub_sftp.SocketDelay(sending, 0, bandwidth=8.0/(1024*1024),
 
396
                                          really_sleep=False)
452
397
        # check that simulated time is charged only per round-trip:
453
 
        t1 = SocketDelay.simulated_time
 
398
        t1 = stub_sftp.SocketDelay.simulated_time
454
399
        receiving.send("connect")
455
400
        self.assertEqual(sending.recv(1024), "connect")
456
401
        sending.send("a" * 100)
457
402
        self.assertEqual(receiving.recv(1024), "a" * 100)
458
 
        t2 = SocketDelay.simulated_time
 
403
        t2 = stub_sftp.SocketDelay.simulated_time
459
404
        self.assertAlmostEqual(t2 - t1, 100 + 7)
460
405
 
461
406
 
 
407
class ReadvFile(object):
 
408
    """An object that acts like Paramiko's SFTPFile when readv() is used"""
 
409
 
 
410
    def __init__(self, data):
 
411
        self._data = data
 
412
 
 
413
    def readv(self, requests):
 
414
        for start, length in requests:
 
415
            yield self._data[start:start+length]
 
416
 
 
417
    def close(self):
 
418
        pass
 
419
 
 
420
 
 
421
def _null_report_activity(*a, **k):
 
422
    pass
 
423
 
 
424
 
 
425
class Test_SFTPReadvHelper(tests.TestCase):
 
426
 
 
427
    def checkGetRequests(self, expected_requests, offsets):
 
428
        self.requireFeature(features.paramiko)
 
429
        helper = _mod_sftp._SFTPReadvHelper(offsets, 'artificial_test',
 
430
            _null_report_activity)
 
431
        self.assertEqual(expected_requests, helper._get_requests())
 
432
 
 
433
    def test__get_requests(self):
 
434
        # Small single requests become a single readv request
 
435
        self.checkGetRequests([(0, 100)],
 
436
                              [(0, 20), (30, 50), (20, 10), (80, 20)])
 
437
        # Non-contiguous ranges are given as multiple requests
 
438
        self.checkGetRequests([(0, 20), (30, 50)],
 
439
                              [(10, 10), (30, 20), (0, 10), (50, 30)])
 
440
        # Ranges larger than _max_request_size (32kB) are broken up into
 
441
        # multiple requests, even if it actually spans multiple logical
 
442
        # requests
 
443
        self.checkGetRequests([(0, 32768), (32768, 32768), (65536, 464)],
 
444
                              [(0, 40000), (40000, 100), (40100, 1900),
 
445
                               (42000, 24000)])
 
446
 
 
447
    def checkRequestAndYield(self, expected, data, offsets):
 
448
        self.requireFeature(features.paramiko)
 
449
        helper = _mod_sftp._SFTPReadvHelper(offsets, 'artificial_test',
 
450
            _null_report_activity)
 
451
        data_f = ReadvFile(data)
 
452
        result = list(helper.request_and_yield_offsets(data_f))
 
453
        self.assertEqual(expected, result)
 
454
 
 
455
    def test_request_and_yield_offsets(self):
 
456
        data = 'abcdefghijklmnopqrstuvwxyz'
 
457
        self.checkRequestAndYield([(0, 'a'), (5, 'f'), (10, 'klm')], data,
 
458
                                  [(0, 1), (5, 1), (10, 3)])
 
459
        # Should combine requests, and split them again
 
460
        self.checkRequestAndYield([(0, 'a'), (1, 'b'), (10, 'klm')], data,
 
461
                                  [(0, 1), (1, 1), (10, 3)])
 
462
        # Out of order requests. The requests should get combined, but then be
 
463
        # yielded out-of-order. We also need one that is at the end of a
 
464
        # previous range. See bug #293746
 
465
        self.checkRequestAndYield([(0, 'a'), (10, 'k'), (4, 'efg'), (1, 'bcd')],
 
466
                                  data, [(0, 1), (10, 1), (4, 3), (1, 3)])
 
467
 
 
468
 
 
469
class TestUsesAuthConfig(TestCaseWithSFTPServer):
 
470
    """Test that AuthenticationConfig can supply default usernames."""
 
471
 
 
472
    def get_transport_for_connection(self, set_config):
 
473
        port = self.get_server().port
 
474
        if set_config:
 
475
            conf = config.AuthenticationConfig()
 
476
            conf._get_config().update(
 
477
                {'sftptest': {'scheme': 'ssh', 'port': port, 'user': 'bar'}})
 
478
            conf._save()
 
479
        t = _mod_transport.get_transport_from_url(
 
480
            'sftp://localhost:%d' % port)
 
481
        # force a connection to be performed.
 
482
        t.has('foo')
 
483
        return t
 
484
 
 
485
    def test_sftp_uses_config(self):
 
486
        t = self.get_transport_for_connection(set_config=True)
 
487
        self.assertEqual('bar', t._get_credentials()[0])
 
488
 
 
489
    def test_sftp_is_none_if_no_config(self):
 
490
        t = self.get_transport_for_connection(set_config=False)
 
491
        self.assertIs(None, t._get_credentials()[0])
 
492
 
 
493
    def test_sftp_doesnt_prompt_username(self):
 
494
        stdout = tests.StringIOWrapper()
 
495
        ui.ui_factory = tests.TestUIFactory(stdin='joe\nfoo\n', stdout=stdout)
 
496
        t = self.get_transport_for_connection(set_config=False)
 
497
        self.assertIs(None, t._get_credentials()[0])
 
498
        # No prompts should've been printed, stdin shouldn't have been read
 
499
        self.assertEqual("", stdout.getvalue())
 
500
        self.assertEqual(0, ui.ui_factory.stdin.tell())