~bzr-pqm/bzr/bzr.dev : revision 1470

1

2

3

# This program is free software; you can redistribute it and/or modify

4

# it under the terms of the GNU General Public License as published by

5

# the Free Software Foundation; either version 2 of the License, or

6

# (at your option) any later version.

7

8

# This program is distributed in the hope that it will be useful,

9

# but WITHOUT ANY WARRANTY; without even the implied warranty of

10

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

11

# GNU General Public License for more details.

12

13

# You should have received a copy of the GNU General Public License

14

# along with this program; if not, write to the Free Software

15

# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA

16

17

"""

18

Implementation of Transport over SFTP, using paramiko.

19

"""

20

21

import getpass

22

import os

23

import re

24

import stat

25

import sys

26

import urllib

27

28

from bzrlib.errors import TransportNotPossible, NoSuchFile, NonRelativePath, TransportError

29

from bzrlib.config import config_dir

30

from bzrlib.trace import mutter, warning, error

31

from bzrlib.transport import Transport, register_transport

32

33

try:

34

import paramiko

35

except ImportError:

36

error('The SFTP plugin requires paramiko.')

37

raise

38

39

40

SYSTEM_HOSTKEYS = {}

41

BZR_HOSTKEYS = {}

42

43

def load_host_keys():

44

"""

45

Load system host keys (probably doesn't work on windows) and any

46

"discovered" keys from previous sessions.

47

"""

48

global SYSTEM_HOSTKEYS, BZR_HOSTKEYS

49

try:

50

SYSTEM_HOSTKEYS = paramiko.util.load_host_keys(os.path.expanduser('~/.ssh/known_hosts'))

51

except Exception, e:

52

mutter('failed to load system host keys: ' + str(e))

53

bzr_hostkey_path = os.path.join(config_dir(), 'ssh_host_keys')

54

try:

55

BZR_HOSTKEYS = paramiko.util.load_host_keys(bzr_hostkey_path)

56

except Exception, e:

57

mutter('failed to load bzr host keys: ' + str(e))

58

save_host_keys()

59

60

def save_host_keys():

61

"""

62

Save "discovered" host keys in $(config)/ssh_host_keys/.

63

"""

64

global SYSTEM_HOSTKEYS, BZR_HOSTKEYS

65

bzr_hostkey_path = os.path.join(config_dir(), 'ssh_host_keys')

66

if not os.path.isdir(config_dir()):

67

os.mkdir(config_dir())

68

try:

69

f = open(bzr_hostkey_path, 'w')

70

f.write('# SSH host keys collected by bzr\n')

71

for hostname, keys in BZR_HOSTKEYS.iteritems():

72

for keytype, key in keys.iteritems():

73

f.write('%s %s %s\n' % (hostname, keytype, key.get_base64()))

74

f.close()

75

except IOError, e:

76

mutter('failed to save bzr host keys: ' + str(e))

77

78

79

80

class SFTPTransportError (TransportError):

81

pass

82

83

84

class SFTPTransport (Transport):

85

"""

86

Transport implementation for SFTP access.

87

"""

88

89

_url_matcher = re.compile(r'^sftp://([^@]*@)?(.*?)(:\d+)?(/.*)?$')

90

91

def __init__(self, base, clone_from=None):

92

assert base.startswith('sftp://')

93

super(SFTPTransport, self).__init__(base)

94

self._parse_url(base)

95

if clone_from is None:

96

self._sftp_connect()

97

else:

98

# use the same ssh connection, etc

99

self._sftp = clone_from._sftp

100

# super saves 'self.base'

101

102

def should_cache(self):

103

"""

104

Return True if the data pulled across should be cached locally.

105

"""

106

return True

107

108

def clone(self, offset=None):

109

"""

110

Return a new SFTPTransport with root at self.base + offset.

111

We share the same SFTP session between such transports, because it's

112

fairly expensive to set them up.

113

"""

114

if offset is None:

115

return SFTPTransport(self.base, self)

116

else:

117

return SFTPTransport(self.abspath(offset), self)

118

119

def abspath(self, relpath):

120

"""

121

Return the full url to the given relative path.

122

123

@param relpath: the relative path or path components

124

@type relpath: str or list

125

"""

126

return self._unparse_url(self._abspath(relpath))

127

128

def _abspath(self, relpath):

129

"""Return the absolute path segment without the SFTP URL."""

130

# FIXME: share the common code across transports

131

assert isinstance(relpath, basestring)

132

relpath = [urllib.unquote(relpath)]

133

basepath = self._path.split('/')

134

if len(basepath) > 0 and basepath[-1] == '':

135

basepath = basepath[:-1]

136

137

for p in relpath:

138

if p == '..':

139

if len(basepath) == 0:

140

# In most filesystems, a request for the parent

141

# of root, just returns root.

142

continue

143

basepath.pop()

144

elif p == '.':

145

continue # No-op

146

else:

147

basepath.append(p)

148

149

path = '/'.join(basepath)

150

if path[0] != '/':

151

path = '/' + path

152

return path

153

154

def relpath(self, abspath):

155

# FIXME: this is identical to HttpTransport -- share it

156

if not abspath.startswith(self.base):

157

raise NonRelativePath('path %r is not under base URL %r'

158

% (abspath, self.base))

159

pl = len(self.base)

160

return abspath[pl:].lstrip('/')

161

162

def has(self, relpath):

163

"""

164

Does the target location exist?

165

"""

166

try:

167

self._sftp.stat(self._abspath(relpath))

168

return True

169

except IOError:

170

return False

171

172

def get(self, relpath, decode=False):

173

"""

174

Get the file at the given relative path.

175

176

:param relpath: The relative path to the file

177

"""

178

try:

179

path = self._abspath(relpath)

180

return self._sftp.file(path)

181

except (IOError, paramiko.SSHException), x:

182

raise NoSuchFile('Error retrieving %s: %s' % (path, str(x)), x)

183

184

def get_partial(self, relpath, start, length=None):

185

"""

186

Get just part of a file.

187

188

:param relpath: Path to the file, relative to base

189

:param start: The starting position to read from

190

:param length: The length to read. A length of None indicates

191

read to the end of the file.

192

:return: A file-like object containing at least the specified bytes.

193

Some implementations may return objects which can be read

194

past this length, but this is not guaranteed.

195

"""

196

f = self.get(relpath)

197

f.seek(start)

198

return f

199

200

def put(self, relpath, f):

201

"""

202

Copy the file-like or string object into the location.

203

204

:param relpath: Location to put the contents, relative to base.

205

:param f: File-like or string object.

206

"""

207

# FIXME: should do something atomic or locking here, this is unsafe

208

try:

209

path = self._abspath(relpath)

210

fout = self._sftp.file(path, 'wb')

211

except (IOError, paramiko.SSHException), x:

212

raise SFTPTransportError('Unable to write file %r' % (path,), x)

213

try:

214

self._pump(f, fout)

215

finally:

216

fout.close()

217

218

def iter_files_recursive(self):

219

"""Walk the relative paths of all files in this transport."""

220

queue = list(self.list_dir('.'))

221

while queue:

222

relpath = queue.pop(0)

223

st = self.stat(relpath)

224

if stat.S_ISDIR(st.st_mode):

225

for i, basename in enumerate(self.list_dir(relpath)):

226

queue.insert(i, relpath+'/'+basename)

227

else:

228

yield relpath

229

230

def mkdir(self, relpath):

231

"""Create a directory at the given path."""

232

try:

233

path = self._abspath(relpath)

234

self._sftp.mkdir(path)

235

except (IOError, paramiko.SSHException), x:

236

raise SFTPTransportError('Unable to mkdir %r' % (path,), x)

237

238

def append(self, relpath, f):

239

"""

240

Append the text in the file-like object into the final

241

location.

242

"""

243

try:

244

path = self._abspath(relpath)

245

fout = self._sftp.file(path, 'ab')

246

self._pump(f, fout)

247

except (IOError, paramiko.SSHException), x:

248

raise SFTPTransportError('Unable to append file %r' % (path,), x)

249

250

def copy(self, rel_from, rel_to):

251

"""Copy the item at rel_from to the location at rel_to"""

252

path_from = self._abspath(rel_from)

253

path_to = self._abspath(rel_to)

254

try:

255

fin = self._sftp.file(path_from, 'rb')

256

try:

257

fout = self._sftp.file(path_to, 'wb')

258

try:

259

fout.set_pipelined(True)

260

self._pump(fin, fout)

261

finally:

262

fout.close()

263

finally:

264

fin.close()

265

except (IOError, paramiko.SSHException), x:

266

raise SFTPTransportError('Unable to copy %r to %r' % (path_from, path_to), x)

267

268

def move(self, rel_from, rel_to):

269

"""Move the item at rel_from to the location at rel_to"""

270

path_from = self._abspath(rel_from)

271

path_to = self._abspath(rel_to)

272

try:

273

self._sftp.rename(path_from, path_to)

274

except (IOError, paramiko.SSHException), x:

275

raise SFTPTransportError('Unable to move %r to %r' % (path_from, path_to), x)

276

277

def delete(self, relpath):

278

"""Delete the item at relpath"""

279

path = self._abspath(relpath)

280

try:

281

self._sftp.remove(path)

282

except (IOError, paramiko.SSHException), x:

283

raise SFTPTransportError('Unable to delete %r' % (path,), x)

284

285

def listable(self):

286

"""Return True if this store supports listing."""

287

return True

288

289

def list_dir(self, relpath):

290

"""

291

Return a list of all files at the given location.

292

"""

293

# does anything actually use this?

294

path = self._abspath(relpath)

295

try:

296

return self._sftp.listdir(path)

297

except (IOError, paramiko.SSHException), x:

298

raise SFTPTransportError('Unable to list folder %r' % (path,), x)

299

300

def stat(self, relpath):

301

"""Return the stat information for a file."""

302

path = self._abspath(relpath)

303

try:

304

return self._sftp.stat(path)

305

except (IOError, paramiko.SSHException), x:

306

raise SFTPTransportError('Unable to stat %r' % (path,), x)

307

308

def lock_read(self, relpath):

309

"""

310

Lock the given file for shared (read) access.

311

:return: A lock object, which should be passed to Transport.unlock()

312

"""

313

# FIXME: there should be something clever i can do here...

314

class BogusLock(object):

315

def __init__(self, path):

316

self.path = path

317

def unlock(self):

318

pass

319

return BogusLock(relpath)

320

321

def lock_write(self, relpath):

322

"""

323

Lock the given file for exclusive (write) access.

324

WARNING: many transports do not support this, so trying avoid using it

325

326

:return: A lock object, which should be passed to Transport.unlock()

327

"""

328

# FIXME: there should be something clever i can do here...

329

class BogusLock(object):

330

def __init__(self, path):

331

self.path = path

332

def unlock(self):

333

pass

334

return BogusLock(relpath)

335

336

337

def _unparse_url(self, path=None):

338

if path is None:

339

path = self._path

340

if self._port == 22:

341

return 'sftp://%s@%s%s' % (self._username, self._host, path)

342

return 'sftp://%s@%s:%d%s' % (self._username, self._host, self._port, path)

343

344

def _parse_url(self, url):

345

assert url[:7] == 'sftp://'

346

m = self._url_matcher.match(url)

347

if m is None:

348

raise SFTPTransportError('Unable to parse SFTP URL %r' % (url,))

349

self._username, self._host, self._port, self._path = m.groups()

350

if self._username is None:

351

self._username = getpass.getuser()

352

else:

353

self._username = self._username[:-1]

354

if self._port is None:

355

self._port = 22

356

else:

357

self._port = int(self._port[1:])

358

if (self._path is None) or (self._path == ''):

359

self._path = '/'

360

361

def _sftp_connect(self):

362

global SYSTEM_HOSTKEYS, BZR_HOSTKEYS

363

364

load_host_keys()

365

366

try:

367

t = paramiko.Transport((self._host, self._port))

368

t.start_client()

369

except paramiko.SSHException:

370

raise SFTPTransportError('Unable to reach SSH host %s:%d' % (self._host, self._port))

371

372

server_key = t.get_remote_server_key()

373

server_key_hex = paramiko.util.hexify(server_key.get_fingerprint())

374

keytype = server_key.get_name()

375

if SYSTEM_HOSTKEYS.has_key(self._host) and SYSTEM_HOSTKEYS[self._host].has_key(keytype):

376

our_server_key = SYSTEM_HOSTKEYS[self._host][keytype]

377

our_server_key_hex = paramiko.util.hexify(our_server_key.get_fingerprint())

378

elif BZR_HOSTKEYS.has_key(self._host) and BZR_HOSTKEYS[self._host].has_key(keytype):

379

our_server_key = BZR_HOSTKEYS[self._host][keytype]

380

our_server_key_hex = paramiko.util.hexify(our_server_key.get_fingerprint())

381

else:

382

warning('Adding %s host key for %s: %s' % (keytype, self._host, server_key_hex))

383

if not BZR_HOSTKEYS.has_key(self._host):

384

BZR_HOSTKEYS[self._host] = {}

385

BZR_HOSTKEYS[self._host][keytype] = server_key

386

our_server_key = server_key

387

our_server_key_hex = paramiko.util.hexify(our_server_key.get_fingerprint())

388

save_host_keys()

389

if server_key != our_server_key:

390

filename1 = os.path.expanduser('~/.ssh/known_hosts')

391

filename2 = os.path.join(config_dir(), 'ssh_host_keys')

392

raise SFTPTransportError('Host keys for %s do not match! %s != %s' % \

393

(self._host, our_server_key_hex, server_key_hex),

394

['Try editing %s or %s' % (filename1, filename2)])

395

396

self._sftp_auth(t, self._username, self._host)

397

398

try:

399

self._sftp = t.open_sftp_client()

400

except paramiko.SSHException:

401

raise BzrError('Unable to find path %s on SFTP server %s' % \

402

(self._path, self._host))

403

404

def _sftp_auth(self, transport, username, host):

405

agent = paramiko.Agent()

406

for key in agent.get_keys():

407

mutter('Trying SSH agent key %s' % paramiko.util.hexify(key.get_fingerprint()))

408

try:

409

transport.auth_publickey(self._username, key)

410

return

411

except paramiko.SSHException, e:

412

pass

413

414

# okay, try finding id_rsa or id_dss? (posix only)

415

if self._try_pkey_auth(transport, paramiko.RSAKey, 'id_rsa'):

416

return

417

if self._try_pkey_auth(transport, paramiko.DSSKey, 'id_dsa'):

418

return

419

420

# give up and ask for a password

421

password = getpass.getpass('SSH %s@%s password: ' % (self._username, self._host))

422

try:

423

transport.auth_password(self._username, password)

424

except paramiko.SSHException:

425

raise SFTPTransportError('Unable to authenticate to SSH host as %s@%s' % \

426

(self._username, self._host))

427

428

def _try_pkey_auth(self, transport, pkey_class, filename):

429

filename = os.path.expanduser('~/.ssh/' + filename)

430

try:

431

key = pkey_class.from_private_key_file(filename)

432

transport.auth_publickey(self._username, key)

433

return True

434

except paramiko.PasswordRequiredException:

435

password = getpass.getpass('SSH %s password: ' % (os.path.basename(filename),))

436

try:

437

key = pkey_class.from_private_key_file(filename, password)

438

transport.auth_publickey(self._username, key)

439

return True

440

except paramiko.SSHException:

441

mutter('SSH authentication via %s key failed.' % (os.path.basename(filename),))

442

except paramiko.SSHException:

443

mutter('SSH authentication via %s key failed.' % (os.path.basename(filename),))

444

except IOError:

445

pass

446

return False

447

448

449

register_transport('sftp://', SFTPTransport)

450