« back to all changes in this revision
Viewing changes to bzrlib/transport/__init__.py
- browse files at revision 1185.11.1
- compare with another revision
- compare with revision 2610.1.1
- download diff
- download tarball
- view history from revision 1185.11.1
- Committer: John Arbash Meinel
- Date: 2005-09-15 21:35:53 UTC
- mfrom: (907.1.57)
- mto: (1393.2.1)
- mto: This revision was merged to the branch mainline in revision 1396.
- Revision ID: john@arbash-meinel.com-20050915213552-a6c83a5ef1e20897
(broken) Transport work is merged in. Tests do not pass yet.
- files added:
- bzrlib/selftest/testtransport.py
- bzrlib/store
- bzrlib/transport
- files renamed:
- bzrlib/store.py => bzrlib/store/__init__.py
- files modified:
- bzrlib/__init__.py
added
removed
bzrlib/transport/__init__.py
1
#!/usr/bin/env python
2
"""\
3
This module contains the basic class handling transport of
4
information.
5
"""
6
7
from bzrlib.trace import mutter
8
from bzrlib.errors import BzrError
9
10
_protocol_handlers = {
11
}
12
13
def register_transport(prefix, klass, override=True):
14
global _protocol_handlers
15
16
if _protocol_handlers.has_key(prefix):
17
if override:
18
mutter('overriding transport: %s => %s' % (prefix, klass.__name__))
19
_protocol_handlers[prefix] = klass
20
else:
21
mutter('registering transport: %s => %s' % (prefix, klass.__name__))
22
_protocol_handlers[prefix] = klass
23
24
class TransportError(BzrError):
25
"""All errors thrown by Transport implementations should derive
26
from this class.
27
"""
28
def __init__(self, msg=None, orig_error=None):
29
if msg is None and orig_error is not None:
30
msg = str(orig_error)
31
BzrError.__init__(self, msg)
32
self.msg = msg
33
self.orig_error = orig_error
34
35
class AsyncError(TransportError):
36
pass
37
38
# A set of semi-meaningful errors which can be thrown
39
class TransportNotPossible(TransportError):
40
"""This is for transports where a specific function is explicitly not
41
possible. Such as pushing files to an HTTP server.
42
"""
43
pass
44
45
class NonRelativePath(TransportError):
46
"""An absolute path was supplied, that could not be decoded into
47
a relative path.
48
"""
49
pass
50
51
class NoSuchFile(TransportError, IOError):
52
"""A get() was issued for a file that doesn't exist."""
53
def __init__(self, msg=None, orig_error=None):
54
import errno
55
TransportError.__init__(self, msg=msg, orig_error=orig_error)
56
IOError.__init__(self, errno.ENOENT, self.msg)
57
58
class FileExists(TransportError, OSError):
59
"""An operation was attempted, which would overwrite an entry,
60
but overwritting is not supported.
61
62
mkdir() can throw this, but put() just overwites existing files.
63
"""
64
def __init__(self, msg=None, orig_error=None):
65
import errno
66
TransportError.__init__(self, msg=msg, orig_error=orig_error)
67
OSError.__init__(self, errno.EEXIST, self.msg)
68
69
class PermissionDenied(TransportError):
70
"""An operation cannot succeed because of a lack of permissions."""
71
pass
72
73
class ConnectionReset(TransportError):
74
"""The connection has been closed."""
75
pass
76
77
class AsyncFile(object):
78
"""This will be returned from a Transport object,
79
whenever an asyncronous get is requested.
80
"""
81
_max_write_buffer = 8192
82
83
def __init__(self):
84
self.read_buffer = []
85
self.write_buffer = []
86
self._finalized = False
87
88
def _total_len(self, buffer):
89
count = 0
90
for b in buffer:
91
count += len(b)
92
return count
93
94
def finalize(self):
95
"""This will block until all data has been buffered for
96
reading, or all data has been flushed for writing.
97
Once this is called, you can no longer write more data.
98
"""
99
raise NotImplementedError
100
101
def can_read(self):
102
"""Can we read some data without blocking"""
103
return len(self.read_buffer) > 0
104
105
def can_write(self):
106
"""Can we write some more data out without blocking?"""
107
return (not self._finalized \
108
and self._total_len(self.write_buffer) < self._max_write_buffer)
109
110
def write(self, chunk):
111
if self._finalized:
112
raise AsyncError('write attempted on finalized file.')
113
self.write_buffer.append(chunk)
114
115
def read(self, size=None):
116
if size is None:
117
return ''.join(self.read_buffer)
118
else:
119
out = ''
120
while True:
121
buf = self.read_buffer.pop(0)
122
if len(buf) + len(out) < size:
123
out += buf
124
else:
125
left = size - len(out)
126
out += buf[:left]
127
buf = buf[left:]
128
self.read_buffer.insert(0, buf)
129
return out
130
131
132
class Transport(object):
133
"""This class encapsulates methods for retrieving or putting a file
134
from/to a storage location.
135
136
Most functions have a _multi variant, which allows you to queue up
137
multiple requests. They generally have a dumb base implementation
138
which just iterates over the arguments, but smart Transport
139
implementations can do pipelining.
140
In general implementations should support having a generator or a list
141
as an argument (ie always iterate, never index)
142
"""
143
144
def __init__(self, base):
145
super(Transport, self).__init__()
146
self.base = base
147
148
def clone(self, offset=None):
149
"""Return a new Transport object, cloned from the current location,
150
using a subdirectory. This allows connections to be pooled,
151
rather than a new one needed for each subdir.
152
"""
153
raise NotImplementedError
154
155
def should_cache(self):
156
"""Return True if the data pulled across should be cached locally.
157
"""
158
return False
159
160
def _pump(self, from_file, to_file):
161
"""Most children will need to copy from one file-like
162
object or string to another one.
163
This just gives them something easy to call.
164
"""
165
if isinstance(from_file, basestring):
166
to_file.write(from_file)
167
else:
168
from bzrlib.osutils import pumpfile
169
pumpfile(from_file, to_file)
170
171
def _get_total(self, multi):
172
"""Try to figure out how many entries are in multi,
173
but if not possible, return None.
174
"""
175
try:
176
return len(multi)
177
except TypeError: # We can't tell how many, because relpaths is a generator
178
return None
179
180
def _update_pb(self, pb, msg, count, total):
181
"""Update the progress bar based on the current count
182
and total available, total may be None if it was
183
not possible to determine.
184
"""
185
if pb is None:
186
return
187
if total is None:
188
pb.update(msg, count, count+1)
189
else:
190
pb.update(msg, count, total)
191
192
def _iterate_over(self, multi, func, pb, msg, expand=True):
193
"""Iterate over all entries in multi, passing them to func,
194
and update the progress bar as you go along.
195
196
:param expand: If True, the entries will be passed to the function
197
by expanding the tuple. If False, it will be passed
198
as a single parameter.
199
"""
200
total = self._get_total(multi)
201
count = 0
202
for entry in multi:
203
self._update_pb(pb, msg, count, total)
204
if expand:
205
func(*entry)
206
else:
207
func(entry)
208
count += 1
209
return count
210
211
def abspath(self, relpath):
212
"""Return the full url to the given relative path.
213
This can be supplied with a string or a list
214
"""
215
raise NotImplementedError
216
217
def relpath(self, abspath):
218
"""Return the local path portion from a given absolute path.
219
"""
220
raise NotImplementedError
221
222
def has(self, relpath):
223
"""Does the target location exist?"""
224
raise NotImplementedError
225
226
def has_multi(self, relpaths, pb=None):
227
"""Return True/False for each entry in relpaths"""
228
total = self._get_total(relpaths)
229
count = 0
230
for relpath in relpaths:
231
self._update_pb(pb, 'has', count, total)
232
yield self.has(relpath)
233
count += 1
234
235
def get(self, relpath):
236
"""Get the file at the given relative path.
237
238
:param relpath: The relative path to the file
239
"""
240
raise NotImplementedError
241
242
def get_partial(self, relpath, portion):
243
"""Get just part of a file.
244
245
:param relpath: Path to the file, relative to base
246
:param portion: A tuple of [start,length).
247
Length can be -1 indicating copy until the end
248
of the file. If the file ends before length bytes,
249
just the set of bytes will be returned (think read())
250
:return: A file-like object containing at least the specified bytes.
251
Some implementations may return objects which can be read
252
past this length, but this is not guaranteed.
253
"""
254
raise NotImplementedError
255
256
def get_partial_multi(self, files, pb=None):
257
"""Put a set of files or strings into the location.
258
259
Requesting multiple portions of the same file can be dangerous.
260
261
:param files: A list of tuples of relpath, portion
262
[(path1, portion1), (path2, portion2),...]
263
:param pb: An optional ProgressBar for indicating percent done.
264
:return: A generator of file-like objects.
265
"""
266
def get_partial(relpath, portion):
267
yield self.get_partial(relpath, portion)
268
self._iterate_over(files, get_partial, pb, 'get_partial', expand=True)
269
270
def get_multi(self, relpaths, pb=None):
271
"""Get a list of file-like objects, one for each entry in relpaths.
272
273
:param relpaths: A list of relative paths.
274
:param pb: An optional ProgressBar for indicating percent done.
275
:return: A list or generator of file-like objects
276
"""
277
# TODO: Consider having this actually buffer the requests,
278
# in the default mode, it probably won't give worse performance,
279
# and all children wouldn't have to implement buffering
280
total = self._get_total(relpaths)
281
count = 0
282
for relpath in relpaths:
283
self._update_pb(pb, 'get', count, total)
284
yield self.get(relpath)
285
count += 1
286
287
def put(self, relpath, f):
288
"""Copy the file-like or string object into the location.
289
290
:param relpath: Location to put the contents, relative to base.
291
:param f: File-like or string object.
292
"""
293
raise NotImplementedError
294
295
def put_multi(self, files, pb=None):
296
"""Put a set of files or strings into the location.
297
298
:param files: A list of tuples of relpath, file object [(path1, file1), (path2, file2),...]
299
:param pb: An optional ProgressBar for indicating percent done.
300
:return: The number of files copied.
301
"""
302
def put(relpath, f):
303
self.put(relpath, f)
304
return self._iterate_over(files, put, pb, 'put', expand=True)
305
306
def mkdir(self, relpath):
307
"""Create a directory at the given path."""
308
raise NotImplementedError
309
310
def mkdir_multi(self, relpaths, pb=None):
311
"""Create a group of directories"""
312
return self._iterate_over(relpaths, self.mkdir, pb, 'mkdir', expand=False)
313
314
def append(self, relpath, f):
315
"""Append the text in the file-like or string object to
316
the supplied location.
317
"""
318
raise NotImplementedError
319
320
def append_multi(self, files):
321
"""Append the text in each file-like or string object to
322
the supplied location.
323
324
:param files: A set of (path, f) entries
325
:param pb: An optional ProgressBar for indicating percent done.
326
"""
327
return self._iterate_over(files, self.append, pb, 'append', expand=True)
328
329
def copy(self, rel_from, rel_to):
330
"""Copy the item at rel_from to the location at rel_to"""
331
raise NotImplementedError
332
333
def copy_multi(self, relpaths, pb=None):
334
"""Copy a bunch of entries.
335
336
:param relpaths: A list of tuples of the form [(from, to), (from, to),...]
337
"""
338
# This is the non-pipelined implementation, so that
339
# implementors don't have to implement everything.
340
return self._iterate_over(relpaths, self.copy, pb, 'copy', expand=True)
341
342
def copy_to(self, relpaths, other, pb=None):
343
"""Copy a set of entries from self into another Transport.
344
345
:param relpaths: A list/generator of entries to be copied.
346
"""
347
# The dummy implementation just does a simple get + put
348
def copy_entry(path):
349
other.put(path, self.get(path))
350
351
return self._iterate_over(relpaths, copy_entry, pb, 'copy_to', expand=False)
352
353
354
def move(self, rel_from, rel_to):
355
"""Move the item at rel_from to the location at rel_to"""
356
raise NotImplementedError
357
358
def move_multi(self, relpaths, pb=None):
359
"""Move a bunch of entries.
360
361
:param relpaths: A list of tuples of the form [(from1, to1), (from2, to2),...]
362
"""
363
return self._iterate_over(relpaths, self.move, pb, 'move', expand=True)
364
365
def move_multi_to(self, relpaths, rel_to):
366
"""Move a bunch of entries to a single location.
367
This differs from move_multi in that you give a list of from, and
368
a single destination, rather than multiple destinations.
369
370
:param relpaths: A list of relative paths [from1, from2, from3, ...]
371
:param rel_to: A directory where each entry should be placed.
372
"""
373
# This is not implemented, because you need to do special tricks to
374
# extract the basename, and add it to rel_to
375
raise NotImplementedError
376
377
def delete(self, relpath):
378
"""Delete the item at relpath"""
379
raise NotImplementedError
380
381
def delete_multi(self, relpaths, pb=None):
382
"""Queue up a bunch of deletes to be done.
383
"""
384
return self._iterate_over(relpaths, self.delete, pb, 'delete', expand=False)
385
386
def stat(self, relpath):
387
"""Return the stat information for a file.
388
WARNING: This may not be implementable for all protocols, so use
389
sparingly.
390
"""
391
raise NotImplementedError
392
393
def stat_multi(self, relpaths, pb=None):
394
"""Stat multiple files and return the information.
395
"""
396
#TODO: Is it worth making this a generator instead of a
397
# returning a list?
398
stats = []
399
def gather(path):
400
stats.append(self.stat(path))
401
402
count = self._iterate_over(relpaths, gather, pb, 'stat', expand=False)
403
return stats
404
405
406
def list_dir(self, relpath):
407
"""Return a list of all files at the given location.
408
WARNING: many transports do not support this, so trying avoid using
409
it if at all possible.
410
"""
411
raise NotImplementedError
412
413
def lock_read(self, relpath):
414
"""Lock the given file for shared (read) access.
415
WARNING: many transports do not support this, so trying avoid using it
416
417
:return: A lock object, which should contain an unlock() function.
418
"""
419
raise NotImplementedError
420
421
def lock_write(self, relpath):
422
"""Lock the given file for exclusive (write) access.
423
WARNING: many transports do not support this, so trying avoid using it
424
425
:return: A lock object, which should contain an unlock() function.
426
"""
427
raise NotImplementedError
428
429
430
def transport(base):
431
global _protocol_handlers
432
if base is None:
433
base = '.'
434
for proto, klass in _protocol_handlers.iteritems():
435
if proto is not None and base.startswith(proto):
436
return klass(base)
437
# The default handler is the filesystem handler
438
# which has a lookup of None
439
return _protocol_handlers[None](base)
440
441
# Local transport should always be initialized
442
import bzrlib.transport.local
Loggerhead is a web-based interface for Breezy
Version: 2.0.1