← Back to branch summary

~abentley/bzrtools/bzrtools.dev

« back to all changes in this revision

Viewing changes to bzrtools.py

  • Committer: Aaron Bentley
  • Date: 2011-06-27 23:07:10 UTC
  • Revision ID: aaron@aaronbentley.com-20110627230710-orth0tzf1kwknfen
Better handling of compound tar names.

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrtools.py
1
 
# Copyright (C) 2005, 2006, 2007 Aaron Bentley <aaron.bentley@utoronto.ca>
 
1
# Copyright (C) 2005, 2006, 2007 Aaron Bentley <aaron@aaronbentley.com>
2
2
# Copyright (C) 2007 John Arbash Meinel
3
3
#
4
4
#    This program is free software; you can redistribute it and/or modify
24
24
import sys
25
25
 
26
26
import bzrlib
 
27
from bzrlib import revision as _mod_revision, trace, urlutils
27
28
import bzrlib.errors
28
 
from bzrlib.errors import (BzrCommandError, NotBranchError, NoSuchFile,
29
 
                           UnsupportedFormatError, TransportError, 
30
 
                           NoWorkingTree, PermissionDenied)
 
29
from bzrlib.errors import (
 
30
    BzrCommandError,
 
31
    BzrError,
 
32
    ConnectionError,
 
33
    NotBranchError,
 
34
    NoSuchFile,
 
35
    NoWorkingTree,
 
36
    PermissionDenied,
 
37
    UnsupportedFormatError,
 
38
    TransportError,
 
39
    )
31
40
from bzrlib.bzrdir import BzrDir, BzrDirFormat
 
41
from bzrlib.transport import get_transport
32
42
 
33
43
def temp_tree():
34
44
    dirname = tempfile.mkdtemp("temp-branch")
55
65
    return not delta.has_changed(), non_source
56
66
 
57
67
def set_push_data(tree, location):
58
 
    tree.branch.control_files.put_utf8("x-push-data", "%s\n" % location)
 
68
    tree.branch._transport.put_bytes("x-push-data", "%s\n" % location)
59
69
 
60
70
def get_push_data(tree):
61
71
    """
68
78
    >>> rm_tree(tree)
69
79
    """
70
80
    try:
71
 
        location = tree.branch.control_files.get_utf8('x-push-data').read()
 
81
        location = tree.branch._transport.get('x-push-data').read()
72
82
    except NoSuchFile:
73
83
        return None
 
84
    location = location.decode('utf-8')
74
85
    return location.rstrip('\n')
75
86
 
76
87
"""
99
110
    def __init__(self, rsync_name):
100
111
        Exception.__init__(self, "%s not found." % rsync_name)
101
112
 
102
 
def rsync(source, target, ssh=False, excludes=(), silent=False, 
 
113
 
 
114
def rsync(source, target, ssh=False, excludes=(), silent=False,
103
115
          rsync_name="rsync"):
104
 
    """
105
 
    >>> new_dir = tempfile.mkdtemp()
106
 
    >>> old_dir = os.getcwd()
107
 
    >>> os.chdir(new_dir)
108
 
    >>> rsync("a", "b", silent=True)
109
 
    Traceback (most recent call last):
110
 
    RsyncNoFile: No such file...
111
 
    >>> rsync(new_dir + "/a", new_dir + "/b", excludes=("*.py",), silent=True)
112
 
    Traceback (most recent call last):
113
 
    RsyncNoFile: No such file...
114
 
    >>> rsync(new_dir + "/a", new_dir + "/b", excludes=("*.py",), silent=True, rsync_name="rsyncc")
115
 
    Traceback (most recent call last):
116
 
    NoRsync: rsyncc not found.
117
 
    >>> os.chdir(old_dir)
118
 
    >>> os.rmdir(new_dir)
119
 
    """
120
116
    cmd = [rsync_name, "-av", "--delete"]
121
117
    if ssh:
122
118
        cmd.extend(('-e', 'ssh'))
134
130
    except OSError, e:
135
131
        if e.errno == errno.ENOENT:
136
132
            raise NoRsync(rsync_name)
137
 
            
 
133
 
138
134
    proc.stdin.write('\n'.join(excludes)+'\n')
139
135
    proc.stdin.close()
140
136
    if silent:
176
172
        raise RsyncUnknownStatus(proc.returncode)
177
173
    return [l.split(' ')[-1].rstrip('\n') for l in result.splitlines(True)]
178
174
 
179
 
exclusions = ('.bzr/x-push-data', '.bzr/branch/x-push/data', '.bzr/parent', 
 
175
exclusions = ('.bzr/x-push-data', '.bzr/branch/x-push/data', '.bzr/parent',
180
176
              '.bzr/branch/parent', '.bzr/x-pull-data', '.bzr/x-pull',
181
177
              '.bzr/pull', '.bzr/stat-cache', '.bzr/x-rsync-data',
182
178
              '.bzr/basis-inventory', '.bzr/inventory.backup.weave')
186
182
    return [l.rstrip('\r\n') for l in
187
183
            codecs.open(fname, 'rb', 'utf-8').readlines()]
188
184
 
 
185
 
 
186
def read_revision_info(path):
 
187
    """Parse a last_revision file to determine revision_info"""
 
188
    line = open(path, 'rb').readlines()[0].strip('\n')
 
189
    revno, revision_id = line.split(' ', 1)
 
190
    revno = int(revno)
 
191
    return revno, revision_id
 
192
 
 
193
 
189
194
class RsyncNoFile(Exception):
190
195
    def __init__(self, path):
191
196
        Exception.__init__(self, "No such file %s" % path)
194
199
    def __init__(self):
195
200
        Exception.__init__(self, "Error in rsync protocol data stream.")
196
201
 
197
 
def get_revision_history(location):
 
202
 
 
203
class NotStandalone(BzrError):
 
204
 
 
205
    _fmt = '%(location)s is not a standalone tree.'
 
206
    _internal = False
 
207
 
 
208
    def __init__(self, location):
 
209
        BzrError.__init__(self, location=location)
 
210
 
 
211
 
 
212
def get_revision_history(location, _rsync):
198
213
    tempdir = tempfile.mkdtemp('push')
 
214
    my_rsync = _rsync
 
215
    if my_rsync is None:
 
216
        my_rsync = rsync
199
217
    try:
200
218
        history_fname = os.path.join(tempdir, 'revision-history')
201
219
        try:
202
 
            cmd = rsync(location+'.bzr/revision-history', history_fname,
 
220
            cmd = my_rsync(location+'.bzr/revision-history', history_fname,
203
221
                        silent=True)
204
222
        except RsyncNoFile:
205
223
            cmd = rsync(location+'.bzr/branch/revision-history', history_fname,
209
227
        shutil.rmtree(tempdir)
210
228
    return history
211
229
 
212
 
def history_subset(location, branch):
213
 
    remote_history = get_revision_history(location)
 
230
 
 
231
def get_revision_info(location, _rsync):
 
232
    """Get the revsision_info for an rsync-able branch"""
 
233
    tempdir = tempfile.mkdtemp('push')
 
234
    my_rsync = _rsync
 
235
    if my_rsync is None:
 
236
        my_rsync = rsync
 
237
    try:
 
238
        info_fname = os.path.join(tempdir, 'last-revision')
 
239
        cmd = rsync(location+'.bzr/branch/last-revision', info_fname,
 
240
                    silent=True)
 
241
        return read_revision_info(info_fname)
 
242
    finally:
 
243
        shutil.rmtree(tempdir)
 
244
 
 
245
 
 
246
def history_subset(location, branch, _rsync=None):
214
247
    local_history = branch.revision_history()
215
 
    if len(remote_history) > len(local_history):
216
 
        return False
217
 
    for local, remote in zip(remote_history, local_history):
218
 
        if local != remote:
219
 
            return False 
220
 
    return True
 
248
    try:
 
249
        remote_history = get_revision_history(location, _rsync)
 
250
    except RsyncNoFile:
 
251
        revno, revision_id = get_revision_info(location, _rsync)
 
252
        if revision_id == _mod_revision.NULL_REVISION:
 
253
            return True
 
254
        return bool(revision_id.decode('utf-8') in local_history)
 
255
    else:
 
256
        if len(remote_history) > len(local_history):
 
257
            return False
 
258
        for local, remote in zip(remote_history, local_history):
 
259
            if local != remote:
 
260
                return False
 
261
        return True
 
262
 
221
263
 
222
264
def empty_or_absent(location):
223
265
    try:
226
268
    except RsyncNoFile:
227
269
        return True
228
270
 
229
 
def rspush(tree, location=None, overwrite=False, working_tree=True):
230
 
    push_location = get_push_data(tree)
231
 
    if location is not None:
232
 
        if not location.endswith('/'):
233
 
            location += '/'
234
 
        push_location = location
235
 
    
236
 
    if push_location is None:
237
 
        raise BzrCommandError("No rspush location known or specified.")
238
 
 
239
 
    if (push_location.find('::') != -1):
240
 
        usessh=False
241
 
    else:
242
 
        usessh=True
243
 
 
244
 
    if (push_location.find('://') != -1 or
245
 
        push_location.find(':') == -1):
246
 
        raise BzrCommandError("Invalid rsync path %r." % push_location)
247
 
 
248
 
    if working_tree:
249
 
        clean, non_source = is_clean(tree)
250
 
        if not clean:
251
 
            print """Error: This tree has uncommitted changes or unknown (?) files.
252
 
    Use "bzr status" to list them."""
253
 
            sys.exit(1)
254
 
        final_exclusions = non_source[:]
255
 
    else:
256
 
        wt = tree
257
 
        final_exclusions = []
258
 
        for path, status, kind, file_id, entry in wt.list_files():
259
 
            final_exclusions.append(path)
260
 
 
261
 
    final_exclusions.extend(exclusions)
262
 
    if not overwrite:
263
 
        try:
264
 
            if not history_subset(push_location, tree.branch):
265
 
                raise bzrlib.errors.BzrCommandError("Local branch is not a"
266
 
                                                    " newer version of remote"
267
 
                                                    " branch.")
268
 
        except RsyncNoFile:
269
 
            if not empty_or_absent(push_location):
270
 
                raise bzrlib.errors.BzrCommandError("Remote location is not a"
271
 
                                                    " bzr branch (or empty"
272
 
                                                    " directory)")
273
 
        except RsyncStreamIO:
274
 
            raise bzrlib.errors.BzrCommandError("Rsync could not use the"
275
 
                " specified location.  Please ensure that"
276
 
                ' "%s" is of the form "machine:/path".' % push_location)
277
 
    print "Pushing to %s" % push_location
278
 
    rsync(tree.basedir+'/', push_location, ssh=usessh, 
279
 
          excludes=final_exclusions)
280
 
 
281
 
    set_push_data(tree, push_location)
 
271
def rspush(tree, location=None, overwrite=False, working_tree=True,
 
272
    _rsync=None):
 
273
    tree.lock_write()
 
274
    try:
 
275
        my_rsync = _rsync
 
276
        if my_rsync is None:
 
277
            my_rsync = rsync
 
278
        if (tree.bzrdir.root_transport.base !=
 
279
            tree.branch.bzrdir.root_transport.base):
 
280
            raise NotStandalone(tree.bzrdir.root_transport.base)
 
281
        if (tree.branch.get_bound_location() is not None):
 
282
            raise NotStandalone(tree.bzrdir.root_transport.base)
 
283
        if (tree.branch.repository.is_shared()):
 
284
            raise NotStandalone(tree.bzrdir.root_transport.base)
 
285
        push_location = get_push_data(tree)
 
286
        if location is not None:
 
287
            if not location.endswith('/'):
 
288
                location += '/'
 
289
            push_location = location
 
290
 
 
291
        if push_location is None:
 
292
            raise BzrCommandError("No rspush location known or specified.")
 
293
 
 
294
        if (push_location.find('::') != -1):
 
295
            usessh=False
 
296
        else:
 
297
            usessh=True
 
298
 
 
299
        if (push_location.find('://') != -1 or
 
300
            push_location.find(':') == -1):
 
301
            raise BzrCommandError("Invalid rsync path %r." % push_location)
 
302
 
 
303
        if working_tree:
 
304
            clean, non_source = is_clean(tree)
 
305
            if not clean:
 
306
                raise bzrlib.errors.BzrCommandError(
 
307
                    'This tree has uncommitted changes or unknown'
 
308
                    ' (?) files.  Use "bzr status" to list them.')
 
309
                sys.exit(1)
 
310
            final_exclusions = non_source[:]
 
311
        else:
 
312
            wt = tree
 
313
            final_exclusions = []
 
314
            for path, status, kind, file_id, entry in wt.list_files():
 
315
                final_exclusions.append(path)
 
316
 
 
317
        final_exclusions.extend(exclusions)
 
318
        if not overwrite:
 
319
            try:
 
320
                if not history_subset(push_location, tree.branch,
 
321
                                      _rsync=my_rsync):
 
322
                    raise bzrlib.errors.BzrCommandError(
 
323
                        "Local branch is not a newer version of remote"
 
324
                        " branch.")
 
325
            except RsyncNoFile:
 
326
                if not empty_or_absent(push_location):
 
327
                    raise bzrlib.errors.BzrCommandError(
 
328
                        "Remote location is not a bzr branch (or empty"
 
329
                        " directory)")
 
330
            except RsyncStreamIO:
 
331
                raise bzrlib.errors.BzrCommandError("Rsync could not use the"
 
332
                    " specified location.  Please ensure that"
 
333
                    ' "%s" is of the form "machine:/path".' % push_location)
 
334
        trace.note("Pushing to %s", push_location)
 
335
        my_rsync(tree.basedir+'/', push_location, ssh=usessh,
 
336
                 excludes=final_exclusions)
 
337
 
 
338
        set_push_data(tree, push_location)
 
339
    finally:
 
340
        tree.unlock()
282
341
 
283
342
 
284
343
def short_committer(committer):
292
351
    """Screen-scrape Apache listings"""
293
352
    apache_dir = '<img border="0" src="/icons/folder.gif" alt="[dir]">'\
294
353
        ' <a href="'
295
 
    lines = t.get('.')
296
 
    expr = re.compile('<a[^>]*href="([^>]*)"[^>]*>', flags=re.I)
 
354
    t = t.clone()
 
355
    t._remote_path = lambda x: t.base
 
356
    try:
 
357
        lines = t.get('')
 
358
    except bzrlib.errors.NoSuchFile:
 
359
        return
 
360
    expr = re.compile('<a[^>]*href="([^>]*)\/"[^>]*>', flags=re.I)
297
361
    for line in lines:
298
362
        match = expr.search(line)
299
363
        if match is None:
306
370
        yield url.rstrip('/')
307
371
 
308
372
 
309
 
def iter_branches(t, lister=None):
310
 
    """Iterate through all the branches under a transport"""
311
 
    for bzrdir in iter_bzrdirs(t, lister):
312
 
        try:
313
 
            branch = bzrdir.open_branch()
314
 
            if branch.bzrdir is bzrdir:
315
 
                yield branch
316
 
        except (NotBranchError, UnsupportedFormatError):
317
 
            pass
 
373
def list_branches(t):
 
374
    def is_inside(branch):
 
375
        return bool(branch.base.startswith(t.base))
 
376
 
 
377
    if t.base.startswith('http://'):
 
378
        def evaluate(bzrdir):
 
379
            try:
 
380
                branch = bzrdir.open_branch()
 
381
                if is_inside(branch):
 
382
                    return True, branch
 
383
                else:
 
384
                    return True, None
 
385
            except NotBranchError:
 
386
                return True, None
 
387
        return [b for b in BzrDir.find_bzrdirs(t, list_current=apache_ls,
 
388
                evaluate=evaluate) if b is not None]
 
389
    elif not t.listable():
 
390
        raise BzrCommandError("Can't list this type of location.")
 
391
    return [b for b in BzrDir.find_branches(t) if is_inside(b)]
 
392
 
 
393
 
 
394
def evaluate_branch_tree(bzrdir):
 
395
    try:
 
396
        tree, branch = bzrdir._get_tree_branch()
 
397
    except NotBranchError:
 
398
        return True, None
 
399
    else:
 
400
        return True, (branch, tree)
318
401
 
319
402
 
320
403
def iter_branch_tree(t, lister=None):
321
 
    for bzrdir in iter_bzrdirs(t, lister):
322
 
        try:
323
 
            wt = bzrdir.open_workingtree()
324
 
            yield wt.branch, wt
325
 
        except NoWorkingTree, UnsupportedFormatError:
326
 
            try:
327
 
                branch = bzrdir.open_branch()
328
 
                if branch.bzrdir is bzrdir:
329
 
                    yield branch, None
330
 
            except (NotBranchError, UnsupportedFormatError):
331
 
                continue
332
 
 
333
 
 
334
 
def iter_bzrdirs(t, lister=None):
335
 
    if lister is None:
336
 
        def lister(t):
337
 
            return t.list_dir('.')
338
 
    try:
339
 
        bzrdir = bzrdir_from_transport(t)
340
 
        yield bzrdir
341
 
    except (NotBranchError, UnsupportedFormatError, TransportError,
342
 
            PermissionDenied):
343
 
        pass
344
 
    try:
345
 
        for directory in lister(t):
346
 
            if directory == ".bzr":
347
 
                continue
348
 
            try:
349
 
                subt = t.clone(directory)
350
 
            except UnicodeDecodeError:
351
 
                continue
352
 
            for bzrdir in iter_bzrdirs(subt, lister):
353
 
                yield bzrdir
354
 
    except (NoSuchFile, PermissionDenied, TransportError):
355
 
        pass
356
 
 
357
 
    
358
 
def bzrdir_from_transport(t):
359
 
    """Open a bzrdir from a transport (not a location)"""
360
 
    format = BzrDirFormat.find_format(t)
361
 
    BzrDir._check_supported(format, False)
362
 
    return format.open(t)
 
404
    return (x for x in BzrDir.find_bzrdirs(t, evaluate=evaluate_branch_tree,
 
405
            list_current=lister) if x is not None)
 
406
 
 
407
 
 
408
def open_from_url(location):
 
409
    location = urlutils.normalize_url(location)
 
410
    dirname, basename = urlutils.split(location)
 
411
    if location.endswith('/') and not basename.endswith('/'):
 
412
        basename += '/'
 
413
    return get_transport(dirname).get(basename)
363
414
 
364
415
 
365
416
def run_tests():