~abentley/bzrtools/bzrtools.dev

« back to all changes in this revision

Viewing changes to baz_import.py

  • Committer: Aaron Bentley
  • Date: 2006-05-03 18:54:10 UTC
  • mto: This revision was merged to the branch mainline in revision 366.
  • Revision ID: abentley@panoramicfeedback.com-20060503185410-2e0cc2db4fad30a0
Merge progress bar updates from robertc

Show diffs side-by-side

added added

removed removed

Lines of Context:
21
21
from bzrlib.errors import (BzrError,
22
22
                           NotBranchError,
23
23
                           NoWorkingTree,
24
 
                           BzrCommandError,
 
24
                           BzrCommandError, 
25
25
                           NoSuchRevision,
26
26
                           NoRepositoryPresent,
27
27
                          )
31
31
from bzrlib.option import _global_option, Option
32
32
from bzrlib.merge import merge_inner
33
33
from bzrlib.revision import NULL_REVISION
 
34
from bzrlib.tree import EmptyTree
34
35
import bzrlib.ui
35
 
import bzrlib.ui.text
36
36
from bzrlib.workingtree import WorkingTree
37
37
from errors import NoPyBaz
38
38
try:
56
56
import email.Utils
57
57
from progress import *
58
58
 
59
 
 
60
 
BAZ_IMPORT_ROOT = 'TREE_ROOT'
61
 
 
62
 
 
63
59
class ImportCommitReporter(NullCommitReporter):
64
60
 
65
61
    def escaped(self, escape_count, message):
84
80
 
85
81
def make_archive(name, location):
86
82
    pb_location = pybaz.ArchiveLocation(location)
87
 
    pb_location.create_master(pybaz.Archive(name),
 
83
    pb_location.create_master(pybaz.Archive(name), 
88
84
                              pybaz.ArchiveLocationParams())
89
85
 
90
86
def test_environ():
275
271
            br_from.bzrdir.clone(to_location, revision_id)
276
272
        except NoSuchRevision:
277
273
            rmtree(to_location)
278
 
            msg = "The branch %s has no revision %s." % (from_location,
 
274
            msg = "The branch %s has no revision %s." % (from_location, 
279
275
                                                         revision_id)
280
276
            raise UserError(msg)
281
277
    finally:
282
278
        br_from.unlock()
283
279
 
284
 
def get_remaining_revisions(output_dir, version, encoding,
285
 
                            reuse_history_from=[]):
 
280
def get_remaining_revisions(output_dir, version, reuse_history_from=[]):
286
281
    last_patch = None
287
282
    old_revno = None
288
283
    output_exists = os.path.exists(output_dir)
290
285
        # We are starting from an existing directory, figure out what
291
286
        # the current version is
292
287
        branch = Branch.open(output_dir)
293
 
        last_patch, last_encoding = get_last_revision(branch)
294
 
        assert encoding == last_encoding
 
288
        last_patch = get_last_revision(branch)
295
289
        if last_patch is None:
296
 
            if branch.last_revision() != None:
297
 
                raise NotPreviousImport(branch.base)
298
 
        elif version is None:
 
290
            raise NotPreviousImport(branch.base)
 
291
        if version is None:
299
292
            version = last_patch.version
300
293
    elif version is None:
301
294
        raise UserError("No version specified, and directory does not exist.")
309
302
                    break
310
303
                # try to grab a copy of ancestor
311
304
                # note that is not optimised: we could look for namespace
312
 
                # transitions and only look for the past after the
 
305
                # transitions and only look for the past after the 
313
306
                # transition.
314
307
                for history_root in reuse_history_from:
315
308
                    possible_source = os.path.join(history_root,
316
309
                        map_namespace(ancestor.version))
317
310
                    try:
318
311
                        source = Branch.open(possible_source)
319
 
                        rev_id = revision_id(ancestor, encoding)
 
312
                        rev_id = revision_id(ancestor)
320
313
                        if rev_id in source.revision_history():
321
314
                            do_branch(source, output_dir, rev_id)
322
315
                            last_patch = ancestor
332
325
                break
333
326
        else:
334
327
            raise UserError("Directory \"%s\" already exists, and the last "
335
 
                "revision (%s) is not in the ancestry of %s" %
 
328
                "revision (%s) is not in the ancestry of %s" % 
336
329
                (output_dir, last_patch, version))
337
330
        # Strip off all of the ancestors which are already present
338
331
        # And get a directory starting with the latest ancestor
344
337
 
345
338
###class Importer(object):
346
339
###    """An importer.
347
 
###
 
340
###    
348
341
###    Currently this is used as a parameter object, though more behaviour is
349
342
###    possible later.
350
343
###    """
351
344
###
352
345
###    def __init__(self, output_dir, version, fast=False,
353
 
###                 verbose=False, dry_run=False, max_count=None,
 
346
###                 verbose=False, dry_run=False, max_count=None, 
354
347
###                   reuse_history_from=[]):
355
348
###        self.output_dir = output_dir
356
349
###        self.version = version
357
350
###        self.
358
351
 
359
352
 
360
 
def import_version(output_dir, version, encoding, fast=False,
 
353
def import_version(output_dir, version, fast=False,
361
354
                   verbose=False, dry_run=False, max_count=None,
362
355
                   reuse_history_from=[], standalone=True):
363
356
    """
364
357
    >>> q = test_environ()
365
 
 
 
358
    
366
359
    Progress bars output to stderr, but doctest does not capture that.
367
360
 
368
361
    >>> old_stderr = sys.stderr
375
368
    >>> bzrlib.ui.ui_factory = bzrlib.ui.text.TextUIFactory(
376
369
    ...     bar_type=bzrlib.progress.DotsProgressBar)
377
370
 
378
 
    >>> import_version('/', version, None, dry_run=True)
 
371
    >>> import_version('/', version, dry_run=True)
379
372
    Traceback (most recent call last):
380
373
    NotPreviousImport: / is not the location of a previous import.
381
 
    >>> import_version(result_path, version, None, dry_run=True)
 
374
    >>> import_version(result_path, version, dry_run=True)
382
375
    Traceback (most recent call last):
383
376
    UserError: The version test@example.com/test--test--0.1 does not exist.
384
377
    >>> version = pybaz.Version("test@example.com/test--test--0")
385
 
    >>> import_version(result_path, version, None, dry_run=True) #doctest: +ELLIPSIS
 
378
    >>> import_version(result_path, version, dry_run=True) #doctest: +ELLIPSIS
386
379
    importing test@example.com/test--test--0 into ...
387
380
    ...
388
381
    revisions: ..........................................
389
382
    Dry run, not modifying output_dir
390
383
    Cleaning up
391
 
    >>> import_version(result_path, version, None) #doctest: +ELLIPSIS
 
384
    >>> import_version(result_path, version) #doctest: +ELLIPSIS
392
385
    importing test@example.com/test--test--0 into ...
393
386
    ...
394
387
    revisions: .....................................................................
395
388
    Cleaning up
396
389
    Import complete.
397
 
    >>> import_version(result_path, version, None) #doctest: +ELLIPSIS
 
390
    >>> import_version(result_path, version) #doctest: +ELLIPSIS
398
391
    Tree is up-to-date with test@example.com/test--test--0--patch-2
399
392
    >>> commit_more_test_revisions()
400
 
    >>> import_version(result_path, version, None) #doctest: +ELLIPSIS
 
393
    >>> import_version(result_path, version) #doctest: +ELLIPSIS
401
394
    importing test@example.com/test--test--0 into ...
402
395
    revisions: ....................................................
403
396
    Cleaning up
410
403
    try:
411
404
        try:
412
405
            ancestors, old_revno = get_remaining_revisions(output_dir, version,
413
 
                                                           encoding,
414
406
                                                           reuse_history_from)
415
407
        except NotBranchError, e:
416
408
            raise NotPreviousImport(e.path)
418
410
            progress_bar.note('Version %s has no revisions.' % version)
419
411
            return
420
412
        if len(ancestors) == 0:
421
 
            last_revision, last_encoding = \
422
 
                get_last_revision(Branch.open(output_dir))
 
413
            last_revision = get_last_revision(Branch.open(output_dir))
423
414
            progress_bar.note('Tree is up-to-date with %s' % last_revision)
424
415
            return
425
416
 
426
417
        progress_bar.note("importing %s into %s" % (version, output_dir))
427
 
 
 
418
    
428
419
        tempdir = tempfile.mkdtemp(prefix="baz2bzr-",
429
420
                                   dir=os.path.dirname(output_dir))
430
421
        try:
431
422
            wt = WorkingTree.open(output_dir)
432
423
        except (NotBranchError, NoWorkingTree):
433
424
            wt = None
 
425
        if wt is None:
 
426
            old_basis = EmptyTree()
 
427
        else:
 
428
            old_basis = wt.basis_tree()
434
429
        try:
435
430
            for result in iter_import_version(output_dir, ancestors, tempdir,
436
 
                    pb=progress_bar, encoding=encoding, fast=fast,
437
 
                    verbose=verbose, dry_run=dry_run, max_count=max_count,
438
 
                    standalone=standalone):
 
431
                    pb=progress_bar,
 
432
                    fast=fast, verbose=verbose, dry_run=dry_run,
 
433
                    max_count=max_count, standalone=standalone):
439
434
                show_progress(progress_bar, result)
440
435
            if dry_run:
441
436
                progress_bar.note('Dry run, not modifying output_dir')
442
437
                return
443
 
 
 
438
    
444
439
            # Update the working tree of the branch
445
440
            try:
446
441
                wt = WorkingTree.open(output_dir)
448
443
                wt = None
449
444
            if wt is not None:
450
445
                wt.set_last_revision(wt.branch.last_revision())
451
 
                wt.set_root_id(BAZ_IMPORT_ROOT)
 
446
                merge_inner(wt.branch, wt.basis_tree(), old_basis, 
 
447
                            ignore_zero=True, this_tree=wt)
452
448
                wt.revert([])
453
 
 
 
449
    
454
450
        finally:
455
 
 
 
451
            
456
452
            progress_bar.note('Cleaning up')
457
453
            shutil.rmtree(tempdir)
458
454
        progress_bar.note("Import complete.")
459
455
    finally:
460
456
        progress_bar.finished()
461
 
 
 
457
            
462
458
class UserError(BzrCommandError):
463
459
    def __init__(self, message):
464
460
        """Exception to throw when a user makes an impossible request
473
469
                           % path)
474
470
 
475
471
 
476
 
def revision_id(arch_revision, encoding):
 
472
def revision_id(arch_revision):
477
473
    """
478
474
    Generate a Bzr revision id from an Arch revision id.  'x' in the id
479
475
    designates a revision imported with an experimental algorithm.  A number
481
477
 
482
478
    :param arch_revision: The Arch revision to generate an ID for.
483
479
 
484
 
    >>> revision_id(pybaz.Revision("you@example.com/cat--br--0--base-0"), None)
 
480
    >>> revision_id(pybaz.Revision("you@example.com/cat--br--0--base-0"))
485
481
    'Arch-1:you@example.com%cat--br--0--base-0'
486
 
    >>> revision_id(pybaz.Revision("you@example.com/cat--br--0--base-0"), 'utf-8')
487
 
    'Arch-1-utf-8:you@example.com%cat--br--0--base-0'
488
482
    """
489
 
    if encoding is None:
490
 
        encoding = ''
491
 
    else:
492
 
        encoding = '-' + encoding
493
 
    return "Arch-1%s:%s" % (encoding, str(arch_revision).replace('/', '%'))
 
483
    return "Arch-1:%s" % str(arch_revision).replace('/', '%')
494
484
 
495
485
class NotArchRevision(Exception):
496
486
    def __init__(self, revision_id):
506
496
    >>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0--base-5"))
507
497
    Traceback (most recent call last):
508
498
    NotArchRevision: The revision id Arch-1:jrandom@example.com%test--test--0--base-5 does not look like it came from Arch.
509
 
    >>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0--patch-5")[0])
510
 
    'jrandom@example.com/test--test--0--patch-5'
511
 
    >>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0--patch-5")[0])
512
 
    'jrandom@example.com/test--test--0--patch-5'
513
 
    >>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0--patch-5")[1])
514
 
    'None'
515
 
    >>> str(arch_revision("Arch-1-utf-8:jrandom@example.com%test--test--0--patch-5")[1])
516
 
    'utf-8'
 
499
    >>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0--patch-5"))
 
500
    'jrandom@example.com/test--test--0--patch-5'
517
501
    """
518
502
    if revision_id is None:
519
 
        return None, None
520
 
    if revision_id[:7] not in ('Arch-1:', 'Arch-1-'):
 
503
        return None
 
504
    if revision_id[:7] != 'Arch-1:':
521
505
        raise NotArchRevision(revision_id)
522
506
    else:
523
507
        try:
524
 
            encoding, arch_name = revision_id[6:].split(':', 1)
525
 
            arch_name = arch_name.replace('%', '/')
526
 
            if encoding == '':
527
 
                encoding = None
528
 
            else:
529
 
                encoding = encoding[1:]
530
 
            return pybaz.Revision(arch_name), encoding
 
508
            return pybaz.Revision(revision_id[7:].replace('%', '/'))
531
509
        except pybaz.errors.NamespaceError, e:
532
510
            raise NotArchRevision(revision_id)
533
511
 
552
530
    if revision_id is None:
553
531
        revision_id = source.last_revision()
554
532
    wt = create_checkout(source, to_location, NULL_REVISION)
555
 
    wt.lock_write()
556
 
    try:
557
 
        wt.set_last_revision(revision_id)
558
 
        wt.flush()
559
 
        if revision_id not in (NULL_REVISION, None):
560
 
            basis = wt.basis_tree()
561
 
            basis.lock_read()
562
 
            try:
563
 
                wt._write_inventory(basis.inventory)
564
 
            finally:
565
 
                basis.unlock()
566
 
    finally:
567
 
        wt.unlock()
 
533
    wt.set_last_revision(revision_id)
 
534
    wt._write_inventory(wt.basis_tree().inventory)
568
535
    return wt
569
536
 
570
537
 
571
 
def iter_import_version(output_dir, ancestors, tempdir, pb, encoding,
572
 
                        fast=False, verbose=False, dry_run=False,
573
 
                        max_count=None, standalone=False):
 
538
def iter_import_version(output_dir, ancestors, tempdir, pb, fast=False,
 
539
                        verbose=False, dry_run=False, max_count=None,
 
540
                        standalone=False):
574
541
    revdir = None
575
 
    log_encoding = 'ascii'
576
 
    if encoding is not None:
577
 
        log_encoding = encoding
578
542
 
579
543
    # Uncomment this for testing, it basically just has baz2bzr only update
580
544
    # 5 patches at a time
609
573
 
610
574
    for i in range(len(ancestors)):
611
575
        revision = ancestors[i]
612
 
        rev_id = revision_id(revision, encoding)
 
576
        rev_id = revision_id(revision)
613
577
        direct_merges = []
614
578
        if verbose:
615
579
            version = str(revision.version)
666
630
        try:
667
631
            if missing_ancestor:
668
632
                # if we want it to be in revision-history, do that here.
669
 
                target_tree.set_parent_ids(
670
 
                    [revision_id(missing_ancestor, encoding)],
671
 
                    allow_leftmost_as_ghost=True)
 
633
                target_tree.add_pending_merge(revision_id(missing_ancestor))
672
634
                missing_ancestor = None
673
635
            for merged_rev in direct_merges:
674
 
                target_tree.add_pending_merge(revision_id(merged_rev,
675
 
                                                          encoding))
676
 
            target_tree.set_root_id(BAZ_IMPORT_ROOT)
677
 
            target_tree.flush()
 
636
                target_tree.add_pending_merge(revision_id(merged_rev))
678
637
            target_tree.set_inventory(baz_inv)
679
638
            commitobj = Commit(reporter=ImportCommitReporter())
680
639
            commitobj.commit(working_tree=target_tree,
681
 
                message=log_message.decode(log_encoding, 'replace'),
682
 
                verbose=False, committer=log_creator, timestamp=timestamp,
683
 
                timezone=0, rev_id=rev_id, revprops={})
 
640
                             message=log_message.decode('ascii', 'replace'), 
 
641
                             verbose=False, committer=log_creator,
 
642
                             timestamp=timestamp, timezone=0, rev_id=rev_id,
 
643
                             revprops={})
684
644
        finally:
685
645
            target_tree.unlock()
686
646
            branch.unlock()
691
651
    previous_version = revision.version
692
652
    if pybaz.WorkingTree(revdir).tree_version != previous_version:
693
653
        pybaz.WorkingTree(revdir).set_tree_version(previous_version)
694
 
    log_path = "%s/{arch}/%s/%s/%s/%s/patch-log/%s" % (revdir,
695
 
        revision.category.nonarch, revision.branch.nonarch,
 
654
    log_path = "%s/{arch}/%s/%s/%s/%s/patch-log/%s" % (revdir, 
 
655
        revision.category.nonarch, revision.branch.nonarch, 
696
656
        revision.version.nonarch, revision.archive, revision.patchlevel)
697
657
    temp_path = tempfile.mktemp(dir=os.path.dirname(revdir))
698
658
    os.rename(log_path, temp_path)
718
678
    tree = revision.get(revdir)
719
679
    log = get_log(tree, revision)
720
680
    try:
721
 
        return tree, bzr_inventory_data(tree), log
 
681
        return tree, bzr_inventory_data(tree), log 
722
682
    except BadFileKind, e:
723
 
        raise UserError("Cannot convert %s because %s is a %s" %
 
683
        raise UserError("Cannot convert %s because %s is a %s" % 
724
684
                        (revision,e.path, e.kind))
725
685
 
726
686
 
730
690
    try:
731
691
        return bzr_inventory_data(tree), log
732
692
    except BadFileKind, e:
733
 
        raise UserError("Cannot convert %s because %s is a %s" %
 
693
        raise UserError("Cannot convert %s because %s is a %s" % 
734
694
                        (revision,e.path, e.kind))
735
695
 
736
696
 
749
709
    inv_map = {}
750
710
    for arch_id, path in inv_iter:
751
711
        bzr_file_id = map_file_id(arch_id)
752
 
        inv_map[path] = bzr_file_id
 
712
        inv_map[path] = bzr_file_id 
753
713
 
754
714
    bzr_inv = []
755
715
    for path, file_id in inv_map.iteritems():
766
726
    bzr_inv.sort()
767
727
    return bzr_inv
768
728
 
 
729
_global_option('max-count', type = int)
 
730
class cmd_baz_import_branch(Command):
 
731
    """Import an Arch or Baz branch into a bzr branch.  <BZRTOOLS>"""
 
732
    takes_args = ['to_location', 'from_branch?', 'reuse_history*']
 
733
    takes_options = ['verbose', 'max-count']
769
734
 
770
 
def baz_import_branch(to_location, from_branch, fast, max_count, verbose,
771
 
                      encoding, dry_run, reuse_history_list):
772
 
    to_location = os.path.realpath(str(to_location))
773
 
    if from_branch is not None:
774
 
        try:
775
 
            from_branch = pybaz.Version(from_branch)
776
 
        except pybaz.errors.NamespaceError:
777
 
            print "%s is not a valid Arch branch." % from_branch
778
 
            return 1
779
 
    if reuse_history_list is None:
780
 
        reuse_history_list = []
781
 
    import_version(to_location, from_branch, encoding, max_count=max_count,
782
 
                   reuse_history_from=reuse_history_list)
 
735
    def run(self, to_location, from_branch=None, fast=False, max_count=None,
 
736
            verbose=False, dry_run=False, reuse_history_list=[]):
 
737
        to_location = os.path.realpath(str(to_location))
 
738
        if from_branch is not None:
 
739
            try:
 
740
                from_branch = pybaz.Version(from_branch)
 
741
            except pybaz.errors.NamespaceError:
 
742
                print "%s is not a valid Arch branch." % from_branch
 
743
                return 1
 
744
        if reuse_history_list is None:
 
745
            reuse_history_list = []
 
746
        import_version(to_location, from_branch, 
 
747
                       max_count=max_count, 
 
748
                       reuse_history_from=reuse_history_list)
783
749
 
784
750
 
785
751
class NotInABranch(Exception):
788
754
        self.path = path
789
755
 
790
756
 
791
 
 
792
 
def baz_import(to_root_dir, from_archive, encoding, verbose=False,
793
 
               reuse_history_list=[], prefixes=None):
794
 
    if reuse_history_list is None:
795
 
        reuse_history_list = []
796
 
    to_root = str(os.path.realpath(to_root_dir))
797
 
    if not os.path.exists(to_root):
798
 
        os.mkdir(to_root)
799
 
    if prefixes is not None:
800
 
        prefixes = prefixes.split(':')
801
 
    import_archive(to_root, from_archive, verbose, encoding,
802
 
                   reuse_history_list, prefixes=prefixes)
 
757
class cmd_baz_import(Command):
 
758
    """Import an Arch or Baz archive into a bzr repository.  <BZRTOOLS>
 
759
 
 
760
    This command should be used on local archives (or mirrors) only.  It is
 
761
    quite slow on remote archives.
 
762
    
 
763
    reuse_history allows you to specify any previous imports you 
 
764
    have done of different archives, which this archive has branches
 
765
    tagged from. This will dramatically reduce the time to convert 
 
766
    the archive as it will not have to convert the history already
 
767
    converted in that other branch.
 
768
 
 
769
    If you specify prefixes, only branches whose names start with that prefix
 
770
    will be imported.  Skipped branches will be listed, so you can import any
 
771
    branches you missed by accident.  Here's an example of doing a partial
 
772
    import from thelove@canonical.com:
 
773
    bzr baz-import thelove thelove@canonical.com --prefixes dists:talloc-except
 
774
    """
 
775
    takes_args = ['to_root_dir', 'from_archive', 'reuse_history*']
 
776
    takes_options = ['verbose', Option('prefixes', type=str,
 
777
                     help="Prefixes of branches to import, colon-separated")]
 
778
 
 
779
    def run(self, to_root_dir, from_archive, verbose=False,
 
780
            reuse_history_list=[], prefixes=None):
 
781
        if reuse_history_list is None:
 
782
            reuse_history_list = []
 
783
        to_root = str(os.path.realpath(to_root_dir))
 
784
        if not os.path.exists(to_root):
 
785
            os.mkdir(to_root)
 
786
        if prefixes is not None:
 
787
            prefixes = prefixes.split(':')
 
788
        import_archive(to_root, from_archive, verbose,
 
789
                       reuse_history_list, prefixes=prefixes)
803
790
 
804
791
 
805
792
def import_archive(to_root, from_archive, verbose,
806
 
                   encoding, reuse_history_from=[], standalone=False,
 
793
                   reuse_history_from=[], standalone=False,
807
794
                   prefixes=None):
808
795
    def selected(version):
809
796
        if prefixes is None:
835
822
            if not os.path.exists(os.path.dirname(target)):
836
823
                os.makedirs(os.path.dirname(target))
837
824
            try:
838
 
                import_version(target, version, encoding,
839
 
                               reuse_history_from=reuse_history_from,
 
825
                import_version(target, version,
 
826
                               reuse_history_from=reuse_history_from, 
840
827
                               standalone=standalone)
841
828
            except pybaz.errors.ExecProblem,e:
842
829
                if str(e).find('The requested revision cannot be built.') != -1: