← Back to branch summary

~abentley/bzrtools/bzrtools.dev

« back to all changes in this revision

Viewing changes to baz_import.py

Merge from ab-baz2bzr

Show diffs side-by-side

added added

removed removed

Lines of Context:
baz_import.py
1
 
# Copyright (C) 2005 by Aaron Bentley
 
1
# Copyright (C) 2005, 2006 by Aaron Bentley
2
2
 
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
16
16
 
17
17
import errno
18
18
 
19
 
from bzrlib.errors import BzrError
20
 
from bzrlib.errors import NotBranchError, BzrCommandError, NoSuchRevision
 
19
from bzrlib.bzrdir import BzrDir
 
20
import bzrlib.bzrdir as bzrdir
 
21
from bzrlib.errors import (BzrError,
 
22
                           NotBranchError,
 
23
                           NoWorkingTree,
 
24
                           BzrCommandError, 
 
25
                           NoSuchRevision,
 
26
                           NoRepositoryPresent,
 
27
                          )
21
28
from bzrlib.branch import Branch
22
29
from bzrlib.commit import Commit, NullCommitReporter
23
30
from bzrlib.commands import Command
24
 
from bzrlib.option import _global_option
 
31
from bzrlib.option import _global_option, Option
 
32
from bzrlib.merge import merge_inner
 
33
from bzrlib.revision import NULL_REVISION
 
34
from bzrlib.tree import EmptyTree
 
35
import bzrlib.ui
25
36
from bzrlib.workingtree import WorkingTree
26
37
from errors import NoPyBaz
27
38
try:
70
81
 
71
82
saved_dir = None
72
83
 
 
84
def make_archive(name, location):
 
85
    pb_location = pybaz.ArchiveLocation(location)
 
86
    pb_location.create_master(pybaz.Archive(name), 
 
87
                              pybaz.ArchiveLocationParams())
 
88
 
73
89
def test_environ():
74
90
    """
75
91
    >>> q = test_environ()
87
103
    os.environ["HOME"] = os.path.join(tdir, "home")
88
104
    os.mkdir(os.environ["HOME"])
89
105
    arch_dir = os.path.join(tdir, "archive_dir")
90
 
    pybaz.make_archive("test@example.com", arch_dir)
 
106
    make_archive("test@example.com", arch_dir)
91
107
    work_dir = os.path.join(tdir, "work_dir")
92
108
    os.mkdir(work_dir)
93
109
    os.chdir(work_dir)
255
271
            else:
256
272
                raise
257
273
        try:
258
 
            br_from.clone(to_location, revision_id, None)
 
274
            br_from.bzrdir.clone(to_location, revision_id)
259
275
        except NoSuchRevision:
260
276
            rmtree(to_location)
261
 
            msg = "The branch %s has no revision %s." % (from_location, revision_id)
 
277
            msg = "The branch %s has no revision %s." % (from_location, 
 
278
                                                         revision_id)
262
279
            raise UserError(msg)
263
280
    finally:
264
281
        br_from.unlock()
338
355
 
339
356
def import_version(output_dir, version, printer, fancy=True, fast=False,
340
357
                   verbose=False, dry_run=False, max_count=None,
341
 
                   reuse_history_from=[]):
 
358
                   reuse_history_from=[], standalone=True):
342
359
    """
343
360
    >>> q = test_environ()
344
361
    >>> result_path = os.path.join(q, "result")
385
402
        print 'Tree is up-to-date with %s' % last_revision
386
403
        return
387
404
 
388
 
    progress_bar = ProgressBar()
 
405
    progress_bar = bzrlib.ui.ui_factory.progress_bar()
389
406
    tempdir = tempfile.mkdtemp(prefix="baz2bzr-",
390
407
                               dir=os.path.dirname(output_dir))
391
408
    try:
 
409
        wt = WorkingTree.open(output_dir)
 
410
    except NotBranchError:
 
411
        wt = None
 
412
    if wt is None:
 
413
        old_basis = EmptyTree()
 
414
    else:
 
415
        old_basis = wt.basis_tree()
 
416
    try:
392
417
        if not fancy:
393
418
            print "not fancy"
394
419
        try:
395
420
            for result in iter_import_version(output_dir, ancestors, tempdir,
396
421
                    progress_bar, fast=fast, verbose=verbose, dry_run=dry_run,
397
 
                    max_count=max_count):
 
422
                    max_count=max_count, standalone=standalone):
398
423
                if fancy:
399
424
                    show_progress(progress_bar, result)
400
425
                else:
408
433
        if dry_run:
409
434
            print 'Dry run, not modifying output_dir'
410
435
            return
411
 
        if os.path.exists(output_dir):
412
 
            # Move the bzr control directory back, and update the working tree
413
 
            revdir = os.path.join(tempdir, "rd")
414
 
            if os.path.exists(revdir):
415
 
                # actual imports were done
416
 
                tmp_bzr_dir = os.path.join(tempdir, '.bzr')
417
 
                
418
 
                bzr_dir = os.path.join(output_dir, '.bzr')
419
 
                new_bzr_dir = os.path.join(tempdir, "rd", '.bzr')
420
 
    
421
 
                os.rename(bzr_dir, tmp_bzr_dir) # Move the original bzr out of the way
422
 
                os.rename(new_bzr_dir, bzr_dir)
423
 
                try:
424
 
                    tree = WorkingTree(output_dir)
425
 
                    tree.revert([])
426
 
                except:
427
 
                    # If something failed, move back the original bzr directory
428
 
                    os.rename(bzr_dir, new_bzr_dir)
429
 
                    os.rename(tmp_bzr_dir, bzr_dir)
430
 
                    raise
431
 
            else:
432
 
                # no imports - perhaps just append_revisions
433
 
                # should not fail:
434
 
                tree = WorkingTree(output_dir)
435
 
                tree.revert([])
436
 
        else:
437
 
            revdir = os.path.join(tempdir, "rd")
438
 
            os.rename(revdir, output_dir)
 
436
 
 
437
        # Update the working tree of the branch
 
438
        try:
 
439
            wt = WorkingTree.open(output_dir)
 
440
        except NoWorkingTree:
 
441
            wt = None
 
442
        if wt is not None:
 
443
            wt.set_last_revision(wt.branch.last_revision())
 
444
            merge_inner(wt.branch, wt.basis_tree(), old_basis, 
 
445
                        ignore_zero=True, this_tree=wt)
 
446
            wt.revert([])
439
447
 
440
448
    finally:
441
449
        printer('Cleaning up')
495
503
            return pybaz.Revision(revision_id[7:].replace('%', '/'))
496
504
        except pybaz.errors.NamespaceError, e:
497
505
            raise NotArchRevision(revision_id)
498
 
            
 
506
 
 
507
 
 
508
def create_shared_repository(output_dir):
 
509
    bd = bzrdir.BzrDirMetaFormat1().initialize(output_dir)
 
510
    bd.create_repository(shared=True)
 
511
 
 
512
def create_branch(output_dir):
 
513
    os.mkdir(output_dir)
 
514
    bd = bzrdir.BzrDirMetaFormat1().initialize(output_dir)
 
515
    return bd.create_branch()
 
516
 
 
517
 
 
518
def create_checkout(source, to_location, revision_id=None):
 
519
    checkout = bzrdir.BzrDirMetaFormat1().initialize(to_location)
 
520
    bzrlib.branch.BranchReferenceFormat().initialize(checkout, source)
 
521
    return checkout.create_workingtree(revision_id)
 
522
 
 
523
 
 
524
def create_checkout_metadata(source, to_location, revision_id=None):
 
525
    if revision_id is None:
 
526
        revision_id = source.last_revision()
 
527
    wt = create_checkout(source, to_location, NULL_REVISION)
 
528
    wt.set_last_revision(revision_id)
 
529
    wt._write_inventory(wt.basis_tree().inventory)
 
530
    return wt
 
531
 
 
532
 
499
533
def iter_import_version(output_dir, ancestors, tempdir, pb, fast=False,
500
 
                        verbose=False, dry_run=False, max_count=None):
 
534
                        verbose=False, dry_run=False, max_count=None,
 
535
                        standalone=False):
501
536
    revdir = None
502
537
 
503
538
    # Uncomment this for testing, it basically just has baz2bzr only update
516
551
 
517
552
    previous_version=None
518
553
    missing_ancestor = None
 
554
    if dry_run:
 
555
        dry_output_dir = os.path.join(tempdir, 'od')
 
556
        if os.path.exists(output_dir):
 
557
            shutil.copytree(output_dir, dry_output_dir)
 
558
        output_dir = dry_output_dir
 
559
 
 
560
    if os.path.exists(output_dir):
 
561
        target_branch = Branch.open(output_dir)
 
562
    else:
 
563
        if standalone:
 
564
            wt = BzrDir.create_standalone_workingtree(output_dir)
 
565
            target_branch = wt.branch
 
566
        else:
 
567
            target_branch = create_branch(output_dir)
519
568
 
520
569
    for i in range(len(ancestors)):
521
570
        revision = ancestors[i]
530
579
            previous_version = version
531
580
        else:
532
581
            yield Progress("revisions", i, len(ancestors))
533
 
        if revdir is None and os.path.exists(output_dir):
534
 
            # check for imported revisions and if present just append immediately
535
 
            branch = Branch.open(output_dir)
536
 
            if branch.repository.has_revision(rev_id):
537
 
                branch.append_revision(rev_id)
538
 
                continue
 
582
 
 
583
        if target_branch.repository.has_revision(rev_id):
 
584
            target_branch.append_revision(rev_id)
 
585
            continue
539
586
        if revdir is None:
540
587
            revdir = os.path.join(tempdir, "rd")
541
588
            try:
548
595
                print ("unable to access ancestor %s, making into a merge."
549
596
                       % missing_ancestor)
550
597
                continue
551
 
            if os.path.exists(output_dir):
552
 
                bzr_dir = os.path.join(output_dir, '.bzr')
553
 
                new_bzr_dir = os.path.join(tempdir, "rd", '.bzr')
554
 
                # This would be much faster with a simple os.rename(), but if
555
 
                # we fail, we have corrupted the original .bzr directory.  Is
556
 
                # that a big problem, as we can just back out the last
557
 
                # revisions in .bzr/revision_history I don't really know
558
 
                # RBC20051024 - yes, it would be a problem as we could not then
559
 
                # apply the corrupted revision.
560
 
                shutil.copytree(bzr_dir, new_bzr_dir)
561
 
                # Now revdir should have a tree with the latest .bzr, and the
562
 
                # next revision of the baz tree
563
 
                branch = Branch.open(revdir)
564
 
            else:
565
 
                branch = Branch.initialize(revdir)
 
598
            target_tree = create_checkout_metadata(target_branch, revdir)
 
599
            branch = target_tree.branch
566
600
        else:
567
601
            old = os.path.join(revdir, ".bzr")
568
602
            new = os.path.join(tempdir, ".bzr")
569
603
            os.rename(old, new)
570
604
            baz_inv, log = apply_revision(tree, revision)
571
605
            os.rename(new, old)
572
 
            branch = Branch.open(revdir)
 
606
            target_tree = WorkingTree.open(revdir)
 
607
            branch = target_tree.branch
573
608
        # cached so we can delete the log
574
609
        log_date = log.date
575
610
        log_summary = log.summary
586
621
            log_message = "\n".join((log_summary, log_description))
587
622
        else:
588
623
            log_message = log_summary
 
624
        target_tree.lock_write()
589
625
        branch.lock_write()
590
 
        target_tree = WorkingTree(revdir ,branch=branch)
591
 
        target_tree.lock_write()
592
626
        try:
593
627
            if missing_ancestor:
594
628
                # if we want it to be in revision-history, do that here.
598
632
                target_tree.add_pending_merge(revision_id(merged_rev))
599
633
            target_tree.set_inventory(baz_inv)
600
634
            commitobj = Commit(reporter=ImportCommitReporter(pb))
601
 
            commitobj.commit(branch, log_message.decode('ascii', 'replace'), 
 
635
            commitobj.commit(working_tree=target_tree,
 
636
                             message=log_message.decode('ascii', 'replace'), 
602
637
                             verbose=False, committer=log_creator,
603
 
                             timestamp=timestamp, timezone=0, rev_id=rev_id)
 
638
                             timestamp=timestamp, timezone=0, rev_id=rev_id,
 
639
                             revprops={'branch-nick': str(revision.version)})
604
640
        finally:
605
641
            target_tree.unlock()
606
642
            branch.unlock()
607
643
    yield Progress("revisions", len(ancestors), len(ancestors))
608
 
    unlink_unversioned(branch, revdir)
609
644
 
610
645
def get_direct_merges(revdir, revision, log):
611
646
    continuation = log.continuation_of
622
657
    os.rename(temp_path, log_path)
623
658
    return direct
624
659
 
625
 
def unlink_unversioned(branch, revdir):
626
 
    for unversioned in branch.working_tree().extras():
627
 
        path = os.path.join(revdir, unversioned)
 
660
def unlink_unversioned(wt):
 
661
    for unversioned in wt.extras():
 
662
        path = wt.abspath(unversioned)
628
663
        if os.path.isdir(path):
629
664
            shutil.rmtree(path)
630
665
        else:
641
676
    try:
642
677
        return tree, bzr_inventory_data(tree), log 
643
678
    except BadFileKind, e:
644
 
        raise UserError("Cannot convert %s because %s is a %s" % (revision,e.path, e.kind) )
 
679
        raise UserError("Cannot convert %s because %s is a %s" % 
 
680
                        (revision,e.path, e.kind))
645
681
 
646
682
 
647
683
def apply_revision(tree, revision):
650
686
    try:
651
687
        return bzr_inventory_data(tree), log
652
688
    except BadFileKind, e:
653
 
        raise UserError("Cannot convert %s because %s is a %s" % (revision,e.path, e.kind) )
 
689
        raise UserError("Cannot convert %s because %s is a %s" % 
 
690
                        (revision,e.path, e.kind))
654
691
 
655
692
 
656
693
class BadFileKind(Exception):
718
755
 
719
756
class cmd_baz_import(Command):
720
757
    """Import an Arch or Baz archive into bzr branches.
 
758
 
 
759
    This command should be used on local archives (or mirrors) only.  It is
 
760
    quite slow on remote archives.
721
761
    
722
762
    reuse_history allows you to specify any previous imports you 
723
763
    have done of different archives, which this archive has branches
724
764
    tagged from. This will dramatically reduce the time to convert 
725
765
    the archive as it will not have to convert the history already
726
766
    converted in that other branch.
 
767
 
 
768
    If you specify prefixes, only branches whose names start with that prefix
 
769
    will be imported.  Skipped branches will be listed, so you can import any
 
770
    branches you missed by accident.  Here's an example of doing a partial
 
771
    import from thelove@canonical.com:
 
772
    bzr baz-import thelove thelove@canonical.com --prefixes dists:talloc-except
727
773
    """
728
774
    takes_args = ['to_root_dir', 'from_archive', 'reuse_history*']
729
 
    takes_options = ['verbose']
 
775
    takes_options = ['verbose', Option('prefixes', type=str,
 
776
                     help="Prefixes of branches to import, colon-separated")]
730
777
 
731
778
    def printer(self, name):
732
779
        print name
733
780
 
734
781
    def run(self, to_root_dir, from_archive, verbose=False,
735
 
            reuse_history_list=[]):
 
782
            reuse_history_list=[], prefixes=None):
736
783
        if reuse_history_list is None:
737
784
            reuse_history_list = []
738
785
        to_root = str(os.path.realpath(to_root_dir))
739
786
        if not os.path.exists(to_root):
740
787
            os.mkdir(to_root)
 
788
        if prefixes is not None:
 
789
            prefixes = prefixes.split(':')
741
790
        import_archive(to_root, from_archive, verbose, self.printer, 
742
 
                       reuse_history_list)
 
791
                       reuse_history_list, prefixes=prefixes)
743
792
 
744
793
 
745
794
def import_archive(to_root, from_archive, verbose, printer,
746
 
                   reuse_history_from=[]):
 
795
                   reuse_history_from=[], standalone=False,
 
796
                   prefixes=None):
 
797
    def selected(version):
 
798
        if prefixes is None:
 
799
            return True
 
800
        else:
 
801
            for prefix in prefixes:
 
802
                if version.nonarch.startswith(prefix):
 
803
                    return True
 
804
            return False
747
805
    real_to = os.path.realpath(to_root)
748
806
    history_locations = [real_to] + reuse_history_from
 
807
    if standalone is False:
 
808
        try:
 
809
            bd = BzrDir.open(to_root)
 
810
            bd.find_repository()
 
811
        except NotBranchError:
 
812
            create_shared_repository(to_root)
 
813
        except NoRepositoryPresent:
 
814
            raise BzrCommandError("Can't create repository at existing branch.")
749
815
    for version in pybaz.Archive(str(from_archive)).iter_versions():
 
816
        if not selected(version):
 
817
            print "Skipping %s" % version
 
818
            continue
750
819
        target = os.path.join(to_root, map_namespace(version))
751
820
        printer("importing %s into %s" % (version, target))
752
821
        if not os.path.exists(os.path.dirname(target)):
753
822
            os.makedirs(os.path.dirname(target))
754
823
        try:
755
824
            import_version(target, version, printer,
756
 
                           reuse_history_from=reuse_history_from)
 
825
                           reuse_history_from=reuse_history_from, 
 
826
                           standalone=standalone)
757
827
        except pybaz.errors.ExecProblem,e:
758
828
            if str(e).find('The requested revision cannot be built.') != -1:
759
829
                printer("Skipping version %s as it cannot be built due"