~abentley/bzrtools/bzrtools.dev

« back to all changes in this revision

Viewing changes to baz_import.py

  • Committer: Aaron Bentley
  • Date: 2007-06-10 17:55:08 UTC
  • mfrom: (531.2.2 bzrtools)
  • Revision ID: aaron.bentley@utoronto.ca-20070610175508-gex1oxvmfv0qoagi
Merge whitespace cleanups

Show diffs side-by-side

added added

removed removed

Lines of Context:
21
21
from bzrlib.errors import (BzrError,
22
22
                           NotBranchError,
23
23
                           NoWorkingTree,
24
 
                           BzrCommandError, 
 
24
                           BzrCommandError,
25
25
                           NoSuchRevision,
26
26
                           NoRepositoryPresent,
27
27
                          )
31
31
from bzrlib.option import _global_option, Option
32
32
from bzrlib.merge import merge_inner
33
33
from bzrlib.revision import NULL_REVISION
34
 
from bzrlib.tree import EmptyTree
35
34
import bzrlib.ui
36
35
import bzrlib.ui.text
37
36
from bzrlib.workingtree import WorkingTree
57
56
import email.Utils
58
57
from progress import *
59
58
 
 
59
 
 
60
BAZ_IMPORT_ROOT = 'TREE_ROOT'
 
61
 
 
62
 
60
63
class ImportCommitReporter(NullCommitReporter):
61
64
 
62
65
    def escaped(self, escape_count, message):
81
84
 
82
85
def make_archive(name, location):
83
86
    pb_location = pybaz.ArchiveLocation(location)
84
 
    pb_location.create_master(pybaz.Archive(name), 
 
87
    pb_location.create_master(pybaz.Archive(name),
85
88
                              pybaz.ArchiveLocationParams())
86
89
 
87
90
def test_environ():
272
275
            br_from.bzrdir.clone(to_location, revision_id)
273
276
        except NoSuchRevision:
274
277
            rmtree(to_location)
275
 
            msg = "The branch %s has no revision %s." % (from_location, 
 
278
            msg = "The branch %s has no revision %s." % (from_location,
276
279
                                                         revision_id)
277
280
            raise UserError(msg)
278
281
    finally:
279
282
        br_from.unlock()
280
283
 
281
 
def get_remaining_revisions(output_dir, version, reuse_history_from=[]):
 
284
def get_remaining_revisions(output_dir, version, encoding,
 
285
                            reuse_history_from=[]):
282
286
    last_patch = None
283
287
    old_revno = None
284
288
    output_exists = os.path.exists(output_dir)
286
290
        # We are starting from an existing directory, figure out what
287
291
        # the current version is
288
292
        branch = Branch.open(output_dir)
289
 
        last_patch = get_last_revision(branch)
 
293
        last_patch, last_encoding = get_last_revision(branch)
 
294
        assert encoding == last_encoding
290
295
        if last_patch is None:
291
296
            if branch.last_revision() != None:
292
297
                raise NotPreviousImport(branch.base)
304
309
                    break
305
310
                # try to grab a copy of ancestor
306
311
                # note that is not optimised: we could look for namespace
307
 
                # transitions and only look for the past after the 
 
312
                # transitions and only look for the past after the
308
313
                # transition.
309
314
                for history_root in reuse_history_from:
310
315
                    possible_source = os.path.join(history_root,
311
316
                        map_namespace(ancestor.version))
312
317
                    try:
313
318
                        source = Branch.open(possible_source)
314
 
                        rev_id = revision_id(ancestor)
 
319
                        rev_id = revision_id(ancestor, encoding)
315
320
                        if rev_id in source.revision_history():
316
321
                            do_branch(source, output_dir, rev_id)
317
322
                            last_patch = ancestor
327
332
                break
328
333
        else:
329
334
            raise UserError("Directory \"%s\" already exists, and the last "
330
 
                "revision (%s) is not in the ancestry of %s" % 
 
335
                "revision (%s) is not in the ancestry of %s" %
331
336
                (output_dir, last_patch, version))
332
337
        # Strip off all of the ancestors which are already present
333
338
        # And get a directory starting with the latest ancestor
339
344
 
340
345
###class Importer(object):
341
346
###    """An importer.
342
 
###    
 
347
###
343
348
###    Currently this is used as a parameter object, though more behaviour is
344
349
###    possible later.
345
350
###    """
346
351
###
347
352
###    def __init__(self, output_dir, version, fast=False,
348
 
###                 verbose=False, dry_run=False, max_count=None, 
 
353
###                 verbose=False, dry_run=False, max_count=None,
349
354
###                   reuse_history_from=[]):
350
355
###        self.output_dir = output_dir
351
356
###        self.version = version
352
357
###        self.
353
358
 
354
359
 
355
 
def import_version(output_dir, version, fast=False,
 
360
def import_version(output_dir, version, encoding, fast=False,
356
361
                   verbose=False, dry_run=False, max_count=None,
357
362
                   reuse_history_from=[], standalone=True):
358
363
    """
359
364
    >>> q = test_environ()
360
 
    
 
365
 
361
366
    Progress bars output to stderr, but doctest does not capture that.
362
367
 
363
368
    >>> old_stderr = sys.stderr
370
375
    >>> bzrlib.ui.ui_factory = bzrlib.ui.text.TextUIFactory(
371
376
    ...     bar_type=bzrlib.progress.DotsProgressBar)
372
377
 
373
 
    >>> import_version('/', version, dry_run=True)
 
378
    >>> import_version('/', version, None, dry_run=True)
374
379
    Traceback (most recent call last):
375
380
    NotPreviousImport: / is not the location of a previous import.
376
 
    >>> import_version(result_path, version, dry_run=True)
 
381
    >>> import_version(result_path, version, None, dry_run=True)
377
382
    Traceback (most recent call last):
378
383
    UserError: The version test@example.com/test--test--0.1 does not exist.
379
384
    >>> version = pybaz.Version("test@example.com/test--test--0")
380
 
    >>> import_version(result_path, version, dry_run=True) #doctest: +ELLIPSIS
 
385
    >>> import_version(result_path, version, None, dry_run=True) #doctest: +ELLIPSIS
381
386
    importing test@example.com/test--test--0 into ...
382
387
    ...
383
388
    revisions: ..........................................
384
389
    Dry run, not modifying output_dir
385
390
    Cleaning up
386
 
    >>> import_version(result_path, version) #doctest: +ELLIPSIS
 
391
    >>> import_version(result_path, version, None) #doctest: +ELLIPSIS
387
392
    importing test@example.com/test--test--0 into ...
388
393
    ...
389
394
    revisions: .....................................................................
390
395
    Cleaning up
391
396
    Import complete.
392
 
    >>> import_version(result_path, version) #doctest: +ELLIPSIS
 
397
    >>> import_version(result_path, version, None) #doctest: +ELLIPSIS
393
398
    Tree is up-to-date with test@example.com/test--test--0--patch-2
394
399
    >>> commit_more_test_revisions()
395
 
    >>> import_version(result_path, version) #doctest: +ELLIPSIS
 
400
    >>> import_version(result_path, version, None) #doctest: +ELLIPSIS
396
401
    importing test@example.com/test--test--0 into ...
397
402
    revisions: ....................................................
398
403
    Cleaning up
405
410
    try:
406
411
        try:
407
412
            ancestors, old_revno = get_remaining_revisions(output_dir, version,
 
413
                                                           encoding,
408
414
                                                           reuse_history_from)
409
415
        except NotBranchError, e:
410
416
            raise NotPreviousImport(e.path)
412
418
            progress_bar.note('Version %s has no revisions.' % version)
413
419
            return
414
420
        if len(ancestors) == 0:
415
 
            last_revision = get_last_revision(Branch.open(output_dir))
 
421
            last_revision, last_encoding = \
 
422
                get_last_revision(Branch.open(output_dir))
416
423
            progress_bar.note('Tree is up-to-date with %s' % last_revision)
417
424
            return
418
425
 
419
426
        progress_bar.note("importing %s into %s" % (version, output_dir))
420
 
    
 
427
 
421
428
        tempdir = tempfile.mkdtemp(prefix="baz2bzr-",
422
429
                                   dir=os.path.dirname(output_dir))
423
430
        try:
424
431
            wt = WorkingTree.open(output_dir)
425
432
        except (NotBranchError, NoWorkingTree):
426
433
            wt = None
427
 
        if wt is None:
428
 
            old_basis = EmptyTree()
429
 
        else:
430
 
            old_basis = wt.basis_tree()
431
434
        try:
432
435
            for result in iter_import_version(output_dir, ancestors, tempdir,
433
 
                    pb=progress_bar,
434
 
                    fast=fast, verbose=verbose, dry_run=dry_run,
435
 
                    max_count=max_count, standalone=standalone):
 
436
                    pb=progress_bar, encoding=encoding, fast=fast,
 
437
                    verbose=verbose, dry_run=dry_run, max_count=max_count,
 
438
                    standalone=standalone):
436
439
                show_progress(progress_bar, result)
437
440
            if dry_run:
438
441
                progress_bar.note('Dry run, not modifying output_dir')
439
442
                return
440
 
    
 
443
 
441
444
            # Update the working tree of the branch
442
445
            try:
443
446
                wt = WorkingTree.open(output_dir)
445
448
                wt = None
446
449
            if wt is not None:
447
450
                wt.set_last_revision(wt.branch.last_revision())
448
 
                merge_inner(wt.branch, wt.basis_tree(), old_basis, 
449
 
                            ignore_zero=True, this_tree=wt)
 
451
                wt.set_root_id(BAZ_IMPORT_ROOT)
450
452
                wt.revert([])
451
 
    
 
453
 
452
454
        finally:
453
 
            
 
455
 
454
456
            progress_bar.note('Cleaning up')
455
457
            shutil.rmtree(tempdir)
456
458
        progress_bar.note("Import complete.")
457
459
    finally:
458
460
        progress_bar.finished()
459
 
            
 
461
 
460
462
class UserError(BzrCommandError):
461
463
    def __init__(self, message):
462
464
        """Exception to throw when a user makes an impossible request
471
473
                           % path)
472
474
 
473
475
 
474
 
def revision_id(arch_revision):
 
476
def revision_id(arch_revision, encoding):
475
477
    """
476
478
    Generate a Bzr revision id from an Arch revision id.  'x' in the id
477
479
    designates a revision imported with an experimental algorithm.  A number
479
481
 
480
482
    :param arch_revision: The Arch revision to generate an ID for.
481
483
 
482
 
    >>> revision_id(pybaz.Revision("you@example.com/cat--br--0--base-0"))
 
484
    >>> revision_id(pybaz.Revision("you@example.com/cat--br--0--base-0"), None)
483
485
    'Arch-1:you@example.com%cat--br--0--base-0'
 
486
    >>> revision_id(pybaz.Revision("you@example.com/cat--br--0--base-0"), 'utf-8')
 
487
    'Arch-1-utf-8:you@example.com%cat--br--0--base-0'
484
488
    """
485
 
    return "Arch-1:%s" % str(arch_revision).replace('/', '%')
 
489
    if encoding is None:
 
490
        encoding = ''
 
491
    else:
 
492
        encoding = '-' + encoding
 
493
    return "Arch-1%s:%s" % (encoding, str(arch_revision).replace('/', '%'))
486
494
 
487
495
class NotArchRevision(Exception):
488
496
    def __init__(self, revision_id):
498
506
    >>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0--base-5"))
499
507
    Traceback (most recent call last):
500
508
    NotArchRevision: The revision id Arch-1:jrandom@example.com%test--test--0--base-5 does not look like it came from Arch.
501
 
    >>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0--patch-5"))
502
 
    'jrandom@example.com/test--test--0--patch-5'
 
509
    >>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0--patch-5")[0])
 
510
    'jrandom@example.com/test--test--0--patch-5'
 
511
    >>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0--patch-5")[0])
 
512
    'jrandom@example.com/test--test--0--patch-5'
 
513
    >>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0--patch-5")[1])
 
514
    'None'
 
515
    >>> str(arch_revision("Arch-1-utf-8:jrandom@example.com%test--test--0--patch-5")[1])
 
516
    'utf-8'
503
517
    """
504
518
    if revision_id is None:
505
 
        return None
506
 
    if revision_id[:7] != 'Arch-1:':
 
519
        return None, None
 
520
    if revision_id[:7] not in ('Arch-1:', 'Arch-1-'):
507
521
        raise NotArchRevision(revision_id)
508
522
    else:
509
523
        try:
510
 
            return pybaz.Revision(revision_id[7:].replace('%', '/'))
 
524
            encoding, arch_name = revision_id[6:].split(':', 1)
 
525
            arch_name = arch_name.replace('%', '/')
 
526
            if encoding == '':
 
527
                encoding = None
 
528
            else:
 
529
                encoding = encoding[1:]
 
530
            return pybaz.Revision(arch_name), encoding
511
531
        except pybaz.errors.NamespaceError, e:
512
532
            raise NotArchRevision(revision_id)
513
533
 
532
552
    if revision_id is None:
533
553
        revision_id = source.last_revision()
534
554
    wt = create_checkout(source, to_location, NULL_REVISION)
535
 
    wt.set_last_revision(revision_id)
536
 
    wt._write_inventory(wt.basis_tree().inventory)
 
555
    wt.lock_write()
 
556
    try:
 
557
        wt.set_last_revision(revision_id)
 
558
        wt.flush()
 
559
        if revision_id not in (NULL_REVISION, None):
 
560
            basis = wt.basis_tree()
 
561
            basis.lock_read()
 
562
            try:
 
563
                wt._write_inventory(basis.inventory)
 
564
            finally:
 
565
                basis.unlock()
 
566
    finally:
 
567
        wt.unlock()
537
568
    return wt
538
569
 
539
570
 
540
 
def iter_import_version(output_dir, ancestors, tempdir, pb, fast=False,
541
 
                        verbose=False, dry_run=False, max_count=None,
542
 
                        standalone=False):
 
571
def iter_import_version(output_dir, ancestors, tempdir, pb, encoding,
 
572
                        fast=False, verbose=False, dry_run=False,
 
573
                        max_count=None, standalone=False):
543
574
    revdir = None
 
575
    log_encoding = 'ascii'
 
576
    if encoding is not None:
 
577
        log_encoding = encoding
544
578
 
545
579
    # Uncomment this for testing, it basically just has baz2bzr only update
546
580
    # 5 patches at a time
575
609
 
576
610
    for i in range(len(ancestors)):
577
611
        revision = ancestors[i]
578
 
        rev_id = revision_id(revision)
 
612
        rev_id = revision_id(revision, encoding)
579
613
        direct_merges = []
580
614
        if verbose:
581
615
            version = str(revision.version)
632
666
        try:
633
667
            if missing_ancestor:
634
668
                # if we want it to be in revision-history, do that here.
635
 
                target_tree.add_pending_merge(revision_id(missing_ancestor))
 
669
                target_tree.set_parent_ids(
 
670
                    [revision_id(missing_ancestor, encoding)],
 
671
                    allow_leftmost_as_ghost=True)
636
672
                missing_ancestor = None
637
673
            for merged_rev in direct_merges:
638
 
                target_tree.add_pending_merge(revision_id(merged_rev))
 
674
                target_tree.add_pending_merge(revision_id(merged_rev,
 
675
                                                          encoding))
 
676
            target_tree.set_root_id(BAZ_IMPORT_ROOT)
 
677
            target_tree.flush()
639
678
            target_tree.set_inventory(baz_inv)
640
679
            commitobj = Commit(reporter=ImportCommitReporter())
641
680
            commitobj.commit(working_tree=target_tree,
642
 
                             message=log_message.decode('ascii', 'replace'), 
643
 
                             verbose=False, committer=log_creator,
644
 
                             timestamp=timestamp, timezone=0, rev_id=rev_id,
645
 
                             revprops={})
 
681
                message=log_message.decode(log_encoding, 'replace'),
 
682
                verbose=False, committer=log_creator, timestamp=timestamp,
 
683
                timezone=0, rev_id=rev_id, revprops={})
646
684
        finally:
647
685
            target_tree.unlock()
648
686
            branch.unlock()
653
691
    previous_version = revision.version
654
692
    if pybaz.WorkingTree(revdir).tree_version != previous_version:
655
693
        pybaz.WorkingTree(revdir).set_tree_version(previous_version)
656
 
    log_path = "%s/{arch}/%s/%s/%s/%s/patch-log/%s" % (revdir, 
657
 
        revision.category.nonarch, revision.branch.nonarch, 
 
694
    log_path = "%s/{arch}/%s/%s/%s/%s/patch-log/%s" % (revdir,
 
695
        revision.category.nonarch, revision.branch.nonarch,
658
696
        revision.version.nonarch, revision.archive, revision.patchlevel)
659
697
    temp_path = tempfile.mktemp(dir=os.path.dirname(revdir))
660
698
    os.rename(log_path, temp_path)
680
718
    tree = revision.get(revdir)
681
719
    log = get_log(tree, revision)
682
720
    try:
683
 
        return tree, bzr_inventory_data(tree), log 
 
721
        return tree, bzr_inventory_data(tree), log
684
722
    except BadFileKind, e:
685
 
        raise UserError("Cannot convert %s because %s is a %s" % 
 
723
        raise UserError("Cannot convert %s because %s is a %s" %
686
724
                        (revision,e.path, e.kind))
687
725
 
688
726
 
692
730
    try:
693
731
        return bzr_inventory_data(tree), log
694
732
    except BadFileKind, e:
695
 
        raise UserError("Cannot convert %s because %s is a %s" % 
 
733
        raise UserError("Cannot convert %s because %s is a %s" %
696
734
                        (revision,e.path, e.kind))
697
735
 
698
736
 
711
749
    inv_map = {}
712
750
    for arch_id, path in inv_iter:
713
751
        bzr_file_id = map_file_id(arch_id)
714
 
        inv_map[path] = bzr_file_id 
 
752
        inv_map[path] = bzr_file_id
715
753
 
716
754
    bzr_inv = []
717
755
    for path, file_id in inv_map.iteritems():
728
766
    bzr_inv.sort()
729
767
    return bzr_inv
730
768
 
731
 
_global_option('max-count', type = int)
732
 
class cmd_baz_import_branch(Command):
733
 
    """Import an Arch or Baz branch into a bzr branch."""
734
 
    takes_args = ['to_location', 'from_branch?', 'reuse_history*']
735
 
    takes_options = ['verbose', 'max-count']
736
769
 
737
 
    def run(self, to_location, from_branch=None, fast=False, max_count=None,
738
 
            verbose=False, dry_run=False, reuse_history_list=[]):
739
 
        to_location = os.path.realpath(str(to_location))
740
 
        if from_branch is not None:
741
 
            try:
742
 
                from_branch = pybaz.Version(from_branch)
743
 
            except pybaz.errors.NamespaceError:
744
 
                print "%s is not a valid Arch branch." % from_branch
745
 
                return 1
746
 
        if reuse_history_list is None:
747
 
            reuse_history_list = []
748
 
        import_version(to_location, from_branch, 
749
 
                       max_count=max_count, 
750
 
                       reuse_history_from=reuse_history_list)
 
770
def baz_import_branch(to_location, from_branch, fast, max_count, verbose,
 
771
                      encoding, dry_run, reuse_history_list):
 
772
    to_location = os.path.realpath(str(to_location))
 
773
    if from_branch is not None:
 
774
        try:
 
775
            from_branch = pybaz.Version(from_branch)
 
776
        except pybaz.errors.NamespaceError:
 
777
            print "%s is not a valid Arch branch." % from_branch
 
778
            return 1
 
779
    if reuse_history_list is None:
 
780
        reuse_history_list = []
 
781
    import_version(to_location, from_branch, encoding, max_count=max_count,
 
782
                   reuse_history_from=reuse_history_list)
751
783
 
752
784
 
753
785
class NotInABranch(Exception):
756
788
        self.path = path
757
789
 
758
790
 
759
 
class cmd_baz_import(Command):
760
 
    """Import an Arch or Baz archive into a bzr repository.
761
 
 
762
 
    This command should be used on local archives (or mirrors) only.  It is
763
 
    quite slow on remote archives.
764
 
    
765
 
    reuse_history allows you to specify any previous imports you 
766
 
    have done of different archives, which this archive has branches
767
 
    tagged from. This will dramatically reduce the time to convert 
768
 
    the archive as it will not have to convert the history already
769
 
    converted in that other branch.
770
 
 
771
 
    If you specify prefixes, only branches whose names start with that prefix
772
 
    will be imported.  Skipped branches will be listed, so you can import any
773
 
    branches you missed by accident.  Here's an example of doing a partial
774
 
    import from thelove@canonical.com:
775
 
    bzr baz-import thelove thelove@canonical.com --prefixes dists:talloc-except
776
 
    """
777
 
    takes_args = ['to_root_dir', 'from_archive', 'reuse_history*']
778
 
    takes_options = ['verbose', Option('prefixes', type=str,
779
 
                     help="Prefixes of branches to import, colon-separated")]
780
 
 
781
 
    def run(self, to_root_dir, from_archive, verbose=False,
782
 
            reuse_history_list=[], prefixes=None):
783
 
        if reuse_history_list is None:
784
 
            reuse_history_list = []
785
 
        to_root = str(os.path.realpath(to_root_dir))
786
 
        if not os.path.exists(to_root):
787
 
            os.mkdir(to_root)
788
 
        if prefixes is not None:
789
 
            prefixes = prefixes.split(':')
790
 
        import_archive(to_root, from_archive, verbose,
791
 
                       reuse_history_list, prefixes=prefixes)
 
791
 
 
792
def baz_import(to_root_dir, from_archive, encoding, verbose=False,
 
793
               reuse_history_list=[], prefixes=None):
 
794
    if reuse_history_list is None:
 
795
        reuse_history_list = []
 
796
    to_root = str(os.path.realpath(to_root_dir))
 
797
    if not os.path.exists(to_root):
 
798
        os.mkdir(to_root)
 
799
    if prefixes is not None:
 
800
        prefixes = prefixes.split(':')
 
801
    import_archive(to_root, from_archive, verbose, encoding,
 
802
                   reuse_history_list, prefixes=prefixes)
792
803
 
793
804
 
794
805
def import_archive(to_root, from_archive, verbose,
795
 
                   reuse_history_from=[], standalone=False,
 
806
                   encoding, reuse_history_from=[], standalone=False,
796
807
                   prefixes=None):
797
808
    def selected(version):
798
809
        if prefixes is None:
824
835
            if not os.path.exists(os.path.dirname(target)):
825
836
                os.makedirs(os.path.dirname(target))
826
837
            try:
827
 
                import_version(target, version,
828
 
                               reuse_history_from=reuse_history_from, 
 
838
                import_version(target, version, encoding,
 
839
                               reuse_history_from=reuse_history_from,
829
840
                               standalone=standalone)
830
841
            except pybaz.errors.ExecProblem,e:
831
842
                if str(e).find('The requested revision cannot be built.') != -1: