~abentley/bzrtools/bzrtools.dev

« back to all changes in this revision

Viewing changes to baz_import.py

  • Committer: Michael Ellerman
  • Date: 2006-06-21 13:51:45 UTC
  • mto: This revision was merged to the branch mainline in revision 405.
  • Revision ID: michael@ellerman.id.au-20060621135145-8691d8247e9fcc80
Parse ~/.colordiffrc for colour information if it's there.

Show diffs side-by-side

added added

removed removed

Lines of Context:
21
21
from bzrlib.errors import (BzrError,
22
22
                           NotBranchError,
23
23
                           NoWorkingTree,
24
 
                           BzrCommandError,
 
24
                           BzrCommandError, 
25
25
                           NoSuchRevision,
26
26
                           NoRepositoryPresent,
27
27
                          )
30
30
from bzrlib.commands import Command
31
31
from bzrlib.option import _global_option, Option
32
32
from bzrlib.merge import merge_inner
33
 
from bzrlib.revision import NULL_REVISION, is_null
 
33
from bzrlib.revision import NULL_REVISION
 
34
from bzrlib.tree import EmptyTree
34
35
import bzrlib.ui
35
36
import bzrlib.ui.text
36
37
from bzrlib.workingtree import WorkingTree
42
43
    from pybaz.backends.baz import null_cmd
43
44
except ImportError:
44
45
    raise NoPyBaz
45
 
from baz_helper import iter_new_merges, direct_merges
 
46
from fai import iter_new_merges, direct_merges
46
47
import tempfile
47
48
import os
48
49
import os.path
56
57
import email.Utils
57
58
from progress import *
58
59
 
59
 
 
60
 
BAZ_IMPORT_ROOT = 'TREE_ROOT'
61
 
 
62
 
 
63
60
class ImportCommitReporter(NullCommitReporter):
64
61
 
65
62
    def escaped(self, escape_count, message):
84
81
 
85
82
def make_archive(name, location):
86
83
    pb_location = pybaz.ArchiveLocation(location)
87
 
    pb_location.create_master(pybaz.Archive(name),
 
84
    pb_location.create_master(pybaz.Archive(name), 
88
85
                              pybaz.ArchiveLocationParams())
89
86
 
90
87
def test_environ():
275
272
            br_from.bzrdir.clone(to_location, revision_id)
276
273
        except NoSuchRevision:
277
274
            rmtree(to_location)
278
 
            msg = "The branch %s has no revision %s." % (from_location,
 
275
            msg = "The branch %s has no revision %s." % (from_location, 
279
276
                                                         revision_id)
280
277
            raise UserError(msg)
281
278
    finally:
282
279
        br_from.unlock()
283
280
 
284
 
def get_remaining_revisions(output_dir, version, encoding,
285
 
                            reuse_history_from=[]):
 
281
def get_remaining_revisions(output_dir, version, reuse_history_from=[]):
286
282
    last_patch = None
287
283
    old_revno = None
288
284
    output_exists = os.path.exists(output_dir)
290
286
        # We are starting from an existing directory, figure out what
291
287
        # the current version is
292
288
        branch = Branch.open(output_dir)
293
 
        last_patch, last_encoding = get_last_revision(branch)
294
 
        assert encoding == last_encoding
 
289
        last_patch = get_last_revision(branch)
295
290
        if last_patch is None:
296
 
            if not is_null(branch.last_revision()):
 
291
            if branch.last_revision() != None:
297
292
                raise NotPreviousImport(branch.base)
298
293
        elif version is None:
299
294
            version = last_patch.version
309
304
                    break
310
305
                # try to grab a copy of ancestor
311
306
                # note that is not optimised: we could look for namespace
312
 
                # transitions and only look for the past after the
 
307
                # transitions and only look for the past after the 
313
308
                # transition.
314
309
                for history_root in reuse_history_from:
315
310
                    possible_source = os.path.join(history_root,
316
311
                        map_namespace(ancestor.version))
317
312
                    try:
318
313
                        source = Branch.open(possible_source)
319
 
                        rev_id = revision_id(ancestor, encoding)
 
314
                        rev_id = revision_id(ancestor)
320
315
                        if rev_id in source.revision_history():
321
316
                            do_branch(source, output_dir, rev_id)
322
317
                            last_patch = ancestor
332
327
                break
333
328
        else:
334
329
            raise UserError("Directory \"%s\" already exists, and the last "
335
 
                "revision (%s) is not in the ancestry of %s" %
 
330
                "revision (%s) is not in the ancestry of %s" % 
336
331
                (output_dir, last_patch, version))
337
332
        # Strip off all of the ancestors which are already present
338
333
        # And get a directory starting with the latest ancestor
344
339
 
345
340
###class Importer(object):
346
341
###    """An importer.
347
 
###
 
342
###    
348
343
###    Currently this is used as a parameter object, though more behaviour is
349
344
###    possible later.
350
345
###    """
351
346
###
352
347
###    def __init__(self, output_dir, version, fast=False,
353
 
###                 verbose=False, dry_run=False, max_count=None,
 
348
###                 verbose=False, dry_run=False, max_count=None, 
354
349
###                   reuse_history_from=[]):
355
350
###        self.output_dir = output_dir
356
351
###        self.version = version
357
352
###        self.
358
353
 
359
354
 
360
 
def import_version(output_dir, version, encoding, fast=False,
 
355
def import_version(output_dir, version, fast=False,
361
356
                   verbose=False, dry_run=False, max_count=None,
362
357
                   reuse_history_from=[], standalone=True):
363
358
    """
364
359
    >>> q = test_environ()
365
 
 
 
360
    
366
361
    Progress bars output to stderr, but doctest does not capture that.
367
362
 
368
363
    >>> old_stderr = sys.stderr
375
370
    >>> bzrlib.ui.ui_factory = bzrlib.ui.text.TextUIFactory(
376
371
    ...     bar_type=bzrlib.progress.DotsProgressBar)
377
372
 
378
 
    >>> import_version('/', version, None, dry_run=True)
 
373
    >>> import_version('/', version, dry_run=True)
379
374
    Traceback (most recent call last):
380
375
    NotPreviousImport: / is not the location of a previous import.
381
 
    >>> import_version(result_path, version, None, dry_run=True)
 
376
    >>> import_version(result_path, version, dry_run=True)
382
377
    Traceback (most recent call last):
383
378
    UserError: The version test@example.com/test--test--0.1 does not exist.
384
379
    >>> version = pybaz.Version("test@example.com/test--test--0")
385
 
    >>> import_version(result_path, version, None, dry_run=True) #doctest: +ELLIPSIS
 
380
    >>> import_version(result_path, version, dry_run=True) #doctest: +ELLIPSIS
386
381
    importing test@example.com/test--test--0 into ...
387
382
    ...
388
383
    revisions: ..........................................
389
384
    Dry run, not modifying output_dir
390
385
    Cleaning up
391
 
    >>> import_version(result_path, version, None) #doctest: +ELLIPSIS
 
386
    >>> import_version(result_path, version) #doctest: +ELLIPSIS
392
387
    importing test@example.com/test--test--0 into ...
393
388
    ...
394
389
    revisions: .....................................................................
395
390
    Cleaning up
396
391
    Import complete.
397
 
    >>> import_version(result_path, version, None) #doctest: +ELLIPSIS
 
392
    >>> import_version(result_path, version) #doctest: +ELLIPSIS
398
393
    Tree is up-to-date with test@example.com/test--test--0--patch-2
399
394
    >>> commit_more_test_revisions()
400
 
    >>> import_version(result_path, version, None) #doctest: +ELLIPSIS
 
395
    >>> import_version(result_path, version) #doctest: +ELLIPSIS
401
396
    importing test@example.com/test--test--0 into ...
402
397
    revisions: ....................................................
403
398
    Cleaning up
410
405
    try:
411
406
        try:
412
407
            ancestors, old_revno = get_remaining_revisions(output_dir, version,
413
 
                                                           encoding,
414
408
                                                           reuse_history_from)
415
409
        except NotBranchError, e:
416
410
            raise NotPreviousImport(e.path)
418
412
            progress_bar.note('Version %s has no revisions.' % version)
419
413
            return
420
414
        if len(ancestors) == 0:
421
 
            last_revision, last_encoding = \
422
 
                get_last_revision(Branch.open(output_dir))
 
415
            last_revision = get_last_revision(Branch.open(output_dir))
423
416
            progress_bar.note('Tree is up-to-date with %s' % last_revision)
424
417
            return
425
418
 
426
419
        progress_bar.note("importing %s into %s" % (version, output_dir))
427
 
 
 
420
    
428
421
        tempdir = tempfile.mkdtemp(prefix="baz2bzr-",
429
422
                                   dir=os.path.dirname(output_dir))
430
423
        try:
431
424
            wt = WorkingTree.open(output_dir)
432
425
        except (NotBranchError, NoWorkingTree):
433
426
            wt = None
 
427
        if wt is None:
 
428
            old_basis = EmptyTree()
 
429
        else:
 
430
            old_basis = wt.basis_tree()
434
431
        try:
435
432
            for result in iter_import_version(output_dir, ancestors, tempdir,
436
 
                    pb=progress_bar, encoding=encoding, fast=fast,
437
 
                    verbose=verbose, dry_run=dry_run, max_count=max_count,
438
 
                    standalone=standalone):
 
433
                    pb=progress_bar,
 
434
                    fast=fast, verbose=verbose, dry_run=dry_run,
 
435
                    max_count=max_count, standalone=standalone):
439
436
                show_progress(progress_bar, result)
440
437
            if dry_run:
441
438
                progress_bar.note('Dry run, not modifying output_dir')
442
439
                return
443
 
 
 
440
    
444
441
            # Update the working tree of the branch
445
442
            try:
446
443
                wt = WorkingTree.open(output_dir)
448
445
                wt = None
449
446
            if wt is not None:
450
447
                wt.set_last_revision(wt.branch.last_revision())
451
 
                wt.set_root_id(BAZ_IMPORT_ROOT)
452
 
                wt.revert()
453
 
 
 
448
                merge_inner(wt.branch, wt.basis_tree(), old_basis, 
 
449
                            ignore_zero=True, this_tree=wt)
 
450
                wt.revert([])
 
451
    
454
452
        finally:
455
 
 
 
453
            
456
454
            progress_bar.note('Cleaning up')
457
455
            shutil.rmtree(tempdir)
458
456
        progress_bar.note("Import complete.")
459
457
    finally:
460
458
        progress_bar.finished()
461
 
 
 
459
            
462
460
class UserError(BzrCommandError):
463
461
    def __init__(self, message):
464
462
        """Exception to throw when a user makes an impossible request
473
471
                           % path)
474
472
 
475
473
 
476
 
def revision_id(arch_revision, encoding):
 
474
def revision_id(arch_revision):
477
475
    """
478
476
    Generate a Bzr revision id from an Arch revision id.  'x' in the id
479
477
    designates a revision imported with an experimental algorithm.  A number
481
479
 
482
480
    :param arch_revision: The Arch revision to generate an ID for.
483
481
 
484
 
    >>> revision_id(pybaz.Revision("you@example.com/cat--br--0--base-0"), None)
 
482
    >>> revision_id(pybaz.Revision("you@example.com/cat--br--0--base-0"))
485
483
    'Arch-1:you@example.com%cat--br--0--base-0'
486
 
    >>> revision_id(pybaz.Revision("you@example.com/cat--br--0--base-0"), 'utf-8')
487
 
    'Arch-1-utf-8:you@example.com%cat--br--0--base-0'
488
484
    """
489
 
    if encoding is None:
490
 
        encoding = ''
491
 
    else:
492
 
        encoding = '-' + encoding
493
 
    return "Arch-1%s:%s" % (encoding, str(arch_revision).replace('/', '%'))
 
485
    return "Arch-1:%s" % str(arch_revision).replace('/', '%')
494
486
 
495
487
class NotArchRevision(Exception):
496
488
    def __init__(self, revision_id):
506
498
    >>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0--base-5"))
507
499
    Traceback (most recent call last):
508
500
    NotArchRevision: The revision id Arch-1:jrandom@example.com%test--test--0--base-5 does not look like it came from Arch.
509
 
    >>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0--patch-5")[0])
510
 
    'jrandom@example.com/test--test--0--patch-5'
511
 
    >>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0--patch-5")[0])
512
 
    'jrandom@example.com/test--test--0--patch-5'
513
 
    >>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0--patch-5")[1])
514
 
    'None'
515
 
    >>> str(arch_revision("Arch-1-utf-8:jrandom@example.com%test--test--0--patch-5")[1])
516
 
    'utf-8'
517
 
    >>> str(arch_revision('null:'))
518
 
    '(None, None)'
 
501
    >>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0--patch-5"))
 
502
    'jrandom@example.com/test--test--0--patch-5'
519
503
    """
520
 
    if is_null(revision_id):
521
 
        return None, None
522
 
    if revision_id[:7] not in ('Arch-1:', 'Arch-1-'):
 
504
    if revision_id is None:
 
505
        return None
 
506
    if revision_id[:7] != 'Arch-1:':
523
507
        raise NotArchRevision(revision_id)
524
508
    else:
525
509
        try:
526
 
            encoding, arch_name = revision_id[6:].split(':', 1)
527
 
            arch_name = arch_name.replace('%', '/')
528
 
            if encoding == '':
529
 
                encoding = None
530
 
            else:
531
 
                encoding = encoding[1:]
532
 
            return pybaz.Revision(arch_name), encoding
 
510
            return pybaz.Revision(revision_id[7:].replace('%', '/'))
533
511
        except pybaz.errors.NamespaceError, e:
534
512
            raise NotArchRevision(revision_id)
535
513
 
554
532
    if revision_id is None:
555
533
        revision_id = source.last_revision()
556
534
    wt = create_checkout(source, to_location, NULL_REVISION)
557
 
    wt.lock_write()
558
 
    try:
559
 
        wt.set_last_revision(revision_id)
560
 
        wt.flush()
561
 
        if revision_id not in (NULL_REVISION, None):
562
 
            basis = wt.basis_tree()
563
 
            basis.lock_read()
564
 
            try:
565
 
                wt._write_inventory(basis.inventory)
566
 
            finally:
567
 
                basis.unlock()
568
 
    finally:
569
 
        wt.unlock()
 
535
    wt.set_last_revision(revision_id)
 
536
    wt._write_inventory(wt.basis_tree().inventory)
570
537
    return wt
571
538
 
572
539
 
573
 
def iter_import_version(output_dir, ancestors, tempdir, pb, encoding,
574
 
                        fast=False, verbose=False, dry_run=False,
575
 
                        max_count=None, standalone=False):
 
540
def iter_import_version(output_dir, ancestors, tempdir, pb, fast=False,
 
541
                        verbose=False, dry_run=False, max_count=None,
 
542
                        standalone=False):
576
543
    revdir = None
577
 
    log_encoding = 'ascii'
578
 
    if encoding is not None:
579
 
        log_encoding = encoding
580
544
 
581
545
    # Uncomment this for testing, it basically just has baz2bzr only update
582
546
    # 5 patches at a time
609
573
        else:
610
574
            target_branch = create_branch(output_dir)
611
575
 
612
 
    for i, revision in enumerate(ancestors):
613
 
        rev_id = revision_id(revision, encoding)
 
576
    for i in range(len(ancestors)):
 
577
        revision = ancestors[i]
 
578
        rev_id = revision_id(revision)
614
579
        direct_merges = []
615
580
        if verbose:
616
581
            version = str(revision.version)
622
587
            yield Progress("revisions", i, len(ancestors))
623
588
 
624
589
        if target_branch.repository.has_revision(rev_id):
625
 
            target_branch.set_last_revision_info(i+1, rev_id)
 
590
            target_branch.append_revision(rev_id)
626
591
            continue
627
592
        if revdir is None:
628
593
            revdir = os.path.join(tempdir, "rd")
667
632
        try:
668
633
            if missing_ancestor:
669
634
                # if we want it to be in revision-history, do that here.
670
 
                target_tree.set_parent_ids(
671
 
                    [revision_id(missing_ancestor, encoding)],
672
 
                    allow_leftmost_as_ghost=True)
 
635
                target_tree.add_pending_merge(revision_id(missing_ancestor))
673
636
                missing_ancestor = None
674
637
            for merged_rev in direct_merges:
675
 
                target_tree.add_pending_merge(revision_id(merged_rev,
676
 
                                                          encoding))
677
 
            target_tree.set_root_id(BAZ_IMPORT_ROOT)
678
 
            target_tree.flush()
 
638
                target_tree.add_pending_merge(revision_id(merged_rev))
679
639
            target_tree.set_inventory(baz_inv)
680
640
            commitobj = Commit(reporter=ImportCommitReporter())
681
641
            commitobj.commit(working_tree=target_tree,
682
 
                message=log_message.decode(log_encoding, 'replace'),
683
 
                verbose=False, committer=log_creator, timestamp=timestamp,
684
 
                timezone=0, rev_id=rev_id, revprops={})
 
642
                             message=log_message.decode('ascii', 'replace'), 
 
643
                             verbose=False, committer=log_creator,
 
644
                             timestamp=timestamp, timezone=0, rev_id=rev_id,
 
645
                             revprops={})
685
646
        finally:
686
647
            target_tree.unlock()
687
648
            branch.unlock()
692
653
    previous_version = revision.version
693
654
    if pybaz.WorkingTree(revdir).tree_version != previous_version:
694
655
        pybaz.WorkingTree(revdir).set_tree_version(previous_version)
695
 
    log_path = "%s/{arch}/%s/%s/%s/%s/patch-log/%s" % (revdir,
696
 
        revision.category.nonarch, revision.branch.nonarch,
 
656
    log_path = "%s/{arch}/%s/%s/%s/%s/patch-log/%s" % (revdir, 
 
657
        revision.category.nonarch, revision.branch.nonarch, 
697
658
        revision.version.nonarch, revision.archive, revision.patchlevel)
698
659
    temp_path = tempfile.mktemp(dir=os.path.dirname(revdir))
699
660
    os.rename(log_path, temp_path)
719
680
    tree = revision.get(revdir)
720
681
    log = get_log(tree, revision)
721
682
    try:
722
 
        return tree, bzr_inventory_data(tree), log
 
683
        return tree, bzr_inventory_data(tree), log 
723
684
    except BadFileKind, e:
724
 
        raise UserError("Cannot convert %s because %s is a %s" %
 
685
        raise UserError("Cannot convert %s because %s is a %s" % 
725
686
                        (revision,e.path, e.kind))
726
687
 
727
688
 
731
692
    try:
732
693
        return bzr_inventory_data(tree), log
733
694
    except BadFileKind, e:
734
 
        raise UserError("Cannot convert %s because %s is a %s" %
 
695
        raise UserError("Cannot convert %s because %s is a %s" % 
735
696
                        (revision,e.path, e.kind))
736
697
 
737
698
 
750
711
    inv_map = {}
751
712
    for arch_id, path in inv_iter:
752
713
        bzr_file_id = map_file_id(arch_id)
753
 
        inv_map[path] = bzr_file_id
 
714
        inv_map[path] = bzr_file_id 
754
715
 
755
716
    bzr_inv = []
756
717
    for path, file_id in inv_map.iteritems():
767
728
    bzr_inv.sort()
768
729
    return bzr_inv
769
730
 
 
731
_global_option('max-count', type = int)
 
732
class cmd_baz_import_branch(Command):
 
733
    """Import an Arch or Baz branch into a bzr branch.  <BZRTOOLS>"""
 
734
    takes_args = ['to_location', 'from_branch?', 'reuse_history*']
 
735
    takes_options = ['verbose', 'max-count']
770
736
 
771
 
def baz_import_branch(to_location, from_branch, fast, max_count, verbose,
772
 
                      encoding, dry_run, reuse_history_list):
773
 
    to_location = os.path.realpath(str(to_location))
774
 
    if from_branch is not None:
775
 
        try:
776
 
            from_branch = pybaz.Version(from_branch)
777
 
        except pybaz.errors.NamespaceError:
778
 
            print "%s is not a valid Arch branch." % from_branch
779
 
            return 1
780
 
    if reuse_history_list is None:
781
 
        reuse_history_list = []
782
 
    import_version(to_location, from_branch, encoding, max_count=max_count,
783
 
                   reuse_history_from=reuse_history_list)
 
737
    def run(self, to_location, from_branch=None, fast=False, max_count=None,
 
738
            verbose=False, dry_run=False, reuse_history_list=[]):
 
739
        to_location = os.path.realpath(str(to_location))
 
740
        if from_branch is not None:
 
741
            try:
 
742
                from_branch = pybaz.Version(from_branch)
 
743
            except pybaz.errors.NamespaceError:
 
744
                print "%s is not a valid Arch branch." % from_branch
 
745
                return 1
 
746
        if reuse_history_list is None:
 
747
            reuse_history_list = []
 
748
        import_version(to_location, from_branch, 
 
749
                       max_count=max_count, 
 
750
                       reuse_history_from=reuse_history_list)
784
751
 
785
752
 
786
753
class NotInABranch(Exception):
789
756
        self.path = path
790
757
 
791
758
 
792
 
 
793
 
def baz_import(to_root_dir, from_archive, encoding, verbose=False,
794
 
               reuse_history_list=[], prefixes=None):
795
 
    if reuse_history_list is None:
796
 
        reuse_history_list = []
797
 
    to_root = str(os.path.realpath(to_root_dir))
798
 
    if not os.path.exists(to_root):
799
 
        os.mkdir(to_root)
800
 
    if prefixes is not None:
801
 
        prefixes = prefixes.split(':')
802
 
    import_archive(to_root, from_archive, verbose, encoding,
803
 
                   reuse_history_list, prefixes=prefixes)
 
759
class cmd_baz_import(Command):
 
760
    """Import an Arch or Baz archive into a bzr repository.  <BZRTOOLS>
 
761
 
 
762
    This command should be used on local archives (or mirrors) only.  It is
 
763
    quite slow on remote archives.
 
764
    
 
765
    reuse_history allows you to specify any previous imports you 
 
766
    have done of different archives, which this archive has branches
 
767
    tagged from. This will dramatically reduce the time to convert 
 
768
    the archive as it will not have to convert the history already
 
769
    converted in that other branch.
 
770
 
 
771
    If you specify prefixes, only branches whose names start with that prefix
 
772
    will be imported.  Skipped branches will be listed, so you can import any
 
773
    branches you missed by accident.  Here's an example of doing a partial
 
774
    import from thelove@canonical.com:
 
775
    bzr baz-import thelove thelove@canonical.com --prefixes dists:talloc-except
 
776
    """
 
777
    takes_args = ['to_root_dir', 'from_archive', 'reuse_history*']
 
778
    takes_options = ['verbose', Option('prefixes', type=str,
 
779
                     help="Prefixes of branches to import, colon-separated")]
 
780
 
 
781
    def run(self, to_root_dir, from_archive, verbose=False,
 
782
            reuse_history_list=[], prefixes=None):
 
783
        if reuse_history_list is None:
 
784
            reuse_history_list = []
 
785
        to_root = str(os.path.realpath(to_root_dir))
 
786
        if not os.path.exists(to_root):
 
787
            os.mkdir(to_root)
 
788
        if prefixes is not None:
 
789
            prefixes = prefixes.split(':')
 
790
        import_archive(to_root, from_archive, verbose,
 
791
                       reuse_history_list, prefixes=prefixes)
804
792
 
805
793
 
806
794
def import_archive(to_root, from_archive, verbose,
807
 
                   encoding, reuse_history_from=[], standalone=False,
 
795
                   reuse_history_from=[], standalone=False,
808
796
                   prefixes=None):
809
797
    def selected(version):
810
798
        if prefixes is None:
836
824
            if not os.path.exists(os.path.dirname(target)):
837
825
                os.makedirs(os.path.dirname(target))
838
826
            try:
839
 
                import_version(target, version, encoding,
840
 
                               reuse_history_from=reuse_history_from,
 
827
                import_version(target, version,
 
828
                               reuse_history_from=reuse_history_from, 
841
829
                               standalone=standalone)
842
830
            except pybaz.errors.ExecProblem,e:
843
831
                if str(e).find('The requested revision cannot be built.') != -1: