~abentley/bzrtools/bzrtools.dev

« back to all changes in this revision

Viewing changes to baz_import.py

  • Committer: Aaron Bentley
  • Date: 2008-03-11 13:25:33 UTC
  • Revision ID: aaron@aaronbentley.com-20080311132533-m4ycn0ck7wqwmkq9
Update for API change

Show diffs side-by-side

added added

removed removed

Lines of Context:
21
21
from bzrlib.errors import (BzrError,
22
22
                           NotBranchError,
23
23
                           NoWorkingTree,
24
 
                           BzrCommandError, 
 
24
                           BzrCommandError,
25
25
                           NoSuchRevision,
26
26
                           NoRepositoryPresent,
27
27
                          )
30
30
from bzrlib.commands import Command
31
31
from bzrlib.option import _global_option, Option
32
32
from bzrlib.merge import merge_inner
33
 
from bzrlib.revision import NULL_REVISION
34
 
from bzrlib.tree import EmptyTree
 
33
from bzrlib.revision import NULL_REVISION, is_null
35
34
import bzrlib.ui
 
35
import bzrlib.ui.text
36
36
from bzrlib.workingtree import WorkingTree
37
37
from errors import NoPyBaz
38
38
try:
42
42
    from pybaz.backends.baz import null_cmd
43
43
except ImportError:
44
44
    raise NoPyBaz
45
 
from fai import iter_new_merges, direct_merges
 
45
from baz_helper import iter_new_merges, direct_merges
46
46
import tempfile
47
47
import os
48
48
import os.path
56
56
import email.Utils
57
57
from progress import *
58
58
 
 
59
 
 
60
BAZ_IMPORT_ROOT = 'TREE_ROOT'
 
61
 
 
62
 
59
63
class ImportCommitReporter(NullCommitReporter):
60
64
 
61
65
    def escaped(self, escape_count, message):
80
84
 
81
85
def make_archive(name, location):
82
86
    pb_location = pybaz.ArchiveLocation(location)
83
 
    pb_location.create_master(pybaz.Archive(name), 
 
87
    pb_location.create_master(pybaz.Archive(name),
84
88
                              pybaz.ArchiveLocationParams())
85
89
 
86
90
def test_environ():
271
275
            br_from.bzrdir.clone(to_location, revision_id)
272
276
        except NoSuchRevision:
273
277
            rmtree(to_location)
274
 
            msg = "The branch %s has no revision %s." % (from_location, 
 
278
            msg = "The branch %s has no revision %s." % (from_location,
275
279
                                                         revision_id)
276
280
            raise UserError(msg)
277
281
    finally:
278
282
        br_from.unlock()
279
283
 
280
 
def get_remaining_revisions(output_dir, version, reuse_history_from=[]):
 
284
def get_remaining_revisions(output_dir, version, encoding,
 
285
                            reuse_history_from=[]):
281
286
    last_patch = None
282
287
    old_revno = None
283
288
    output_exists = os.path.exists(output_dir)
285
290
        # We are starting from an existing directory, figure out what
286
291
        # the current version is
287
292
        branch = Branch.open(output_dir)
288
 
        last_patch = get_last_revision(branch)
 
293
        last_patch, last_encoding = get_last_revision(branch)
 
294
        assert encoding == last_encoding
289
295
        if last_patch is None:
290
 
            if branch.last_revision() != None:
 
296
            if not is_null(branch.last_revision()):
291
297
                raise NotPreviousImport(branch.base)
292
298
        elif version is None:
293
299
            version = last_patch.version
303
309
                    break
304
310
                # try to grab a copy of ancestor
305
311
                # note that is not optimised: we could look for namespace
306
 
                # transitions and only look for the past after the 
 
312
                # transitions and only look for the past after the
307
313
                # transition.
308
314
                for history_root in reuse_history_from:
309
315
                    possible_source = os.path.join(history_root,
310
316
                        map_namespace(ancestor.version))
311
317
                    try:
312
318
                        source = Branch.open(possible_source)
313
 
                        rev_id = revision_id(ancestor)
 
319
                        rev_id = revision_id(ancestor, encoding)
314
320
                        if rev_id in source.revision_history():
315
321
                            do_branch(source, output_dir, rev_id)
316
322
                            last_patch = ancestor
326
332
                break
327
333
        else:
328
334
            raise UserError("Directory \"%s\" already exists, and the last "
329
 
                "revision (%s) is not in the ancestry of %s" % 
 
335
                "revision (%s) is not in the ancestry of %s" %
330
336
                (output_dir, last_patch, version))
331
337
        # Strip off all of the ancestors which are already present
332
338
        # And get a directory starting with the latest ancestor
338
344
 
339
345
###class Importer(object):
340
346
###    """An importer.
341
 
###    
 
347
###
342
348
###    Currently this is used as a parameter object, though more behaviour is
343
349
###    possible later.
344
350
###    """
345
351
###
346
352
###    def __init__(self, output_dir, version, fast=False,
347
 
###                 verbose=False, dry_run=False, max_count=None, 
 
353
###                 verbose=False, dry_run=False, max_count=None,
348
354
###                   reuse_history_from=[]):
349
355
###        self.output_dir = output_dir
350
356
###        self.version = version
351
357
###        self.
352
358
 
353
359
 
354
 
def import_version(output_dir, version, fast=False,
 
360
def import_version(output_dir, version, encoding, fast=False,
355
361
                   verbose=False, dry_run=False, max_count=None,
356
362
                   reuse_history_from=[], standalone=True):
357
363
    """
358
364
    >>> q = test_environ()
359
 
    
 
365
 
360
366
    Progress bars output to stderr, but doctest does not capture that.
361
367
 
362
368
    >>> old_stderr = sys.stderr
369
375
    >>> bzrlib.ui.ui_factory = bzrlib.ui.text.TextUIFactory(
370
376
    ...     bar_type=bzrlib.progress.DotsProgressBar)
371
377
 
372
 
    >>> import_version('/', version, dry_run=True)
 
378
    >>> import_version('/', version, None, dry_run=True)
373
379
    Traceback (most recent call last):
374
380
    NotPreviousImport: / is not the location of a previous import.
375
 
    >>> import_version(result_path, version, dry_run=True)
 
381
    >>> import_version(result_path, version, None, dry_run=True)
376
382
    Traceback (most recent call last):
377
383
    UserError: The version test@example.com/test--test--0.1 does not exist.
378
384
    >>> version = pybaz.Version("test@example.com/test--test--0")
379
 
    >>> import_version(result_path, version, dry_run=True) #doctest: +ELLIPSIS
 
385
    >>> import_version(result_path, version, None, dry_run=True) #doctest: +ELLIPSIS
380
386
    importing test@example.com/test--test--0 into ...
381
387
    ...
382
388
    revisions: ..........................................
383
389
    Dry run, not modifying output_dir
384
390
    Cleaning up
385
 
    >>> import_version(result_path, version) #doctest: +ELLIPSIS
 
391
    >>> import_version(result_path, version, None) #doctest: +ELLIPSIS
386
392
    importing test@example.com/test--test--0 into ...
387
393
    ...
388
394
    revisions: .....................................................................
389
395
    Cleaning up
390
396
    Import complete.
391
 
    >>> import_version(result_path, version) #doctest: +ELLIPSIS
 
397
    >>> import_version(result_path, version, None) #doctest: +ELLIPSIS
392
398
    Tree is up-to-date with test@example.com/test--test--0--patch-2
393
399
    >>> commit_more_test_revisions()
394
 
    >>> import_version(result_path, version) #doctest: +ELLIPSIS
 
400
    >>> import_version(result_path, version, None) #doctest: +ELLIPSIS
395
401
    importing test@example.com/test--test--0 into ...
396
402
    revisions: ....................................................
397
403
    Cleaning up
404
410
    try:
405
411
        try:
406
412
            ancestors, old_revno = get_remaining_revisions(output_dir, version,
 
413
                                                           encoding,
407
414
                                                           reuse_history_from)
408
415
        except NotBranchError, e:
409
416
            raise NotPreviousImport(e.path)
411
418
            progress_bar.note('Version %s has no revisions.' % version)
412
419
            return
413
420
        if len(ancestors) == 0:
414
 
            last_revision = get_last_revision(Branch.open(output_dir))
 
421
            last_revision, last_encoding = \
 
422
                get_last_revision(Branch.open(output_dir))
415
423
            progress_bar.note('Tree is up-to-date with %s' % last_revision)
416
424
            return
417
425
 
418
426
        progress_bar.note("importing %s into %s" % (version, output_dir))
419
 
    
 
427
 
420
428
        tempdir = tempfile.mkdtemp(prefix="baz2bzr-",
421
429
                                   dir=os.path.dirname(output_dir))
422
430
        try:
423
431
            wt = WorkingTree.open(output_dir)
424
432
        except (NotBranchError, NoWorkingTree):
425
433
            wt = None
426
 
        if wt is None:
427
 
            old_basis = EmptyTree()
428
 
        else:
429
 
            old_basis = wt.basis_tree()
430
434
        try:
431
435
            for result in iter_import_version(output_dir, ancestors, tempdir,
432
 
                    pb=progress_bar,
433
 
                    fast=fast, verbose=verbose, dry_run=dry_run,
434
 
                    max_count=max_count, standalone=standalone):
 
436
                    pb=progress_bar, encoding=encoding, fast=fast,
 
437
                    verbose=verbose, dry_run=dry_run, max_count=max_count,
 
438
                    standalone=standalone):
435
439
                show_progress(progress_bar, result)
436
440
            if dry_run:
437
441
                progress_bar.note('Dry run, not modifying output_dir')
438
442
                return
439
 
    
 
443
 
440
444
            # Update the working tree of the branch
441
445
            try:
442
446
                wt = WorkingTree.open(output_dir)
444
448
                wt = None
445
449
            if wt is not None:
446
450
                wt.set_last_revision(wt.branch.last_revision())
447
 
                merge_inner(wt.branch, wt.basis_tree(), old_basis, 
448
 
                            ignore_zero=True, this_tree=wt)
449
 
                wt.revert([])
450
 
    
 
451
                wt.set_root_id(BAZ_IMPORT_ROOT)
 
452
                wt.revert()
 
453
 
451
454
        finally:
452
 
            
 
455
 
453
456
            progress_bar.note('Cleaning up')
454
457
            shutil.rmtree(tempdir)
455
458
        progress_bar.note("Import complete.")
456
459
    finally:
457
460
        progress_bar.finished()
458
 
            
 
461
 
459
462
class UserError(BzrCommandError):
460
463
    def __init__(self, message):
461
464
        """Exception to throw when a user makes an impossible request
470
473
                           % path)
471
474
 
472
475
 
473
 
def revision_id(arch_revision):
 
476
def revision_id(arch_revision, encoding):
474
477
    """
475
478
    Generate a Bzr revision id from an Arch revision id.  'x' in the id
476
479
    designates a revision imported with an experimental algorithm.  A number
478
481
 
479
482
    :param arch_revision: The Arch revision to generate an ID for.
480
483
 
481
 
    >>> revision_id(pybaz.Revision("you@example.com/cat--br--0--base-0"))
 
484
    >>> revision_id(pybaz.Revision("you@example.com/cat--br--0--base-0"), None)
482
485
    'Arch-1:you@example.com%cat--br--0--base-0'
 
486
    >>> revision_id(pybaz.Revision("you@example.com/cat--br--0--base-0"), 'utf-8')
 
487
    'Arch-1-utf-8:you@example.com%cat--br--0--base-0'
483
488
    """
484
 
    return "Arch-1:%s" % str(arch_revision).replace('/', '%')
 
489
    if encoding is None:
 
490
        encoding = ''
 
491
    else:
 
492
        encoding = '-' + encoding
 
493
    return "Arch-1%s:%s" % (encoding, str(arch_revision).replace('/', '%'))
485
494
 
486
495
class NotArchRevision(Exception):
487
496
    def __init__(self, revision_id):
497
506
    >>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0--base-5"))
498
507
    Traceback (most recent call last):
499
508
    NotArchRevision: The revision id Arch-1:jrandom@example.com%test--test--0--base-5 does not look like it came from Arch.
500
 
    >>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0--patch-5"))
501
 
    'jrandom@example.com/test--test--0--patch-5'
 
509
    >>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0--patch-5")[0])
 
510
    'jrandom@example.com/test--test--0--patch-5'
 
511
    >>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0--patch-5")[0])
 
512
    'jrandom@example.com/test--test--0--patch-5'
 
513
    >>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0--patch-5")[1])
 
514
    'None'
 
515
    >>> str(arch_revision("Arch-1-utf-8:jrandom@example.com%test--test--0--patch-5")[1])
 
516
    'utf-8'
 
517
    >>> str(arch_revision('null:'))
 
518
    '(None, None)'
502
519
    """
503
 
    if revision_id is None:
504
 
        return None
505
 
    if revision_id[:7] != 'Arch-1:':
 
520
    if is_null(revision_id):
 
521
        return None, None
 
522
    if revision_id[:7] not in ('Arch-1:', 'Arch-1-'):
506
523
        raise NotArchRevision(revision_id)
507
524
    else:
508
525
        try:
509
 
            return pybaz.Revision(revision_id[7:].replace('%', '/'))
 
526
            encoding, arch_name = revision_id[6:].split(':', 1)
 
527
            arch_name = arch_name.replace('%', '/')
 
528
            if encoding == '':
 
529
                encoding = None
 
530
            else:
 
531
                encoding = encoding[1:]
 
532
            return pybaz.Revision(arch_name), encoding
510
533
        except pybaz.errors.NamespaceError, e:
511
534
            raise NotArchRevision(revision_id)
512
535
 
531
554
    if revision_id is None:
532
555
        revision_id = source.last_revision()
533
556
    wt = create_checkout(source, to_location, NULL_REVISION)
534
 
    wt.set_last_revision(revision_id)
535
 
    wt._write_inventory(wt.basis_tree().inventory)
 
557
    wt.lock_write()
 
558
    try:
 
559
        wt.set_last_revision(revision_id)
 
560
        wt.flush()
 
561
        if revision_id not in (NULL_REVISION, None):
 
562
            basis = wt.basis_tree()
 
563
            basis.lock_read()
 
564
            try:
 
565
                wt._write_inventory(basis.inventory)
 
566
            finally:
 
567
                basis.unlock()
 
568
    finally:
 
569
        wt.unlock()
536
570
    return wt
537
571
 
538
572
 
539
 
def iter_import_version(output_dir, ancestors, tempdir, pb, fast=False,
540
 
                        verbose=False, dry_run=False, max_count=None,
541
 
                        standalone=False):
 
573
def iter_import_version(output_dir, ancestors, tempdir, pb, encoding,
 
574
                        fast=False, verbose=False, dry_run=False,
 
575
                        max_count=None, standalone=False):
542
576
    revdir = None
 
577
    log_encoding = 'ascii'
 
578
    if encoding is not None:
 
579
        log_encoding = encoding
543
580
 
544
581
    # Uncomment this for testing, it basically just has baz2bzr only update
545
582
    # 5 patches at a time
572
609
        else:
573
610
            target_branch = create_branch(output_dir)
574
611
 
575
 
    for i in range(len(ancestors)):
576
 
        revision = ancestors[i]
577
 
        rev_id = revision_id(revision)
 
612
    for i, revision in enumerate(ancestors):
 
613
        rev_id = revision_id(revision, encoding)
578
614
        direct_merges = []
579
615
        if verbose:
580
616
            version = str(revision.version)
586
622
            yield Progress("revisions", i, len(ancestors))
587
623
 
588
624
        if target_branch.repository.has_revision(rev_id):
589
 
            target_branch.append_revision(rev_id)
 
625
            target_branch.set_last_revision_info(i+1, rev_id)
590
626
            continue
591
627
        if revdir is None:
592
628
            revdir = os.path.join(tempdir, "rd")
631
667
        try:
632
668
            if missing_ancestor:
633
669
                # if we want it to be in revision-history, do that here.
634
 
                target_tree.add_pending_merge(revision_id(missing_ancestor))
 
670
                target_tree.set_parent_ids(
 
671
                    [revision_id(missing_ancestor, encoding)],
 
672
                    allow_leftmost_as_ghost=True)
635
673
                missing_ancestor = None
636
674
            for merged_rev in direct_merges:
637
 
                target_tree.add_pending_merge(revision_id(merged_rev))
 
675
                target_tree.add_pending_merge(revision_id(merged_rev,
 
676
                                                          encoding))
 
677
            target_tree.set_root_id(BAZ_IMPORT_ROOT)
 
678
            target_tree.flush()
638
679
            target_tree.set_inventory(baz_inv)
639
680
            commitobj = Commit(reporter=ImportCommitReporter())
640
681
            commitobj.commit(working_tree=target_tree,
641
 
                             message=log_message.decode('ascii', 'replace'), 
642
 
                             verbose=False, committer=log_creator,
643
 
                             timestamp=timestamp, timezone=0, rev_id=rev_id,
644
 
                             revprops={})
 
682
                message=log_message.decode(log_encoding, 'replace'),
 
683
                verbose=False, committer=log_creator, timestamp=timestamp,
 
684
                timezone=0, rev_id=rev_id, revprops={})
645
685
        finally:
646
686
            target_tree.unlock()
647
687
            branch.unlock()
652
692
    previous_version = revision.version
653
693
    if pybaz.WorkingTree(revdir).tree_version != previous_version:
654
694
        pybaz.WorkingTree(revdir).set_tree_version(previous_version)
655
 
    log_path = "%s/{arch}/%s/%s/%s/%s/patch-log/%s" % (revdir, 
656
 
        revision.category.nonarch, revision.branch.nonarch, 
 
695
    log_path = "%s/{arch}/%s/%s/%s/%s/patch-log/%s" % (revdir,
 
696
        revision.category.nonarch, revision.branch.nonarch,
657
697
        revision.version.nonarch, revision.archive, revision.patchlevel)
658
698
    temp_path = tempfile.mktemp(dir=os.path.dirname(revdir))
659
699
    os.rename(log_path, temp_path)
679
719
    tree = revision.get(revdir)
680
720
    log = get_log(tree, revision)
681
721
    try:
682
 
        return tree, bzr_inventory_data(tree), log 
 
722
        return tree, bzr_inventory_data(tree), log
683
723
    except BadFileKind, e:
684
 
        raise UserError("Cannot convert %s because %s is a %s" % 
 
724
        raise UserError("Cannot convert %s because %s is a %s" %
685
725
                        (revision,e.path, e.kind))
686
726
 
687
727
 
691
731
    try:
692
732
        return bzr_inventory_data(tree), log
693
733
    except BadFileKind, e:
694
 
        raise UserError("Cannot convert %s because %s is a %s" % 
 
734
        raise UserError("Cannot convert %s because %s is a %s" %
695
735
                        (revision,e.path, e.kind))
696
736
 
697
737
 
710
750
    inv_map = {}
711
751
    for arch_id, path in inv_iter:
712
752
        bzr_file_id = map_file_id(arch_id)
713
 
        inv_map[path] = bzr_file_id 
 
753
        inv_map[path] = bzr_file_id
714
754
 
715
755
    bzr_inv = []
716
756
    for path, file_id in inv_map.iteritems():
727
767
    bzr_inv.sort()
728
768
    return bzr_inv
729
769
 
730
 
_global_option('max-count', type = int)
731
 
class cmd_baz_import_branch(Command):
732
 
    """Import an Arch or Baz branch into a bzr branch.  <BZRTOOLS>"""
733
 
    takes_args = ['to_location', 'from_branch?', 'reuse_history*']
734
 
    takes_options = ['verbose', 'max-count']
735
770
 
736
 
    def run(self, to_location, from_branch=None, fast=False, max_count=None,
737
 
            verbose=False, dry_run=False, reuse_history_list=[]):
738
 
        to_location = os.path.realpath(str(to_location))
739
 
        if from_branch is not None:
740
 
            try:
741
 
                from_branch = pybaz.Version(from_branch)
742
 
            except pybaz.errors.NamespaceError:
743
 
                print "%s is not a valid Arch branch." % from_branch
744
 
                return 1
745
 
        if reuse_history_list is None:
746
 
            reuse_history_list = []
747
 
        import_version(to_location, from_branch, 
748
 
                       max_count=max_count, 
749
 
                       reuse_history_from=reuse_history_list)
 
771
def baz_import_branch(to_location, from_branch, fast, max_count, verbose,
 
772
                      encoding, dry_run, reuse_history_list):
 
773
    to_location = os.path.realpath(str(to_location))
 
774
    if from_branch is not None:
 
775
        try:
 
776
            from_branch = pybaz.Version(from_branch)
 
777
        except pybaz.errors.NamespaceError:
 
778
            print "%s is not a valid Arch branch." % from_branch
 
779
            return 1
 
780
    if reuse_history_list is None:
 
781
        reuse_history_list = []
 
782
    import_version(to_location, from_branch, encoding, max_count=max_count,
 
783
                   reuse_history_from=reuse_history_list)
750
784
 
751
785
 
752
786
class NotInABranch(Exception):
755
789
        self.path = path
756
790
 
757
791
 
758
 
class cmd_baz_import(Command):
759
 
    """Import an Arch or Baz archive into a bzr repository.  <BZRTOOLS>
760
 
 
761
 
    This command should be used on local archives (or mirrors) only.  It is
762
 
    quite slow on remote archives.
763
 
    
764
 
    reuse_history allows you to specify any previous imports you 
765
 
    have done of different archives, which this archive has branches
766
 
    tagged from. This will dramatically reduce the time to convert 
767
 
    the archive as it will not have to convert the history already
768
 
    converted in that other branch.
769
 
 
770
 
    If you specify prefixes, only branches whose names start with that prefix
771
 
    will be imported.  Skipped branches will be listed, so you can import any
772
 
    branches you missed by accident.  Here's an example of doing a partial
773
 
    import from thelove@canonical.com:
774
 
    bzr baz-import thelove thelove@canonical.com --prefixes dists:talloc-except
775
 
    """
776
 
    takes_args = ['to_root_dir', 'from_archive', 'reuse_history*']
777
 
    takes_options = ['verbose', Option('prefixes', type=str,
778
 
                     help="Prefixes of branches to import, colon-separated")]
779
 
 
780
 
    def run(self, to_root_dir, from_archive, verbose=False,
781
 
            reuse_history_list=[], prefixes=None):
782
 
        if reuse_history_list is None:
783
 
            reuse_history_list = []
784
 
        to_root = str(os.path.realpath(to_root_dir))
785
 
        if not os.path.exists(to_root):
786
 
            os.mkdir(to_root)
787
 
        if prefixes is not None:
788
 
            prefixes = prefixes.split(':')
789
 
        import_archive(to_root, from_archive, verbose,
790
 
                       reuse_history_list, prefixes=prefixes)
 
792
 
 
793
def baz_import(to_root_dir, from_archive, encoding, verbose=False,
 
794
               reuse_history_list=[], prefixes=None):
 
795
    if reuse_history_list is None:
 
796
        reuse_history_list = []
 
797
    to_root = str(os.path.realpath(to_root_dir))
 
798
    if not os.path.exists(to_root):
 
799
        os.mkdir(to_root)
 
800
    if prefixes is not None:
 
801
        prefixes = prefixes.split(':')
 
802
    import_archive(to_root, from_archive, verbose, encoding,
 
803
                   reuse_history_list, prefixes=prefixes)
791
804
 
792
805
 
793
806
def import_archive(to_root, from_archive, verbose,
794
 
                   reuse_history_from=[], standalone=False,
 
807
                   encoding, reuse_history_from=[], standalone=False,
795
808
                   prefixes=None):
796
809
    def selected(version):
797
810
        if prefixes is None:
823
836
            if not os.path.exists(os.path.dirname(target)):
824
837
                os.makedirs(os.path.dirname(target))
825
838
            try:
826
 
                import_version(target, version,
827
 
                               reuse_history_from=reuse_history_from, 
 
839
                import_version(target, version, encoding,
 
840
                               reuse_history_from=reuse_history_from,
828
841
                               standalone=standalone)
829
842
            except pybaz.errors.ExecProblem,e:
830
843
                if str(e).find('The requested revision cannot be built.') != -1: