~abentley/bzrtools/bzrtools.dev

« back to all changes in this revision

Viewing changes to baz_import.py

  • Committer: Aaron Bentley
  • Date: 2008-05-12 18:06:07 UTC
  • Revision ID: aaron@aaronbentley.com-20080512180607-dn6a55if3pk4zdju
Update to avoid deprecated API

Show diffs side-by-side

added added

removed removed

Lines of Context:
21
21
from bzrlib.errors import (BzrError,
22
22
                           NotBranchError,
23
23
                           NoWorkingTree,
24
 
                           BzrCommandError, 
 
24
                           BzrCommandError,
25
25
                           NoSuchRevision,
26
26
                           NoRepositoryPresent,
27
27
                          )
30
30
from bzrlib.commands import Command
31
31
from bzrlib.option import _global_option, Option
32
32
from bzrlib.merge import merge_inner
33
 
from bzrlib.revision import NULL_REVISION
34
 
from bzrlib.tree import EmptyTree
 
33
from bzrlib.revision import NULL_REVISION, is_null
35
34
import bzrlib.ui
36
35
import bzrlib.ui.text
37
36
from bzrlib.workingtree import WorkingTree
43
42
    from pybaz.backends.baz import null_cmd
44
43
except ImportError:
45
44
    raise NoPyBaz
46
 
from fai import iter_new_merges, direct_merges
 
45
from baz_helper import iter_new_merges, direct_merges
47
46
import tempfile
48
47
import os
49
48
import os.path
57
56
import email.Utils
58
57
from progress import *
59
58
 
 
59
 
 
60
BAZ_IMPORT_ROOT = 'TREE_ROOT'
 
61
 
 
62
 
60
63
class ImportCommitReporter(NullCommitReporter):
61
64
 
62
65
    def escaped(self, escape_count, message):
81
84
 
82
85
def make_archive(name, location):
83
86
    pb_location = pybaz.ArchiveLocation(location)
84
 
    pb_location.create_master(pybaz.Archive(name), 
 
87
    pb_location.create_master(pybaz.Archive(name),
85
88
                              pybaz.ArchiveLocationParams())
86
89
 
87
90
def test_environ():
272
275
            br_from.bzrdir.clone(to_location, revision_id)
273
276
        except NoSuchRevision:
274
277
            rmtree(to_location)
275
 
            msg = "The branch %s has no revision %s." % (from_location, 
 
278
            msg = "The branch %s has no revision %s." % (from_location,
276
279
                                                         revision_id)
277
280
            raise UserError(msg)
278
281
    finally:
279
282
        br_from.unlock()
280
283
 
281
 
def get_remaining_revisions(output_dir, version, reuse_history_from=[]):
 
284
def get_remaining_revisions(output_dir, version, encoding,
 
285
                            reuse_history_from=[]):
282
286
    last_patch = None
283
287
    old_revno = None
284
288
    output_exists = os.path.exists(output_dir)
286
290
        # We are starting from an existing directory, figure out what
287
291
        # the current version is
288
292
        branch = Branch.open(output_dir)
289
 
        last_patch = get_last_revision(branch)
 
293
        last_patch, last_encoding = get_last_revision(branch)
 
294
        assert encoding == last_encoding
290
295
        if last_patch is None:
291
 
            if branch.last_revision() != None:
 
296
            if not is_null(branch.last_revision()):
292
297
                raise NotPreviousImport(branch.base)
293
298
        elif version is None:
294
299
            version = last_patch.version
304
309
                    break
305
310
                # try to grab a copy of ancestor
306
311
                # note that is not optimised: we could look for namespace
307
 
                # transitions and only look for the past after the 
 
312
                # transitions and only look for the past after the
308
313
                # transition.
309
314
                for history_root in reuse_history_from:
310
315
                    possible_source = os.path.join(history_root,
311
316
                        map_namespace(ancestor.version))
312
317
                    try:
313
318
                        source = Branch.open(possible_source)
314
 
                        rev_id = revision_id(ancestor)
 
319
                        rev_id = revision_id(ancestor, encoding)
315
320
                        if rev_id in source.revision_history():
316
321
                            do_branch(source, output_dir, rev_id)
317
322
                            last_patch = ancestor
327
332
                break
328
333
        else:
329
334
            raise UserError("Directory \"%s\" already exists, and the last "
330
 
                "revision (%s) is not in the ancestry of %s" % 
 
335
                "revision (%s) is not in the ancestry of %s" %
331
336
                (output_dir, last_patch, version))
332
337
        # Strip off all of the ancestors which are already present
333
338
        # And get a directory starting with the latest ancestor
339
344
 
340
345
###class Importer(object):
341
346
###    """An importer.
342
 
###    
 
347
###
343
348
###    Currently this is used as a parameter object, though more behaviour is
344
349
###    possible later.
345
350
###    """
346
351
###
347
352
###    def __init__(self, output_dir, version, fast=False,
348
 
###                 verbose=False, dry_run=False, max_count=None, 
 
353
###                 verbose=False, dry_run=False, max_count=None,
349
354
###                   reuse_history_from=[]):
350
355
###        self.output_dir = output_dir
351
356
###        self.version = version
352
357
###        self.
353
358
 
354
359
 
355
 
def import_version(output_dir, version, fast=False,
 
360
def import_version(output_dir, version, encoding, fast=False,
356
361
                   verbose=False, dry_run=False, max_count=None,
357
362
                   reuse_history_from=[], standalone=True):
358
363
    """
359
364
    >>> q = test_environ()
360
 
    
 
365
 
361
366
    Progress bars output to stderr, but doctest does not capture that.
362
367
 
363
368
    >>> old_stderr = sys.stderr
370
375
    >>> bzrlib.ui.ui_factory = bzrlib.ui.text.TextUIFactory(
371
376
    ...     bar_type=bzrlib.progress.DotsProgressBar)
372
377
 
373
 
    >>> import_version('/', version, dry_run=True)
 
378
    >>> import_version('/', version, None, dry_run=True)
374
379
    Traceback (most recent call last):
375
380
    NotPreviousImport: / is not the location of a previous import.
376
 
    >>> import_version(result_path, version, dry_run=True)
 
381
    >>> import_version(result_path, version, None, dry_run=True)
377
382
    Traceback (most recent call last):
378
383
    UserError: The version test@example.com/test--test--0.1 does not exist.
379
384
    >>> version = pybaz.Version("test@example.com/test--test--0")
380
 
    >>> import_version(result_path, version, dry_run=True) #doctest: +ELLIPSIS
 
385
    >>> import_version(result_path, version, None, dry_run=True) #doctest: +ELLIPSIS
381
386
    importing test@example.com/test--test--0 into ...
382
387
    ...
383
388
    revisions: ..........................................
384
389
    Dry run, not modifying output_dir
385
390
    Cleaning up
386
 
    >>> import_version(result_path, version) #doctest: +ELLIPSIS
 
391
    >>> import_version(result_path, version, None) #doctest: +ELLIPSIS
387
392
    importing test@example.com/test--test--0 into ...
388
393
    ...
389
394
    revisions: .....................................................................
390
395
    Cleaning up
391
396
    Import complete.
392
 
    >>> import_version(result_path, version) #doctest: +ELLIPSIS
 
397
    >>> import_version(result_path, version, None) #doctest: +ELLIPSIS
393
398
    Tree is up-to-date with test@example.com/test--test--0--patch-2
394
399
    >>> commit_more_test_revisions()
395
 
    >>> import_version(result_path, version) #doctest: +ELLIPSIS
 
400
    >>> import_version(result_path, version, None) #doctest: +ELLIPSIS
396
401
    importing test@example.com/test--test--0 into ...
397
402
    revisions: ....................................................
398
403
    Cleaning up
405
410
    try:
406
411
        try:
407
412
            ancestors, old_revno = get_remaining_revisions(output_dir, version,
 
413
                                                           encoding,
408
414
                                                           reuse_history_from)
409
415
        except NotBranchError, e:
410
416
            raise NotPreviousImport(e.path)
412
418
            progress_bar.note('Version %s has no revisions.' % version)
413
419
            return
414
420
        if len(ancestors) == 0:
415
 
            last_revision = get_last_revision(Branch.open(output_dir))
 
421
            last_revision, last_encoding = \
 
422
                get_last_revision(Branch.open(output_dir))
416
423
            progress_bar.note('Tree is up-to-date with %s' % last_revision)
417
424
            return
418
425
 
419
426
        progress_bar.note("importing %s into %s" % (version, output_dir))
420
 
    
 
427
 
421
428
        tempdir = tempfile.mkdtemp(prefix="baz2bzr-",
422
429
                                   dir=os.path.dirname(output_dir))
423
430
        try:
424
431
            wt = WorkingTree.open(output_dir)
425
432
        except (NotBranchError, NoWorkingTree):
426
433
            wt = None
427
 
        if wt is None:
428
 
            old_basis = EmptyTree()
429
 
        else:
430
 
            old_basis = wt.basis_tree()
431
434
        try:
432
435
            for result in iter_import_version(output_dir, ancestors, tempdir,
433
 
                    pb=progress_bar,
434
 
                    fast=fast, verbose=verbose, dry_run=dry_run,
435
 
                    max_count=max_count, standalone=standalone):
 
436
                    pb=progress_bar, encoding=encoding, fast=fast,
 
437
                    verbose=verbose, dry_run=dry_run, max_count=max_count,
 
438
                    standalone=standalone):
436
439
                show_progress(progress_bar, result)
437
440
            if dry_run:
438
441
                progress_bar.note('Dry run, not modifying output_dir')
439
442
                return
440
 
    
 
443
 
441
444
            # Update the working tree of the branch
442
445
            try:
443
446
                wt = WorkingTree.open(output_dir)
445
448
                wt = None
446
449
            if wt is not None:
447
450
                wt.set_last_revision(wt.branch.last_revision())
448
 
                merge_inner(wt.branch, wt.basis_tree(), old_basis, 
449
 
                            ignore_zero=True, this_tree=wt)
450
 
                wt.revert([])
451
 
    
 
451
                wt.set_root_id(BAZ_IMPORT_ROOT)
 
452
                wt.revert()
 
453
 
452
454
        finally:
453
 
            
 
455
 
454
456
            progress_bar.note('Cleaning up')
455
457
            shutil.rmtree(tempdir)
456
458
        progress_bar.note("Import complete.")
457
459
    finally:
458
460
        progress_bar.finished()
459
 
            
 
461
 
460
462
class UserError(BzrCommandError):
461
463
    def __init__(self, message):
462
464
        """Exception to throw when a user makes an impossible request
471
473
                           % path)
472
474
 
473
475
 
474
 
def revision_id(arch_revision):
 
476
def revision_id(arch_revision, encoding):
475
477
    """
476
478
    Generate a Bzr revision id from an Arch revision id.  'x' in the id
477
479
    designates a revision imported with an experimental algorithm.  A number
479
481
 
480
482
    :param arch_revision: The Arch revision to generate an ID for.
481
483
 
482
 
    >>> revision_id(pybaz.Revision("you@example.com/cat--br--0--base-0"))
 
484
    >>> revision_id(pybaz.Revision("you@example.com/cat--br--0--base-0"), None)
483
485
    'Arch-1:you@example.com%cat--br--0--base-0'
 
486
    >>> revision_id(pybaz.Revision("you@example.com/cat--br--0--base-0"), 'utf-8')
 
487
    'Arch-1-utf-8:you@example.com%cat--br--0--base-0'
484
488
    """
485
 
    return "Arch-1:%s" % str(arch_revision).replace('/', '%')
 
489
    if encoding is None:
 
490
        encoding = ''
 
491
    else:
 
492
        encoding = '-' + encoding
 
493
    return "Arch-1%s:%s" % (encoding, str(arch_revision).replace('/', '%'))
486
494
 
487
495
class NotArchRevision(Exception):
488
496
    def __init__(self, revision_id):
498
506
    >>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0--base-5"))
499
507
    Traceback (most recent call last):
500
508
    NotArchRevision: The revision id Arch-1:jrandom@example.com%test--test--0--base-5 does not look like it came from Arch.
501
 
    >>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0--patch-5"))
502
 
    'jrandom@example.com/test--test--0--patch-5'
 
509
    >>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0--patch-5")[0])
 
510
    'jrandom@example.com/test--test--0--patch-5'
 
511
    >>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0--patch-5")[0])
 
512
    'jrandom@example.com/test--test--0--patch-5'
 
513
    >>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0--patch-5")[1])
 
514
    'None'
 
515
    >>> str(arch_revision("Arch-1-utf-8:jrandom@example.com%test--test--0--patch-5")[1])
 
516
    'utf-8'
 
517
    >>> str(arch_revision('null:'))
 
518
    '(None, None)'
503
519
    """
504
 
    if revision_id is None:
505
 
        return None
506
 
    if revision_id[:7] != 'Arch-1:':
 
520
    if is_null(revision_id):
 
521
        return None, None
 
522
    if revision_id[:7] not in ('Arch-1:', 'Arch-1-'):
507
523
        raise NotArchRevision(revision_id)
508
524
    else:
509
525
        try:
510
 
            return pybaz.Revision(revision_id[7:].replace('%', '/'))
 
526
            encoding, arch_name = revision_id[6:].split(':', 1)
 
527
            arch_name = arch_name.replace('%', '/')
 
528
            if encoding == '':
 
529
                encoding = None
 
530
            else:
 
531
                encoding = encoding[1:]
 
532
            return pybaz.Revision(arch_name), encoding
511
533
        except pybaz.errors.NamespaceError, e:
512
534
            raise NotArchRevision(revision_id)
513
535
 
532
554
    if revision_id is None:
533
555
        revision_id = source.last_revision()
534
556
    wt = create_checkout(source, to_location, NULL_REVISION)
535
 
    wt.set_last_revision(revision_id)
536
 
    wt._write_inventory(wt.basis_tree().inventory)
 
557
    wt.lock_write()
 
558
    try:
 
559
        wt.set_last_revision(revision_id)
 
560
        wt.flush()
 
561
        if revision_id not in (NULL_REVISION, None):
 
562
            basis = wt.basis_tree()
 
563
            basis.lock_read()
 
564
            try:
 
565
                wt._write_inventory(basis.inventory)
 
566
            finally:
 
567
                basis.unlock()
 
568
    finally:
 
569
        wt.unlock()
537
570
    return wt
538
571
 
539
572
 
540
 
def iter_import_version(output_dir, ancestors, tempdir, pb, fast=False,
541
 
                        verbose=False, dry_run=False, max_count=None,
542
 
                        standalone=False):
 
573
def iter_import_version(output_dir, ancestors, tempdir, pb, encoding,
 
574
                        fast=False, verbose=False, dry_run=False,
 
575
                        max_count=None, standalone=False):
543
576
    revdir = None
 
577
    log_encoding = 'ascii'
 
578
    if encoding is not None:
 
579
        log_encoding = encoding
544
580
 
545
581
    # Uncomment this for testing, it basically just has baz2bzr only update
546
582
    # 5 patches at a time
573
609
        else:
574
610
            target_branch = create_branch(output_dir)
575
611
 
576
 
    for i in range(len(ancestors)):
577
 
        revision = ancestors[i]
578
 
        rev_id = revision_id(revision)
 
612
    for i, revision in enumerate(ancestors):
 
613
        rev_id = revision_id(revision, encoding)
579
614
        direct_merges = []
580
615
        if verbose:
581
616
            version = str(revision.version)
587
622
            yield Progress("revisions", i, len(ancestors))
588
623
 
589
624
        if target_branch.repository.has_revision(rev_id):
590
 
            target_branch.append_revision(rev_id)
 
625
            target_branch.set_last_revision_info(i+1, rev_id)
591
626
            continue
592
627
        if revdir is None:
593
628
            revdir = os.path.join(tempdir, "rd")
632
667
        try:
633
668
            if missing_ancestor:
634
669
                # if we want it to be in revision-history, do that here.
635
 
                target_tree.add_pending_merge(revision_id(missing_ancestor))
 
670
                target_tree.set_parent_ids(
 
671
                    [revision_id(missing_ancestor, encoding)],
 
672
                    allow_leftmost_as_ghost=True)
636
673
                missing_ancestor = None
637
674
            for merged_rev in direct_merges:
638
 
                target_tree.add_pending_merge(revision_id(merged_rev))
 
675
                target_tree.add_pending_merge(revision_id(merged_rev,
 
676
                                                          encoding))
 
677
            target_tree.set_root_id(BAZ_IMPORT_ROOT)
 
678
            target_tree.flush()
639
679
            target_tree.set_inventory(baz_inv)
640
680
            commitobj = Commit(reporter=ImportCommitReporter())
641
681
            commitobj.commit(working_tree=target_tree,
642
 
                             message=log_message.decode('ascii', 'replace'), 
643
 
                             verbose=False, committer=log_creator,
644
 
                             timestamp=timestamp, timezone=0, rev_id=rev_id,
645
 
                             revprops={})
 
682
                message=log_message.decode(log_encoding, 'replace'),
 
683
                verbose=False, committer=log_creator, timestamp=timestamp,
 
684
                timezone=0, rev_id=rev_id, revprops={})
646
685
        finally:
647
686
            target_tree.unlock()
648
687
            branch.unlock()
653
692
    previous_version = revision.version
654
693
    if pybaz.WorkingTree(revdir).tree_version != previous_version:
655
694
        pybaz.WorkingTree(revdir).set_tree_version(previous_version)
656
 
    log_path = "%s/{arch}/%s/%s/%s/%s/patch-log/%s" % (revdir, 
657
 
        revision.category.nonarch, revision.branch.nonarch, 
 
695
    log_path = "%s/{arch}/%s/%s/%s/%s/patch-log/%s" % (revdir,
 
696
        revision.category.nonarch, revision.branch.nonarch,
658
697
        revision.version.nonarch, revision.archive, revision.patchlevel)
659
698
    temp_path = tempfile.mktemp(dir=os.path.dirname(revdir))
660
699
    os.rename(log_path, temp_path)
680
719
    tree = revision.get(revdir)
681
720
    log = get_log(tree, revision)
682
721
    try:
683
 
        return tree, bzr_inventory_data(tree), log 
 
722
        return tree, bzr_inventory_data(tree), log
684
723
    except BadFileKind, e:
685
 
        raise UserError("Cannot convert %s because %s is a %s" % 
 
724
        raise UserError("Cannot convert %s because %s is a %s" %
686
725
                        (revision,e.path, e.kind))
687
726
 
688
727
 
692
731
    try:
693
732
        return bzr_inventory_data(tree), log
694
733
    except BadFileKind, e:
695
 
        raise UserError("Cannot convert %s because %s is a %s" % 
 
734
        raise UserError("Cannot convert %s because %s is a %s" %
696
735
                        (revision,e.path, e.kind))
697
736
 
698
737
 
711
750
    inv_map = {}
712
751
    for arch_id, path in inv_iter:
713
752
        bzr_file_id = map_file_id(arch_id)
714
 
        inv_map[path] = bzr_file_id 
 
753
        inv_map[path] = bzr_file_id
715
754
 
716
755
    bzr_inv = []
717
756
    for path, file_id in inv_map.iteritems():
728
767
    bzr_inv.sort()
729
768
    return bzr_inv
730
769
 
731
 
_global_option('max-count', type = int)
732
 
class cmd_baz_import_branch(Command):
733
 
    """Import an Arch or Baz branch into a bzr branch."""
734
 
    takes_args = ['to_location', 'from_branch?', 'reuse_history*']
735
 
    takes_options = ['verbose', 'max-count']
736
770
 
737
 
    def run(self, to_location, from_branch=None, fast=False, max_count=None,
738
 
            verbose=False, dry_run=False, reuse_history_list=[]):
739
 
        to_location = os.path.realpath(str(to_location))
740
 
        if from_branch is not None:
741
 
            try:
742
 
                from_branch = pybaz.Version(from_branch)
743
 
            except pybaz.errors.NamespaceError:
744
 
                print "%s is not a valid Arch branch." % from_branch
745
 
                return 1
746
 
        if reuse_history_list is None:
747
 
            reuse_history_list = []
748
 
        import_version(to_location, from_branch, 
749
 
                       max_count=max_count, 
750
 
                       reuse_history_from=reuse_history_list)
 
771
def baz_import_branch(to_location, from_branch, fast, max_count, verbose,
 
772
                      encoding, dry_run, reuse_history_list):
 
773
    to_location = os.path.realpath(str(to_location))
 
774
    if from_branch is not None:
 
775
        try:
 
776
            from_branch = pybaz.Version(from_branch)
 
777
        except pybaz.errors.NamespaceError:
 
778
            print "%s is not a valid Arch branch." % from_branch
 
779
            return 1
 
780
    if reuse_history_list is None:
 
781
        reuse_history_list = []
 
782
    import_version(to_location, from_branch, encoding, max_count=max_count,
 
783
                   reuse_history_from=reuse_history_list)
751
784
 
752
785
 
753
786
class NotInABranch(Exception):
756
789
        self.path = path
757
790
 
758
791
 
759
 
class cmd_baz_import(Command):
760
 
    """Import an Arch or Baz archive into a bzr repository.
761
 
 
762
 
    This command should be used on local archives (or mirrors) only.  It is
763
 
    quite slow on remote archives.
764
 
    
765
 
    reuse_history allows you to specify any previous imports you 
766
 
    have done of different archives, which this archive has branches
767
 
    tagged from. This will dramatically reduce the time to convert 
768
 
    the archive as it will not have to convert the history already
769
 
    converted in that other branch.
770
 
 
771
 
    If you specify prefixes, only branches whose names start with that prefix
772
 
    will be imported.  Skipped branches will be listed, so you can import any
773
 
    branches you missed by accident.  Here's an example of doing a partial
774
 
    import from thelove@canonical.com:
775
 
    bzr baz-import thelove thelove@canonical.com --prefixes dists:talloc-except
776
 
    """
777
 
    takes_args = ['to_root_dir', 'from_archive', 'reuse_history*']
778
 
    takes_options = ['verbose', Option('prefixes', type=str,
779
 
                     help="Prefixes of branches to import, colon-separated")]
780
 
 
781
 
    def run(self, to_root_dir, from_archive, verbose=False,
782
 
            reuse_history_list=[], prefixes=None):
783
 
        if reuse_history_list is None:
784
 
            reuse_history_list = []
785
 
        to_root = str(os.path.realpath(to_root_dir))
786
 
        if not os.path.exists(to_root):
787
 
            os.mkdir(to_root)
788
 
        if prefixes is not None:
789
 
            prefixes = prefixes.split(':')
790
 
        import_archive(to_root, from_archive, verbose,
791
 
                       reuse_history_list, prefixes=prefixes)
 
792
 
 
793
def baz_import(to_root_dir, from_archive, encoding, verbose=False,
 
794
               reuse_history_list=[], prefixes=None):
 
795
    if reuse_history_list is None:
 
796
        reuse_history_list = []
 
797
    to_root = str(os.path.realpath(to_root_dir))
 
798
    if not os.path.exists(to_root):
 
799
        os.mkdir(to_root)
 
800
    if prefixes is not None:
 
801
        prefixes = prefixes.split(':')
 
802
    import_archive(to_root, from_archive, verbose, encoding,
 
803
                   reuse_history_list, prefixes=prefixes)
792
804
 
793
805
 
794
806
def import_archive(to_root, from_archive, verbose,
795
 
                   reuse_history_from=[], standalone=False,
 
807
                   encoding, reuse_history_from=[], standalone=False,
796
808
                   prefixes=None):
797
809
    def selected(version):
798
810
        if prefixes is None:
824
836
            if not os.path.exists(os.path.dirname(target)):
825
837
                os.makedirs(os.path.dirname(target))
826
838
            try:
827
 
                import_version(target, version,
828
 
                               reuse_history_from=reuse_history_from, 
 
839
                import_version(target, version, encoding,
 
840
                               reuse_history_from=reuse_history_from,
829
841
                               standalone=standalone)
830
842
            except pybaz.errors.ExecProblem,e:
831
843
                if str(e).find('The requested revision cannot be built.') != -1: