~abentley/bzrtools/bzrtools.dev

« back to all changes in this revision

Viewing changes to baz_import.py

  • Committer: Aaron Bentley
  • Date: 2007-06-11 05:08:34 UTC
  • Revision ID: aaron.bentley@utoronto.ca-20070611050834-wcbta2pfitcuopku
fix long-line detection

Show diffs side-by-side

added added

removed removed

Lines of Context:
21
21
from bzrlib.errors import (BzrError,
22
22
                           NotBranchError,
23
23
                           NoWorkingTree,
24
 
                           BzrCommandError, 
 
24
                           BzrCommandError,
25
25
                           NoSuchRevision,
26
26
                           NoRepositoryPresent,
27
27
                          )
31
31
from bzrlib.option import _global_option, Option
32
32
from bzrlib.merge import merge_inner
33
33
from bzrlib.revision import NULL_REVISION
34
 
from bzrlib.tree import EmptyTree
35
34
import bzrlib.ui
 
35
import bzrlib.ui.text
36
36
from bzrlib.workingtree import WorkingTree
37
37
from errors import NoPyBaz
38
38
try:
56
56
import email.Utils
57
57
from progress import *
58
58
 
 
59
 
 
60
BAZ_IMPORT_ROOT = 'TREE_ROOT'
 
61
 
 
62
 
59
63
class ImportCommitReporter(NullCommitReporter):
60
64
 
61
65
    def escaped(self, escape_count, message):
80
84
 
81
85
def make_archive(name, location):
82
86
    pb_location = pybaz.ArchiveLocation(location)
83
 
    pb_location.create_master(pybaz.Archive(name), 
 
87
    pb_location.create_master(pybaz.Archive(name),
84
88
                              pybaz.ArchiveLocationParams())
85
89
 
86
90
def test_environ():
271
275
            br_from.bzrdir.clone(to_location, revision_id)
272
276
        except NoSuchRevision:
273
277
            rmtree(to_location)
274
 
            msg = "The branch %s has no revision %s." % (from_location, 
 
278
            msg = "The branch %s has no revision %s." % (from_location,
275
279
                                                         revision_id)
276
280
            raise UserError(msg)
277
281
    finally:
278
282
        br_from.unlock()
279
283
 
280
 
def get_remaining_revisions(output_dir, version, reuse_history_from=[]):
 
284
def get_remaining_revisions(output_dir, version, encoding,
 
285
                            reuse_history_from=[]):
281
286
    last_patch = None
282
287
    old_revno = None
283
288
    output_exists = os.path.exists(output_dir)
285
290
        # We are starting from an existing directory, figure out what
286
291
        # the current version is
287
292
        branch = Branch.open(output_dir)
288
 
        last_patch = get_last_revision(branch)
 
293
        last_patch, last_encoding = get_last_revision(branch)
 
294
        assert encoding == last_encoding
289
295
        if last_patch is None:
290
296
            if branch.last_revision() != None:
291
297
                raise NotPreviousImport(branch.base)
303
309
                    break
304
310
                # try to grab a copy of ancestor
305
311
                # note that is not optimised: we could look for namespace
306
 
                # transitions and only look for the past after the 
 
312
                # transitions and only look for the past after the
307
313
                # transition.
308
314
                for history_root in reuse_history_from:
309
315
                    possible_source = os.path.join(history_root,
310
316
                        map_namespace(ancestor.version))
311
317
                    try:
312
318
                        source = Branch.open(possible_source)
313
 
                        rev_id = revision_id(ancestor)
 
319
                        rev_id = revision_id(ancestor, encoding)
314
320
                        if rev_id in source.revision_history():
315
321
                            do_branch(source, output_dir, rev_id)
316
322
                            last_patch = ancestor
326
332
                break
327
333
        else:
328
334
            raise UserError("Directory \"%s\" already exists, and the last "
329
 
                "revision (%s) is not in the ancestry of %s" % 
 
335
                "revision (%s) is not in the ancestry of %s" %
330
336
                (output_dir, last_patch, version))
331
337
        # Strip off all of the ancestors which are already present
332
338
        # And get a directory starting with the latest ancestor
338
344
 
339
345
###class Importer(object):
340
346
###    """An importer.
341
 
###    
 
347
###
342
348
###    Currently this is used as a parameter object, though more behaviour is
343
349
###    possible later.
344
350
###    """
345
351
###
346
352
###    def __init__(self, output_dir, version, fast=False,
347
 
###                 verbose=False, dry_run=False, max_count=None, 
 
353
###                 verbose=False, dry_run=False, max_count=None,
348
354
###                   reuse_history_from=[]):
349
355
###        self.output_dir = output_dir
350
356
###        self.version = version
351
357
###        self.
352
358
 
353
359
 
354
 
def import_version(output_dir, version, fast=False,
 
360
def import_version(output_dir, version, encoding, fast=False,
355
361
                   verbose=False, dry_run=False, max_count=None,
356
362
                   reuse_history_from=[], standalone=True):
357
363
    """
358
364
    >>> q = test_environ()
359
 
    
 
365
 
360
366
    Progress bars output to stderr, but doctest does not capture that.
361
367
 
362
368
    >>> old_stderr = sys.stderr
369
375
    >>> bzrlib.ui.ui_factory = bzrlib.ui.text.TextUIFactory(
370
376
    ...     bar_type=bzrlib.progress.DotsProgressBar)
371
377
 
372
 
    >>> import_version('/', version, dry_run=True)
 
378
    >>> import_version('/', version, None, dry_run=True)
373
379
    Traceback (most recent call last):
374
380
    NotPreviousImport: / is not the location of a previous import.
375
 
    >>> import_version(result_path, version, dry_run=True)
 
381
    >>> import_version(result_path, version, None, dry_run=True)
376
382
    Traceback (most recent call last):
377
383
    UserError: The version test@example.com/test--test--0.1 does not exist.
378
384
    >>> version = pybaz.Version("test@example.com/test--test--0")
379
 
    >>> import_version(result_path, version, dry_run=True) #doctest: +ELLIPSIS
 
385
    >>> import_version(result_path, version, None, dry_run=True) #doctest: +ELLIPSIS
380
386
    importing test@example.com/test--test--0 into ...
381
387
    ...
382
388
    revisions: ..........................................
383
389
    Dry run, not modifying output_dir
384
390
    Cleaning up
385
 
    >>> import_version(result_path, version) #doctest: +ELLIPSIS
 
391
    >>> import_version(result_path, version, None) #doctest: +ELLIPSIS
386
392
    importing test@example.com/test--test--0 into ...
387
393
    ...
388
394
    revisions: .....................................................................
389
395
    Cleaning up
390
396
    Import complete.
391
 
    >>> import_version(result_path, version) #doctest: +ELLIPSIS
 
397
    >>> import_version(result_path, version, None) #doctest: +ELLIPSIS
392
398
    Tree is up-to-date with test@example.com/test--test--0--patch-2
393
399
    >>> commit_more_test_revisions()
394
 
    >>> import_version(result_path, version) #doctest: +ELLIPSIS
 
400
    >>> import_version(result_path, version, None) #doctest: +ELLIPSIS
395
401
    importing test@example.com/test--test--0 into ...
396
402
    revisions: ....................................................
397
403
    Cleaning up
404
410
    try:
405
411
        try:
406
412
            ancestors, old_revno = get_remaining_revisions(output_dir, version,
 
413
                                                           encoding,
407
414
                                                           reuse_history_from)
408
415
        except NotBranchError, e:
409
416
            raise NotPreviousImport(e.path)
411
418
            progress_bar.note('Version %s has no revisions.' % version)
412
419
            return
413
420
        if len(ancestors) == 0:
414
 
            last_revision = get_last_revision(Branch.open(output_dir))
 
421
            last_revision, last_encoding = \
 
422
                get_last_revision(Branch.open(output_dir))
415
423
            progress_bar.note('Tree is up-to-date with %s' % last_revision)
416
424
            return
417
425
 
418
426
        progress_bar.note("importing %s into %s" % (version, output_dir))
419
 
    
 
427
 
420
428
        tempdir = tempfile.mkdtemp(prefix="baz2bzr-",
421
429
                                   dir=os.path.dirname(output_dir))
422
430
        try:
423
431
            wt = WorkingTree.open(output_dir)
424
432
        except (NotBranchError, NoWorkingTree):
425
433
            wt = None
426
 
        if wt is None:
427
 
            old_basis = EmptyTree()
428
 
        else:
429
 
            old_basis = wt.basis_tree()
430
434
        try:
431
435
            for result in iter_import_version(output_dir, ancestors, tempdir,
432
 
                    pb=progress_bar,
433
 
                    fast=fast, verbose=verbose, dry_run=dry_run,
434
 
                    max_count=max_count, standalone=standalone):
 
436
                    pb=progress_bar, encoding=encoding, fast=fast,
 
437
                    verbose=verbose, dry_run=dry_run, max_count=max_count,
 
438
                    standalone=standalone):
435
439
                show_progress(progress_bar, result)
436
440
            if dry_run:
437
441
                progress_bar.note('Dry run, not modifying output_dir')
438
442
                return
439
 
    
 
443
 
440
444
            # Update the working tree of the branch
441
445
            try:
442
446
                wt = WorkingTree.open(output_dir)
444
448
                wt = None
445
449
            if wt is not None:
446
450
                wt.set_last_revision(wt.branch.last_revision())
447
 
                merge_inner(wt.branch, wt.basis_tree(), old_basis, 
448
 
                            ignore_zero=True, this_tree=wt)
 
451
                wt.set_root_id(BAZ_IMPORT_ROOT)
449
452
                wt.revert([])
450
 
    
 
453
 
451
454
        finally:
452
 
            
 
455
 
453
456
            progress_bar.note('Cleaning up')
454
457
            shutil.rmtree(tempdir)
455
458
        progress_bar.note("Import complete.")
456
459
    finally:
457
460
        progress_bar.finished()
458
 
            
 
461
 
459
462
class UserError(BzrCommandError):
460
463
    def __init__(self, message):
461
464
        """Exception to throw when a user makes an impossible request
470
473
                           % path)
471
474
 
472
475
 
473
 
def revision_id(arch_revision):
 
476
def revision_id(arch_revision, encoding):
474
477
    """
475
478
    Generate a Bzr revision id from an Arch revision id.  'x' in the id
476
479
    designates a revision imported with an experimental algorithm.  A number
478
481
 
479
482
    :param arch_revision: The Arch revision to generate an ID for.
480
483
 
481
 
    >>> revision_id(pybaz.Revision("you@example.com/cat--br--0--base-0"))
 
484
    >>> revision_id(pybaz.Revision("you@example.com/cat--br--0--base-0"), None)
482
485
    'Arch-1:you@example.com%cat--br--0--base-0'
 
486
    >>> revision_id(pybaz.Revision("you@example.com/cat--br--0--base-0"), 'utf-8')
 
487
    'Arch-1-utf-8:you@example.com%cat--br--0--base-0'
483
488
    """
484
 
    return "Arch-1:%s" % str(arch_revision).replace('/', '%')
 
489
    if encoding is None:
 
490
        encoding = ''
 
491
    else:
 
492
        encoding = '-' + encoding
 
493
    return "Arch-1%s:%s" % (encoding, str(arch_revision).replace('/', '%'))
485
494
 
486
495
class NotArchRevision(Exception):
487
496
    def __init__(self, revision_id):
497
506
    >>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0--base-5"))
498
507
    Traceback (most recent call last):
499
508
    NotArchRevision: The revision id Arch-1:jrandom@example.com%test--test--0--base-5 does not look like it came from Arch.
500
 
    >>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0--patch-5"))
501
 
    'jrandom@example.com/test--test--0--patch-5'
 
509
    >>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0--patch-5")[0])
 
510
    'jrandom@example.com/test--test--0--patch-5'
 
511
    >>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0--patch-5")[0])
 
512
    'jrandom@example.com/test--test--0--patch-5'
 
513
    >>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0--patch-5")[1])
 
514
    'None'
 
515
    >>> str(arch_revision("Arch-1-utf-8:jrandom@example.com%test--test--0--patch-5")[1])
 
516
    'utf-8'
502
517
    """
503
518
    if revision_id is None:
504
 
        return None
505
 
    if revision_id[:7] != 'Arch-1:':
 
519
        return None, None
 
520
    if revision_id[:7] not in ('Arch-1:', 'Arch-1-'):
506
521
        raise NotArchRevision(revision_id)
507
522
    else:
508
523
        try:
509
 
            return pybaz.Revision(revision_id[7:].replace('%', '/'))
 
524
            encoding, arch_name = revision_id[6:].split(':', 1)
 
525
            arch_name = arch_name.replace('%', '/')
 
526
            if encoding == '':
 
527
                encoding = None
 
528
            else:
 
529
                encoding = encoding[1:]
 
530
            return pybaz.Revision(arch_name), encoding
510
531
        except pybaz.errors.NamespaceError, e:
511
532
            raise NotArchRevision(revision_id)
512
533
 
531
552
    if revision_id is None:
532
553
        revision_id = source.last_revision()
533
554
    wt = create_checkout(source, to_location, NULL_REVISION)
534
 
    wt.set_last_revision(revision_id)
535
 
    wt._write_inventory(wt.basis_tree().inventory)
 
555
    wt.lock_write()
 
556
    try:
 
557
        wt.set_last_revision(revision_id)
 
558
        wt.flush()
 
559
        if revision_id not in (NULL_REVISION, None):
 
560
            basis = wt.basis_tree()
 
561
            basis.lock_read()
 
562
            try:
 
563
                wt._write_inventory(basis.inventory)
 
564
            finally:
 
565
                basis.unlock()
 
566
    finally:
 
567
        wt.unlock()
536
568
    return wt
537
569
 
538
570
 
539
 
def iter_import_version(output_dir, ancestors, tempdir, pb, fast=False,
540
 
                        verbose=False, dry_run=False, max_count=None,
541
 
                        standalone=False):
 
571
def iter_import_version(output_dir, ancestors, tempdir, pb, encoding,
 
572
                        fast=False, verbose=False, dry_run=False,
 
573
                        max_count=None, standalone=False):
542
574
    revdir = None
 
575
    log_encoding = 'ascii'
 
576
    if encoding is not None:
 
577
        log_encoding = encoding
543
578
 
544
579
    # Uncomment this for testing, it basically just has baz2bzr only update
545
580
    # 5 patches at a time
574
609
 
575
610
    for i in range(len(ancestors)):
576
611
        revision = ancestors[i]
577
 
        rev_id = revision_id(revision)
 
612
        rev_id = revision_id(revision, encoding)
578
613
        direct_merges = []
579
614
        if verbose:
580
615
            version = str(revision.version)
631
666
        try:
632
667
            if missing_ancestor:
633
668
                # if we want it to be in revision-history, do that here.
634
 
                target_tree.add_pending_merge(revision_id(missing_ancestor))
 
669
                target_tree.set_parent_ids(
 
670
                    [revision_id(missing_ancestor, encoding)],
 
671
                    allow_leftmost_as_ghost=True)
635
672
                missing_ancestor = None
636
673
            for merged_rev in direct_merges:
637
 
                target_tree.add_pending_merge(revision_id(merged_rev))
 
674
                target_tree.add_pending_merge(revision_id(merged_rev,
 
675
                                                          encoding))
 
676
            target_tree.set_root_id(BAZ_IMPORT_ROOT)
 
677
            target_tree.flush()
638
678
            target_tree.set_inventory(baz_inv)
639
679
            commitobj = Commit(reporter=ImportCommitReporter())
640
680
            commitobj.commit(working_tree=target_tree,
641
 
                             message=log_message.decode('ascii', 'replace'), 
642
 
                             verbose=False, committer=log_creator,
643
 
                             timestamp=timestamp, timezone=0, rev_id=rev_id,
644
 
                             revprops={})
 
681
                message=log_message.decode(log_encoding, 'replace'),
 
682
                verbose=False, committer=log_creator, timestamp=timestamp,
 
683
                timezone=0, rev_id=rev_id, revprops={})
645
684
        finally:
646
685
            target_tree.unlock()
647
686
            branch.unlock()
652
691
    previous_version = revision.version
653
692
    if pybaz.WorkingTree(revdir).tree_version != previous_version:
654
693
        pybaz.WorkingTree(revdir).set_tree_version(previous_version)
655
 
    log_path = "%s/{arch}/%s/%s/%s/%s/patch-log/%s" % (revdir, 
656
 
        revision.category.nonarch, revision.branch.nonarch, 
 
694
    log_path = "%s/{arch}/%s/%s/%s/%s/patch-log/%s" % (revdir,
 
695
        revision.category.nonarch, revision.branch.nonarch,
657
696
        revision.version.nonarch, revision.archive, revision.patchlevel)
658
697
    temp_path = tempfile.mktemp(dir=os.path.dirname(revdir))
659
698
    os.rename(log_path, temp_path)
679
718
    tree = revision.get(revdir)
680
719
    log = get_log(tree, revision)
681
720
    try:
682
 
        return tree, bzr_inventory_data(tree), log 
 
721
        return tree, bzr_inventory_data(tree), log
683
722
    except BadFileKind, e:
684
 
        raise UserError("Cannot convert %s because %s is a %s" % 
 
723
        raise UserError("Cannot convert %s because %s is a %s" %
685
724
                        (revision,e.path, e.kind))
686
725
 
687
726
 
691
730
    try:
692
731
        return bzr_inventory_data(tree), log
693
732
    except BadFileKind, e:
694
 
        raise UserError("Cannot convert %s because %s is a %s" % 
 
733
        raise UserError("Cannot convert %s because %s is a %s" %
695
734
                        (revision,e.path, e.kind))
696
735
 
697
736
 
710
749
    inv_map = {}
711
750
    for arch_id, path in inv_iter:
712
751
        bzr_file_id = map_file_id(arch_id)
713
 
        inv_map[path] = bzr_file_id 
 
752
        inv_map[path] = bzr_file_id
714
753
 
715
754
    bzr_inv = []
716
755
    for path, file_id in inv_map.iteritems():
727
766
    bzr_inv.sort()
728
767
    return bzr_inv
729
768
 
730
 
_global_option('max-count', type = int)
731
 
class cmd_baz_import_branch(Command):
732
 
    """Import an Arch or Baz branch into a bzr branch.  <BZRTOOLS>"""
733
 
    takes_args = ['to_location', 'from_branch?', 'reuse_history*']
734
 
    takes_options = ['verbose', 'max-count']
735
769
 
736
 
    def run(self, to_location, from_branch=None, fast=False, max_count=None,
737
 
            verbose=False, dry_run=False, reuse_history_list=[]):
738
 
        to_location = os.path.realpath(str(to_location))
739
 
        if from_branch is not None:
740
 
            try:
741
 
                from_branch = pybaz.Version(from_branch)
742
 
            except pybaz.errors.NamespaceError:
743
 
                print "%s is not a valid Arch branch." % from_branch
744
 
                return 1
745
 
        if reuse_history_list is None:
746
 
            reuse_history_list = []
747
 
        import_version(to_location, from_branch, 
748
 
                       max_count=max_count, 
749
 
                       reuse_history_from=reuse_history_list)
 
770
def baz_import_branch(to_location, from_branch, fast, max_count, verbose,
 
771
                      encoding, dry_run, reuse_history_list):
 
772
    to_location = os.path.realpath(str(to_location))
 
773
    if from_branch is not None:
 
774
        try:
 
775
            from_branch = pybaz.Version(from_branch)
 
776
        except pybaz.errors.NamespaceError:
 
777
            print "%s is not a valid Arch branch." % from_branch
 
778
            return 1
 
779
    if reuse_history_list is None:
 
780
        reuse_history_list = []
 
781
    import_version(to_location, from_branch, encoding, max_count=max_count,
 
782
                   reuse_history_from=reuse_history_list)
750
783
 
751
784
 
752
785
class NotInABranch(Exception):
755
788
        self.path = path
756
789
 
757
790
 
758
 
class cmd_baz_import(Command):
759
 
    """Import an Arch or Baz archive into a bzr repository.  <BZRTOOLS>
760
 
 
761
 
    This command should be used on local archives (or mirrors) only.  It is
762
 
    quite slow on remote archives.
763
 
    
764
 
    reuse_history allows you to specify any previous imports you 
765
 
    have done of different archives, which this archive has branches
766
 
    tagged from. This will dramatically reduce the time to convert 
767
 
    the archive as it will not have to convert the history already
768
 
    converted in that other branch.
769
 
 
770
 
    If you specify prefixes, only branches whose names start with that prefix
771
 
    will be imported.  Skipped branches will be listed, so you can import any
772
 
    branches you missed by accident.  Here's an example of doing a partial
773
 
    import from thelove@canonical.com:
774
 
    bzr baz-import thelove thelove@canonical.com --prefixes dists:talloc-except
775
 
    """
776
 
    takes_args = ['to_root_dir', 'from_archive', 'reuse_history*']
777
 
    takes_options = ['verbose', Option('prefixes', type=str,
778
 
                     help="Prefixes of branches to import, colon-separated")]
779
 
 
780
 
    def run(self, to_root_dir, from_archive, verbose=False,
781
 
            reuse_history_list=[], prefixes=None):
782
 
        if reuse_history_list is None:
783
 
            reuse_history_list = []
784
 
        to_root = str(os.path.realpath(to_root_dir))
785
 
        if not os.path.exists(to_root):
786
 
            os.mkdir(to_root)
787
 
        if prefixes is not None:
788
 
            prefixes = prefixes.split(':')
789
 
        import_archive(to_root, from_archive, verbose,
790
 
                       reuse_history_list, prefixes=prefixes)
 
791
 
 
792
def baz_import(to_root_dir, from_archive, encoding, verbose=False,
 
793
               reuse_history_list=[], prefixes=None):
 
794
    if reuse_history_list is None:
 
795
        reuse_history_list = []
 
796
    to_root = str(os.path.realpath(to_root_dir))
 
797
    if not os.path.exists(to_root):
 
798
        os.mkdir(to_root)
 
799
    if prefixes is not None:
 
800
        prefixes = prefixes.split(':')
 
801
    import_archive(to_root, from_archive, verbose, encoding,
 
802
                   reuse_history_list, prefixes=prefixes)
791
803
 
792
804
 
793
805
def import_archive(to_root, from_archive, verbose,
794
 
                   reuse_history_from=[], standalone=False,
 
806
                   encoding, reuse_history_from=[], standalone=False,
795
807
                   prefixes=None):
796
808
    def selected(version):
797
809
        if prefixes is None:
823
835
            if not os.path.exists(os.path.dirname(target)):
824
836
                os.makedirs(os.path.dirname(target))
825
837
            try:
826
 
                import_version(target, version,
827
 
                               reuse_history_from=reuse_history_from, 
 
838
                import_version(target, version, encoding,
 
839
                               reuse_history_from=reuse_history_from,
828
840
                               standalone=standalone)
829
841
            except pybaz.errors.ExecProblem,e:
830
842
                if str(e).find('The requested revision cannot be built.') != -1: