~abentley/bzrtools/bzrtools.dev

« back to all changes in this revision

Viewing changes to baz_import.py

  • Committer: Aaron Bentley
  • Date: 2006-05-10 01:29:41 UTC
  • mfrom: (364.1.4 bzrtools)
  • Revision ID: aaron.bentley@utoronto.ca-20060510012941-c46cf94b001d8d3e
Merge

Show diffs side-by-side

added added

removed removed

Lines of Context:
21
21
from bzrlib.errors import (BzrError,
22
22
                           NotBranchError,
23
23
                           NoWorkingTree,
24
 
                           BzrCommandError,
 
24
                           BzrCommandError, 
25
25
                           NoSuchRevision,
26
26
                           NoRepositoryPresent,
27
27
                          )
30
30
from bzrlib.commands import Command
31
31
from bzrlib.option import _global_option, Option
32
32
from bzrlib.merge import merge_inner
33
 
from bzrlib.revision import NULL_REVISION, is_null
 
33
from bzrlib.revision import NULL_REVISION
 
34
from bzrlib.tree import EmptyTree
34
35
import bzrlib.ui
35
 
import bzrlib.ui.text
36
36
from bzrlib.workingtree import WorkingTree
37
37
from errors import NoPyBaz
38
38
try:
42
42
    from pybaz.backends.baz import null_cmd
43
43
except ImportError:
44
44
    raise NoPyBaz
45
 
from baz_helper import iter_new_merges, direct_merges
 
45
from fai import iter_new_merges, direct_merges
46
46
import tempfile
47
47
import os
48
48
import os.path
56
56
import email.Utils
57
57
from progress import *
58
58
 
59
 
 
60
 
BAZ_IMPORT_ROOT = 'TREE_ROOT'
61
 
 
62
 
 
63
59
class ImportCommitReporter(NullCommitReporter):
64
60
 
65
61
    def escaped(self, escape_count, message):
84
80
 
85
81
def make_archive(name, location):
86
82
    pb_location = pybaz.ArchiveLocation(location)
87
 
    pb_location.create_master(pybaz.Archive(name),
 
83
    pb_location.create_master(pybaz.Archive(name), 
88
84
                              pybaz.ArchiveLocationParams())
89
85
 
90
86
def test_environ():
275
271
            br_from.bzrdir.clone(to_location, revision_id)
276
272
        except NoSuchRevision:
277
273
            rmtree(to_location)
278
 
            msg = "The branch %s has no revision %s." % (from_location,
 
274
            msg = "The branch %s has no revision %s." % (from_location, 
279
275
                                                         revision_id)
280
276
            raise UserError(msg)
281
277
    finally:
282
278
        br_from.unlock()
283
279
 
284
 
def get_remaining_revisions(output_dir, version, encoding,
285
 
                            reuse_history_from=[]):
 
280
def get_remaining_revisions(output_dir, version, reuse_history_from=[]):
286
281
    last_patch = None
287
282
    old_revno = None
288
283
    output_exists = os.path.exists(output_dir)
290
285
        # We are starting from an existing directory, figure out what
291
286
        # the current version is
292
287
        branch = Branch.open(output_dir)
293
 
        last_patch, last_encoding = get_last_revision(branch)
294
 
        assert encoding == last_encoding
 
288
        last_patch = get_last_revision(branch)
295
289
        if last_patch is None:
296
 
            if not is_null(branch.last_revision()):
 
290
            if branch.last_revision() != None:
297
291
                raise NotPreviousImport(branch.base)
298
292
        elif version is None:
299
293
            version = last_patch.version
309
303
                    break
310
304
                # try to grab a copy of ancestor
311
305
                # note that is not optimised: we could look for namespace
312
 
                # transitions and only look for the past after the
 
306
                # transitions and only look for the past after the 
313
307
                # transition.
314
308
                for history_root in reuse_history_from:
315
309
                    possible_source = os.path.join(history_root,
316
310
                        map_namespace(ancestor.version))
317
311
                    try:
318
312
                        source = Branch.open(possible_source)
319
 
                        rev_id = revision_id(ancestor, encoding)
 
313
                        rev_id = revision_id(ancestor)
320
314
                        if rev_id in source.revision_history():
321
315
                            do_branch(source, output_dir, rev_id)
322
316
                            last_patch = ancestor
332
326
                break
333
327
        else:
334
328
            raise UserError("Directory \"%s\" already exists, and the last "
335
 
                "revision (%s) is not in the ancestry of %s" %
 
329
                "revision (%s) is not in the ancestry of %s" % 
336
330
                (output_dir, last_patch, version))
337
331
        # Strip off all of the ancestors which are already present
338
332
        # And get a directory starting with the latest ancestor
344
338
 
345
339
###class Importer(object):
346
340
###    """An importer.
347
 
###
 
341
###    
348
342
###    Currently this is used as a parameter object, though more behaviour is
349
343
###    possible later.
350
344
###    """
351
345
###
352
346
###    def __init__(self, output_dir, version, fast=False,
353
 
###                 verbose=False, dry_run=False, max_count=None,
 
347
###                 verbose=False, dry_run=False, max_count=None, 
354
348
###                   reuse_history_from=[]):
355
349
###        self.output_dir = output_dir
356
350
###        self.version = version
357
351
###        self.
358
352
 
359
353
 
360
 
def import_version(output_dir, version, encoding, fast=False,
 
354
def import_version(output_dir, version, fast=False,
361
355
                   verbose=False, dry_run=False, max_count=None,
362
356
                   reuse_history_from=[], standalone=True):
363
357
    """
364
358
    >>> q = test_environ()
365
 
 
 
359
    
366
360
    Progress bars output to stderr, but doctest does not capture that.
367
361
 
368
362
    >>> old_stderr = sys.stderr
375
369
    >>> bzrlib.ui.ui_factory = bzrlib.ui.text.TextUIFactory(
376
370
    ...     bar_type=bzrlib.progress.DotsProgressBar)
377
371
 
378
 
    >>> import_version('/', version, None, dry_run=True)
 
372
    >>> import_version('/', version, dry_run=True)
379
373
    Traceback (most recent call last):
380
374
    NotPreviousImport: / is not the location of a previous import.
381
 
    >>> import_version(result_path, version, None, dry_run=True)
 
375
    >>> import_version(result_path, version, dry_run=True)
382
376
    Traceback (most recent call last):
383
377
    UserError: The version test@example.com/test--test--0.1 does not exist.
384
378
    >>> version = pybaz.Version("test@example.com/test--test--0")
385
 
    >>> import_version(result_path, version, None, dry_run=True) #doctest: +ELLIPSIS
 
379
    >>> import_version(result_path, version, dry_run=True) #doctest: +ELLIPSIS
386
380
    importing test@example.com/test--test--0 into ...
387
381
    ...
388
382
    revisions: ..........................................
389
383
    Dry run, not modifying output_dir
390
384
    Cleaning up
391
 
    >>> import_version(result_path, version, None) #doctest: +ELLIPSIS
 
385
    >>> import_version(result_path, version) #doctest: +ELLIPSIS
392
386
    importing test@example.com/test--test--0 into ...
393
387
    ...
394
388
    revisions: .....................................................................
395
389
    Cleaning up
396
390
    Import complete.
397
 
    >>> import_version(result_path, version, None) #doctest: +ELLIPSIS
 
391
    >>> import_version(result_path, version) #doctest: +ELLIPSIS
398
392
    Tree is up-to-date with test@example.com/test--test--0--patch-2
399
393
    >>> commit_more_test_revisions()
400
 
    >>> import_version(result_path, version, None) #doctest: +ELLIPSIS
 
394
    >>> import_version(result_path, version) #doctest: +ELLIPSIS
401
395
    importing test@example.com/test--test--0 into ...
402
396
    revisions: ....................................................
403
397
    Cleaning up
410
404
    try:
411
405
        try:
412
406
            ancestors, old_revno = get_remaining_revisions(output_dir, version,
413
 
                                                           encoding,
414
407
                                                           reuse_history_from)
415
408
        except NotBranchError, e:
416
409
            raise NotPreviousImport(e.path)
418
411
            progress_bar.note('Version %s has no revisions.' % version)
419
412
            return
420
413
        if len(ancestors) == 0:
421
 
            last_revision, last_encoding = \
422
 
                get_last_revision(Branch.open(output_dir))
 
414
            last_revision = get_last_revision(Branch.open(output_dir))
423
415
            progress_bar.note('Tree is up-to-date with %s' % last_revision)
424
416
            return
425
417
 
426
418
        progress_bar.note("importing %s into %s" % (version, output_dir))
427
 
 
 
419
    
428
420
        tempdir = tempfile.mkdtemp(prefix="baz2bzr-",
429
421
                                   dir=os.path.dirname(output_dir))
430
422
        try:
431
423
            wt = WorkingTree.open(output_dir)
432
424
        except (NotBranchError, NoWorkingTree):
433
425
            wt = None
 
426
        if wt is None:
 
427
            old_basis = EmptyTree()
 
428
        else:
 
429
            old_basis = wt.basis_tree()
434
430
        try:
435
431
            for result in iter_import_version(output_dir, ancestors, tempdir,
436
 
                    pb=progress_bar, encoding=encoding, fast=fast,
437
 
                    verbose=verbose, dry_run=dry_run, max_count=max_count,
438
 
                    standalone=standalone):
 
432
                    pb=progress_bar,
 
433
                    fast=fast, verbose=verbose, dry_run=dry_run,
 
434
                    max_count=max_count, standalone=standalone):
439
435
                show_progress(progress_bar, result)
440
436
            if dry_run:
441
437
                progress_bar.note('Dry run, not modifying output_dir')
442
438
                return
443
 
 
 
439
    
444
440
            # Update the working tree of the branch
445
441
            try:
446
442
                wt = WorkingTree.open(output_dir)
448
444
                wt = None
449
445
            if wt is not None:
450
446
                wt.set_last_revision(wt.branch.last_revision())
451
 
                wt.set_root_id(BAZ_IMPORT_ROOT)
452
 
                wt.revert()
453
 
 
 
447
                merge_inner(wt.branch, wt.basis_tree(), old_basis, 
 
448
                            ignore_zero=True, this_tree=wt)
 
449
                wt.revert([])
 
450
    
454
451
        finally:
455
 
 
 
452
            
456
453
            progress_bar.note('Cleaning up')
457
454
            shutil.rmtree(tempdir)
458
455
        progress_bar.note("Import complete.")
459
456
    finally:
460
457
        progress_bar.finished()
461
 
 
 
458
            
462
459
class UserError(BzrCommandError):
463
460
    def __init__(self, message):
464
461
        """Exception to throw when a user makes an impossible request
473
470
                           % path)
474
471
 
475
472
 
476
 
def revision_id(arch_revision, encoding):
 
473
def revision_id(arch_revision):
477
474
    """
478
475
    Generate a Bzr revision id from an Arch revision id.  'x' in the id
479
476
    designates a revision imported with an experimental algorithm.  A number
481
478
 
482
479
    :param arch_revision: The Arch revision to generate an ID for.
483
480
 
484
 
    >>> revision_id(pybaz.Revision("you@example.com/cat--br--0--base-0"), None)
 
481
    >>> revision_id(pybaz.Revision("you@example.com/cat--br--0--base-0"))
485
482
    'Arch-1:you@example.com%cat--br--0--base-0'
486
 
    >>> revision_id(pybaz.Revision("you@example.com/cat--br--0--base-0"), 'utf-8')
487
 
    'Arch-1-utf-8:you@example.com%cat--br--0--base-0'
488
483
    """
489
 
    if encoding is None:
490
 
        encoding = ''
491
 
    else:
492
 
        encoding = '-' + encoding
493
 
    return "Arch-1%s:%s" % (encoding, str(arch_revision).replace('/', '%'))
 
484
    return "Arch-1:%s" % str(arch_revision).replace('/', '%')
494
485
 
495
486
class NotArchRevision(Exception):
496
487
    def __init__(self, revision_id):
506
497
    >>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0--base-5"))
507
498
    Traceback (most recent call last):
508
499
    NotArchRevision: The revision id Arch-1:jrandom@example.com%test--test--0--base-5 does not look like it came from Arch.
509
 
    >>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0--patch-5")[0])
510
 
    'jrandom@example.com/test--test--0--patch-5'
511
 
    >>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0--patch-5")[0])
512
 
    'jrandom@example.com/test--test--0--patch-5'
513
 
    >>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0--patch-5")[1])
514
 
    'None'
515
 
    >>> str(arch_revision("Arch-1-utf-8:jrandom@example.com%test--test--0--patch-5")[1])
516
 
    'utf-8'
517
 
    >>> str(arch_revision('null:'))
518
 
    '(None, None)'
 
500
    >>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0--patch-5"))
 
501
    'jrandom@example.com/test--test--0--patch-5'
519
502
    """
520
 
    if is_null(revision_id):
521
 
        return None, None
522
 
    if revision_id[:7] not in ('Arch-1:', 'Arch-1-'):
 
503
    if revision_id is None:
 
504
        return None
 
505
    if revision_id[:7] != 'Arch-1:':
523
506
        raise NotArchRevision(revision_id)
524
507
    else:
525
508
        try:
526
 
            encoding, arch_name = revision_id[6:].split(':', 1)
527
 
            arch_name = arch_name.replace('%', '/')
528
 
            if encoding == '':
529
 
                encoding = None
530
 
            else:
531
 
                encoding = encoding[1:]
532
 
            return pybaz.Revision(arch_name), encoding
 
509
            return pybaz.Revision(revision_id[7:].replace('%', '/'))
533
510
        except pybaz.errors.NamespaceError, e:
534
511
            raise NotArchRevision(revision_id)
535
512
 
554
531
    if revision_id is None:
555
532
        revision_id = source.last_revision()
556
533
    wt = create_checkout(source, to_location, NULL_REVISION)
557
 
    wt.lock_write()
558
 
    try:
559
 
        wt.set_last_revision(revision_id)
560
 
        wt.flush()
561
 
        if revision_id not in (NULL_REVISION, None):
562
 
            basis = wt.basis_tree()
563
 
            basis.lock_read()
564
 
            try:
565
 
                wt._write_inventory(basis.inventory)
566
 
            finally:
567
 
                basis.unlock()
568
 
    finally:
569
 
        wt.unlock()
 
534
    wt.set_last_revision(revision_id)
 
535
    wt._write_inventory(wt.basis_tree().inventory)
570
536
    return wt
571
537
 
572
538
 
573
 
def iter_import_version(output_dir, ancestors, tempdir, pb, encoding,
574
 
                        fast=False, verbose=False, dry_run=False,
575
 
                        max_count=None, standalone=False):
 
539
def iter_import_version(output_dir, ancestors, tempdir, pb, fast=False,
 
540
                        verbose=False, dry_run=False, max_count=None,
 
541
                        standalone=False):
576
542
    revdir = None
577
 
    log_encoding = 'ascii'
578
 
    if encoding is not None:
579
 
        log_encoding = encoding
580
543
 
581
544
    # Uncomment this for testing, it basically just has baz2bzr only update
582
545
    # 5 patches at a time
609
572
        else:
610
573
            target_branch = create_branch(output_dir)
611
574
 
612
 
    for i, revision in enumerate(ancestors):
613
 
        rev_id = revision_id(revision, encoding)
 
575
    for i in range(len(ancestors)):
 
576
        revision = ancestors[i]
 
577
        rev_id = revision_id(revision)
614
578
        direct_merges = []
615
579
        if verbose:
616
580
            version = str(revision.version)
622
586
            yield Progress("revisions", i, len(ancestors))
623
587
 
624
588
        if target_branch.repository.has_revision(rev_id):
625
 
            target_branch.set_last_revision_info(i+1, rev_id)
 
589
            target_branch.append_revision(rev_id)
626
590
            continue
627
591
        if revdir is None:
628
592
            revdir = os.path.join(tempdir, "rd")
667
631
        try:
668
632
            if missing_ancestor:
669
633
                # if we want it to be in revision-history, do that here.
670
 
                target_tree.set_parent_ids(
671
 
                    [revision_id(missing_ancestor, encoding)],
672
 
                    allow_leftmost_as_ghost=True)
 
634
                target_tree.add_pending_merge(revision_id(missing_ancestor))
673
635
                missing_ancestor = None
674
636
            for merged_rev in direct_merges:
675
 
                target_tree.add_pending_merge(revision_id(merged_rev,
676
 
                                                          encoding))
677
 
            target_tree.set_root_id(BAZ_IMPORT_ROOT)
678
 
            target_tree.flush()
 
637
                target_tree.add_pending_merge(revision_id(merged_rev))
679
638
            target_tree.set_inventory(baz_inv)
680
639
            commitobj = Commit(reporter=ImportCommitReporter())
681
640
            commitobj.commit(working_tree=target_tree,
682
 
                message=log_message.decode(log_encoding, 'replace'),
683
 
                verbose=False, committer=log_creator, timestamp=timestamp,
684
 
                timezone=0, rev_id=rev_id, revprops={})
 
641
                             message=log_message.decode('ascii', 'replace'), 
 
642
                             verbose=False, committer=log_creator,
 
643
                             timestamp=timestamp, timezone=0, rev_id=rev_id,
 
644
                             revprops={})
685
645
        finally:
686
646
            target_tree.unlock()
687
647
            branch.unlock()
692
652
    previous_version = revision.version
693
653
    if pybaz.WorkingTree(revdir).tree_version != previous_version:
694
654
        pybaz.WorkingTree(revdir).set_tree_version(previous_version)
695
 
    log_path = "%s/{arch}/%s/%s/%s/%s/patch-log/%s" % (revdir,
696
 
        revision.category.nonarch, revision.branch.nonarch,
 
655
    log_path = "%s/{arch}/%s/%s/%s/%s/patch-log/%s" % (revdir, 
 
656
        revision.category.nonarch, revision.branch.nonarch, 
697
657
        revision.version.nonarch, revision.archive, revision.patchlevel)
698
658
    temp_path = tempfile.mktemp(dir=os.path.dirname(revdir))
699
659
    os.rename(log_path, temp_path)
719
679
    tree = revision.get(revdir)
720
680
    log = get_log(tree, revision)
721
681
    try:
722
 
        return tree, bzr_inventory_data(tree), log
 
682
        return tree, bzr_inventory_data(tree), log 
723
683
    except BadFileKind, e:
724
 
        raise UserError("Cannot convert %s because %s is a %s" %
 
684
        raise UserError("Cannot convert %s because %s is a %s" % 
725
685
                        (revision,e.path, e.kind))
726
686
 
727
687
 
731
691
    try:
732
692
        return bzr_inventory_data(tree), log
733
693
    except BadFileKind, e:
734
 
        raise UserError("Cannot convert %s because %s is a %s" %
 
694
        raise UserError("Cannot convert %s because %s is a %s" % 
735
695
                        (revision,e.path, e.kind))
736
696
 
737
697
 
750
710
    inv_map = {}
751
711
    for arch_id, path in inv_iter:
752
712
        bzr_file_id = map_file_id(arch_id)
753
 
        inv_map[path] = bzr_file_id
 
713
        inv_map[path] = bzr_file_id 
754
714
 
755
715
    bzr_inv = []
756
716
    for path, file_id in inv_map.iteritems():
767
727
    bzr_inv.sort()
768
728
    return bzr_inv
769
729
 
 
730
_global_option('max-count', type = int)
 
731
class cmd_baz_import_branch(Command):
 
732
    """Import an Arch or Baz branch into a bzr branch.  <BZRTOOLS>"""
 
733
    takes_args = ['to_location', 'from_branch?', 'reuse_history*']
 
734
    takes_options = ['verbose', 'max-count']
770
735
 
771
 
def baz_import_branch(to_location, from_branch, fast, max_count, verbose,
772
 
                      encoding, dry_run, reuse_history_list):
773
 
    to_location = os.path.realpath(str(to_location))
774
 
    if from_branch is not None:
775
 
        try:
776
 
            from_branch = pybaz.Version(from_branch)
777
 
        except pybaz.errors.NamespaceError:
778
 
            print "%s is not a valid Arch branch." % from_branch
779
 
            return 1
780
 
    if reuse_history_list is None:
781
 
        reuse_history_list = []
782
 
    import_version(to_location, from_branch, encoding, max_count=max_count,
783
 
                   reuse_history_from=reuse_history_list)
 
736
    def run(self, to_location, from_branch=None, fast=False, max_count=None,
 
737
            verbose=False, dry_run=False, reuse_history_list=[]):
 
738
        to_location = os.path.realpath(str(to_location))
 
739
        if from_branch is not None:
 
740
            try:
 
741
                from_branch = pybaz.Version(from_branch)
 
742
            except pybaz.errors.NamespaceError:
 
743
                print "%s is not a valid Arch branch." % from_branch
 
744
                return 1
 
745
        if reuse_history_list is None:
 
746
            reuse_history_list = []
 
747
        import_version(to_location, from_branch, 
 
748
                       max_count=max_count, 
 
749
                       reuse_history_from=reuse_history_list)
784
750
 
785
751
 
786
752
class NotInABranch(Exception):
789
755
        self.path = path
790
756
 
791
757
 
792
 
 
793
 
def baz_import(to_root_dir, from_archive, encoding, verbose=False,
794
 
               reuse_history_list=[], prefixes=None):
795
 
    if reuse_history_list is None:
796
 
        reuse_history_list = []
797
 
    to_root = str(os.path.realpath(to_root_dir))
798
 
    if not os.path.exists(to_root):
799
 
        os.mkdir(to_root)
800
 
    if prefixes is not None:
801
 
        prefixes = prefixes.split(':')
802
 
    import_archive(to_root, from_archive, verbose, encoding,
803
 
                   reuse_history_list, prefixes=prefixes)
 
758
class cmd_baz_import(Command):
 
759
    """Import an Arch or Baz archive into a bzr repository.  <BZRTOOLS>
 
760
 
 
761
    This command should be used on local archives (or mirrors) only.  It is
 
762
    quite slow on remote archives.
 
763
    
 
764
    reuse_history allows you to specify any previous imports you 
 
765
    have done of different archives, which this archive has branches
 
766
    tagged from. This will dramatically reduce the time to convert 
 
767
    the archive as it will not have to convert the history already
 
768
    converted in that other branch.
 
769
 
 
770
    If you specify prefixes, only branches whose names start with that prefix
 
771
    will be imported.  Skipped branches will be listed, so you can import any
 
772
    branches you missed by accident.  Here's an example of doing a partial
 
773
    import from thelove@canonical.com:
 
774
    bzr baz-import thelove thelove@canonical.com --prefixes dists:talloc-except
 
775
    """
 
776
    takes_args = ['to_root_dir', 'from_archive', 'reuse_history*']
 
777
    takes_options = ['verbose', Option('prefixes', type=str,
 
778
                     help="Prefixes of branches to import, colon-separated")]
 
779
 
 
780
    def run(self, to_root_dir, from_archive, verbose=False,
 
781
            reuse_history_list=[], prefixes=None):
 
782
        if reuse_history_list is None:
 
783
            reuse_history_list = []
 
784
        to_root = str(os.path.realpath(to_root_dir))
 
785
        if not os.path.exists(to_root):
 
786
            os.mkdir(to_root)
 
787
        if prefixes is not None:
 
788
            prefixes = prefixes.split(':')
 
789
        import_archive(to_root, from_archive, verbose,
 
790
                       reuse_history_list, prefixes=prefixes)
804
791
 
805
792
 
806
793
def import_archive(to_root, from_archive, verbose,
807
 
                   encoding, reuse_history_from=[], standalone=False,
 
794
                   reuse_history_from=[], standalone=False,
808
795
                   prefixes=None):
809
796
    def selected(version):
810
797
        if prefixes is None:
836
823
            if not os.path.exists(os.path.dirname(target)):
837
824
                os.makedirs(os.path.dirname(target))
838
825
            try:
839
 
                import_version(target, version, encoding,
840
 
                               reuse_history_from=reuse_history_from,
 
826
                import_version(target, version,
 
827
                               reuse_history_from=reuse_history_from, 
841
828
                               standalone=standalone)
842
829
            except pybaz.errors.ExecProblem,e:
843
830
                if str(e).find('The requested revision cannot be built.') != -1: