~abentley/bzrtools/bzrtools.dev

« back to all changes in this revision

Viewing changes to baz_import.py

  • Committer: Aaron Bentley
  • Date: 2007-06-10 17:55:08 UTC
  • mfrom: (531.2.2 bzrtools)
  • Revision ID: aaron.bentley@utoronto.ca-20070610175508-gex1oxvmfv0qoagi
Merge whitespace cleanups

Show diffs side-by-side

added added

removed removed

Lines of Context:
21
21
from bzrlib.errors import (BzrError,
22
22
                           NotBranchError,
23
23
                           NoWorkingTree,
24
 
                           BzrCommandError, 
 
24
                           BzrCommandError,
25
25
                           NoSuchRevision,
26
26
                           NoRepositoryPresent,
27
27
                          )
56
56
import email.Utils
57
57
from progress import *
58
58
 
 
59
 
 
60
BAZ_IMPORT_ROOT = 'TREE_ROOT'
 
61
 
 
62
 
59
63
class ImportCommitReporter(NullCommitReporter):
60
64
 
61
65
    def escaped(self, escape_count, message):
80
84
 
81
85
def make_archive(name, location):
82
86
    pb_location = pybaz.ArchiveLocation(location)
83
 
    pb_location.create_master(pybaz.Archive(name), 
 
87
    pb_location.create_master(pybaz.Archive(name),
84
88
                              pybaz.ArchiveLocationParams())
85
89
 
86
90
def test_environ():
271
275
            br_from.bzrdir.clone(to_location, revision_id)
272
276
        except NoSuchRevision:
273
277
            rmtree(to_location)
274
 
            msg = "The branch %s has no revision %s." % (from_location, 
 
278
            msg = "The branch %s has no revision %s." % (from_location,
275
279
                                                         revision_id)
276
280
            raise UserError(msg)
277
281
    finally:
278
282
        br_from.unlock()
279
283
 
280
 
def get_remaining_revisions(output_dir, version, reuse_history_from=[]):
 
284
def get_remaining_revisions(output_dir, version, encoding,
 
285
                            reuse_history_from=[]):
281
286
    last_patch = None
282
287
    old_revno = None
283
288
    output_exists = os.path.exists(output_dir)
285
290
        # We are starting from an existing directory, figure out what
286
291
        # the current version is
287
292
        branch = Branch.open(output_dir)
288
 
        last_patch = get_last_revision(branch)
 
293
        last_patch, last_encoding = get_last_revision(branch)
 
294
        assert encoding == last_encoding
289
295
        if last_patch is None:
290
296
            if branch.last_revision() != None:
291
297
                raise NotPreviousImport(branch.base)
303
309
                    break
304
310
                # try to grab a copy of ancestor
305
311
                # note that is not optimised: we could look for namespace
306
 
                # transitions and only look for the past after the 
 
312
                # transitions and only look for the past after the
307
313
                # transition.
308
314
                for history_root in reuse_history_from:
309
315
                    possible_source = os.path.join(history_root,
310
316
                        map_namespace(ancestor.version))
311
317
                    try:
312
318
                        source = Branch.open(possible_source)
313
 
                        rev_id = revision_id(ancestor)
 
319
                        rev_id = revision_id(ancestor, encoding)
314
320
                        if rev_id in source.revision_history():
315
321
                            do_branch(source, output_dir, rev_id)
316
322
                            last_patch = ancestor
326
332
                break
327
333
        else:
328
334
            raise UserError("Directory \"%s\" already exists, and the last "
329
 
                "revision (%s) is not in the ancestry of %s" % 
 
335
                "revision (%s) is not in the ancestry of %s" %
330
336
                (output_dir, last_patch, version))
331
337
        # Strip off all of the ancestors which are already present
332
338
        # And get a directory starting with the latest ancestor
338
344
 
339
345
###class Importer(object):
340
346
###    """An importer.
341
 
###    
 
347
###
342
348
###    Currently this is used as a parameter object, though more behaviour is
343
349
###    possible later.
344
350
###    """
345
351
###
346
352
###    def __init__(self, output_dir, version, fast=False,
347
 
###                 verbose=False, dry_run=False, max_count=None, 
 
353
###                 verbose=False, dry_run=False, max_count=None,
348
354
###                   reuse_history_from=[]):
349
355
###        self.output_dir = output_dir
350
356
###        self.version = version
351
357
###        self.
352
358
 
353
359
 
354
 
def import_version(output_dir, version, fast=False,
 
360
def import_version(output_dir, version, encoding, fast=False,
355
361
                   verbose=False, dry_run=False, max_count=None,
356
362
                   reuse_history_from=[], standalone=True):
357
363
    """
358
364
    >>> q = test_environ()
359
 
    
 
365
 
360
366
    Progress bars output to stderr, but doctest does not capture that.
361
367
 
362
368
    >>> old_stderr = sys.stderr
369
375
    >>> bzrlib.ui.ui_factory = bzrlib.ui.text.TextUIFactory(
370
376
    ...     bar_type=bzrlib.progress.DotsProgressBar)
371
377
 
372
 
    >>> import_version('/', version, dry_run=True)
 
378
    >>> import_version('/', version, None, dry_run=True)
373
379
    Traceback (most recent call last):
374
380
    NotPreviousImport: / is not the location of a previous import.
375
 
    >>> import_version(result_path, version, dry_run=True)
 
381
    >>> import_version(result_path, version, None, dry_run=True)
376
382
    Traceback (most recent call last):
377
383
    UserError: The version test@example.com/test--test--0.1 does not exist.
378
384
    >>> version = pybaz.Version("test@example.com/test--test--0")
379
 
    >>> import_version(result_path, version, dry_run=True) #doctest: +ELLIPSIS
 
385
    >>> import_version(result_path, version, None, dry_run=True) #doctest: +ELLIPSIS
380
386
    importing test@example.com/test--test--0 into ...
381
387
    ...
382
388
    revisions: ..........................................
383
389
    Dry run, not modifying output_dir
384
390
    Cleaning up
385
 
    >>> import_version(result_path, version) #doctest: +ELLIPSIS
 
391
    >>> import_version(result_path, version, None) #doctest: +ELLIPSIS
386
392
    importing test@example.com/test--test--0 into ...
387
393
    ...
388
394
    revisions: .....................................................................
389
395
    Cleaning up
390
396
    Import complete.
391
 
    >>> import_version(result_path, version) #doctest: +ELLIPSIS
 
397
    >>> import_version(result_path, version, None) #doctest: +ELLIPSIS
392
398
    Tree is up-to-date with test@example.com/test--test--0--patch-2
393
399
    >>> commit_more_test_revisions()
394
 
    >>> import_version(result_path, version) #doctest: +ELLIPSIS
 
400
    >>> import_version(result_path, version, None) #doctest: +ELLIPSIS
395
401
    importing test@example.com/test--test--0 into ...
396
402
    revisions: ....................................................
397
403
    Cleaning up
404
410
    try:
405
411
        try:
406
412
            ancestors, old_revno = get_remaining_revisions(output_dir, version,
 
413
                                                           encoding,
407
414
                                                           reuse_history_from)
408
415
        except NotBranchError, e:
409
416
            raise NotPreviousImport(e.path)
411
418
            progress_bar.note('Version %s has no revisions.' % version)
412
419
            return
413
420
        if len(ancestors) == 0:
414
 
            last_revision = get_last_revision(Branch.open(output_dir))
 
421
            last_revision, last_encoding = \
 
422
                get_last_revision(Branch.open(output_dir))
415
423
            progress_bar.note('Tree is up-to-date with %s' % last_revision)
416
424
            return
417
425
 
418
426
        progress_bar.note("importing %s into %s" % (version, output_dir))
419
 
    
 
427
 
420
428
        tempdir = tempfile.mkdtemp(prefix="baz2bzr-",
421
429
                                   dir=os.path.dirname(output_dir))
422
430
        try:
425
433
            wt = None
426
434
        try:
427
435
            for result in iter_import_version(output_dir, ancestors, tempdir,
428
 
                    pb=progress_bar,
429
 
                    fast=fast, verbose=verbose, dry_run=dry_run,
430
 
                    max_count=max_count, standalone=standalone):
 
436
                    pb=progress_bar, encoding=encoding, fast=fast,
 
437
                    verbose=verbose, dry_run=dry_run, max_count=max_count,
 
438
                    standalone=standalone):
431
439
                show_progress(progress_bar, result)
432
440
            if dry_run:
433
441
                progress_bar.note('Dry run, not modifying output_dir')
434
442
                return
435
 
    
 
443
 
436
444
            # Update the working tree of the branch
437
445
            try:
438
446
                wt = WorkingTree.open(output_dir)
440
448
                wt = None
441
449
            if wt is not None:
442
450
                wt.set_last_revision(wt.branch.last_revision())
 
451
                wt.set_root_id(BAZ_IMPORT_ROOT)
443
452
                wt.revert([])
444
 
    
 
453
 
445
454
        finally:
446
 
            
 
455
 
447
456
            progress_bar.note('Cleaning up')
448
457
            shutil.rmtree(tempdir)
449
458
        progress_bar.note("Import complete.")
450
459
    finally:
451
460
        progress_bar.finished()
452
 
            
 
461
 
453
462
class UserError(BzrCommandError):
454
463
    def __init__(self, message):
455
464
        """Exception to throw when a user makes an impossible request
464
473
                           % path)
465
474
 
466
475
 
467
 
def revision_id(arch_revision):
 
476
def revision_id(arch_revision, encoding):
468
477
    """
469
478
    Generate a Bzr revision id from an Arch revision id.  'x' in the id
470
479
    designates a revision imported with an experimental algorithm.  A number
472
481
 
473
482
    :param arch_revision: The Arch revision to generate an ID for.
474
483
 
475
 
    >>> revision_id(pybaz.Revision("you@example.com/cat--br--0--base-0"))
 
484
    >>> revision_id(pybaz.Revision("you@example.com/cat--br--0--base-0"), None)
476
485
    'Arch-1:you@example.com%cat--br--0--base-0'
 
486
    >>> revision_id(pybaz.Revision("you@example.com/cat--br--0--base-0"), 'utf-8')
 
487
    'Arch-1-utf-8:you@example.com%cat--br--0--base-0'
477
488
    """
478
 
    return "Arch-1:%s" % str(arch_revision).replace('/', '%')
 
489
    if encoding is None:
 
490
        encoding = ''
 
491
    else:
 
492
        encoding = '-' + encoding
 
493
    return "Arch-1%s:%s" % (encoding, str(arch_revision).replace('/', '%'))
479
494
 
480
495
class NotArchRevision(Exception):
481
496
    def __init__(self, revision_id):
491
506
    >>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0--base-5"))
492
507
    Traceback (most recent call last):
493
508
    NotArchRevision: The revision id Arch-1:jrandom@example.com%test--test--0--base-5 does not look like it came from Arch.
494
 
    >>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0--patch-5"))
495
 
    'jrandom@example.com/test--test--0--patch-5'
 
509
    >>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0--patch-5")[0])
 
510
    'jrandom@example.com/test--test--0--patch-5'
 
511
    >>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0--patch-5")[0])
 
512
    'jrandom@example.com/test--test--0--patch-5'
 
513
    >>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0--patch-5")[1])
 
514
    'None'
 
515
    >>> str(arch_revision("Arch-1-utf-8:jrandom@example.com%test--test--0--patch-5")[1])
 
516
    'utf-8'
496
517
    """
497
518
    if revision_id is None:
498
 
        return None
499
 
    if revision_id[:7] != 'Arch-1:':
 
519
        return None, None
 
520
    if revision_id[:7] not in ('Arch-1:', 'Arch-1-'):
500
521
        raise NotArchRevision(revision_id)
501
522
    else:
502
523
        try:
503
 
            return pybaz.Revision(revision_id[7:].replace('%', '/'))
 
524
            encoding, arch_name = revision_id[6:].split(':', 1)
 
525
            arch_name = arch_name.replace('%', '/')
 
526
            if encoding == '':
 
527
                encoding = None
 
528
            else:
 
529
                encoding = encoding[1:]
 
530
            return pybaz.Revision(arch_name), encoding
504
531
        except pybaz.errors.NamespaceError, e:
505
532
            raise NotArchRevision(revision_id)
506
533
 
525
552
    if revision_id is None:
526
553
        revision_id = source.last_revision()
527
554
    wt = create_checkout(source, to_location, NULL_REVISION)
528
 
    wt.set_last_revision(revision_id)
529
 
    wt._write_inventory(wt.basis_tree().inventory)
 
555
    wt.lock_write()
 
556
    try:
 
557
        wt.set_last_revision(revision_id)
 
558
        wt.flush()
 
559
        if revision_id not in (NULL_REVISION, None):
 
560
            basis = wt.basis_tree()
 
561
            basis.lock_read()
 
562
            try:
 
563
                wt._write_inventory(basis.inventory)
 
564
            finally:
 
565
                basis.unlock()
 
566
    finally:
 
567
        wt.unlock()
530
568
    return wt
531
569
 
532
570
 
533
 
def iter_import_version(output_dir, ancestors, tempdir, pb, fast=False,
534
 
                        verbose=False, dry_run=False, max_count=None,
535
 
                        standalone=False):
 
571
def iter_import_version(output_dir, ancestors, tempdir, pb, encoding,
 
572
                        fast=False, verbose=False, dry_run=False,
 
573
                        max_count=None, standalone=False):
536
574
    revdir = None
 
575
    log_encoding = 'ascii'
 
576
    if encoding is not None:
 
577
        log_encoding = encoding
537
578
 
538
579
    # Uncomment this for testing, it basically just has baz2bzr only update
539
580
    # 5 patches at a time
568
609
 
569
610
    for i in range(len(ancestors)):
570
611
        revision = ancestors[i]
571
 
        rev_id = revision_id(revision)
 
612
        rev_id = revision_id(revision, encoding)
572
613
        direct_merges = []
573
614
        if verbose:
574
615
            version = str(revision.version)
625
666
        try:
626
667
            if missing_ancestor:
627
668
                # if we want it to be in revision-history, do that here.
628
 
                target_tree.set_parent_ids([revision_id(missing_ancestor)],
629
 
                                           allow_leftmost_as_ghost=True)
 
669
                target_tree.set_parent_ids(
 
670
                    [revision_id(missing_ancestor, encoding)],
 
671
                    allow_leftmost_as_ghost=True)
630
672
                missing_ancestor = None
631
673
            for merged_rev in direct_merges:
632
 
                target_tree.add_pending_merge(revision_id(merged_rev))
 
674
                target_tree.add_pending_merge(revision_id(merged_rev,
 
675
                                                          encoding))
 
676
            target_tree.set_root_id(BAZ_IMPORT_ROOT)
 
677
            target_tree.flush()
633
678
            target_tree.set_inventory(baz_inv)
634
679
            commitobj = Commit(reporter=ImportCommitReporter())
635
680
            commitobj.commit(working_tree=target_tree,
636
 
                             message=log_message.decode('ascii', 'replace'), 
637
 
                             verbose=False, committer=log_creator,
638
 
                             timestamp=timestamp, timezone=0, rev_id=rev_id,
639
 
                             revprops={})
 
681
                message=log_message.decode(log_encoding, 'replace'),
 
682
                verbose=False, committer=log_creator, timestamp=timestamp,
 
683
                timezone=0, rev_id=rev_id, revprops={})
640
684
        finally:
641
685
            target_tree.unlock()
642
686
            branch.unlock()
647
691
    previous_version = revision.version
648
692
    if pybaz.WorkingTree(revdir).tree_version != previous_version:
649
693
        pybaz.WorkingTree(revdir).set_tree_version(previous_version)
650
 
    log_path = "%s/{arch}/%s/%s/%s/%s/patch-log/%s" % (revdir, 
651
 
        revision.category.nonarch, revision.branch.nonarch, 
 
694
    log_path = "%s/{arch}/%s/%s/%s/%s/patch-log/%s" % (revdir,
 
695
        revision.category.nonarch, revision.branch.nonarch,
652
696
        revision.version.nonarch, revision.archive, revision.patchlevel)
653
697
    temp_path = tempfile.mktemp(dir=os.path.dirname(revdir))
654
698
    os.rename(log_path, temp_path)
674
718
    tree = revision.get(revdir)
675
719
    log = get_log(tree, revision)
676
720
    try:
677
 
        return tree, bzr_inventory_data(tree), log 
 
721
        return tree, bzr_inventory_data(tree), log
678
722
    except BadFileKind, e:
679
 
        raise UserError("Cannot convert %s because %s is a %s" % 
 
723
        raise UserError("Cannot convert %s because %s is a %s" %
680
724
                        (revision,e.path, e.kind))
681
725
 
682
726
 
686
730
    try:
687
731
        return bzr_inventory_data(tree), log
688
732
    except BadFileKind, e:
689
 
        raise UserError("Cannot convert %s because %s is a %s" % 
 
733
        raise UserError("Cannot convert %s because %s is a %s" %
690
734
                        (revision,e.path, e.kind))
691
735
 
692
736
 
705
749
    inv_map = {}
706
750
    for arch_id, path in inv_iter:
707
751
        bzr_file_id = map_file_id(arch_id)
708
 
        inv_map[path] = bzr_file_id 
 
752
        inv_map[path] = bzr_file_id
709
753
 
710
754
    bzr_inv = []
711
755
    for path, file_id in inv_map.iteritems():
723
767
    return bzr_inv
724
768
 
725
769
 
726
 
def baz_import_branch(to_location, from_branch, fast, max_count, verbose, 
727
 
                      dry_run, reuse_history_list):
 
770
def baz_import_branch(to_location, from_branch, fast, max_count, verbose,
 
771
                      encoding, dry_run, reuse_history_list):
728
772
    to_location = os.path.realpath(str(to_location))
729
773
    if from_branch is not None:
730
774
        try:
734
778
            return 1
735
779
    if reuse_history_list is None:
736
780
        reuse_history_list = []
737
 
    import_version(to_location, from_branch, 
738
 
                   max_count=max_count, 
 
781
    import_version(to_location, from_branch, encoding, max_count=max_count,
739
782
                   reuse_history_from=reuse_history_list)
740
783
 
741
784
 
746
789
 
747
790
 
748
791
 
749
 
def baz_import(to_root_dir, from_archive, verbose=False, reuse_history_list=[],
750
 
               prefixes=None):
 
792
def baz_import(to_root_dir, from_archive, encoding, verbose=False,
 
793
               reuse_history_list=[], prefixes=None):
751
794
    if reuse_history_list is None:
752
795
        reuse_history_list = []
753
796
    to_root = str(os.path.realpath(to_root_dir))
755
798
        os.mkdir(to_root)
756
799
    if prefixes is not None:
757
800
        prefixes = prefixes.split(':')
758
 
    import_archive(to_root, from_archive, verbose,
 
801
    import_archive(to_root, from_archive, verbose, encoding,
759
802
                   reuse_history_list, prefixes=prefixes)
760
803
 
761
804
 
762
805
def import_archive(to_root, from_archive, verbose,
763
 
                   reuse_history_from=[], standalone=False,
 
806
                   encoding, reuse_history_from=[], standalone=False,
764
807
                   prefixes=None):
765
808
    def selected(version):
766
809
        if prefixes is None:
792
835
            if not os.path.exists(os.path.dirname(target)):
793
836
                os.makedirs(os.path.dirname(target))
794
837
            try:
795
 
                import_version(target, version,
796
 
                               reuse_history_from=reuse_history_from, 
 
838
                import_version(target, version, encoding,
 
839
                               reuse_history_from=reuse_history_from,
797
840
                               standalone=standalone)
798
841
            except pybaz.errors.ExecProblem,e:
799
842
                if str(e).find('The requested revision cannot be built.') != -1: