« back to all changes in this revision
Viewing changes to baz_import.py
- browse files at revision 354
- compare with another revision
- download diff
- download tarball
- view history from revision 354
- Committer: Aaron Bentley
- Date: 2006-03-24 19:01:30 UTC
- Revision ID: abentley@panoramicfeedback.com-20060324190130-2208c693486a8b33
Added apache index scraping to the branches command
- files added:
- debian
- testdata
- files removed:
- .be/bugs/9312b4cf-379e-490b-bfb5-dfb81357ee25
- .be/bugs/9312b4cf-379e-490b-bfb5-dfb81357ee25/comments
- .be/bugs/9312b4cf-379e-490b-bfb5-dfb81357ee25/comments/906938f4-be68-4d65-9b75-def07743b043
- .be/bugs/c59acbbf-25f4-450a-9a8f-7a6a45b3985e
- .be/bugs/fdc2b62a-9622-4f08-ac78-bd2603a4e0af
- .be/bugs/fdc2b62a-9622-4f08-ac78-bd2603a4e0af/comments
- .be/bugs/fdc2b62a-9622-4f08-ac78-bd2603a4e0af/comments/f13e3358-1a65-4a45-9472-781548bc3fc3
- files renamed:
- rspush.py => push.py
- files modified:
- .be/bugs/dbd10145-51f8-4bda-96d9-06b5d3ef80b8/values
added
removed
baz_import.py
31
31
from bzrlib.option import _global_option, Option
32
32
from bzrlib.merge import merge_inner
33
33
from bzrlib.revision import NULL_REVISION
34
from bzrlib.tree import EmptyTree
34
35
import bzrlib.ui
35
import bzrlib.ui.text
36
36
from bzrlib.workingtree import WorkingTree
37
37
from errors import NoPyBaz
38
38
try:
56
56
import email.Utils
57
57
from progress import *
58
58
59
60
BAZ_IMPORT_ROOT = 'TREE_ROOT'
61
62
63
59
class ImportCommitReporter(NullCommitReporter):
60
def __init__(self, pb):
61
self.pb = pb
64
62
65
63
def escaped(self, escape_count, message):
64
self.pb.clear()
66
65
bzrlib.trace.warning("replaced %d control characters in message" %
67
66
escape_count)
68
67
281
280
finally:
282
281
br_from.unlock()
283
282
284
def get_remaining_revisions(output_dir, version, encoding,
285
reuse_history_from=[]):
283
def get_remaining_revisions(output_dir, version, reuse_history_from=[]):
286
284
last_patch = None
287
285
old_revno = None
288
286
output_exists = os.path.exists(output_dir)
290
288
# We are starting from an existing directory, figure out what
291
289
# the current version is
292
290
branch = Branch.open(output_dir)
293
last_patch, last_encoding = get_last_revision(branch)
294
assert encoding == last_encoding
291
last_patch = get_last_revision(branch)
295
292
if last_patch is None:
296
if branch.last_revision() != None:
297
raise NotPreviousImport(branch.base)
298
elif version is None:
293
raise NotPreviousImport(branch.base)
294
if version is None:
299
295
version = last_patch.version
300
296
elif version is None:
301
297
raise UserError("No version specified, and directory does not exist.")
316
312
map_namespace(ancestor.version))
317
313
try:
318
314
source = Branch.open(possible_source)
319
rev_id = revision_id(ancestor, encoding)
315
rev_id = revision_id(ancestor)
320
316
if rev_id in source.revision_history():
321
317
do_branch(source, output_dir, rev_id)
322
318
last_patch = ancestor
349
345
### possible later.
350
346
### """
351
347
###
352
### def __init__(self, output_dir, version, fast=False,
348
### def __init__(self, output_dir, version, printer, fancy=True, fast=False,
353
349
### verbose=False, dry_run=False, max_count=None,
354
350
### reuse_history_from=[]):
355
351
### self.output_dir = output_dir
357
353
### self.
358
354
359
355
360
def import_version(output_dir, version, encoding, fast=False,
356
def import_version(output_dir, version, printer, fancy=True, fast=False,
361
357
verbose=False, dry_run=False, max_count=None,
362
358
reuse_history_from=[], standalone=True):
363
359
"""
364
360
>>> q = test_environ()
365
366
Progress bars output to stderr, but doctest does not capture that.
367
368
>>> old_stderr = sys.stderr
369
>>> sys.stderr = sys.stdout
370
371
361
>>> result_path = os.path.join(q, "result")
372
362
>>> commit_test_revisions()
373
363
>>> version = pybaz.Version("test@example.com/test--test--0.1")
374
>>> old_ui = bzrlib.ui.ui_factory
375
>>> bzrlib.ui.ui_factory = bzrlib.ui.text.TextUIFactory(
376
... bar_type=bzrlib.progress.DotsProgressBar)
377
378
>>> import_version('/', version, None, dry_run=True)
364
>>> def printer(message): print message
365
>>> import_version('/', version, printer, fancy=False, dry_run=True)
379
366
Traceback (most recent call last):
380
367
NotPreviousImport: / is not the location of a previous import.
381
>>> import_version(result_path, version, None, dry_run=True)
368
>>> import_version(result_path, version, printer, fancy=False, dry_run=True)
382
369
Traceback (most recent call last):
383
370
UserError: The version test@example.com/test--test--0.1 does not exist.
384
371
>>> version = pybaz.Version("test@example.com/test--test--0")
385
>>> import_version(result_path, version, None, dry_run=True) #doctest: +ELLIPSIS
386
importing test@example.com/test--test--0 into ...
387
...
388
revisions: ..........................................
372
>>> import_version(result_path, version, printer, fancy=False, dry_run=True)
373
not fancy
374
....
389
375
Dry run, not modifying output_dir
390
376
Cleaning up
391
>>> import_version(result_path, version, None) #doctest: +ELLIPSIS
392
importing test@example.com/test--test--0 into ...
393
...
394
revisions: .....................................................................
377
>>> import_version(result_path, version, printer, fancy=False)
378
not fancy
379
....
395
380
Cleaning up
396
381
Import complete.
397
>>> import_version(result_path, version, None) #doctest: +ELLIPSIS
382
>>> import_version(result_path, version, printer, fancy=False)
398
383
Tree is up-to-date with test@example.com/test--test--0--patch-2
399
384
>>> commit_more_test_revisions()
400
>>> import_version(result_path, version, None) #doctest: +ELLIPSIS
401
importing test@example.com/test--test--0 into ...
402
revisions: ....................................................
385
>>> import_version(result_path, version, printer, fancy=False)
386
not fancy
387
..
403
388
Cleaning up
404
389
Import complete.
405
>>> bzrlib.ui.ui_factory = old_ui
406
>>> sys.stderr = old_stderr
407
390
>>> teardown_environ(q)
408
391
"""
392
try:
393
ancestors, old_revno = get_remaining_revisions(output_dir, version,
394
reuse_history_from)
395
except NotBranchError, e:
396
raise NotPreviousImport(e.path)
397
if old_revno is None and len(ancestors) == 0:
398
print 'Version %s has no revisions.' % version
399
return
400
if len(ancestors) == 0:
401
last_revision = get_last_revision(Branch.open(output_dir))
402
print 'Tree is up-to-date with %s' % last_revision
403
return
404
409
405
progress_bar = bzrlib.ui.ui_factory.nested_progress_bar()
410
try:
406
tempdir = tempfile.mkdtemp(prefix="baz2bzr-",
407
dir=os.path.dirname(output_dir))
408
try:
409
wt = WorkingTree.open(output_dir)
410
except (NotBranchError, NoWorkingTree):
411
wt = None
412
if wt is None:
413
old_basis = EmptyTree()
414
else:
415
old_basis = wt.basis_tree()
416
try:
417
if not fancy:
418
print "not fancy"
411
419
try:
412
ancestors, old_revno = get_remaining_revisions(output_dir, version,
413
encoding,
414
reuse_history_from)
415
except NotBranchError, e:
416
raise NotPreviousImport(e.path)
417
if old_revno is None and len(ancestors) == 0:
418
progress_bar.note('Version %s has no revisions.' % version)
419
return
420
if len(ancestors) == 0:
421
last_revision, last_encoding = \
422
get_last_revision(Branch.open(output_dir))
423
progress_bar.note('Tree is up-to-date with %s' % last_revision)
424
return
425
426
progress_bar.note("importing %s into %s" % (version, output_dir))
427
428
tempdir = tempfile.mkdtemp(prefix="baz2bzr-",
429
dir=os.path.dirname(output_dir))
420
for result in iter_import_version(output_dir, ancestors, tempdir,
421
progress_bar, fast=fast, verbose=verbose, dry_run=dry_run,
422
max_count=max_count, standalone=standalone):
423
if fancy:
424
show_progress(progress_bar, result)
425
else:
426
sys.stdout.write('.')
427
finally:
428
if fancy:
429
progress_bar.finished()
430
else:
431
sys.stdout.write('\n')
432
433
if dry_run:
434
print 'Dry run, not modifying output_dir'
435
return
436
437
# Update the working tree of the branch
430
438
try:
431
439
wt = WorkingTree.open(output_dir)
432
except (NotBranchError, NoWorkingTree):
440
except NoWorkingTree:
433
441
wt = None
434
try:
435
for result in iter_import_version(output_dir, ancestors, tempdir,
436
pb=progress_bar, encoding=encoding, fast=fast,
437
verbose=verbose, dry_run=dry_run, max_count=max_count,
438
standalone=standalone):
439
show_progress(progress_bar, result)
440
if dry_run:
441
progress_bar.note('Dry run, not modifying output_dir')
442
return
443
444
# Update the working tree of the branch
445
try:
446
wt = WorkingTree.open(output_dir)
447
except NoWorkingTree:
448
wt = None
449
if wt is not None:
450
wt.set_last_revision(wt.branch.last_revision())
451
wt.set_root_id(BAZ_IMPORT_ROOT)
452
wt.revert([])
453
454
finally:
455
456
progress_bar.note('Cleaning up')
457
shutil.rmtree(tempdir)
458
progress_bar.note("Import complete.")
442
if wt is not None:
443
wt.set_last_revision(wt.branch.last_revision())
444
merge_inner(wt.branch, wt.basis_tree(), old_basis,
445
ignore_zero=True, this_tree=wt)
446
wt.revert([])
447
459
448
finally:
460
progress_bar.finished()
449
printer('Cleaning up')
450
shutil.rmtree(tempdir)
451
printer("Import complete.")
461
452
462
453
class UserError(BzrCommandError):
463
454
def __init__(self, message):
473
464
% path)
474
465
475
466
476
def revision_id(arch_revision, encoding):
467
def revision_id(arch_revision):
477
468
"""
478
469
Generate a Bzr revision id from an Arch revision id. 'x' in the id
479
470
designates a revision imported with an experimental algorithm. A number
481
472
482
473
:param arch_revision: The Arch revision to generate an ID for.
483
474
484
>>> revision_id(pybaz.Revision("you@example.com/cat--br--0--base-0"), None)
475
>>> revision_id(pybaz.Revision("you@example.com/cat--br--0--base-0"))
485
476
'Arch-1:you@example.com%cat--br--0--base-0'
486
>>> revision_id(pybaz.Revision("you@example.com/cat--br--0--base-0"), 'utf-8')
487
'Arch-1-utf-8:you@example.com%cat--br--0--base-0'
488
477
"""
489
if encoding is None:
490
encoding = ''
491
else:
492
encoding = '-' + encoding
493
return "Arch-1%s:%s" % (encoding, str(arch_revision).replace('/', '%'))
478
return "Arch-1:%s" % str(arch_revision).replace('/', '%')
494
479
495
480
class NotArchRevision(Exception):
496
481
def __init__(self, revision_id):
506
491
>>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0--base-5"))
507
492
Traceback (most recent call last):
508
493
NotArchRevision: The revision id Arch-1:jrandom@example.com%test--test--0--base-5 does not look like it came from Arch.
509
>>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0--patch-5")[0])
510
'jrandom@example.com/test--test--0--patch-5'
511
>>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0--patch-5")[0])
512
'jrandom@example.com/test--test--0--patch-5'
513
>>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0--patch-5")[1])
514
'None'
515
>>> str(arch_revision("Arch-1-utf-8:jrandom@example.com%test--test--0--patch-5")[1])
516
'utf-8'
494
>>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0--patch-5"))
495
'jrandom@example.com/test--test--0--patch-5'
517
496
"""
518
497
if revision_id is None:
519
return None, None
520
if revision_id[:7] not in ('Arch-1:', 'Arch-1-'):
498
return None
499
if revision_id[:7] != 'Arch-1:':
521
500
raise NotArchRevision(revision_id)
522
501
else:
523
502
try:
524
encoding, arch_name = revision_id[6:].split(':', 1)
525
arch_name = arch_name.replace('%', '/')
526
if encoding == '':
527
encoding = None
528
else:
529
encoding = encoding[1:]
530
return pybaz.Revision(arch_name), encoding
503
return pybaz.Revision(revision_id[7:].replace('%', '/'))
531
504
except pybaz.errors.NamespaceError, e:
532
505
raise NotArchRevision(revision_id)
533
506
553
526
revision_id = source.last_revision()
554
527
wt = create_checkout(source, to_location, NULL_REVISION)
555
528
wt.set_last_revision(revision_id)
556
if revision_id not in (NULL_REVISION, None):
557
wt._write_inventory(wt.basis_tree().inventory)
529
wt._write_inventory(wt.basis_tree().inventory)
558
530
return wt
559
531
560
532
561
def iter_import_version(output_dir, ancestors, tempdir, pb, encoding,
562
fast=False, verbose=False, dry_run=False,
563
max_count=None, standalone=False):
533
def iter_import_version(output_dir, ancestors, tempdir, pb, fast=False,
534
verbose=False, dry_run=False, max_count=None,
535
standalone=False):
564
536
revdir = None
565
log_encoding = 'ascii'
566
if encoding is not None:
567
log_encoding = encoding
568
537
569
538
# Uncomment this for testing, it basically just has baz2bzr only update
570
539
# 5 patches at a time
599
568
600
569
for i in range(len(ancestors)):
601
570
revision = ancestors[i]
602
rev_id = revision_id(revision, encoding)
571
rev_id = revision_id(revision)
603
572
direct_merges = []
604
573
if verbose:
605
574
version = str(revision.version)
606
575
if version != previous_version:
607
pb.note('On version: %s' % version)
576
clear_progress_bar()
577
print '\rOn version: %s' % version
608
578
yield Progress(str(revision.patchlevel), i, len(ancestors))
609
579
previous_version = version
610
580
else:
622
592
raise
623
593
missing_ancestor = revision
624
594
revdir = None
625
pb.note("unable to access ancestor %s, making into a merge."
595
print ("unable to access ancestor %s, making into a merge."
626
596
% missing_ancestor)
627
597
continue
628
598
target_tree = create_checkout_metadata(target_branch, revdir)
656
626
try:
657
627
if missing_ancestor:
658
628
# if we want it to be in revision-history, do that here.
659
target_tree.set_parent_ids(
660
[revision_id(missing_ancestor, encoding)],
661
allow_leftmost_as_ghost=True)
629
target_tree.add_pending_merge(revision_id(missing_ancestor))
662
630
missing_ancestor = None
663
631
for merged_rev in direct_merges:
664
target_tree.add_pending_merge(revision_id(merged_rev,
665
encoding))
666
target_tree.set_root_id(BAZ_IMPORT_ROOT)
632
target_tree.add_pending_merge(revision_id(merged_rev))
667
633
target_tree.set_inventory(baz_inv)
668
commitobj = Commit(reporter=ImportCommitReporter())
634
commitobj = Commit(reporter=ImportCommitReporter(pb))
669
635
commitobj.commit(working_tree=target_tree,
670
message=log_message.decode(log_encoding, 'replace'),
671
verbose=False, committer=log_creator, timestamp=timestamp,
672
timezone=0, rev_id=rev_id, revprops={})
636
message=log_message.decode('ascii', 'replace'),
637
verbose=False, committer=log_creator,
638
timestamp=timestamp, timezone=0, rev_id=rev_id,
639
revprops={})
673
640
finally:
674
641
target_tree.unlock()
675
642
branch.unlock()
755
722
bzr_inv.sort()
756
723
return bzr_inv
757
724
758
759
def baz_import_branch(to_location, from_branch, fast, max_count, verbose,
760
encoding, dry_run, reuse_history_list):
761
to_location = os.path.realpath(str(to_location))
762
if from_branch is not None:
763
try:
764
from_branch = pybaz.Version(from_branch)
765
except pybaz.errors.NamespaceError:
766
print "%s is not a valid Arch branch." % from_branch
767
return 1
768
if reuse_history_list is None:
769
reuse_history_list = []
770
import_version(to_location, from_branch, encoding, max_count=max_count,
771
reuse_history_from=reuse_history_list)
725
_global_option('max-count', type = int)
726
class cmd_baz_import_branch(Command):
727
"""Import an Arch or Baz branch into a bzr branch. <BZRTOOLS>"""
728
takes_args = ['to_location', 'from_branch?', 'reuse_history*']
729
takes_options = ['verbose', 'max-count']
730
731
def printer(self, name):
732
print name
733
734
def run(self, to_location, from_branch=None, fast=False, max_count=None,
735
verbose=False, dry_run=False, reuse_history_list=[]):
736
to_location = os.path.realpath(str(to_location))
737
if from_branch is not None:
738
try:
739
from_branch = pybaz.Version(from_branch)
740
except pybaz.errors.NamespaceError:
741
print "%s is not a valid Arch branch." % from_branch
742
return 1
743
if reuse_history_list is None:
744
reuse_history_list = []
745
import_version(to_location, from_branch, self.printer,
746
max_count=max_count,
747
reuse_history_from=reuse_history_list)
772
748
773
749
774
750
class NotInABranch(Exception):
777
753
self.path = path
778
754
779
755
780
781
def baz_import(to_root_dir, from_archive, encoding, verbose=False,
782
reuse_history_list=[], prefixes=None):
783
if reuse_history_list is None:
784
reuse_history_list = []
785
to_root = str(os.path.realpath(to_root_dir))
786
if not os.path.exists(to_root):
787
os.mkdir(to_root)
788
if prefixes is not None:
789
prefixes = prefixes.split(':')
790
import_archive(to_root, from_archive, verbose, encoding,
791
reuse_history_list, prefixes=prefixes)
792
793
794
def import_archive(to_root, from_archive, verbose,
795
encoding, reuse_history_from=[], standalone=False,
756
class cmd_baz_import(Command):
757
"""Import an Arch or Baz archive into bzr branches. <BZRTOOLS>
758
759
This command should be used on local archives (or mirrors) only. It is
760
quite slow on remote archives.
761
762
reuse_history allows you to specify any previous imports you
763
have done of different archives, which this archive has branches
764
tagged from. This will dramatically reduce the time to convert
765
the archive as it will not have to convert the history already
766
converted in that other branch.
767
768
If you specify prefixes, only branches whose names start with that prefix
769
will be imported. Skipped branches will be listed, so you can import any
770
branches you missed by accident. Here's an example of doing a partial
771
import from thelove@canonical.com:
772
bzr baz-import thelove thelove@canonical.com --prefixes dists:talloc-except
773
"""
774
takes_args = ['to_root_dir', 'from_archive', 'reuse_history*']
775
takes_options = ['verbose', Option('prefixes', type=str,
776
help="Prefixes of branches to import, colon-separated")]
777
778
def printer(self, name):
779
print name
780
781
def run(self, to_root_dir, from_archive, verbose=False,
782
reuse_history_list=[], prefixes=None):
783
if reuse_history_list is None:
784
reuse_history_list = []
785
to_root = str(os.path.realpath(to_root_dir))
786
if not os.path.exists(to_root):
787
os.mkdir(to_root)
788
if prefixes is not None:
789
prefixes = prefixes.split(':')
790
import_archive(to_root, from_archive, verbose, self.printer,
791
reuse_history_list, prefixes=prefixes)
792
793
794
def import_archive(to_root, from_archive, verbose, printer,
795
reuse_history_from=[], standalone=False,
796
796
prefixes=None):
797
797
def selected(version):
798
798
if prefixes is None:
813
813
except NoRepositoryPresent:
814
814
raise BzrCommandError("Can't create repository at existing branch.")
815
815
versions = list(pybaz.Archive(str(from_archive)).iter_versions())
816
progress_bar = bzrlib.ui.ui_factory.nested_progress_bar()
816
# progress_bar = bzrlib.ui.ui_factory.nested_progress_bar()
817
817
try:
818
818
for num, version in enumerate(versions):
819
progress_bar.update("Branch", num, len(versions))
819
# progress_bar.update("Branch", num, len(versions))
820
820
if not selected(version):
821
821
print "Skipping %s" % version
822
822
continue
823
823
target = os.path.join(to_root, map_namespace(version))
824
printer("importing %s into %s" % (version, target))
824
825
if not os.path.exists(os.path.dirname(target)):
825
826
os.makedirs(os.path.dirname(target))
826
827
try:
827
import_version(target, version, encoding,
828
import_version(target, version, printer,
828
829
reuse_history_from=reuse_history_from,
829
830
standalone=standalone)
830
831
except pybaz.errors.ExecProblem,e:
831
832
if str(e).find('The requested revision cannot be built.') != -1:
832
progress_bar.note(
833
"Skipping version %s as it cannot be built due"
834
" to a missing parent archive." % version)
833
printer("Skipping version %s as it cannot be built due"
834
" to a missing parent archive." % version)
835
835
else:
836
836
raise
837
837
except UserError, e:
838
838
if str(e).find('already exists, and the last revision ') != -1:
839
progress_bar.note(
840
"Skipping version %s as it has had commits made"
841
" since it was converted to bzr." % version)
839
printer("Skipping version %s as it has had commits made"
840
" since it was converted to bzr." % version)
842
841
else:
843
842
raise
844
843
finally:
845
progress_bar.finished()
846
844
pass
845
# progress_bar.finished()
847
846
848
847
def map_namespace(a_version):
849
848
a_version = pybaz.Version("%s" % a_version)
857
856
return "%s/%s" % (category, branch)
858
857
return "%s/%s/%s" % (category, version, branch)
859
858
860
861
859
def map_file_id(file_id):
862
860
"""Convert a baz file id to a bzr one."""
863
861
return file_id.replace('%', '%25').replace('/', '%2f')
Loggerhead is a web-based interface for Breezy
Version: 2.0.1