1
# Copyright (C) 2005 by Aaron Bentley
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19
from bzrlib.errors import BzrError
20
from bzrlib.errors import NotBranchError, BzrCommandError, NoSuchRevision
21
from bzrlib.branch import Branch
22
from bzrlib.clone import copy_branch
23
from bzrlib.commit import Commit, NullCommitReporter
24
from bzrlib.commands import Command
25
from bzrlib.option import _global_option
26
from bzrlib.workingtree import WorkingTree
27
from errors import NoPyBaz
31
from pybaz import NameParser as NameParser
32
from pybaz.backends.baz import null_cmd
35
from fai import iter_new_merges, direct_merges
43
import bzrlib.inventory
47
from progress import *
49
class ImportCommitReporter(NullCommitReporter):
50
def __init__(self, pb):
53
def escaped(self, escape_count, message):
55
bzrlib.trace.warning("replaced %d control characters in message" %
58
def add_id(files, id=None):
59
"""Adds an explicit id to a list of files.
61
:param files: the name of the file to add an id to
62
:type files: list of str
63
:param id: tag one file using the specified id, instead of generating id
68
args.extend(["--id", id])
76
>>> q = test_environ()
79
>>> os.path.exists(os.path.join(q, "home", ".arch-params"))
81
>>> teardown_environ(q)
86
saved_dir = os.getcwdu()
87
tdir = tempfile.mkdtemp(prefix="testdir-")
88
os.environ["HOME"] = os.path.join(tdir, "home")
89
os.mkdir(os.environ["HOME"])
90
arch_dir = os.path.join(tdir, "archive_dir")
91
pybaz.make_archive("test@example.com", arch_dir)
92
work_dir = os.path.join(tdir, "work_dir")
95
pybaz.init_tree(work_dir, "test@example.com/test--test--0")
96
lib_dir = os.path.join(tdir, "lib_dir")
98
pybaz.register_revision_library(lib_dir)
99
pybaz.set_my_id("Test User<test@example.org>")
102
def add_file(path, text, id):
104
>>> q = test_environ()
105
>>> add_file("path with space", "text", "lalala")
106
>>> tree = pybaz.tree_root(".")
107
>>> inv = list(tree.iter_inventory_ids(source=True, both=True))
108
>>> ("x_lalala", "path with space") in inv
110
>>> teardown_environ(q)
112
file(path, "wb").write(text)
116
def add_dir(path, id):
118
>>> q = test_environ()
119
>>> add_dir("path with\(sp) space", "lalala")
120
>>> tree = pybaz.tree_root(".")
121
>>> inv = list(tree.iter_inventory_ids(source=True, both=True))
122
>>> ("x_lalala", "path with\(sp) space") in inv
124
>>> teardown_environ(q)
129
def teardown_environ(tdir):
133
def timport(tree, summary):
134
msg = tree.log_message()
135
msg["summary"] = summary
138
def commit(tree, summary):
140
>>> q = test_environ()
141
>>> tree = pybaz.tree_root(".")
142
>>> timport(tree, "import")
143
>>> commit(tree, "commit")
144
>>> logs = [str(l.revision) for l in tree.iter_logs()]
148
'test@example.com/test--test--0--base-0'
150
'test@example.com/test--test--0--patch-1'
151
>>> teardown_environ(q)
153
msg = tree.log_message()
154
msg["summary"] = summary
157
def commit_test_revisions():
159
>>> q = test_environ()
160
>>> commit_test_revisions()
161
>>> a = pybaz.Archive("test@example.com")
162
>>> revisions = list(a.iter_revisions("test--test--0"))
165
>>> str(revisions[2])
166
'test@example.com/test--test--0--base-0'
167
>>> str(revisions[1])
168
'test@example.com/test--test--0--patch-1'
169
>>> str(revisions[0])
170
'test@example.com/test--test--0--patch-2'
171
>>> teardown_environ(q)
173
tree = pybaz.tree_root(".")
174
add_file("mainfile", "void main(void){}", "mainfile by aaron")
175
timport(tree, "Created mainfile")
176
file("mainfile", "wb").write("or something like that")
177
commit(tree, "altered mainfile")
178
add_file("ofile", "this is another file", "ofile by aaron")
179
commit(tree, "altered mainfile")
182
def commit_more_test_revisions():
184
>>> q = test_environ()
185
>>> commit_test_revisions()
186
>>> commit_more_test_revisions()
187
>>> a = pybaz.Archive("test@example.com")
188
>>> revisions = list(a.iter_revisions("test--test--0"))
191
>>> str(revisions[0])
192
'test@example.com/test--test--0--patch-3'
193
>>> teardown_environ(q)
195
tree = pybaz.tree_root(".")
196
add_file("trainfile", "void train(void){}", "trainfile by aaron")
197
commit(tree, "altered trainfile")
199
class NoSuchVersion(Exception):
200
def __init__(self, version):
201
Exception.__init__(self, "The version %s does not exist." % version)
202
self.version = version
204
def version_ancestry(version):
206
>>> q = test_environ()
207
>>> commit_test_revisions()
208
>>> version = pybaz.Version("test@example.com/test--test--0")
209
>>> ancestors = version_ancestry(version)
210
>>> str(ancestors[0])
211
'test@example.com/test--test--0--base-0'
212
>>> str(ancestors[1])
213
'test@example.com/test--test--0--patch-1'
214
>>> version = pybaz.Version("test@example.com/test--test--0.5")
215
>>> ancestors = version_ancestry(version)
216
Traceback (most recent call last):
217
NoSuchVersion: The version test@example.com/test--test--0.5 does not exist.
218
>>> teardown_environ(q)
221
revision = version.iter_revisions(reverse=True).next()
222
except StopIteration:
226
if not version.exists():
227
raise NoSuchVersion(version)
230
ancestors = list(revision.iter_ancestors(metoo=True))
234
def get_last_revision(branch):
235
last_patch = branch.last_revision()
237
return arch_revision(last_patch)
238
except NotArchRevision:
240
"Directory \"%s\" already exists, and the last revision is not"
241
" an Arch revision (%s)" % (branch.base, last_patch))
243
def do_branch(br_from, to_location, revision_id):
244
"""Derived from branch in builtins."""
248
os.mkdir(to_location)
250
if e.errno == errno.EEXIST:
251
raise UserError('Target directory "%s" already'
252
' exists.' % to_location)
253
if e.errno == errno.ENOENT:
254
raise UserError('Parent of "%s" does not exist.' %
259
copy_branch(br_from, to_location, revision_id, None)
260
except NoSuchRevision:
262
msg = "The branch %s has no revision %s." % (from_location, revision_id)
267
def get_remaining_revisions(output_dir, version, reuse_history_from=[]):
270
output_exists = os.path.exists(output_dir)
272
# We are starting from an existing directory, figure out what
273
# the current version is
274
branch = Branch.open(output_dir)
275
last_patch = get_last_revision(branch)
276
if last_patch is None:
277
raise NotPreviousImport(branch.base)
279
version = last_patch.version
280
elif version is None:
281
raise UserError("No version specified, and directory does not exist.")
284
ancestors = version_ancestry(version)
285
if not output_exists and reuse_history_from != []:
286
for ancestor in reversed(ancestors):
287
if last_patch is not None:
288
# found something to copy
290
# try to grab a copy of ancestor
291
# note that is not optimised: we could look for namespace
292
# transitions and only look for the past after the
294
for history_root in reuse_history_from:
295
possible_source = os.path.join(history_root,
296
map_namespace(ancestor.version))
298
source = Branch.open(possible_source)
299
rev_id = revision_id(ancestor)
300
if rev_id in source.revision_history():
301
do_branch(source, output_dir, rev_id)
302
last_patch = ancestor
304
except NotBranchError:
306
except NoSuchVersion, e:
307
raise UserError(str(e))
310
for i in range(len(ancestors)):
311
if ancestors[i] == last_patch:
314
raise UserError("Directory \"%s\" already exists, and the last "
315
"revision (%s) is not in the ancestry of %s" %
316
(output_dir, last_patch, version))
317
# Strip off all of the ancestors which are already present
318
# And get a directory starting with the latest ancestor
319
latest_ancestor = ancestors[i]
320
old_revno = Branch.open(output_dir).revno()
321
ancestors = ancestors[i+1:]
322
return ancestors, old_revno
325
###class Importer(object):
328
### Currently this is used as a parameter object, though more behaviour is
332
### def __init__(self, output_dir, version, printer, fancy=True, fast=False,
333
### verbose=False, dry_run=False, max_count=None,
334
### reuse_history_from=[]):
335
### self.output_dir = output_dir
336
### self.version = version
340
def import_version(output_dir, version, printer, fancy=True, fast=False,
341
verbose=False, dry_run=False, max_count=None,
342
reuse_history_from=[]):
344
>>> q = test_environ()
345
>>> result_path = os.path.join(q, "result")
346
>>> commit_test_revisions()
347
>>> version = pybaz.Version("test@example.com/test--test--0.1")
348
>>> def printer(message): print message
349
>>> import_version('/', version, printer, fancy=False, dry_run=True)
350
Traceback (most recent call last):
351
NotPreviousImport: / is not the location of a previous import.
352
>>> import_version(result_path, version, printer, fancy=False, dry_run=True)
353
Traceback (most recent call last):
354
UserError: The version test@example.com/test--test--0.1 does not exist.
355
>>> version = pybaz.Version("test@example.com/test--test--0")
356
>>> import_version(result_path, version, printer, fancy=False, dry_run=True)
359
Dry run, not modifying output_dir
361
>>> import_version(result_path, version, printer, fancy=False)
366
>>> import_version(result_path, version, printer, fancy=False)
367
Tree is up-to-date with test@example.com/test--test--0--patch-2
368
>>> commit_more_test_revisions()
369
>>> import_version(result_path, version, printer, fancy=False)
374
>>> teardown_environ(q)
377
ancestors, old_revno = get_remaining_revisions(output_dir, version,
379
except NotBranchError, e:
380
raise NotPreviousImport(e.path)
381
if old_revno is None and len(ancestors) == 0:
382
print 'Version %s has no revisions.' % version
384
if len(ancestors) == 0:
385
last_revision = get_last_revision(Branch.open(output_dir))
386
print 'Tree is up-to-date with %s' % last_revision
389
progress_bar = ProgressBar()
390
tempdir = tempfile.mkdtemp(prefix="baz2bzr-",
391
dir=os.path.dirname(output_dir))
396
for result in iter_import_version(output_dir, ancestors, tempdir,
397
progress_bar, fast=fast, verbose=verbose, dry_run=dry_run,
398
max_count=max_count):
400
show_progress(progress_bar, result)
402
sys.stdout.write('.')
407
sys.stdout.write('\n')
410
print 'Dry run, not modifying output_dir'
412
if os.path.exists(output_dir):
413
# Move the bzr control directory back, and update the working tree
414
revdir = os.path.join(tempdir, "rd")
415
if os.path.exists(revdir):
416
# actual imports were done
417
tmp_bzr_dir = os.path.join(tempdir, '.bzr')
419
bzr_dir = os.path.join(output_dir, '.bzr')
420
new_bzr_dir = os.path.join(tempdir, "rd", '.bzr')
422
os.rename(bzr_dir, tmp_bzr_dir) # Move the original bzr out of the way
423
os.rename(new_bzr_dir, bzr_dir)
425
bzrlib.merge.merge((output_dir, -1), (output_dir, None), # old_revno),
426
check_clean=False, this_dir=output_dir,
429
# If something failed, move back the original bzr directory
430
os.rename(bzr_dir, new_bzr_dir)
431
os.rename(tmp_bzr_dir, bzr_dir)
434
# no imports - perhaps just append_revisions
436
bzrlib.merge.merge((output_dir, -1), (output_dir, None), # old_revno),
437
check_clean=False, this_dir=output_dir,
440
revdir = os.path.join(tempdir, "rd")
441
os.rename(revdir, output_dir)
444
printer('Cleaning up')
445
shutil.rmtree(tempdir)
446
printer("Import complete.")
448
class UserError(BzrCommandError):
449
def __init__(self, message):
450
"""Exception to throw when a user makes an impossible request
451
:param message: The message to emit when printing this exception
452
:type message: string
454
BzrCommandError.__init__(self, message)
456
class NotPreviousImport(UserError):
457
def __init__(self, path):
458
UserError.__init__(self, "%s is not the location of a previous import."
462
def revision_id(arch_revision):
464
Generate a Bzr revision id from an Arch revision id. 'x' in the id
465
designates a revision imported with an experimental algorithm. A number
466
would indicate a particular standardized version.
468
:param arch_revision: The Arch revision to generate an ID for.
470
>>> revision_id(pybaz.Revision("you@example.com/cat--br--0--base-0"))
471
'Arch-1:you@example.com%cat--br--0--base-0'
473
return "Arch-1:%s" % str(arch_revision).replace('/', '%')
475
class NotArchRevision(Exception):
476
def __init__(self, revision_id):
477
msg = "The revision id %s does not look like it came from Arch."\
479
Exception.__init__(self, msg)
481
def arch_revision(revision_id):
483
>>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0"))
484
Traceback (most recent call last):
485
NotArchRevision: The revision id Arch-1:jrandom@example.com%test--test--0 does not look like it came from Arch.
486
>>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0--base-5"))
487
Traceback (most recent call last):
488
NotArchRevision: The revision id Arch-1:jrandom@example.com%test--test--0--base-5 does not look like it came from Arch.
489
>>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0--patch-5"))
490
'jrandom@example.com/test--test--0--patch-5'
492
if revision_id is None:
494
if revision_id[:7] != 'Arch-1:':
495
raise NotArchRevision(revision_id)
498
return pybaz.Revision(revision_id[7:].replace('%', '/'))
499
except pybaz.errors.NamespaceError, e:
500
raise NotArchRevision(revision_id)
502
def iter_import_version(output_dir, ancestors, tempdir, pb, fast=False,
503
verbose=False, dry_run=False, max_count=None):
506
# Uncomment this for testing, it basically just has baz2bzr only update
507
# 5 patches at a time
509
ancestors = ancestors[:max_count]
511
# Not sure if I want this output. basically it tells you ahead of time
512
# what it is going to do, but then later it tells you as it is doing it.
513
# what probably would be best would be to collapse it into ranges, so that
514
# this gives the simple view, and then later it gives the blow by blow.
516
# print 'Adding the following revisions:'
517
# for a in ancestors:
520
previous_version=None
521
missing_ancestor = None
523
for i in range(len(ancestors)):
524
revision = ancestors[i]
525
rev_id = revision_id(revision)
528
version = str(revision.version)
529
if version != previous_version:
531
print '\rOn version: %s' % version
532
yield Progress(str(revision.patchlevel), i, len(ancestors))
533
previous_version = version
535
yield Progress("revisions", i, len(ancestors))
536
if revdir is None and os.path.exists(output_dir):
537
# check for imported revisions and if present just append immediately
538
branch = Branch.open(output_dir)
539
if branch.has_revision(rev_id):
540
branch.append_revision(rev_id)
543
revdir = os.path.join(tempdir, "rd")
545
tree, baz_inv, log = get_revision(revdir, revision)
546
except pybaz.errors.ExecProblem, e:
547
if ("%s" % e.args).find('could not connect') == -1:
549
missing_ancestor = revision
551
print ("unable to access ancestor %s, making into a merge."
554
if os.path.exists(output_dir):
555
bzr_dir = os.path.join(output_dir, '.bzr')
556
new_bzr_dir = os.path.join(tempdir, "rd", '.bzr')
557
# This would be much faster with a simple os.rename(), but if
558
# we fail, we have corrupted the original .bzr directory. Is
559
# that a big problem, as we can just back out the last
560
# revisions in .bzr/revision_history I don't really know
561
# RBC20051024 - yes, it would be a problem as we could not then
562
# apply the corrupted revision.
563
shutil.copytree(bzr_dir, new_bzr_dir)
564
# Now revdir should have a tree with the latest .bzr, and the
565
# next revision of the baz tree
566
branch = Branch.open(revdir)
568
branch = Branch.initialize(revdir)
570
old = os.path.join(revdir, ".bzr")
571
new = os.path.join(tempdir, ".bzr")
573
baz_inv, log = apply_revision(tree, revision)
575
branch = Branch.open(revdir)
576
# cached so we can delete the log
578
log_summary = log.summary
579
log_description = log.description
580
is_continuation = log.continuation_of is not None
581
log_creator = log.creator
582
direct_merges = get_direct_merges(revdir, revision, log)
584
timestamp = email.Utils.mktime_tz(log_date + (0,))
585
if log_summary is None:
587
# log_descriptions of None and "" are ignored.
588
if not is_continuation and log_description:
589
log_message = "\n".join((log_summary, log_description))
591
log_message = log_summary
593
target_tree = WorkingTree(revdir ,branch=branch)
594
target_tree.lock_write()
597
# if we want it to be in revision-history, do that here.
598
target_tree.add_pending_merge(revision_id(missing_ancestor))
599
missing_ancestor = None
600
for merged_rev in direct_merges:
601
target_tree.add_pending_merge(revision_id(merged_rev))
602
target_tree.set_inventory(baz_inv)
603
commitobj = Commit(reporter=ImportCommitReporter(pb))
604
commitobj.commit(branch, log_message.decode('ascii', 'replace'),
605
verbose=False, committer=log_creator,
606
timestamp=timestamp, timezone=0, rev_id=rev_id)
610
yield Progress("revisions", len(ancestors), len(ancestors))
611
unlink_unversioned(branch, revdir)
613
def get_direct_merges(revdir, revision, log):
614
continuation = log.continuation_of
615
previous_version = revision.version
616
if pybaz.WorkingTree(revdir).tree_version != previous_version:
617
pybaz.WorkingTree(revdir).set_tree_version(previous_version)
618
log_path = "%s/{arch}/%s/%s/%s/%s/patch-log/%s" % (revdir,
619
revision.category.nonarch, revision.branch.nonarch,
620
revision.version.nonarch, revision.archive, revision.patchlevel)
621
temp_path = tempfile.mktemp(dir=os.path.dirname(revdir))
622
os.rename(log_path, temp_path)
623
merges = list(iter_new_merges(revdir, revision.version))
624
direct = direct_merges(merges, [continuation])
625
os.rename(temp_path, log_path)
628
def unlink_unversioned(branch, revdir):
629
for unversioned in branch.working_tree().extras():
630
path = os.path.join(revdir, unversioned)
631
if os.path.isdir(path):
636
def get_log(tree, revision):
637
log = pybaz.Patchlog(revision, tree=tree)
638
assert str(log.revision) == str(revision), (log.revision, revision)
641
def get_revision(revdir, revision):
642
tree = revision.get(revdir)
643
log = get_log(tree, revision)
645
return tree, bzr_inventory_data(tree), log
646
except BadFileKind, e:
647
raise UserError("Cannot convert %s because %s is a %s" % (revision,e.path, e.kind) )
650
def apply_revision(tree, revision):
652
log = get_log(tree, revision)
654
return bzr_inventory_data(tree), log
655
except BadFileKind, e:
656
raise UserError("Cannot convert %s because %s is a %s" % (revision,e.path, e.kind) )
659
class BadFileKind(Exception):
660
"""The file kind is not permitted in bzr inventories"""
661
def __init__(self, tree_root, path, kind):
662
self.tree_root = tree_root
665
Exception.__init__(self, "File %s is of forbidden type %s" %
666
(os.path.join(tree_root, path), kind))
669
def bzr_inventory_data(tree):
670
inv_iter = tree.iter_inventory_ids(source=True, both=True)
672
for arch_id, path in inv_iter:
673
bzr_file_id = map_file_id(arch_id)
674
inv_map[path] = bzr_file_id
677
for path, file_id in inv_map.iteritems():
678
full_path = os.path.join(tree, path)
679
kind = bzrlib.osutils.file_kind(full_path)
680
if kind not in ("file", "directory", "symlink"):
681
raise BadFileKind(tree, path, kind)
682
parent_dir = os.path.dirname(path)
684
parent_id = inv_map[parent_dir]
686
parent_id = bzrlib.inventory.ROOT_ID
687
bzr_inv.append((path, file_id, parent_id, kind))
691
_global_option('max-count', type = int)
692
class cmd_baz_import_branch(Command):
693
"""Import an Arch or Baz branch into a bzr branch"""
694
takes_args = ['to_location', 'from_branch?', 'reuse_history*']
695
takes_options = ['verbose', 'max-count']
697
def printer(self, name):
700
def run(self, to_location, from_branch=None, fast=False, max_count=None,
701
verbose=False, dry_run=False, reuse_history_list=[]):
702
to_location = os.path.realpath(str(to_location))
703
if from_branch is not None:
705
from_branch = pybaz.Version(from_branch)
706
except pybaz.errors.NamespaceError:
707
print "%s is not a valid Arch branch." % from_branch
709
if reuse_history_list is None:
710
reuse_history_list = []
711
import_version(to_location, from_branch, self.printer,
713
reuse_history_from=reuse_history_list)
716
class NotInABranch(Exception):
717
def __init__(self, path):
718
Exception.__init__(self, "%s is not in a branch." % path)
722
class cmd_baz_import(Command):
723
"""Import an Arch or Baz archive into bzr branches.
725
reuse_history allows you to specify any previous imports you
726
have done of different archives, which this archive has branches
727
tagged from. This will dramatically reduce the time to convert
728
the archive as it will not have to convert the history already
729
converted in that other branch.
731
takes_args = ['to_root_dir', 'from_archive', 'reuse_history*']
732
takes_options = ['verbose']
734
def printer(self, name):
737
def run(self, to_root_dir, from_archive, verbose=False,
738
reuse_history_list=[]):
739
if reuse_history_list is None:
740
reuse_history_list = []
741
to_root = str(os.path.realpath(to_root_dir))
742
if not os.path.exists(to_root):
744
import_archive(to_root, from_archive, verbose, self.printer,
748
def import_archive(to_root, from_archive, verbose, printer,
749
reuse_history_from=[]):
750
real_to = os.path.realpath(to_root)
751
history_locations = [real_to] + reuse_history_from
752
for version in pybaz.Archive(str(from_archive)).iter_versions():
753
target = os.path.join(to_root, map_namespace(version))
754
printer("importing %s into %s" % (version, target))
755
if not os.path.exists(os.path.dirname(target)):
756
os.makedirs(os.path.dirname(target))
758
import_version(target, version, printer,
759
reuse_history_from=reuse_history_from)
760
except pybaz.errors.ExecProblem,e:
761
if str(e).find('The requested revision cannot be built.') != -1:
762
printer("Skipping version %s as it cannot be built due"
763
" to a missing parent archive." % version)
767
if str(e).find('already exists, and the last revision ') != -1:
768
printer("Skipping version %s as it has had commits made"
769
" since it was converted to bzr." % version)
774
def map_namespace(a_version):
775
a_version = pybaz.Version("%s" % a_version)
776
parser = NameParser(a_version)
777
version = parser.get_version()
778
branch = parser.get_branch()
779
category = parser.get_category()
780
if branch is None or branch == '':
783
return "%s/%s" % (category, branch)
784
return "%s/%s/%s" % (category, version, branch)
786
def map_file_id(file_id):
787
"""Convert a baz file id to a bzr one."""
788
return file_id.replace('%', '%25').replace('/', '%2f')