1
# Copyright (C) 2005 by Aaron Bentley
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19
from bzrlib.errors import BzrError
20
from bzrlib.errors import NotBranchError, BzrCommandError, NoSuchRevision
21
from bzrlib.branch import Branch
22
from bzrlib.clone import copy_branch
23
from bzrlib.commit import Commit, NullCommitReporter
24
from bzrlib.commands import Command
25
from bzrlib.option import _global_option
26
from errors import NoPyBaz
30
from pybaz import NameParser as NameParser
31
from pybaz.backends.baz import null_cmd
34
from fai import iter_new_merges, direct_merges
42
import bzrlib.inventory
46
from progress import *
48
class ImportCommitReporter(NullCommitReporter):
49
def __init__(self, pb):
52
def escaped(self, escape_count, message):
54
bzrlib.trace.warning("replaced %d control characters in message" %
57
def add_id(files, id=None):
58
"""Adds an explicit id to a list of files.
60
:param files: the name of the file to add an id to
61
:type files: list of str
62
:param id: tag one file using the specified id, instead of generating id
67
args.extend(["--id", id])
75
>>> q = test_environ()
78
>>> os.path.exists(os.path.join(q, "home", ".arch-params"))
80
>>> teardown_environ(q)
85
saved_dir = os.getcwdu()
86
tdir = tempfile.mkdtemp(prefix="testdir-")
87
os.environ["HOME"] = os.path.join(tdir, "home")
88
os.mkdir(os.environ["HOME"])
89
arch_dir = os.path.join(tdir, "archive_dir")
90
pybaz.make_archive("test@example.com", arch_dir)
91
work_dir = os.path.join(tdir, "work_dir")
94
pybaz.init_tree(work_dir, "test@example.com/test--test--0")
95
lib_dir = os.path.join(tdir, "lib_dir")
97
pybaz.register_revision_library(lib_dir)
98
pybaz.set_my_id("Test User<test@example.org>")
101
def add_file(path, text, id):
103
>>> q = test_environ()
104
>>> add_file("path with space", "text", "lalala")
105
>>> tree = pybaz.tree_root(".")
106
>>> inv = list(tree.iter_inventory_ids(source=True, both=True))
107
>>> ("x_lalala", "path with space") in inv
109
>>> teardown_environ(q)
111
file(path, "wb").write(text)
115
def add_dir(path, id):
117
>>> q = test_environ()
118
>>> add_dir("path with\(sp) space", "lalala")
119
>>> tree = pybaz.tree_root(".")
120
>>> inv = list(tree.iter_inventory_ids(source=True, both=True))
121
>>> ("x_lalala", "path with\(sp) space") in inv
123
>>> teardown_environ(q)
128
def teardown_environ(tdir):
132
def timport(tree, summary):
133
msg = tree.log_message()
134
msg["summary"] = summary
137
def commit(tree, summary):
139
>>> q = test_environ()
140
>>> tree = pybaz.tree_root(".")
141
>>> timport(tree, "import")
142
>>> commit(tree, "commit")
143
>>> logs = [str(l.revision) for l in tree.iter_logs()]
147
'test@example.com/test--test--0--base-0'
149
'test@example.com/test--test--0--patch-1'
150
>>> teardown_environ(q)
152
msg = tree.log_message()
153
msg["summary"] = summary
156
def commit_test_revisions():
158
>>> q = test_environ()
159
>>> commit_test_revisions()
160
>>> a = pybaz.Archive("test@example.com")
161
>>> revisions = list(a.iter_revisions("test--test--0"))
164
>>> str(revisions[2])
165
'test@example.com/test--test--0--base-0'
166
>>> str(revisions[1])
167
'test@example.com/test--test--0--patch-1'
168
>>> str(revisions[0])
169
'test@example.com/test--test--0--patch-2'
170
>>> teardown_environ(q)
172
tree = pybaz.tree_root(".")
173
add_file("mainfile", "void main(void){}", "mainfile by aaron")
174
timport(tree, "Created mainfile")
175
file("mainfile", "wb").write("or something like that")
176
commit(tree, "altered mainfile")
177
add_file("ofile", "this is another file", "ofile by aaron")
178
commit(tree, "altered mainfile")
181
def commit_more_test_revisions():
183
>>> q = test_environ()
184
>>> commit_test_revisions()
185
>>> commit_more_test_revisions()
186
>>> a = pybaz.Archive("test@example.com")
187
>>> revisions = list(a.iter_revisions("test--test--0"))
190
>>> str(revisions[0])
191
'test@example.com/test--test--0--patch-3'
192
>>> teardown_environ(q)
194
tree = pybaz.tree_root(".")
195
add_file("trainfile", "void train(void){}", "trainfile by aaron")
196
commit(tree, "altered trainfile")
198
class NoSuchVersion(Exception):
199
def __init__(self, version):
200
Exception.__init__(self, "The version %s does not exist." % version)
201
self.version = version
203
def version_ancestry(version):
205
>>> q = test_environ()
206
>>> commit_test_revisions()
207
>>> version = pybaz.Version("test@example.com/test--test--0")
208
>>> ancestors = version_ancestry(version)
209
>>> str(ancestors[0])
210
'test@example.com/test--test--0--base-0'
211
>>> str(ancestors[1])
212
'test@example.com/test--test--0--patch-1'
213
>>> version = pybaz.Version("test@example.com/test--test--0.5")
214
>>> ancestors = version_ancestry(version)
215
Traceback (most recent call last):
216
NoSuchVersion: The version test@example.com/test--test--0.5 does not exist.
217
>>> teardown_environ(q)
220
revision = version.iter_revisions(reverse=True).next()
221
except StopIteration:
225
if not version.exists():
226
raise NoSuchVersion(version)
229
ancestors = list(revision.iter_ancestors(metoo=True))
233
def get_last_revision(branch):
234
last_patch = branch.last_revision()
236
return arch_revision(last_patch)
237
except NotArchRevision:
239
"Directory \"%s\" already exists, and the last revision is not"
240
" an Arch revision (%s)" % (branch.base, last_patch))
242
def do_branch(br_from, to_location, revision_id):
243
"""Derived from branch in builtins."""
247
os.mkdir(to_location)
249
if e.errno == errno.EEXIST:
250
raise UserError('Target directory "%s" already'
251
' exists.' % to_location)
252
if e.errno == errno.ENOENT:
253
raise UserError('Parent of "%s" does not exist.' %
258
copy_branch(br_from, to_location, revision_id, None)
259
except NoSuchRevision:
261
msg = "The branch %s has no revision %s." % (from_location, revision_id)
266
def get_remaining_revisions(output_dir, version, reuse_history_from=[]):
269
output_exists = os.path.exists(output_dir)
271
# We are starting from an existing directory, figure out what
272
# the current version is
273
branch = Branch.open(output_dir)
274
last_patch = get_last_revision(branch)
276
version = last_patch.version
277
elif version is None:
278
raise UserError("No version specified, and directory does not exist.")
281
ancestors = version_ancestry(version)
282
if not output_exists and reuse_history_from != []:
283
for ancestor in reversed(ancestors):
284
if last_patch is not None:
285
# found something to copy
287
# try to grab a copy of ancestor
288
# note that is not optimised: we could look for namespace
289
# transitions and only look for the past after the
291
for history_root in reuse_history_from:
292
possible_source = os.path.join(history_root,
293
map_namespace(ancestor.version))
295
source = Branch.open(possible_source)
296
rev_id = revision_id(ancestor)
297
if rev_id in source.revision_history():
298
do_branch(source, output_dir, rev_id)
299
last_patch = ancestor
301
except NotBranchError:
303
except NoSuchVersion, e:
304
raise UserError(str(e))
307
for i in range(len(ancestors)):
308
if ancestors[i] == last_patch:
311
raise UserError("Directory \"%s\" already exists, and the last "
312
"revision (%s) is not in the ancestry of %s" %
313
(output_dir, last_patch, version))
314
# Strip off all of the ancestors which are already present
315
# And get a directory starting with the latest ancestor
316
latest_ancestor = ancestors[i]
317
old_revno = Branch.open(output_dir).revno()
318
ancestors = ancestors[i+1:]
319
return ancestors, old_revno
322
###class Importer(object):
325
### Currently this is used as a parameter object, though more behaviour is
329
### def __init__(self, output_dir, version, printer, fancy=True, fast=False,
330
### verbose=False, dry_run=False, max_count=None,
331
### reuse_history_from=[]):
332
### self.output_dir = output_dir
333
### self.version = version
337
def import_version(output_dir, version, printer, fancy=True, fast=False,
338
verbose=False, dry_run=False, max_count=None,
339
reuse_history_from=[]):
341
>>> q = test_environ()
342
>>> result_path = os.path.join(q, "result")
343
>>> commit_test_revisions()
344
>>> version = pybaz.Version("test@example.com/test--test--0.1")
345
>>> def printer(message): print message
346
>>> import_version('/', version, printer, fancy=False, dry_run=True)
347
Traceback (most recent call last):
348
UserError: / exists, but is not a bzr branch.
349
>>> import_version(result_path, version, printer, fancy=False, dry_run=True)
350
Traceback (most recent call last):
351
UserError: The version test@example.com/test--test--0.1 does not exist.
352
>>> version = pybaz.Version("test@example.com/test--test--0")
353
>>> import_version(result_path, version, printer, fancy=False, dry_run=True)
356
Dry run, not modifying output_dir
358
>>> import_version(result_path, version, printer, fancy=False)
363
>>> import_version(result_path, version, printer, fancy=False)
364
Tree is up-to-date with test@example.com/test--test--0--patch-2
365
>>> commit_more_test_revisions()
366
>>> import_version(result_path, version, printer, fancy=False)
371
>>> teardown_environ(q)
374
ancestors, old_revno = get_remaining_revisions(output_dir, version,
376
except NotBranchError, e:
377
raise UserError("%s exists, but is not a bzr branch." % output_dir)
378
if old_revno is None and len(ancestors) == 0:
379
print 'Version %s has no revisions.' % version
381
if len(ancestors) == 0:
382
last_revision = get_last_revision(Branch.open(output_dir))
383
print 'Tree is up-to-date with %s' % last_revision
386
progress_bar = ProgressBar()
387
tempdir = tempfile.mkdtemp(prefix="baz2bzr-",
388
dir=os.path.dirname(output_dir))
393
for result in iter_import_version(output_dir, ancestors, tempdir,
394
progress_bar, fast=fast, verbose=verbose, dry_run=dry_run,
395
max_count=max_count):
397
show_progress(progress_bar, result)
399
sys.stdout.write('.')
404
sys.stdout.write('\n')
407
print 'Dry run, not modifying output_dir'
409
if os.path.exists(output_dir):
410
# Move the bzr control directory back, and update the working tree
411
revdir = os.path.join(tempdir, "rd")
412
if os.path.exists(revdir):
413
# actual imports were done
414
tmp_bzr_dir = os.path.join(tempdir, '.bzr')
416
bzr_dir = os.path.join(output_dir, '.bzr')
417
new_bzr_dir = os.path.join(tempdir, "rd", '.bzr')
419
os.rename(bzr_dir, tmp_bzr_dir) # Move the original bzr out of the way
420
os.rename(new_bzr_dir, bzr_dir)
422
bzrlib.merge.merge((output_dir, -1), (output_dir, None), # old_revno),
423
check_clean=False, this_dir=output_dir,
426
# If something failed, move back the original bzr directory
427
os.rename(bzr_dir, new_bzr_dir)
428
os.rename(tmp_bzr_dir, bzr_dir)
431
# no imports - perhaps just append_revisions
433
bzrlib.merge.merge((output_dir, -1), (output_dir, None), # old_revno),
434
check_clean=False, this_dir=output_dir,
437
revdir = os.path.join(tempdir, "rd")
438
os.rename(revdir, output_dir)
441
printer('Cleaning up')
442
shutil.rmtree(tempdir)
443
printer("Import complete.")
445
class UserError(BzrCommandError):
446
def __init__(self, message):
447
"""Exception to throw when a user makes an impossible request
448
:param message: The message to emit when printing this exception
449
:type message: string
451
BzrCommandError.__init__(self, message)
453
def revision_id(arch_revision):
455
Generate a Bzr revision id from an Arch revision id. 'x' in the id
456
designates a revision imported with an experimental algorithm. A number
457
would indicate a particular standardized version.
459
:param arch_revision: The Arch revision to generate an ID for.
461
>>> revision_id(pybaz.Revision("you@example.com/cat--br--0--base-0"))
462
'Arch-1:you@example.com%cat--br--0--base-0'
464
return "Arch-1:%s" % str(arch_revision).replace('/', '%')
466
class NotArchRevision(Exception):
467
def __init__(self, revision_id):
468
msg = "The revision id %s does not look like it came from Arch."\
470
Exception.__init__(self, msg)
472
def arch_revision(revision_id):
474
>>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0"))
475
Traceback (most recent call last):
476
NotArchRevision: The revision id Arch-1:jrandom@example.com%test--test--0 does not look like it came from Arch.
477
>>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0--base-5"))
478
Traceback (most recent call last):
479
NotArchRevision: The revision id Arch-1:jrandom@example.com%test--test--0--base-5 does not look like it came from Arch.
480
>>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0--patch-5"))
481
'jrandom@example.com/test--test--0--patch-5'
483
if revision_id is None:
485
if revision_id[:7] != 'Arch-1:':
486
raise NotArchRevision(revision_id)
489
return pybaz.Revision(revision_id[7:].replace('%', '/'))
490
except pybaz.errors.NamespaceError, e:
491
raise NotArchRevision(revision_id)
493
def iter_import_version(output_dir, ancestors, tempdir, pb, fast=False,
494
verbose=False, dry_run=False, max_count=None):
497
# Uncomment this for testing, it basically just has baz2bzr only update
498
# 5 patches at a time
500
ancestors = ancestors[:max_count]
502
# Not sure if I want this output. basically it tells you ahead of time
503
# what it is going to do, but then later it tells you as it is doing it.
504
# what probably would be best would be to collapse it into ranges, so that
505
# this gives the simple view, and then later it gives the blow by blow.
507
# print 'Adding the following revisions:'
508
# for a in ancestors:
511
previous_version=None
512
missing_ancestor = None
514
for i in range(len(ancestors)):
515
revision = ancestors[i]
516
rev_id = revision_id(revision)
519
version = str(revision.version)
520
if version != previous_version:
522
print '\rOn version: %s' % version
523
yield Progress(str(revision.patchlevel), i, len(ancestors))
524
previous_version = version
526
yield Progress("revisions", i, len(ancestors))
527
if revdir is None and os.path.exists(output_dir):
528
# check for imported revisions and if present just append immediately
529
branch = Branch.open(output_dir)
530
if branch.has_revision(rev_id):
531
branch.append_revision(rev_id)
534
revdir = os.path.join(tempdir, "rd")
536
tree, baz_inv, log = get_revision(revdir, revision)
537
except pybaz.errors.ExecProblem, e:
538
if ("%s" % e.args).find('could not connect') == -1:
540
missing_ancestor = revision
542
print ("unable to access ancestor %s, making into a merge."
545
if os.path.exists(output_dir):
546
bzr_dir = os.path.join(output_dir, '.bzr')
547
new_bzr_dir = os.path.join(tempdir, "rd", '.bzr')
548
# This would be much faster with a simple os.rename(), but if
549
# we fail, we have corrupted the original .bzr directory. Is
550
# that a big problem, as we can just back out the last
551
# revisions in .bzr/revision_history I don't really know
552
# RBC20051024 - yes, it would be a problem as we could not then
553
# apply the corrupted revision.
554
shutil.copytree(bzr_dir, new_bzr_dir)
555
# Now revdir should have a tree with the latest .bzr, and the
556
# next revision of the baz tree
557
branch = Branch.open(revdir)
559
branch = Branch.initialize(revdir)
561
old = os.path.join(revdir, ".bzr")
562
new = os.path.join(tempdir, ".bzr")
564
baz_inv, log = apply_revision(tree, revision)
566
branch = Branch.open(revdir)
567
# cached so we can delete the log
569
log_summary = log.summary
570
log_description = log.description
571
is_continuation = log.continuation_of is not None
572
log_creator = log.creator
573
direct_merges = get_direct_merges(revdir, revision, log)
575
timestamp = email.Utils.mktime_tz(log_date + (0,))
576
if log_summary is None:
578
# log_descriptions of None and "" are ignored.
579
if not is_continuation and log_description:
580
log_message = "\n".join((log_summary, log_description))
582
log_message = log_summary
586
# if we want it to be in revision-history, do that here.
587
branch.add_pending_merge(revision_id(missing_ancestor))
588
missing_ancestor = None
589
for merged_rev in direct_merges:
590
branch.add_pending_merge(revision_id(merged_rev))
591
branch.set_inventory(baz_inv)
592
commitobj = Commit(reporter=ImportCommitReporter(pb))
593
commitobj.commit(branch, log_message.decode('ascii', 'replace'),
594
verbose=False, committer=log_creator,
595
timestamp=timestamp, timezone=0, rev_id=rev_id)
598
yield Progress("revisions", len(ancestors), len(ancestors))
599
unlink_unversioned(branch, revdir)
601
def get_direct_merges(revdir, revision, log):
602
continuation = log.continuation_of
603
previous_version = revision.version
604
if pybaz.WorkingTree(revdir).tree_version != previous_version:
605
pybaz.WorkingTree(revdir).set_tree_version(previous_version)
606
log_path = "%s/{arch}/%s/%s/%s/%s/patch-log/%s" % (revdir,
607
revision.category.nonarch, revision.branch.nonarch,
608
revision.version.nonarch, revision.archive, revision.patchlevel)
609
temp_path = tempfile.mktemp(dir=os.path.dirname(revdir))
610
os.rename(log_path, temp_path)
611
merges = list(iter_new_merges(revdir, revision.version))
612
direct = direct_merges(merges, [continuation])
613
os.rename(temp_path, log_path)
616
def unlink_unversioned(branch, revdir):
617
for unversioned in branch.working_tree().extras():
618
path = os.path.join(revdir, unversioned)
619
if os.path.isdir(path):
624
def get_log(tree, revision):
625
log = pybaz.Patchlog(revision, tree=tree)
626
assert str(log.revision) == str(revision), (log.revision, revision)
629
def get_revision(revdir, revision):
630
tree = revision.get(revdir)
631
log = get_log(tree, revision)
633
return tree, bzr_inventory_data(tree), log
634
except BadFileKind, e:
635
raise UserError("Cannot convert %s because %s is a %s" % (revision,e.path, e.kind) )
638
def apply_revision(tree, revision):
640
log = get_log(tree, revision)
642
return bzr_inventory_data(tree), log
643
except BadFileKind, e:
644
raise UserError("Cannot convert %s because %s is a %s" % (revision,e.path, e.kind) )
647
class BadFileKind(Exception):
648
"""The file kind is not permitted in bzr inventories"""
649
def __init__(self, tree_root, path, kind):
650
self.tree_root = tree_root
653
Exception.__init__(self, "File %s is of forbidden type %s" %
654
(os.path.join(tree_root, path), kind))
657
def bzr_inventory_data(tree):
658
inv_iter = tree.iter_inventory_ids(source=True, both=True)
660
for arch_id, path in inv_iter:
661
bzr_file_id = map_file_id(arch_id)
662
inv_map[path] = bzr_file_id
665
for path, file_id in inv_map.iteritems():
666
full_path = os.path.join(tree, path)
667
kind = bzrlib.osutils.file_kind(full_path)
668
if kind not in ("file", "directory", "symlink"):
669
raise BadFileKind(tree, path, kind)
670
parent_dir = os.path.dirname(path)
672
parent_id = inv_map[parent_dir]
674
parent_id = bzrlib.inventory.ROOT_ID
675
bzr_inv.append((path, file_id, parent_id, kind))
679
_global_option('max-count', type = int)
680
class cmd_baz_import_branch(Command):
681
"""Import an Arch or Baz branch into a bzr branch"""
682
takes_args = ['to_location', 'from_branch?', 'reuse_history*']
683
takes_options = ['verbose', 'max-count']
685
def printer(self, name):
688
def run(self, to_location, from_branch=None, fast=False, max_count=None,
689
verbose=False, dry_run=False, reuse_history_list=[]):
690
to_location = os.path.realpath(str(to_location))
691
if from_branch is not None:
693
from_branch = pybaz.Version(from_branch)
694
except pybaz.errors.NamespaceError:
695
print "%s is not a valid Arch branch." % from_branch
697
if reuse_history_list is None:
698
reuse_history_list = []
699
import_version(to_location, from_branch, self.printer,
700
max_count=max_count, reuse_history_from=reuse_history_list)
703
class cmd_baz_import(Command):
704
"""Import an Arch or Baz archive into bzr branches.
706
reuse_history allows you to specify any previous imports you
707
have done of different archives, which this archive has branches
708
tagged from. This will dramatically reduce the time to convert
709
the archive as it will not have to convert the history already
710
converted in that other branch.
712
takes_args = ['to_root_dir', 'from_archive', 'reuse_history*']
713
takes_options = ['verbose']
715
def printer(self, name):
718
def run(self, to_root_dir, from_archive, verbose=False,
719
reuse_history_list=[]):
720
if reuse_history_list is None:
721
reuse_history_list = []
722
to_root = str(os.path.realpath(to_root_dir))
723
if not os.path.exists(to_root):
725
import_archive(to_root, from_archive, verbose, self.printer,
729
def import_archive(to_root, from_archive, verbose, printer,
730
reuse_history_from=[]):
731
real_to = os.path.realpath(to_root)
732
history_locations = [real_to] + reuse_history_from
733
for version in pybaz.Archive(str(from_archive)).iter_versions():
734
target = os.path.join(to_root, map_namespace(version))
735
printer("importing %s into %s" % (version, target))
736
if not os.path.exists(os.path.dirname(target)):
737
os.makedirs(os.path.dirname(target))
739
import_version(target, version, printer,
740
reuse_history_from=reuse_history_from)
741
except pybaz.errors.ExecProblem,e:
742
if str(e).find('The requested revision cannot be built.') != -1:
743
printer("Skipping version %s as it cannot be built due"
744
" to a missing parent archive." % version)
748
if str(e).find('already exists, and the last revision ') != -1:
749
printer("Skipping version %s as it has had commits made"
750
" since it was converted to bzr." % version)
755
def map_namespace(a_version):
756
a_version = pybaz.Version("%s" % a_version)
757
parser = NameParser(a_version)
758
version = parser.get_version()
759
branch = parser.get_branch()
760
category = parser.get_category()
761
if branch is None or branch == '':
764
return "%s/%s" % (category, branch)
765
return "%s/%s/%s" % (category, version, branch)
767
def map_file_id(file_id):
768
"""Convert a baz file id to a bzr one."""
769
return file_id.replace('%', '%25').replace('/', '%2f')
added
removed
baz_import.py
