1
# Copyright (C) 2005 by Aaron Bentley
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19
from bzrlib.bzrdir import BzrDir
20
from bzrlib.errors import BzrError
21
from bzrlib.errors import NotBranchError, BzrCommandError, NoSuchRevision
22
from bzrlib.branch import Branch
23
from bzrlib.commit import Commit, NullCommitReporter
24
from bzrlib.commands import Command
25
from bzrlib.option import _global_option
26
from bzrlib.workingtree import WorkingTree
27
from errors import NoPyBaz
31
from pybaz import NameParser as NameParser
32
from pybaz.backends.baz import null_cmd
35
from fai import iter_new_merges, direct_merges
43
import bzrlib.inventory
47
from progress import *
49
class ImportCommitReporter(NullCommitReporter):
50
def __init__(self, pb):
53
def escaped(self, escape_count, message):
55
bzrlib.trace.warning("replaced %d control characters in message" %
58
def add_id(files, id=None):
59
"""Adds an explicit id to a list of files.
61
:param files: the name of the file to add an id to
62
:type files: list of str
63
:param id: tag one file using the specified id, instead of generating id
68
args.extend(["--id", id])
74
def make_archive(name, location):
75
pb_location = pybaz.ArchiveLocation(location)
76
pb_location.create_master(pybaz.Archive(name),
77
pybaz.ArchiveLocationParams())
81
>>> q = test_environ()
84
>>> os.path.exists(os.path.join(q, "home", ".arch-params"))
86
>>> teardown_environ(q)
91
saved_dir = os.getcwdu()
92
tdir = tempfile.mkdtemp(prefix="testdir-")
93
os.environ["HOME"] = os.path.join(tdir, "home")
94
os.mkdir(os.environ["HOME"])
95
arch_dir = os.path.join(tdir, "archive_dir")
96
make_archive("test@example.com", arch_dir)
97
work_dir = os.path.join(tdir, "work_dir")
100
pybaz.init_tree(work_dir, "test@example.com/test--test--0")
101
lib_dir = os.path.join(tdir, "lib_dir")
103
pybaz.register_revision_library(lib_dir)
104
pybaz.set_my_id("Test User<test@example.org>")
107
def add_file(path, text, id):
109
>>> q = test_environ()
110
>>> add_file("path with space", "text", "lalala")
111
>>> tree = pybaz.tree_root(".")
112
>>> inv = list(tree.iter_inventory_ids(source=True, both=True))
113
>>> ("x_lalala", "path with space") in inv
115
>>> teardown_environ(q)
117
file(path, "wb").write(text)
121
def add_dir(path, id):
123
>>> q = test_environ()
124
>>> add_dir("path with\(sp) space", "lalala")
125
>>> tree = pybaz.tree_root(".")
126
>>> inv = list(tree.iter_inventory_ids(source=True, both=True))
127
>>> ("x_lalala", "path with\(sp) space") in inv
129
>>> teardown_environ(q)
134
def teardown_environ(tdir):
138
def timport(tree, summary):
139
msg = tree.log_message()
140
msg["summary"] = summary
143
def commit(tree, summary):
145
>>> q = test_environ()
146
>>> tree = pybaz.tree_root(".")
147
>>> timport(tree, "import")
148
>>> commit(tree, "commit")
149
>>> logs = [str(l.revision) for l in tree.iter_logs()]
153
'test@example.com/test--test--0--base-0'
155
'test@example.com/test--test--0--patch-1'
156
>>> teardown_environ(q)
158
msg = tree.log_message()
159
msg["summary"] = summary
162
def commit_test_revisions():
164
>>> q = test_environ()
165
>>> commit_test_revisions()
166
>>> a = pybaz.Archive("test@example.com")
167
>>> revisions = list(a.iter_revisions("test--test--0"))
170
>>> str(revisions[2])
171
'test@example.com/test--test--0--base-0'
172
>>> str(revisions[1])
173
'test@example.com/test--test--0--patch-1'
174
>>> str(revisions[0])
175
'test@example.com/test--test--0--patch-2'
176
>>> teardown_environ(q)
178
tree = pybaz.tree_root(".")
179
add_file("mainfile", "void main(void){}", "mainfile by aaron")
180
timport(tree, "Created mainfile")
181
file("mainfile", "wb").write("or something like that")
182
commit(tree, "altered mainfile")
183
add_file("ofile", "this is another file", "ofile by aaron")
184
commit(tree, "altered mainfile")
187
def commit_more_test_revisions():
189
>>> q = test_environ()
190
>>> commit_test_revisions()
191
>>> commit_more_test_revisions()
192
>>> a = pybaz.Archive("test@example.com")
193
>>> revisions = list(a.iter_revisions("test--test--0"))
196
>>> str(revisions[0])
197
'test@example.com/test--test--0--patch-3'
198
>>> teardown_environ(q)
200
tree = pybaz.tree_root(".")
201
add_file("trainfile", "void train(void){}", "trainfile by aaron")
202
commit(tree, "altered trainfile")
204
class NoSuchVersion(Exception):
205
def __init__(self, version):
206
Exception.__init__(self, "The version %s does not exist." % version)
207
self.version = version
209
def version_ancestry(version):
211
>>> q = test_environ()
212
>>> commit_test_revisions()
213
>>> version = pybaz.Version("test@example.com/test--test--0")
214
>>> ancestors = version_ancestry(version)
215
>>> str(ancestors[0])
216
'test@example.com/test--test--0--base-0'
217
>>> str(ancestors[1])
218
'test@example.com/test--test--0--patch-1'
219
>>> version = pybaz.Version("test@example.com/test--test--0.5")
220
>>> ancestors = version_ancestry(version)
221
Traceback (most recent call last):
222
NoSuchVersion: The version test@example.com/test--test--0.5 does not exist.
223
>>> teardown_environ(q)
226
revision = version.iter_revisions(reverse=True).next()
227
except StopIteration:
231
if not version.exists():
232
raise NoSuchVersion(version)
235
ancestors = list(revision.iter_ancestors(metoo=True))
239
def get_last_revision(branch):
240
last_patch = branch.last_revision()
242
return arch_revision(last_patch)
243
except NotArchRevision:
245
"Directory \"%s\" already exists, and the last revision is not"
246
" an Arch revision (%s)" % (branch.base, last_patch))
248
def do_branch(br_from, to_location, revision_id):
249
"""Derived from branch in builtins."""
253
os.mkdir(to_location)
255
if e.errno == errno.EEXIST:
256
raise UserError('Target directory "%s" already'
257
' exists.' % to_location)
258
if e.errno == errno.ENOENT:
259
raise UserError('Parent of "%s" does not exist.' %
264
br_from.bzrdir.clone(to_location, revision_id)
265
except NoSuchRevision:
267
msg = "The branch %s has no revision %s." % (from_location,
273
def get_remaining_revisions(output_dir, version, reuse_history_from=[]):
276
output_exists = os.path.exists(output_dir)
278
# We are starting from an existing directory, figure out what
279
# the current version is
280
branch = Branch.open(output_dir)
281
last_patch = get_last_revision(branch)
282
if last_patch is None:
283
raise NotPreviousImport(branch.base)
285
version = last_patch.version
286
elif version is None:
287
raise UserError("No version specified, and directory does not exist.")
290
ancestors = version_ancestry(version)
291
if not output_exists and reuse_history_from != []:
292
for ancestor in reversed(ancestors):
293
if last_patch is not None:
294
# found something to copy
296
# try to grab a copy of ancestor
297
# note that is not optimised: we could look for namespace
298
# transitions and only look for the past after the
300
for history_root in reuse_history_from:
301
possible_source = os.path.join(history_root,
302
map_namespace(ancestor.version))
304
source = Branch.open(possible_source)
305
rev_id = revision_id(ancestor)
306
if rev_id in source.revision_history():
307
do_branch(source, output_dir, rev_id)
308
last_patch = ancestor
310
except NotBranchError:
312
except NoSuchVersion, e:
313
raise UserError(str(e))
316
for i in range(len(ancestors)):
317
if ancestors[i] == last_patch:
320
raise UserError("Directory \"%s\" already exists, and the last "
321
"revision (%s) is not in the ancestry of %s" %
322
(output_dir, last_patch, version))
323
# Strip off all of the ancestors which are already present
324
# And get a directory starting with the latest ancestor
325
latest_ancestor = ancestors[i]
326
old_revno = Branch.open(output_dir).revno()
327
ancestors = ancestors[i+1:]
328
return ancestors, old_revno
331
###class Importer(object):
334
### Currently this is used as a parameter object, though more behaviour is
338
### def __init__(self, output_dir, version, printer, fancy=True, fast=False,
339
### verbose=False, dry_run=False, max_count=None,
340
### reuse_history_from=[]):
341
### self.output_dir = output_dir
342
### self.version = version
346
def import_version(output_dir, version, printer, fancy=True, fast=False,
347
verbose=False, dry_run=False, max_count=None,
348
reuse_history_from=[]):
350
>>> q = test_environ()
351
>>> result_path = os.path.join(q, "result")
352
>>> commit_test_revisions()
353
>>> version = pybaz.Version("test@example.com/test--test--0.1")
354
>>> def printer(message): print message
355
>>> import_version('/', version, printer, fancy=False, dry_run=True)
356
Traceback (most recent call last):
357
NotPreviousImport: / is not the location of a previous import.
358
>>> import_version(result_path, version, printer, fancy=False, dry_run=True)
359
Traceback (most recent call last):
360
UserError: The version test@example.com/test--test--0.1 does not exist.
361
>>> version = pybaz.Version("test@example.com/test--test--0")
362
>>> import_version(result_path, version, printer, fancy=False, dry_run=True)
365
Dry run, not modifying output_dir
367
>>> import_version(result_path, version, printer, fancy=False)
372
>>> import_version(result_path, version, printer, fancy=False)
373
Tree is up-to-date with test@example.com/test--test--0--patch-2
374
>>> commit_more_test_revisions()
375
>>> import_version(result_path, version, printer, fancy=False)
380
>>> teardown_environ(q)
383
ancestors, old_revno = get_remaining_revisions(output_dir, version,
385
except NotBranchError, e:
386
raise NotPreviousImport(e.path)
387
if old_revno is None and len(ancestors) == 0:
388
print 'Version %s has no revisions.' % version
390
if len(ancestors) == 0:
391
last_revision = get_last_revision(Branch.open(output_dir))
392
print 'Tree is up-to-date with %s' % last_revision
395
progress_bar = ProgressBar()
396
tempdir = tempfile.mkdtemp(prefix="baz2bzr-",
397
dir=os.path.dirname(output_dir))
402
for result in iter_import_version(output_dir, ancestors, tempdir,
403
progress_bar, fast=fast, verbose=verbose, dry_run=dry_run,
404
max_count=max_count):
406
show_progress(progress_bar, result)
408
sys.stdout.write('.')
413
sys.stdout.write('\n')
416
print 'Dry run, not modifying output_dir'
418
if os.path.exists(output_dir):
419
# Move the bzr control directory back, and update the working tree
420
revdir = os.path.join(tempdir, "rd")
421
if os.path.exists(revdir):
422
# actual imports were done
423
tmp_bzr_dir = os.path.join(tempdir, '.bzr')
425
bzr_dir = os.path.join(output_dir, '.bzr')
426
new_bzr_dir = os.path.join(tempdir, "rd", '.bzr')
428
# Move the original bzr out of the way
429
os.rename(bzr_dir, tmp_bzr_dir)
430
os.rename(new_bzr_dir, bzr_dir)
432
WorkingTree.open_containing(output_dir)[0].revert([])
434
# If something failed, move back the original bzr directory
435
os.rename(bzr_dir, new_bzr_dir)
436
os.rename(tmp_bzr_dir, bzr_dir)
439
# no imports - perhaps just append_revisions
441
WorkingTree.open_containing(output_dir)[0].revert([])
443
revdir = os.path.join(tempdir, "rd")
444
os.rename(revdir, output_dir)
447
printer('Cleaning up')
448
shutil.rmtree(tempdir)
449
printer("Import complete.")
451
class UserError(BzrCommandError):
452
def __init__(self, message):
453
"""Exception to throw when a user makes an impossible request
454
:param message: The message to emit when printing this exception
455
:type message: string
457
BzrCommandError.__init__(self, message)
459
class NotPreviousImport(UserError):
460
def __init__(self, path):
461
UserError.__init__(self, "%s is not the location of a previous import."
465
def revision_id(arch_revision):
467
Generate a Bzr revision id from an Arch revision id. 'x' in the id
468
designates a revision imported with an experimental algorithm. A number
469
would indicate a particular standardized version.
471
:param arch_revision: The Arch revision to generate an ID for.
473
>>> revision_id(pybaz.Revision("you@example.com/cat--br--0--base-0"))
474
'Arch-1:you@example.com%cat--br--0--base-0'
476
return "Arch-1:%s" % str(arch_revision).replace('/', '%')
478
class NotArchRevision(Exception):
479
def __init__(self, revision_id):
480
msg = "The revision id %s does not look like it came from Arch."\
482
Exception.__init__(self, msg)
484
def arch_revision(revision_id):
486
>>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0"))
487
Traceback (most recent call last):
488
NotArchRevision: The revision id Arch-1:jrandom@example.com%test--test--0 does not look like it came from Arch.
489
>>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0--base-5"))
490
Traceback (most recent call last):
491
NotArchRevision: The revision id Arch-1:jrandom@example.com%test--test--0--base-5 does not look like it came from Arch.
492
>>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0--patch-5"))
493
'jrandom@example.com/test--test--0--patch-5'
495
if revision_id is None:
497
if revision_id[:7] != 'Arch-1:':
498
raise NotArchRevision(revision_id)
501
return pybaz.Revision(revision_id[7:].replace('%', '/'))
502
except pybaz.errors.NamespaceError, e:
503
raise NotArchRevision(revision_id)
505
def iter_import_version(output_dir, ancestors, tempdir, pb, fast=False,
506
verbose=False, dry_run=False, max_count=None):
509
# Uncomment this for testing, it basically just has baz2bzr only update
510
# 5 patches at a time
512
ancestors = ancestors[:max_count]
514
# Not sure if I want this output. basically it tells you ahead of time
515
# what it is going to do, but then later it tells you as it is doing it.
516
# what probably would be best would be to collapse it into ranges, so that
517
# this gives the simple view, and then later it gives the blow by blow.
519
# print 'Adding the following revisions:'
520
# for a in ancestors:
523
previous_version=None
524
missing_ancestor = None
526
for i in range(len(ancestors)):
527
revision = ancestors[i]
528
rev_id = revision_id(revision)
531
version = str(revision.version)
532
if version != previous_version:
534
print '\rOn version: %s' % version
535
yield Progress(str(revision.patchlevel), i, len(ancestors))
536
previous_version = version
538
yield Progress("revisions", i, len(ancestors))
539
if revdir is None and os.path.exists(output_dir):
540
# check for imported revisions and if present just append
542
branch = Branch.open(output_dir)
543
if branch.repository.has_revision(rev_id):
544
branch.append_revision(rev_id)
547
revdir = os.path.join(tempdir, "rd")
549
tree, baz_inv, log = get_revision(revdir, revision)
550
except pybaz.errors.ExecProblem, e:
551
if ("%s" % e.args).find('could not connect') == -1:
553
missing_ancestor = revision
555
print ("unable to access ancestor %s, making into a merge."
558
if os.path.exists(output_dir):
559
bzr_dir = os.path.join(output_dir, '.bzr')
560
new_bzr_dir = os.path.join(tempdir, "rd", '.bzr')
561
# This would be much faster with a simple os.rename(), but if
562
# we fail, we have corrupted the original .bzr directory. Is
563
# that a big problem, as we can just back out the last
564
# revisions in .bzr/revision_history I don't really know
565
# RBC20051024 - yes, it would be a problem as we could not then
566
# apply the corrupted revision.
567
shutil.copytree(bzr_dir, new_bzr_dir)
568
# Now revdir should have a tree with the latest .bzr, and the
569
# next revision of the baz tree
570
branch = Branch.open(revdir)
572
branch = BzrDir.create_standalone_workingtree(revdir).branch
574
old = os.path.join(revdir, ".bzr")
575
new = os.path.join(tempdir, ".bzr")
577
baz_inv, log = apply_revision(tree, revision)
579
branch = Branch.open(revdir)
580
# cached so we can delete the log
582
log_summary = log.summary
583
log_description = log.description
584
is_continuation = log.continuation_of is not None
585
log_creator = log.creator
586
direct_merges = get_direct_merges(revdir, revision, log)
588
timestamp = email.Utils.mktime_tz(log_date + (0,))
589
if log_summary is None:
591
# log_descriptions of None and "" are ignored.
592
if not is_continuation and log_description:
593
log_message = "\n".join((log_summary, log_description))
595
log_message = log_summary
597
target_tree = WorkingTree(revdir ,branch=branch)
598
target_tree.lock_write()
601
# if we want it to be in revision-history, do that here.
602
target_tree.add_pending_merge(revision_id(missing_ancestor))
603
missing_ancestor = None
604
for merged_rev in direct_merges:
605
target_tree.add_pending_merge(revision_id(merged_rev))
606
target_tree.set_inventory(baz_inv)
607
commitobj = Commit(reporter=ImportCommitReporter(pb))
608
commitobj.commit(branch, log_message.decode('ascii', 'replace'),
609
verbose=False, committer=log_creator,
610
timestamp=timestamp, timezone=0, rev_id=rev_id)
614
yield Progress("revisions", len(ancestors), len(ancestors))
615
unlink_unversioned(branch, revdir)
617
def get_direct_merges(revdir, revision, log):
618
continuation = log.continuation_of
619
previous_version = revision.version
620
if pybaz.WorkingTree(revdir).tree_version != previous_version:
621
pybaz.WorkingTree(revdir).set_tree_version(previous_version)
622
log_path = "%s/{arch}/%s/%s/%s/%s/patch-log/%s" % (revdir,
623
revision.category.nonarch, revision.branch.nonarch,
624
revision.version.nonarch, revision.archive, revision.patchlevel)
625
temp_path = tempfile.mktemp(dir=os.path.dirname(revdir))
626
os.rename(log_path, temp_path)
627
merges = list(iter_new_merges(revdir, revision.version))
628
direct = direct_merges(merges, [continuation])
629
os.rename(temp_path, log_path)
632
def unlink_unversioned(branch, revdir):
633
for unversioned in branch.working_tree().extras():
634
path = os.path.join(revdir, unversioned)
635
if os.path.isdir(path):
640
def get_log(tree, revision):
641
log = pybaz.Patchlog(revision, tree=tree)
642
assert str(log.revision) == str(revision), (log.revision, revision)
645
def get_revision(revdir, revision):
646
tree = revision.get(revdir)
647
log = get_log(tree, revision)
649
return tree, bzr_inventory_data(tree), log
650
except BadFileKind, e:
651
raise UserError("Cannot convert %s because %s is a %s" %
652
(revision,e.path, e.kind))
655
def apply_revision(tree, revision):
657
log = get_log(tree, revision)
659
return bzr_inventory_data(tree), log
660
except BadFileKind, e:
661
raise UserError("Cannot convert %s because %s is a %s" %
662
(revision,e.path, e.kind))
665
class BadFileKind(Exception):
666
"""The file kind is not permitted in bzr inventories"""
667
def __init__(self, tree_root, path, kind):
668
self.tree_root = tree_root
671
Exception.__init__(self, "File %s is of forbidden type %s" %
672
(os.path.join(tree_root, path), kind))
675
def bzr_inventory_data(tree):
676
inv_iter = tree.iter_inventory_ids(source=True, both=True)
678
for arch_id, path in inv_iter:
679
bzr_file_id = map_file_id(arch_id)
680
inv_map[path] = bzr_file_id
683
for path, file_id in inv_map.iteritems():
684
full_path = os.path.join(tree, path)
685
kind = bzrlib.osutils.file_kind(full_path)
686
if kind not in ("file", "directory", "symlink"):
687
raise BadFileKind(tree, path, kind)
688
parent_dir = os.path.dirname(path)
690
parent_id = inv_map[parent_dir]
692
parent_id = bzrlib.inventory.ROOT_ID
693
bzr_inv.append((path, file_id, parent_id, kind))
697
_global_option('max-count', type = int)
698
class cmd_baz_import_branch(Command):
699
"""Import an Arch or Baz branch into a bzr branch"""
700
takes_args = ['to_location', 'from_branch?', 'reuse_history*']
701
takes_options = ['verbose', 'max-count']
703
def printer(self, name):
706
def run(self, to_location, from_branch=None, fast=False, max_count=None,
707
verbose=False, dry_run=False, reuse_history_list=[]):
708
to_location = os.path.realpath(str(to_location))
709
if from_branch is not None:
711
from_branch = pybaz.Version(from_branch)
712
except pybaz.errors.NamespaceError:
713
print "%s is not a valid Arch branch." % from_branch
715
if reuse_history_list is None:
716
reuse_history_list = []
717
import_version(to_location, from_branch, self.printer,
719
reuse_history_from=reuse_history_list)
722
class NotInABranch(Exception):
723
def __init__(self, path):
724
Exception.__init__(self, "%s is not in a branch." % path)
728
class cmd_baz_import(Command):
729
"""Import an Arch or Baz archive into bzr branches.
731
reuse_history allows you to specify any previous imports you
732
have done of different archives, which this archive has branches
733
tagged from. This will dramatically reduce the time to convert
734
the archive as it will not have to convert the history already
735
converted in that other branch.
737
takes_args = ['to_root_dir', 'from_archive', 'reuse_history*']
738
takes_options = ['verbose']
740
def printer(self, name):
743
def run(self, to_root_dir, from_archive, verbose=False,
744
reuse_history_list=[]):
745
if reuse_history_list is None:
746
reuse_history_list = []
747
to_root = str(os.path.realpath(to_root_dir))
748
if not os.path.exists(to_root):
750
import_archive(to_root, from_archive, verbose, self.printer,
754
def import_archive(to_root, from_archive, verbose, printer,
755
reuse_history_from=[]):
756
real_to = os.path.realpath(to_root)
757
history_locations = [real_to] + reuse_history_from
758
for version in pybaz.Archive(str(from_archive)).iter_versions():
759
target = os.path.join(to_root, map_namespace(version))
760
printer("importing %s into %s" % (version, target))
761
if not os.path.exists(os.path.dirname(target)):
762
os.makedirs(os.path.dirname(target))
764
import_version(target, version, printer,
765
reuse_history_from=reuse_history_from)
766
except pybaz.errors.ExecProblem,e:
767
if str(e).find('The requested revision cannot be built.') != -1:
768
printer("Skipping version %s as it cannot be built due"
769
" to a missing parent archive." % version)
773
if str(e).find('already exists, and the last revision ') != -1:
774
printer("Skipping version %s as it has had commits made"
775
" since it was converted to bzr." % version)
780
def map_namespace(a_version):
781
a_version = pybaz.Version("%s" % a_version)
782
parser = NameParser(a_version)
783
version = parser.get_version()
784
branch = parser.get_branch()
785
category = parser.get_category()
786
if branch is None or branch == '':
789
return "%s/%s" % (category, branch)
790
return "%s/%s/%s" % (category, version, branch)
792
def map_file_id(file_id):
793
"""Convert a baz file id to a bzr one."""
794
return file_id.replace('%', '%25').replace('/', '%2f')
added
removed
baz_import.py
