1
# Copyright (C) 2005, 2006 by Aaron Bentley
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19
from bzrlib.bzrdir import BzrDir
20
import bzrlib.bzrdir as bzrdir
21
from bzrlib.errors import (BzrError,
28
from bzrlib.branch import Branch
29
from bzrlib.commit import Commit, NullCommitReporter
30
from bzrlib.commands import Command
31
from bzrlib.option import _global_option, Option
32
from bzrlib.merge import merge_inner
33
from bzrlib.revision import NULL_REVISION
36
from bzrlib.workingtree import WorkingTree
37
from errors import NoPyBaz
41
from pybaz import NameParser as NameParser
42
from pybaz.backends.baz import null_cmd
45
from fai import iter_new_merges, direct_merges
53
import bzrlib.inventory
57
from progress import *
59
class ImportCommitReporter(NullCommitReporter):
61
def escaped(self, escape_count, message):
62
bzrlib.trace.warning("replaced %d control characters in message" %
65
def add_id(files, id=None):
66
"""Adds an explicit id to a list of files.
68
:param files: the name of the file to add an id to
69
:type files: list of str
70
:param id: tag one file using the specified id, instead of generating id
75
args.extend(["--id", id])
81
def make_archive(name, location):
82
pb_location = pybaz.ArchiveLocation(location)
83
pb_location.create_master(pybaz.Archive(name),
84
pybaz.ArchiveLocationParams())
88
>>> q = test_environ()
91
>>> os.path.exists(os.path.join(q, "home", ".arch-params"))
93
>>> teardown_environ(q)
98
saved_dir = os.getcwdu()
99
tdir = tempfile.mkdtemp(prefix="testdir-")
100
os.environ["HOME"] = os.path.join(tdir, "home")
101
os.mkdir(os.environ["HOME"])
102
arch_dir = os.path.join(tdir, "archive_dir")
103
make_archive("test@example.com", arch_dir)
104
work_dir = os.path.join(tdir, "work_dir")
107
pybaz.init_tree(work_dir, "test@example.com/test--test--0")
108
lib_dir = os.path.join(tdir, "lib_dir")
110
pybaz.register_revision_library(lib_dir)
111
pybaz.set_my_id("Test User<test@example.org>")
114
def add_file(path, text, id):
116
>>> q = test_environ()
117
>>> add_file("path with space", "text", "lalala")
118
>>> tree = pybaz.tree_root(".")
119
>>> inv = list(tree.iter_inventory_ids(source=True, both=True))
120
>>> ("x_lalala", "path with space") in inv
122
>>> teardown_environ(q)
124
file(path, "wb").write(text)
128
def add_dir(path, id):
130
>>> q = test_environ()
131
>>> add_dir("path with\(sp) space", "lalala")
132
>>> tree = pybaz.tree_root(".")
133
>>> inv = list(tree.iter_inventory_ids(source=True, both=True))
134
>>> ("x_lalala", "path with\(sp) space") in inv
136
>>> teardown_environ(q)
141
def teardown_environ(tdir):
145
def timport(tree, summary):
146
msg = tree.log_message()
147
msg["summary"] = summary
150
def commit(tree, summary):
152
>>> q = test_environ()
153
>>> tree = pybaz.tree_root(".")
154
>>> timport(tree, "import")
155
>>> commit(tree, "commit")
156
>>> logs = [str(l.revision) for l in tree.iter_logs()]
160
'test@example.com/test--test--0--base-0'
162
'test@example.com/test--test--0--patch-1'
163
>>> teardown_environ(q)
165
msg = tree.log_message()
166
msg["summary"] = summary
169
def commit_test_revisions():
171
>>> q = test_environ()
172
>>> commit_test_revisions()
173
>>> a = pybaz.Archive("test@example.com")
174
>>> revisions = list(a.iter_revisions("test--test--0"))
177
>>> str(revisions[2])
178
'test@example.com/test--test--0--base-0'
179
>>> str(revisions[1])
180
'test@example.com/test--test--0--patch-1'
181
>>> str(revisions[0])
182
'test@example.com/test--test--0--patch-2'
183
>>> teardown_environ(q)
185
tree = pybaz.tree_root(".")
186
add_file("mainfile", "void main(void){}", "mainfile by aaron")
187
timport(tree, "Created mainfile")
188
file("mainfile", "wb").write("or something like that")
189
commit(tree, "altered mainfile")
190
add_file("ofile", "this is another file", "ofile by aaron")
191
commit(tree, "altered mainfile")
194
def commit_more_test_revisions():
196
>>> q = test_environ()
197
>>> commit_test_revisions()
198
>>> commit_more_test_revisions()
199
>>> a = pybaz.Archive("test@example.com")
200
>>> revisions = list(a.iter_revisions("test--test--0"))
203
>>> str(revisions[0])
204
'test@example.com/test--test--0--patch-3'
205
>>> teardown_environ(q)
207
tree = pybaz.tree_root(".")
208
add_file("trainfile", "void train(void){}", "trainfile by aaron")
209
commit(tree, "altered trainfile")
211
class NoSuchVersion(Exception):
212
def __init__(self, version):
213
Exception.__init__(self, "The version %s does not exist." % version)
214
self.version = version
216
def version_ancestry(version):
218
>>> q = test_environ()
219
>>> commit_test_revisions()
220
>>> version = pybaz.Version("test@example.com/test--test--0")
221
>>> ancestors = version_ancestry(version)
222
>>> str(ancestors[0])
223
'test@example.com/test--test--0--base-0'
224
>>> str(ancestors[1])
225
'test@example.com/test--test--0--patch-1'
226
>>> version = pybaz.Version("test@example.com/test--test--0.5")
227
>>> ancestors = version_ancestry(version)
228
Traceback (most recent call last):
229
NoSuchVersion: The version test@example.com/test--test--0.5 does not exist.
230
>>> teardown_environ(q)
233
revision = version.iter_revisions(reverse=True).next()
234
except StopIteration:
238
if not version.exists():
239
raise NoSuchVersion(version)
242
ancestors = list(revision.iter_ancestors(metoo=True))
246
def get_last_revision(branch):
247
last_patch = branch.last_revision()
249
return arch_revision(last_patch)
250
except NotArchRevision:
252
"Directory \"%s\" already exists, and the last revision is not"
253
" an Arch revision (%s)" % (branch.base, last_patch))
255
def do_branch(br_from, to_location, revision_id):
256
"""Derived from branch in builtins."""
260
os.mkdir(to_location)
262
if e.errno == errno.EEXIST:
263
raise UserError('Target directory "%s" already'
264
' exists.' % to_location)
265
if e.errno == errno.ENOENT:
266
raise UserError('Parent of "%s" does not exist.' %
271
br_from.bzrdir.clone(to_location, revision_id)
272
except NoSuchRevision:
274
msg = "The branch %s has no revision %s." % (from_location,
280
def get_remaining_revisions(output_dir, version, reuse_history_from=[]):
283
output_exists = os.path.exists(output_dir)
285
# We are starting from an existing directory, figure out what
286
# the current version is
287
branch = Branch.open(output_dir)
288
last_patch = get_last_revision(branch)
289
if last_patch is None:
290
if branch.last_revision() != None:
291
raise NotPreviousImport(branch.base)
292
elif version is None:
293
version = last_patch.version
294
elif version is None:
295
raise UserError("No version specified, and directory does not exist.")
298
ancestors = version_ancestry(version)
299
if not output_exists and reuse_history_from != []:
300
for ancestor in reversed(ancestors):
301
if last_patch is not None:
302
# found something to copy
304
# try to grab a copy of ancestor
305
# note that is not optimised: we could look for namespace
306
# transitions and only look for the past after the
308
for history_root in reuse_history_from:
309
possible_source = os.path.join(history_root,
310
map_namespace(ancestor.version))
312
source = Branch.open(possible_source)
313
rev_id = revision_id(ancestor)
314
if rev_id in source.revision_history():
315
do_branch(source, output_dir, rev_id)
316
last_patch = ancestor
318
except NotBranchError:
320
except NoSuchVersion, e:
321
raise UserError(str(e))
324
for i in range(len(ancestors)):
325
if ancestors[i] == last_patch:
328
raise UserError("Directory \"%s\" already exists, and the last "
329
"revision (%s) is not in the ancestry of %s" %
330
(output_dir, last_patch, version))
331
# Strip off all of the ancestors which are already present
332
# And get a directory starting with the latest ancestor
333
latest_ancestor = ancestors[i]
334
old_revno = Branch.open(output_dir).revno()
335
ancestors = ancestors[i+1:]
336
return ancestors, old_revno
339
###class Importer(object):
342
### Currently this is used as a parameter object, though more behaviour is
346
### def __init__(self, output_dir, version, fast=False,
347
### verbose=False, dry_run=False, max_count=None,
348
### reuse_history_from=[]):
349
### self.output_dir = output_dir
350
### self.version = version
354
def import_version(output_dir, version, fast=False,
355
verbose=False, dry_run=False, max_count=None,
356
reuse_history_from=[], standalone=True):
358
>>> q = test_environ()
360
Progress bars output to stderr, but doctest does not capture that.
362
>>> old_stderr = sys.stderr
363
>>> sys.stderr = sys.stdout
365
>>> result_path = os.path.join(q, "result")
366
>>> commit_test_revisions()
367
>>> version = pybaz.Version("test@example.com/test--test--0.1")
368
>>> old_ui = bzrlib.ui.ui_factory
369
>>> bzrlib.ui.ui_factory = bzrlib.ui.text.TextUIFactory(
370
... bar_type=bzrlib.progress.DotsProgressBar)
372
>>> import_version('/', version, dry_run=True)
373
Traceback (most recent call last):
374
NotPreviousImport: / is not the location of a previous import.
375
>>> import_version(result_path, version, dry_run=True)
376
Traceback (most recent call last):
377
UserError: The version test@example.com/test--test--0.1 does not exist.
378
>>> version = pybaz.Version("test@example.com/test--test--0")
379
>>> import_version(result_path, version, dry_run=True) #doctest: +ELLIPSIS
380
importing test@example.com/test--test--0 into ...
382
revisions: ..........................................
383
Dry run, not modifying output_dir
385
>>> import_version(result_path, version) #doctest: +ELLIPSIS
386
importing test@example.com/test--test--0 into ...
388
revisions: .....................................................................
391
>>> import_version(result_path, version) #doctest: +ELLIPSIS
392
Tree is up-to-date with test@example.com/test--test--0--patch-2
393
>>> commit_more_test_revisions()
394
>>> import_version(result_path, version) #doctest: +ELLIPSIS
395
importing test@example.com/test--test--0 into ...
396
revisions: ....................................................
399
>>> bzrlib.ui.ui_factory = old_ui
400
>>> sys.stderr = old_stderr
401
>>> teardown_environ(q)
403
progress_bar = bzrlib.ui.ui_factory.nested_progress_bar()
406
ancestors, old_revno = get_remaining_revisions(output_dir, version,
408
except NotBranchError, e:
409
raise NotPreviousImport(e.path)
410
if old_revno is None and len(ancestors) == 0:
411
progress_bar.note('Version %s has no revisions.' % version)
413
if len(ancestors) == 0:
414
last_revision = get_last_revision(Branch.open(output_dir))
415
progress_bar.note('Tree is up-to-date with %s' % last_revision)
418
progress_bar.note("importing %s into %s" % (version, output_dir))
420
tempdir = tempfile.mkdtemp(prefix="baz2bzr-",
421
dir=os.path.dirname(output_dir))
423
wt = WorkingTree.open(output_dir)
424
except (NotBranchError, NoWorkingTree):
427
for result in iter_import_version(output_dir, ancestors, tempdir,
429
fast=fast, verbose=verbose, dry_run=dry_run,
430
max_count=max_count, standalone=standalone):
431
show_progress(progress_bar, result)
433
progress_bar.note('Dry run, not modifying output_dir')
436
# Update the working tree of the branch
438
wt = WorkingTree.open(output_dir)
439
except NoWorkingTree:
442
wt.set_last_revision(wt.branch.last_revision())
447
progress_bar.note('Cleaning up')
448
shutil.rmtree(tempdir)
449
progress_bar.note("Import complete.")
451
progress_bar.finished()
453
class UserError(BzrCommandError):
454
def __init__(self, message):
455
"""Exception to throw when a user makes an impossible request
456
:param message: The message to emit when printing this exception
457
:type message: string
459
BzrCommandError.__init__(self, message)
461
class NotPreviousImport(UserError):
462
def __init__(self, path):
463
UserError.__init__(self, "%s is not the location of a previous import."
467
def revision_id(arch_revision):
469
Generate a Bzr revision id from an Arch revision id. 'x' in the id
470
designates a revision imported with an experimental algorithm. A number
471
would indicate a particular standardized version.
473
:param arch_revision: The Arch revision to generate an ID for.
475
>>> revision_id(pybaz.Revision("you@example.com/cat--br--0--base-0"))
476
'Arch-1:you@example.com%cat--br--0--base-0'
478
return "Arch-1:%s" % str(arch_revision).replace('/', '%')
480
class NotArchRevision(Exception):
481
def __init__(self, revision_id):
482
msg = "The revision id %s does not look like it came from Arch."\
484
Exception.__init__(self, msg)
486
def arch_revision(revision_id):
488
>>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0"))
489
Traceback (most recent call last):
490
NotArchRevision: The revision id Arch-1:jrandom@example.com%test--test--0 does not look like it came from Arch.
491
>>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0--base-5"))
492
Traceback (most recent call last):
493
NotArchRevision: The revision id Arch-1:jrandom@example.com%test--test--0--base-5 does not look like it came from Arch.
494
>>> str(arch_revision("Arch-1:jrandom@example.com%test--test--0--patch-5"))
495
'jrandom@example.com/test--test--0--patch-5'
497
if revision_id is None:
499
if revision_id[:7] != 'Arch-1:':
500
raise NotArchRevision(revision_id)
503
return pybaz.Revision(revision_id[7:].replace('%', '/'))
504
except pybaz.errors.NamespaceError, e:
505
raise NotArchRevision(revision_id)
508
def create_shared_repository(output_dir):
509
bd = bzrdir.BzrDirMetaFormat1().initialize(output_dir)
510
bd.create_repository(shared=True)
512
def create_branch(output_dir):
514
bd = bzrdir.BzrDirMetaFormat1().initialize(output_dir)
515
return bd.create_branch()
518
def create_checkout(source, to_location, revision_id=None):
519
checkout = bzrdir.BzrDirMetaFormat1().initialize(to_location)
520
bzrlib.branch.BranchReferenceFormat().initialize(checkout, source)
521
return checkout.create_workingtree(revision_id)
524
def create_checkout_metadata(source, to_location, revision_id=None):
525
if revision_id is None:
526
revision_id = source.last_revision()
527
wt = create_checkout(source, to_location, NULL_REVISION)
528
wt.set_last_revision(revision_id)
529
wt._write_inventory(wt.basis_tree().inventory)
533
def iter_import_version(output_dir, ancestors, tempdir, pb, fast=False,
534
verbose=False, dry_run=False, max_count=None,
538
# Uncomment this for testing, it basically just has baz2bzr only update
539
# 5 patches at a time
541
ancestors = ancestors[:max_count]
543
# Not sure if I want this output. basically it tells you ahead of time
544
# what it is going to do, but then later it tells you as it is doing it.
545
# what probably would be best would be to collapse it into ranges, so that
546
# this gives the simple view, and then later it gives the blow by blow.
548
# print 'Adding the following revisions:'
549
# for a in ancestors:
552
previous_version=None
553
missing_ancestor = None
555
dry_output_dir = os.path.join(tempdir, 'od')
556
if os.path.exists(output_dir):
557
shutil.copytree(output_dir, dry_output_dir)
558
output_dir = dry_output_dir
560
if os.path.exists(output_dir):
561
target_branch = Branch.open(output_dir)
564
wt = BzrDir.create_standalone_workingtree(output_dir)
565
target_branch = wt.branch
567
target_branch = create_branch(output_dir)
569
for i in range(len(ancestors)):
570
revision = ancestors[i]
571
rev_id = revision_id(revision)
574
version = str(revision.version)
575
if version != previous_version:
576
pb.note('On version: %s' % version)
577
yield Progress(str(revision.patchlevel), i, len(ancestors))
578
previous_version = version
580
yield Progress("revisions", i, len(ancestors))
582
if target_branch.repository.has_revision(rev_id):
583
target_branch.append_revision(rev_id)
586
revdir = os.path.join(tempdir, "rd")
588
tree, baz_inv, log = get_revision(revdir, revision)
589
except pybaz.errors.ExecProblem, e:
590
if ("%s" % e.args).find('could not connect') == -1:
592
missing_ancestor = revision
594
pb.note("unable to access ancestor %s, making into a merge."
597
target_tree = create_checkout_metadata(target_branch, revdir)
598
branch = target_tree.branch
600
old = os.path.join(revdir, ".bzr")
601
new = os.path.join(tempdir, ".bzr")
603
baz_inv, log = apply_revision(tree, revision)
605
target_tree = WorkingTree.open(revdir)
606
branch = target_tree.branch
607
# cached so we can delete the log
609
log_summary = log.summary
610
log_description = log.description
611
is_continuation = log.continuation_of is not None
612
log_creator = log.creator
613
direct_merges = get_direct_merges(revdir, revision, log)
615
timestamp = email.Utils.mktime_tz(log_date + (0,))
616
if log_summary is None:
618
# log_descriptions of None and "" are ignored.
619
if not is_continuation and log_description:
620
log_message = "\n".join((log_summary, log_description))
622
log_message = log_summary
623
target_tree.lock_write()
627
# if we want it to be in revision-history, do that here.
628
target_tree.add_pending_merge(revision_id(missing_ancestor))
629
missing_ancestor = None
630
for merged_rev in direct_merges:
631
target_tree.add_pending_merge(revision_id(merged_rev))
632
target_tree.set_inventory(baz_inv)
633
commitobj = Commit(reporter=ImportCommitReporter())
634
commitobj.commit(working_tree=target_tree,
635
message=log_message.decode('ascii', 'replace'),
636
verbose=False, committer=log_creator,
637
timestamp=timestamp, timezone=0, rev_id=rev_id,
642
yield Progress("revisions", len(ancestors), len(ancestors))
644
def get_direct_merges(revdir, revision, log):
645
continuation = log.continuation_of
646
previous_version = revision.version
647
if pybaz.WorkingTree(revdir).tree_version != previous_version:
648
pybaz.WorkingTree(revdir).set_tree_version(previous_version)
649
log_path = "%s/{arch}/%s/%s/%s/%s/patch-log/%s" % (revdir,
650
revision.category.nonarch, revision.branch.nonarch,
651
revision.version.nonarch, revision.archive, revision.patchlevel)
652
temp_path = tempfile.mktemp(dir=os.path.dirname(revdir))
653
os.rename(log_path, temp_path)
654
merges = list(iter_new_merges(revdir, revision.version))
655
direct = direct_merges(merges, [continuation])
656
os.rename(temp_path, log_path)
659
def unlink_unversioned(wt):
660
for unversioned in wt.extras():
661
path = wt.abspath(unversioned)
662
if os.path.isdir(path):
667
def get_log(tree, revision):
668
log = pybaz.Patchlog(revision, tree=tree)
669
assert str(log.revision) == str(revision), (log.revision, revision)
672
def get_revision(revdir, revision):
673
tree = revision.get(revdir)
674
log = get_log(tree, revision)
676
return tree, bzr_inventory_data(tree), log
677
except BadFileKind, e:
678
raise UserError("Cannot convert %s because %s is a %s" %
679
(revision,e.path, e.kind))
682
def apply_revision(tree, revision):
684
log = get_log(tree, revision)
686
return bzr_inventory_data(tree), log
687
except BadFileKind, e:
688
raise UserError("Cannot convert %s because %s is a %s" %
689
(revision,e.path, e.kind))
692
class BadFileKind(Exception):
693
"""The file kind is not permitted in bzr inventories"""
694
def __init__(self, tree_root, path, kind):
695
self.tree_root = tree_root
698
Exception.__init__(self, "File %s is of forbidden type %s" %
699
(os.path.join(tree_root, path), kind))
702
def bzr_inventory_data(tree):
703
inv_iter = tree.iter_inventory_ids(source=True, both=True)
705
for arch_id, path in inv_iter:
706
bzr_file_id = map_file_id(arch_id)
707
inv_map[path] = bzr_file_id
710
for path, file_id in inv_map.iteritems():
711
full_path = os.path.join(tree, path)
712
kind = bzrlib.osutils.file_kind(full_path)
713
if kind not in ("file", "directory", "symlink"):
714
raise BadFileKind(tree, path, kind)
715
parent_dir = os.path.dirname(path)
717
parent_id = inv_map[parent_dir]
719
parent_id = bzrlib.inventory.ROOT_ID
720
bzr_inv.append((path, file_id, parent_id, kind))
725
def baz_import_branch(to_location, from_branch, fast, max_count, verbose,
726
dry_run, reuse_history_list):
727
to_location = os.path.realpath(str(to_location))
728
if from_branch is not None:
730
from_branch = pybaz.Version(from_branch)
731
except pybaz.errors.NamespaceError:
732
print "%s is not a valid Arch branch." % from_branch
734
if reuse_history_list is None:
735
reuse_history_list = []
736
import_version(to_location, from_branch,
738
reuse_history_from=reuse_history_list)
741
class NotInABranch(Exception):
742
def __init__(self, path):
743
Exception.__init__(self, "%s is not in a branch." % path)
748
def baz_import(to_root_dir, from_archive, verbose=False, reuse_history_list=[],
750
if reuse_history_list is None:
751
reuse_history_list = []
752
to_root = str(os.path.realpath(to_root_dir))
753
if not os.path.exists(to_root):
755
if prefixes is not None:
756
prefixes = prefixes.split(':')
757
import_archive(to_root, from_archive, verbose,
758
reuse_history_list, prefixes=prefixes)
761
def import_archive(to_root, from_archive, verbose,
762
reuse_history_from=[], standalone=False,
764
def selected(version):
768
for prefix in prefixes:
769
if version.nonarch.startswith(prefix):
772
real_to = os.path.realpath(to_root)
773
history_locations = [real_to] + reuse_history_from
774
if standalone is False:
776
bd = BzrDir.open(to_root)
778
except NotBranchError:
779
create_shared_repository(to_root)
780
except NoRepositoryPresent:
781
raise BzrCommandError("Can't create repository at existing branch.")
782
versions = list(pybaz.Archive(str(from_archive)).iter_versions())
783
progress_bar = bzrlib.ui.ui_factory.nested_progress_bar()
785
for num, version in enumerate(versions):
786
progress_bar.update("Branch", num, len(versions))
787
if not selected(version):
788
print "Skipping %s" % version
790
target = os.path.join(to_root, map_namespace(version))
791
if not os.path.exists(os.path.dirname(target)):
792
os.makedirs(os.path.dirname(target))
794
import_version(target, version,
795
reuse_history_from=reuse_history_from,
796
standalone=standalone)
797
except pybaz.errors.ExecProblem,e:
798
if str(e).find('The requested revision cannot be built.') != -1:
800
"Skipping version %s as it cannot be built due"
801
" to a missing parent archive." % version)
805
if str(e).find('already exists, and the last revision ') != -1:
807
"Skipping version %s as it has had commits made"
808
" since it was converted to bzr." % version)
812
progress_bar.finished()
815
def map_namespace(a_version):
816
a_version = pybaz.Version("%s" % a_version)
817
parser = NameParser(a_version)
818
version = parser.get_version()
819
branch = parser.get_branch()
820
category = parser.get_category()
821
if branch is None or branch == '':
824
return "%s/%s" % (category, branch)
825
return "%s/%s/%s" % (category, version, branch)
828
def map_file_id(file_id):
829
"""Convert a baz file id to a bzr one."""
830
return file_id.replace('%', '%25').replace('/', '%2f')
added
removed
baz_import.py
