16
14
# along with this program; if not, write to the Free Software
17
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19
"""Experiment in converting existing bzr branches to weaves."""
21
# To make this properly useful
23
# 1. assign text version ids, and put those text versions into
24
# the inventory as they're converted.
26
# 2. keep track of the previous version of each file, rather than
27
# just using the last one imported
29
# 3. assign entry versions when files are added, renamed or moved.
31
# 4. when merged-in versions are observed, walk down through them
32
# to discover everything, then commit bottom-up
34
# 5. track ancestry as things are merged in, and commit that in each
37
# Perhaps it's best to first walk the whole graph and make a plan for
38
# what should be imported in what order? Need a kind of topological
39
# sort of all revisions. (Or do we, can we just before doing a revision
40
# see that all its parents have either been converted or abandoned?)
43
# Cannot import a revision until all its parents have been
44
# imported. in other words, we can only import revisions whose
45
# parents have all been imported. the first step must be to
46
# import a revision with no parents, of which there must be at
47
# least one. (So perhaps it's useful to store forward pointers
48
# from a list of parents to their children?)
50
# Another (equivalent?) approach is to build up the ordered
51
# ancestry list for the last revision, and walk through that. We
52
# are going to need that.
54
# We don't want to have to recurse all the way back down the list.
56
# Suppose we keep a queue of the revisions able to be processed at
57
# any point. This starts out with all the revisions having no
60
# This seems like a generally useful algorithm...
62
# The current algorithm is dumb (O(n**2)?) but will do the job, and
63
# takes less than a second on the bzr.dev branch.
65
# This currently does a kind of lazy conversion of file texts, where a
66
# new text is written in every version. That's unnecessary but for
67
# the moment saves us having to worry about when files need new
71
# TODO: Don't create a progress bar here, have it passed by the caller.
72
# At least do it from the UI factory.
88
from bzrlib.branch import Branch, find_branch, BZR_BRANCH_FORMAT_5
89
from bzrlib.revfile import Revfile
90
from bzrlib.weave import Weave
91
from bzrlib.weavefile import read_weave, write_weave
92
from bzrlib.ui import ui_factory
93
from bzrlib.atomicfile import AtomicFile
94
from bzrlib.xml4 import serializer_v4
95
from bzrlib.xml5 import serializer_v5
96
from bzrlib.trace import mutter, note, warning, enable_default_logging
97
from bzrlib.osutils import sha_strings, sha_string
98
from bzrlib.commit import merge_ancestry_lines
17
"""bzr upgrade logic."""
19
# change upgrade from .bzr to create a '.bzr-new', then do a bait and switch.
22
from bzrlib.bzrdir import ConvertBzrDir4To5, ConvertBzrDir5To6, BzrDir, BzrDirFormat4, BzrDirFormat5, BzrDirFormat
23
import bzrlib.errors as errors
24
from bzrlib.transport import get_transport
25
import bzrlib.ui as ui
101
28
class Convert(object):
102
def __init__(self, base_dir):
104
self.converted_revs = set()
105
self.absent_revisions = set()
30
def __init__(self, url, format):
32
self.bzrdir = BzrDir.open_unsupported(url)
33
if self.bzrdir.root_transport.is_readonly():
34
raise errors.UpgradeReadonly
35
self.transport = self.bzrdir.root_transport
36
self.pb = ui.ui_factory.nested_progress_bar()
111
42
def convert(self):
112
if not self._open_branch():
114
note('starting upgrade of %s', os.path.abspath(self.base))
44
branch = self.bzrdir.open_branch()
45
if branch.bzrdir.root_transport.base != \
46
self.bzrdir.root_transport.base:
47
self.pb.note("This is a checkout. The branch (%s) needs to be "
48
"upgraded separately.",
49
branch.bzrdir.root_transport.base)
50
except errors.NotBranchError:
52
if not self.bzrdir.needs_format_conversion(self.format):
53
raise errors.UpToDateFormat(self.bzrdir._format)
54
if not self.bzrdir.can_convert_format():
55
raise errors.BzrError("cannot upgrade from branch format %s" %
57
if self.format is None:
58
target_format = BzrDirFormat.get_default_format()
60
target_format = self.format
61
self.bzrdir.check_conversion_target(target_format)
62
self.pb.note('starting upgrade of %s', self.transport.base)
115
63
self._backup_control_dir()
116
note('starting upgrade')
117
note('note: upgrade may be faster if all store files are ungzipped first')
118
self.pb = ui_factory.progress_bar()
119
if not os.path.isdir(self.base + '/.bzr/weaves'):
120
os.mkdir(self.base + '/.bzr/weaves')
121
self.inv_weave = Weave('inventory')
122
self.anc_weave = Weave('ancestry')
124
# holds in-memory weaves for all files
125
self.text_weaves = {}
126
os.remove(self.branch.controlfilename('branch-format'))
127
self._convert_working_inv()
128
rev_history = self.branch.revision_history()
129
# to_read is a stack holding the revisions we still need to process;
130
# appending to it adds new highest-priority revisions
131
self.known_revisions = set(rev_history)
132
self.to_read = [rev_history[-1]]
134
rev_id = self.to_read.pop()
135
if (rev_id not in self.revisions
136
and rev_id not in self.absent_revisions):
137
self._load_one_rev(rev_id)
139
to_import = self._make_order()
140
for i, rev_id in enumerate(to_import):
141
self.pb.update('converting revision', i, len(to_import))
142
self._convert_one_rev(rev_id)
144
note('upgraded to weaves:')
145
note(' %6d revisions and inventories' % len(self.revisions))
146
note(' %6d revisions not present' % len(self.absent_revisions))
147
note(' %6d texts' % self.text_count)
148
self._write_all_weaves()
149
self._write_all_revs()
150
self._set_new_format()
151
self._cleanup_spare_files()
154
def _open_branch(self):
155
self.branch = Branch.open_downlevel(self.base)
156
if self.branch._branch_format == 5:
157
note('this branch is already in the most current format')
159
if self.branch._branch_format != 4:
160
raise BzrError("cannot upgrade from branch format %r" %
161
self.branch._branch_format)
165
def _set_new_format(self):
166
self.branch.put_controlfile('branch-format', BZR_BRANCH_FORMAT_5)
169
def _cleanup_spare_files(self):
170
for n in 'merged-patches', 'pending-merged-patches':
171
p = self.branch.controlfilename(n)
172
if not os.path.exists(p):
174
## assert os.path.getsize(p) == 0
176
shutil.rmtree(self.base + '/.bzr/inventory-store')
177
shutil.rmtree(self.base + '/.bzr/text-store')
64
while self.bzrdir.needs_format_conversion(self.format):
65
converter = self.bzrdir._format.get_converter(self.format)
66
self.bzrdir = converter.convert(self.bzrdir, self.pb)
67
self.pb.note("finished")
180
69
def _backup_control_dir(self):
181
orig = self.base + '/.bzr'
182
backup = orig + '.backup'
183
note('making backup of tree history')
184
shutil.copytree(orig, backup)
185
note('%s has been backed up to %s', orig, backup)
186
note('if conversion fails, you can move this directory back to .bzr')
187
note('if it succeeds, you can remove this directory if you wish')
190
def _convert_working_inv(self):
192
inv = serializer_v4.read_inventory(branch.controlfile('inventory', 'rb'))
193
new_inv_xml = serializer_v5.write_inventory_to_string(inv)
194
branch.put_controlfile('inventory', new_inv_xml)
198
def _write_all_weaves(self):
199
write_a_weave(self.inv_weave, self.base + '/.bzr/inventory.weave')
200
write_a_weave(self.anc_weave, self.base + '/.bzr/ancestry.weave')
203
for file_id, file_weave in self.text_weaves.items():
204
self.pb.update('writing weave', i, len(self.text_weaves))
205
write_a_weave(file_weave, self.base + '/.bzr/weaves/%s.weave' % file_id)
211
def _write_all_revs(self):
212
"""Write all revisions out in new form."""
213
shutil.rmtree(self.base + '/.bzr/revision-store')
214
os.mkdir(self.base + '/.bzr/revision-store')
216
for i, rev_id in enumerate(self.converted_revs):
217
self.pb.update('write revision', i, len(self.converted_revs))
218
f = file(self.base + '/.bzr/revision-store/%s' % rev_id, 'wb')
220
serializer_v5.write_revision(self.revisions[rev_id], f)
227
def _load_one_rev(self, rev_id):
228
"""Load a revision object into memory.
230
Any parents not either loaded or abandoned get queued to be
232
self.pb.update('loading revision',
234
len(self.known_revisions))
235
if rev_id not in self.branch.revision_store:
237
note('revision {%s} not present in branch; '
238
'will be converted as a ghost',
240
self.absent_revisions.add(rev_id)
242
rev_xml = self.branch.revision_store[rev_id].read()
243
rev = serializer_v4.read_revision_from_string(rev_xml)
244
for parent_id in rev.parent_ids:
245
self.known_revisions.add(parent_id)
246
self.to_read.append(parent_id)
247
self.revisions[rev_id] = rev
250
def _load_old_inventory(self, rev_id):
251
assert rev_id not in self.converted_revs
252
old_inv_xml = self.branch.inventory_store[rev_id].read()
253
inv = serializer_v4.read_inventory_from_string(old_inv_xml)
254
rev = self.revisions[rev_id]
255
if rev.inventory_sha1:
256
assert rev.inventory_sha1 == sha_string(old_inv_xml), \
257
'inventory sha mismatch for {%s}' % rev_id
261
def _load_updated_inventory(self, rev_id):
262
assert rev_id in self.converted_revs
263
inv_xml = self.inv_weave.get_text(rev_id)
264
inv = serializer_v5.read_inventory_from_string(inv_xml)
268
def _convert_one_rev(self, rev_id):
269
"""Convert revision and all referenced objects to new format."""
270
rev = self.revisions[rev_id]
271
inv = self._load_old_inventory(rev_id)
272
present_parents = [p for p in rev.parent_ids
273
if p not in self.absent_revisions]
274
self._convert_revision_contents(rev, inv, present_parents)
275
self._store_new_weave(rev, inv, present_parents)
276
self._make_rev_ancestry(rev, present_parents)
277
self.converted_revs.add(rev_id)
280
def _store_new_weave(self, rev, inv, present_parents):
281
# the XML is now updated with text versions
285
if ie.kind == 'root_directory':
287
assert hasattr(ie, 'revision'), \
288
'no revision on {%s} in {%s}' % \
289
(file_id, rev.revision_id)
290
new_inv_xml = serializer_v5.write_inventory_to_string(inv)
291
new_inv_sha1 = sha_string(new_inv_xml)
292
self.inv_weave.add(rev.revision_id,
294
new_inv_xml.splitlines(True),
296
rev.inventory_sha1 = new_inv_sha1
299
def _make_rev_ancestry(self, rev, present_parents):
300
rev_id = rev.revision_id
301
for parent_id in present_parents:
302
assert parent_id in self.converted_revs
304
lines = list(self.anc_weave.mash_iter(present_parents))
307
lines.append(rev_id + '\n')
309
parent_ancestries = [self.ancestries[p] for p in present_parents]
310
new_lines = merge_ancestry_lines(rev_id, parent_ancestries)
311
assert set(lines) == set(new_lines)
312
self.ancestries[rev_id] = new_lines
313
self.anc_weave.add(rev_id, present_parents, lines)
316
def _convert_revision_contents(self, rev, inv, present_parents):
317
"""Convert all the files within a revision.
319
Also upgrade the inventory to refer to the text revision ids."""
320
rev_id = rev.revision_id
321
mutter('converting texts of revision {%s}',
323
parent_invs = map(self._load_updated_inventory, present_parents)
326
self._set_revision(rev, ie, parent_invs)
327
if not ie.has_text():
329
self._convert_file_version(rev, ie, parent_invs)
332
def _set_revision(self, rev, ie, parent_invs):
333
"""Set name version for a file.
336
if ie.kind == 'root_directory':
338
if len(parent_invs) == 0:
339
ie.revision = rev.revision_id
342
for p_inv in parent_invs:
343
if not p_inv.has_id(file_id):
344
ie.revision = rev.revision_id
346
old_ie = p_inv[file_id]
347
if (old_ie.parent_id != ie.parent_id
348
or old_ie.name != ie.name
349
or old_ie.text_sha1 != ie.text_sha1
350
or old_ie.text_size != ie.text_size):
351
ie.revision = rev.revision_id
353
if matched_id is None:
354
matched_id = old_ie.revision
355
elif matched_id != old_ie.revision:
356
ie.revision = rev.revision_id
359
assert matched_id is not None
360
ie.revision = matched_id
362
def _convert_file_version(self, rev, ie, parent_invs):
363
"""Convert one version of one file.
365
The file needs to be added into the weave if it is a merge
366
of >=2 parents or if it's changed from its parent.
369
rev_id = rev.revision_id
370
w = self.text_weaves.get(file_id)
373
self.text_weaves[file_id] = w
376
for parent_inv in parent_invs:
377
if parent_inv.has_id(file_id):
378
parent_ie = parent_inv[file_id]
379
old_revision = parent_ie.revision
380
# if this fails, its a ghost ?
381
assert old_revision in self.converted_revs
382
if old_revision not in file_parents:
383
file_parents.append(old_revision)
384
if parent_ie.text_sha1 != ie.text_sha1:
386
if len(file_parents) != 1 or text_changed:
387
file_lines = self.branch.text_store[ie.text_id].readlines()
388
assert sha_strings(file_lines) == ie.text_sha1
389
assert sum(map(len, file_lines)) == ie.text_size
390
w.add(rev_id, file_parents, file_lines, ie.text_sha1)
393
##mutter('import text {%s} of {%s}',
394
## ie.text_id, file_id)
396
##mutter('text of {%s} unchanged from parent', file_id)
397
ie.revision = file_parents[0]
402
def _make_order(self):
403
"""Return a suitable order for importing revisions.
405
The order must be such that an revision is imported after all
406
its (present) parents.
408
todo = set(self.revisions.keys())
409
done = self.absent_revisions.copy()
412
# scan through looking for a revision whose parents
414
for rev_id in sorted(list(todo)):
415
rev = self.revisions[rev_id]
416
parent_ids = set(rev.parent_ids)
417
if parent_ids.issubset(done):
418
# can take this one now
425
def write_a_weave(weave, filename):
426
inv_wf = file(filename, 'wb')
428
write_weave(weave, inv_wf)
433
def upgrade(base_dir):
70
self.pb.note('making backup of tree history')
71
self.transport.copy_tree('.bzr', 'backup.bzr')
72
self.pb.note('%s.bzr has been backed up to %sbackup.bzr',
75
self.pb.note('if conversion fails, you can move this directory back to .bzr')
76
self.pb.note('if it succeeds, you can remove this directory if you wish')
78
def upgrade(url, format=None):
79
"""Upgrade to format, or the default bzrdir format if not supplied."""
added
removed
bzrlib/upgrade.py
