3
# Copyright (C) 2005 Canonical Ltd
5
# This program is free software; you can redistribute it and/or modify
6
# it under the terms of the GNU General Public License as published by
7
# the Free Software Foundation; either version 2 of the License, or
8
# (at your option) any later version.
10
# This program is distributed in the hope that it will be useful,
11
# but WITHOUT ANY WARRANTY; without even the implied warranty of
12
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
# GNU General Public License for more details.
15
# You should have received a copy of the GNU General Public License
16
# along with this program; if not, write to the Free Software
17
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19
"""Experiment in converting existing bzr branches to weaves."""
21
# To make this properly useful
23
# 1. assign text version ids, and put those text versions into
24
# the inventory as they're converted.
26
# 2. keep track of the previous version of each file, rather than
27
# just using the last one imported
29
# 3. assign entry versions when files are added, renamed or moved.
31
# 4. when merged-in versions are observed, walk down through them
32
# to discover everything, then commit bottom-up
34
# 5. track ancestry as things are merged in, and commit that in each
37
# Perhaps it's best to first walk the whole graph and make a plan for
38
# what should be imported in what order? Need a kind of topological
39
# sort of all revisions. (Or do we, can we just before doing a revision
40
# see that all its parents have either been converted or abandoned?)
43
# Cannot import a revision until all its parents have been
44
# imported. in other words, we can only import revisions whose
45
# parents have all been imported. the first step must be to
46
# import a revision with no parents, of which there must be at
47
# least one. (So perhaps it's useful to store forward pointers
48
# from a list of parents to their children?)
50
# Another (equivalent?) approach is to build up the ordered
51
# ancestry list for the last revision, and walk through that. We
52
# are going to need that.
54
# We don't want to have to recurse all the way back down the list.
56
# Suppose we keep a queue of the revisions able to be processed at
57
# any point. This starts out with all the revisions having no
60
# This seems like a generally useful algorithm...
62
# The current algorithm is dumb (O(n**2)?) but will do the job, and
63
# takes less than a second on the bzr.dev branch.
65
# This currently does a kind of lazy conversion of file texts, where a
66
# new text is written in every version. That's unnecessary but for
67
# the moment saves us having to worry about when files need new
71
# TODO: Don't create a progress bar here, have it passed by the caller.
72
# At least do it from the UI factory.
88
from bzrlib.branch import Branch, find_branch
89
from bzrlib.branch import BZR_BRANCH_FORMAT_5, BZR_BRANCH_FORMAT_6
90
from bzrlib.revfile import Revfile
91
from bzrlib.weave import Weave
92
from bzrlib.weavefile import read_weave, write_weave
93
from bzrlib.ui import ui_factory
94
from bzrlib.atomicfile import AtomicFile
95
from bzrlib.xml4 import serializer_v4
96
from bzrlib.xml5 import serializer_v5
97
from bzrlib.trace import mutter, note, warning, enable_default_logging
98
from bzrlib.osutils import sha_strings, sha_string
101
class Convert(object):
102
def __init__(self, base_dir):
104
self.converted_revs = set()
105
self.absent_revisions = set()
112
if not self._open_branch():
114
note('starting upgrade of %s', os.path.abspath(self.base))
115
self._backup_control_dir()
116
self.pb = ui_factory.progress_bar()
117
if self.old_format == 4:
118
note('starting upgrade from format 4 to 5')
119
self._convert_to_weaves()
121
if self.old_format == 5:
122
note('starting upgrade from format 5 to 6')
123
self._convert_to_prefixed()
128
def _convert_to_prefixed(self):
129
from bzrlib.store import hash_prefix
130
for store_name in ["weaves", "revision-store"]:
131
note("adding prefixes to %s" % store_name)
132
store_dir = os.path.join(self.base, ".bzr", store_name)
133
for filename in os.listdir(store_dir):
134
if filename.endswith(".weave") or filename.endswith(".gz"):
135
file_id = os.path.splitext(filename)[0]
138
prefix_dir = os.path.join(store_dir, hash_prefix(file_id))
139
if not os.path.isdir(prefix_dir):
141
os.rename(os.path.join(store_dir, filename),
142
os.path.join(prefix_dir, filename))
143
self._set_new_format(BZR_BRANCH_FORMAT_6)
146
def _convert_to_weaves(self):
147
note('note: upgrade may be faster if all store files are ungzipped first')
148
if not os.path.isdir(self.base + '/.bzr/weaves'):
149
os.mkdir(self.base + '/.bzr/weaves')
150
self.inv_weave = Weave('inventory')
151
# holds in-memory weaves for all files
152
self.text_weaves = {}
153
os.remove(self.branch.controlfilename('branch-format'))
154
self._convert_working_inv()
155
rev_history = self.branch.revision_history()
156
# to_read is a stack holding the revisions we still need to process;
157
# appending to it adds new highest-priority revisions
158
self.known_revisions = set(rev_history)
159
self.to_read = [rev_history[-1]]
161
rev_id = self.to_read.pop()
162
if (rev_id not in self.revisions
163
and rev_id not in self.absent_revisions):
164
self._load_one_rev(rev_id)
166
to_import = self._make_order()
167
for i, rev_id in enumerate(to_import):
168
self.pb.update('converting revision', i, len(to_import))
169
self._convert_one_rev(rev_id)
171
note('upgraded to weaves:')
172
note(' %6d revisions and inventories' % len(self.revisions))
173
note(' %6d revisions not present' % len(self.absent_revisions))
174
note(' %6d texts' % self.text_count)
175
self._write_all_weaves()
176
self._write_all_revs()
177
self._cleanup_spare_files()
178
self._set_new_format(BZR_BRANCH_FORMAT_5)
181
def _open_branch(self):
182
self.branch = Branch.open_downlevel(self.base)
183
self.old_format = self.branch._branch_format
184
if self.old_format == 6:
185
note('this branch is in the most current format')
187
if self.old_format not in (4, 5):
188
raise BzrError("cannot upgrade from branch format %r" %
189
self.branch._branch_format)
193
def _set_new_format(self, format):
194
self.branch.put_controlfile('branch-format', format)
197
def _cleanup_spare_files(self):
198
for n in 'merged-patches', 'pending-merged-patches':
199
p = self.branch.controlfilename(n)
200
if not os.path.exists(p):
202
## assert os.path.getsize(p) == 0
204
shutil.rmtree(self.base + '/.bzr/inventory-store')
205
shutil.rmtree(self.base + '/.bzr/text-store')
208
def _backup_control_dir(self):
209
orig = self.base + '/.bzr'
210
backup = orig + '.backup'
211
note('making backup of tree history')
212
shutil.copytree(orig, backup)
213
note('%s has been backed up to %s', orig, backup)
214
note('if conversion fails, you can move this directory back to .bzr')
215
note('if it succeeds, you can remove this directory if you wish')
218
def _convert_working_inv(self):
220
inv = serializer_v4.read_inventory(branch.controlfile('inventory', 'rb'))
221
new_inv_xml = serializer_v5.write_inventory_to_string(inv)
222
branch.put_controlfile('inventory', new_inv_xml)
226
def _write_all_weaves(self):
227
write_a_weave(self.inv_weave, self.base + '/.bzr/inventory.weave')
230
for file_id, file_weave in self.text_weaves.items():
231
self.pb.update('writing weave', i, len(self.text_weaves))
232
write_a_weave(file_weave, self.base + '/.bzr/weaves/%s.weave' % file_id)
238
def _write_all_revs(self):
239
"""Write all revisions out in new form."""
240
shutil.rmtree(self.base + '/.bzr/revision-store')
241
os.mkdir(self.base + '/.bzr/revision-store')
243
for i, rev_id in enumerate(self.converted_revs):
244
self.pb.update('write revision', i, len(self.converted_revs))
245
f = file(self.base + '/.bzr/revision-store/%s' % rev_id, 'wb')
247
serializer_v5.write_revision(self.revisions[rev_id], f)
254
def _load_one_rev(self, rev_id):
255
"""Load a revision object into memory.
257
Any parents not either loaded or abandoned get queued to be
259
self.pb.update('loading revision',
261
len(self.known_revisions))
262
if rev_id not in self.branch.revision_store:
264
note('revision {%s} not present in branch; '
265
'will be converted as a ghost',
267
self.absent_revisions.add(rev_id)
269
rev_xml = self.branch.revision_store[rev_id].read()
270
rev = serializer_v4.read_revision_from_string(rev_xml)
271
for parent_id in rev.parent_ids:
272
self.known_revisions.add(parent_id)
273
self.to_read.append(parent_id)
274
self.revisions[rev_id] = rev
277
def _load_old_inventory(self, rev_id):
278
assert rev_id not in self.converted_revs
279
old_inv_xml = self.branch.inventory_store[rev_id].read()
280
inv = serializer_v4.read_inventory_from_string(old_inv_xml)
281
rev = self.revisions[rev_id]
282
if rev.inventory_sha1:
283
assert rev.inventory_sha1 == sha_string(old_inv_xml), \
284
'inventory sha mismatch for {%s}' % rev_id
288
def _load_updated_inventory(self, rev_id):
289
assert rev_id in self.converted_revs
290
inv_xml = self.inv_weave.get_text(rev_id)
291
inv = serializer_v5.read_inventory_from_string(inv_xml)
295
def _convert_one_rev(self, rev_id):
296
"""Convert revision and all referenced objects to new format."""
297
rev = self.revisions[rev_id]
298
inv = self._load_old_inventory(rev_id)
299
present_parents = [p for p in rev.parent_ids
300
if p not in self.absent_revisions]
301
self._convert_revision_contents(rev, inv, present_parents)
302
self._store_new_weave(rev, inv, present_parents)
303
self.converted_revs.add(rev_id)
306
def _store_new_weave(self, rev, inv, present_parents):
307
# the XML is now updated with text versions
311
if ie.kind == 'root_directory':
313
assert hasattr(ie, 'revision'), \
314
'no revision on {%s} in {%s}' % \
315
(file_id, rev.revision_id)
316
new_inv_xml = serializer_v5.write_inventory_to_string(inv)
317
new_inv_sha1 = sha_string(new_inv_xml)
318
self.inv_weave.add(rev.revision_id,
320
new_inv_xml.splitlines(True),
322
rev.inventory_sha1 = new_inv_sha1
324
def _convert_revision_contents(self, rev, inv, present_parents):
325
"""Convert all the files within a revision.
327
Also upgrade the inventory to refer to the text revision ids."""
328
rev_id = rev.revision_id
329
mutter('converting texts of revision {%s}',
331
parent_invs = map(self._load_updated_inventory, present_parents)
334
self._convert_file_version(rev, ie, parent_invs)
336
def _convert_file_version(self, rev, ie, parent_invs):
337
"""Convert one version of one file.
339
The file needs to be added into the weave if it is a merge
340
of >=2 parents or if it's changed from its parent.
342
if ie.kind == 'root_directory':
345
rev_id = rev.revision_id
346
w = self.text_weaves.get(file_id)
349
self.text_weaves[file_id] = w
351
previous_entries = ie.find_previous_heads(parent_invs, w)
352
for old_revision in previous_entries:
353
# if this fails, its a ghost ?
354
assert old_revision in self.converted_revs
355
self.snapshot_ie(previous_entries, ie, w, rev_id)
357
assert getattr(ie, 'revision', None) is not None
359
def snapshot_ie(self, previous_revisions, ie, w, rev_id):
360
# TODO: convert this logic, which is ~= snapshot to
361
# a call to:. This needs the path figured out. rather than a work_tree
362
# a v4 revision_tree can be given, or something that looks enough like
363
# one to give the file content to the entry if it needs it.
364
# and we need something that looks like a weave store for snapshot to
366
#ie.snapshot(rev, PATH, previous_revisions, REVISION_TREE, InMemoryWeaveStore(self.text_weaves))
367
if len(previous_revisions) == 1:
368
previous_ie = previous_revisions.values()[0]
369
if ie._unchanged(previous_ie):
370
ie.revision = previous_ie.revision
372
parent_indexes = map(w.lookup, previous_revisions)
374
file_lines = self.branch.text_store[ie.text_id].readlines()
375
assert sha_strings(file_lines) == ie.text_sha1
376
assert sum(map(len, file_lines)) == ie.text_size
377
w.add(rev_id, parent_indexes, file_lines, ie.text_sha1)
380
w.add(rev_id, parent_indexes, [], None)
382
##mutter('import text {%s} of {%s}',
383
## ie.text_id, file_id)
385
def _make_order(self):
386
"""Return a suitable order for importing revisions.
388
The order must be such that an revision is imported after all
389
its (present) parents.
391
todo = set(self.revisions.keys())
392
done = self.absent_revisions.copy()
395
# scan through looking for a revision whose parents
397
for rev_id in sorted(list(todo)):
398
rev = self.revisions[rev_id]
399
parent_ids = set(rev.parent_ids)
400
if parent_ids.issubset(done):
401
# can take this one now
408
def write_a_weave(weave, filename):
409
inv_wf = file(filename, 'wb')
411
write_weave(weave, inv_wf)
416
def upgrade(base_dir):
added
removed
bzrlib/upgrade.py
