1
3
# Copyright (C) 2005 Canonical Ltd
3
5
# This program is free software; you can redistribute it and/or modify
4
6
# it under the terms of the GNU General Public License as published by
5
7
# the Free Software Foundation; either version 2 of the License, or
6
8
# (at your option) any later version.
8
10
# This program is distributed in the hope that it will be useful,
9
11
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
12
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
13
# GNU General Public License for more details.
13
15
# You should have received a copy of the GNU General Public License
14
16
# along with this program; if not, write to the Free Software
15
17
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
19
"""Experiment in converting existing bzr branches to weaves."""
19
# To make this properly useful
21
# 1. assign text version ids, and put those text versions into
22
# the inventory as they're converted.
24
# 2. keep track of the previous version of each file, rather than
25
# just using the last one imported
27
# 3. assign entry versions when files are added, renamed or moved.
29
# 4. when merged-in versions are observed, walk down through them
30
# to discover everything, then commit bottom-up
32
# 5. track ancestry as things are merged in, and commit that in each
35
# Perhaps it's best to first walk the whole graph and make a plan for
36
# what should be imported in what order? Need a kind of topological
37
# sort of all revisions. (Or do we, can we just before doing a revision
38
# see that all its parents have either been converted or abandoned?)
41
# Cannot import a revision until all its parents have been
42
# imported. in other words, we can only import revisions whose
43
# parents have all been imported. the first step must be to
44
# import a revision with no parents, of which there must be at
45
# least one. (So perhaps it's useful to store forward pointers
46
# from a list of parents to their children?)
48
# Another (equivalent?) approach is to build up the ordered
49
# ancestry list for the last revision, and walk through that. We
50
# are going to need that.
52
# We don't want to have to recurse all the way back down the list.
54
# Suppose we keep a queue of the revisions able to be processed at
55
# any point. This starts out with all the revisions having no
58
# This seems like a generally useful algorithm...
60
# The current algorithm is dumb (O(n**2)?) but will do the job, and
61
# takes less than a second on the bzr.dev branch.
63
# This currently does a kind of lazy conversion of file texts, where a
64
# new text is written in every version. That's unnecessary but for
65
# the moment saves us having to worry about when files need new
74
from bzrlib.branch import Branch, find_branch
75
from bzrlib.branch import BZR_BRANCH_FORMAT_5, BZR_BRANCH_FORMAT_6
76
import bzrlib.hashcache as hashcache
31
from bzrlib.revfile import Revfile
77
32
from bzrlib.weave import Weave
78
33
from bzrlib.weavefile import read_weave, write_weave
79
from bzrlib.ui import ui_factory
34
from bzrlib.progress import ProgressBar
80
35
from bzrlib.atomicfile import AtomicFile
81
from bzrlib.xml4 import serializer_v4
82
from bzrlib.xml5 import serializer_v5
83
from bzrlib.trace import mutter, note, warning
84
from bzrlib.osutils import sha_strings, sha_string, pathjoin, abspath
87
class Convert(object):
88
def __init__(self, base_dir):
90
self.converted_revs = set()
91
self.absent_revisions = set()
98
if not self._open_branch():
100
note('starting upgrade of %s', os.path.abspath(self.base))
101
self._backup_control_dir()
102
self.pb = ui_factory.progress_bar()
103
if self.old_format == 4:
104
note('starting upgrade from format 4 to 5')
105
self.branch.lock_write()
107
self._convert_to_weaves()
111
if self.old_format == 5:
112
note('starting upgrade from format 5 to 6')
113
self.branch.lock_write()
115
self._convert_to_prefixed()
119
cache = hashcache.HashCache(abspath(self.base))
125
def _convert_to_prefixed(self):
126
from bzrlib.store import hash_prefix
127
for store_name in ["weaves", "revision-store"]:
128
note("adding prefixes to %s" % store_name)
129
store_dir = pathjoin(self.base, ".bzr", store_name)
130
for filename in os.listdir(store_dir):
131
if filename.endswith(".weave") or filename.endswith(".gz"):
132
file_id = os.path.splitext(filename)[0]
135
prefix_dir = pathjoin(store_dir, hash_prefix(file_id))
136
if not os.path.isdir(prefix_dir):
138
os.rename(pathjoin(store_dir, filename),
139
pathjoin(prefix_dir, filename))
140
self._set_new_format(BZR_BRANCH_FORMAT_6)
143
def _convert_to_weaves(self):
144
note('note: upgrade may be faster if all store files are ungzipped first')
145
if not os.path.isdir(self.base + '/.bzr/weaves'):
146
os.mkdir(self.base + '/.bzr/weaves')
147
self.inv_weave = Weave('inventory')
148
# holds in-memory weaves for all files
149
self.text_weaves = {}
150
os.remove(self.branch.control_files.controlfilename('branch-format'))
151
self._convert_working_inv()
152
rev_history = self.branch.revision_history()
153
# to_read is a stack holding the revisions we still need to process;
154
# appending to it adds new highest-priority revisions
155
self.known_revisions = set(rev_history)
156
self.to_read = rev_history[-1:]
158
rev_id = self.to_read.pop()
159
if (rev_id not in self.revisions
160
and rev_id not in self.absent_revisions):
161
self._load_one_rev(rev_id)
163
to_import = self._make_order()
164
for i, rev_id in enumerate(to_import):
165
self.pb.update('converting revision', i, len(to_import))
166
self._convert_one_rev(rev_id)
168
note('upgraded to weaves:')
169
note(' %6d revisions and inventories' % len(self.revisions))
170
note(' %6d revisions not present' % len(self.absent_revisions))
171
note(' %6d texts' % self.text_count)
172
self._write_all_weaves()
173
self._write_all_revs()
174
self._cleanup_spare_files()
175
self._set_new_format(BZR_BRANCH_FORMAT_5)
178
def _open_branch(self):
179
self.branch = Branch.open_downlevel(self.base)
180
self.old_format = self.branch._branch_format
181
if self.old_format == 6:
182
note('this branch is in the most current format')
184
if self.old_format not in (4, 5):
185
raise BzrError("cannot upgrade from branch format %r" %
186
self.branch._branch_format)
189
def _set_new_format(self, format):
190
self.branch.control_files.put_utf8('branch-format', format)
192
def _cleanup_spare_files(self):
193
for n in 'merged-patches', 'pending-merged-patches':
194
p = self.branch.control_files.controlfilename(n)
195
if not os.path.exists(p):
197
## assert os.path.getsize(p) == 0
199
shutil.rmtree(self.base + '/.bzr/inventory-store')
200
shutil.rmtree(self.base + '/.bzr/text-store')
202
def _backup_control_dir(self):
203
orig = self.base + '/.bzr'
204
backup = orig + '.backup'
205
note('making backup of tree history')
206
shutil.copytree(orig, backup)
207
note('%s has been backed up to %s', orig, backup)
208
note('if conversion fails, you can move this directory back to .bzr')
209
note('if it succeeds, you can remove this directory if you wish')
211
def _convert_working_inv(self):
213
inv = serializer_v4.read_inventory(branch.control_files.controlfile('inventory', 'rb'))
214
new_inv_xml = serializer_v5.write_inventory_to_string(inv)
215
branch.control_files.put_utf8('inventory', new_inv_xml)
217
def _write_all_weaves(self):
218
write_a_weave(self.inv_weave, self.base + '/.bzr/inventory.weave')
221
for file_id, file_weave in self.text_weaves.items():
222
self.pb.update('writing weave', i, len(self.text_weaves))
223
write_a_weave(file_weave, self.base + '/.bzr/weaves/%s.weave' % file_id)
229
def _write_all_revs(self):
230
"""Write all revisions out in new form."""
231
shutil.rmtree(self.base + '/.bzr/revision-store')
232
os.mkdir(self.base + '/.bzr/revision-store')
234
for i, rev_id in enumerate(self.converted_revs):
235
self.pb.update('write revision', i, len(self.converted_revs))
236
f = file(self.base + '/.bzr/revision-store/%s' % rev_id, 'wb')
238
serializer_v5.write_revision(self.revisions[rev_id], f)
245
def _load_one_rev(self, rev_id):
246
"""Load a revision object into memory.
248
Any parents not either loaded or abandoned get queued to be
250
self.pb.update('loading revision',
252
len(self.known_revisions))
253
if not self.branch.repository.revision_store.has_id(rev_id):
255
note('revision {%s} not present in branch; '
256
'will be converted as a ghost',
258
self.absent_revisions.add(rev_id)
260
rev_xml = self.branch.repository.revision_store.get(rev_id).read()
261
rev = serializer_v4.read_revision_from_string(rev_xml)
262
for parent_id in rev.parent_ids:
263
self.known_revisions.add(parent_id)
264
self.to_read.append(parent_id)
265
self.revisions[rev_id] = rev
268
def _load_old_inventory(self, rev_id):
269
assert rev_id not in self.converted_revs
270
old_inv_xml = self.branch.repository.inventory_store.get(rev_id).read()
271
inv = serializer_v4.read_inventory_from_string(old_inv_xml)
272
rev = self.revisions[rev_id]
273
if rev.inventory_sha1:
274
assert rev.inventory_sha1 == sha_string(old_inv_xml), \
275
'inventory sha mismatch for {%s}' % rev_id
38
import hotshot, hotshot.stats
42
bzrlib.trace.enable_default_logging()
50
# holds in-memory weaves for all files
53
b = bzrlib.branch.find_branch('.')
56
rev_history = b.revision_history()
61
for rev_id in rev_history:
62
pb.update('converting revision', revno, len(rev_history))
279
def _load_updated_inventory(self, rev_id):
280
assert rev_id in self.converted_revs
281
inv_xml = self.inv_weave.get_text(rev_id)
282
inv = serializer_v5.read_inventory_from_string(inv_xml)
286
def _convert_one_rev(self, rev_id):
287
"""Convert revision and all referenced objects to new format."""
288
rev = self.revisions[rev_id]
289
inv = self._load_old_inventory(rev_id)
290
present_parents = [p for p in rev.parent_ids
291
if p not in self.absent_revisions]
292
self._convert_revision_contents(rev, inv, present_parents)
293
self._store_new_weave(rev, inv, present_parents)
294
self.converted_revs.add(rev_id)
297
def _store_new_weave(self, rev, inv, present_parents):
298
# the XML is now updated with text versions
302
if ie.kind == 'root_directory':
304
assert hasattr(ie, 'revision'), \
305
'no revision on {%s} in {%s}' % \
306
(file_id, rev.revision_id)
307
new_inv_xml = serializer_v5.write_inventory_to_string(inv)
308
new_inv_sha1 = sha_string(new_inv_xml)
309
self.inv_weave.add(rev.revision_id,
311
new_inv_xml.splitlines(True),
313
rev.inventory_sha1 = new_inv_sha1
315
def _convert_revision_contents(self, rev, inv, present_parents):
316
"""Convert all the files within a revision.
318
Also upgrade the inventory to refer to the text revision ids."""
319
rev_id = rev.revision_id
320
mutter('converting texts of revision {%s}',
322
parent_invs = map(self._load_updated_inventory, present_parents)
64
inv_xml = b.get_inventory_xml(rev_id).readlines()
66
new_idx = inv_weave.add(rev_id, inv_parents, inv_xml)
67
inv_parents = [new_idx]
69
tree = b.revision_tree(rev_id)
72
# for each file in the inventory, put it into its own revfile
323
73
for file_id in inv:
325
self._convert_file_version(rev, ie, parent_invs)
327
def _convert_file_version(self, rev, ie, parent_invs):
328
"""Convert one version of one file.
330
The file needs to be added into the weave if it is a merge
331
of >=2 parents or if it's changed from its parent.
333
if ie.kind == 'root_directory':
336
rev_id = rev.revision_id
337
w = self.text_weaves.get(file_id)
340
self.text_weaves[file_id] = w
342
previous_entries = ie.find_previous_heads(parent_invs, w)
343
for old_revision in previous_entries:
344
# if this fails, its a ghost ?
345
assert old_revision in self.converted_revs
346
self.snapshot_ie(previous_entries, ie, w, rev_id)
348
assert getattr(ie, 'revision', None) is not None
350
def snapshot_ie(self, previous_revisions, ie, w, rev_id):
351
# TODO: convert this logic, which is ~= snapshot to
352
# a call to:. This needs the path figured out. rather than a work_tree
353
# a v4 revision_tree can be given, or something that looks enough like
354
# one to give the file content to the entry if it needs it.
355
# and we need something that looks like a weave store for snapshot to
357
#ie.snapshot(rev, PATH, previous_revisions, REVISION_TREE, InMemoryWeaveStore(self.text_weaves))
358
if len(previous_revisions) == 1:
359
previous_ie = previous_revisions.values()[0]
360
if ie._unchanged(previous_ie):
361
ie.revision = previous_ie.revision
363
parent_indexes = map(w.lookup, previous_revisions)
365
text = self.branch.repository.text_store.get(ie.text_id)
366
file_lines = text.readlines()
367
assert sha_strings(file_lines) == ie.text_sha1
368
assert sum(map(len, file_lines)) == ie.text_size
369
w.add(rev_id, parent_indexes, file_lines, ie.text_sha1)
372
w.add(rev_id, parent_indexes, [], None)
374
##mutter('import text {%s} of {%s}',
375
## ie.text_id, file_id)
377
def _make_order(self):
378
"""Return a suitable order for importing revisions.
380
The order must be such that an revision is imported after all
381
its (present) parents.
383
todo = set(self.revisions.keys())
384
done = self.absent_revisions.copy()
387
# scan through looking for a revision whose parents
389
for rev_id in sorted(list(todo)):
390
rev = self.revisions[rev_id]
391
parent_ids = set(rev.parent_ids)
392
if parent_ids.issubset(done):
393
# can take this one now
400
def write_a_weave(weave, filename):
401
inv_wf = file(filename, 'wb')
77
if last_text_sha.get(file_id) == ie.text_sha1:
80
last_text_sha[file_id] = ie.text_sha1
82
# new text (though possibly already stored); need to store it
83
text_lines = tree.get_file(file_id).readlines()
85
# if the file's created for the first time in this
86
# revision then make a new weave; else find the old one
87
if file_id not in text_weaves:
88
text_weaves[file_id] = Weave()
90
w = text_weaves[file_id]
92
# base the new text version off whatever was last
93
# (actually it'd be better to track this, to allow for
94
# files that are deleted and then reappear)
101
w.add(rev_id, parents, text_lines)
107
print '%6d revisions and inventories' % revno
108
print '%6d texts' % text_count
111
# TODO: commit them all atomically at the end, not one by one
112
write_atomic_weave(inv_weave, 'weaves/inventory.weave')
113
for file_id, file_weave in text_weaves.items():
114
pb.update('writing weave', i, len(text_weaves))
115
write_atomic_weave(file_weave, 'weaves/%s.weave' % file_id)
121
def write_atomic_weave(weave, filename):
122
inv_wf = AtomicFile(filename)
403
124
write_weave(weave, inv_wf)
408
def upgrade(base_dir):
132
def profile_convert():
133
prof_f = tempfile.NamedTemporaryFile()
135
prof = hotshot.Profile(prof_f.name)
137
prof.runcall(convert)
140
stats = hotshot.stats.load(prof_f.name)
142
stats.sort_stats('time')
143
## XXX: Might like to write to stderr or the trace file instead but
144
## print_stats seems hardcoded to stdout
145
stats.print_stats(20)
148
if '-p' in sys.argv[1:]:
added
removed
tools/history2weaves.py
