« back to all changes in this revision
Viewing changes to tools/history2weaves.py
- browse files at revision 1185.11.1
- compare with another revision
- download diff
- download tarball
- view history from revision 1185.11.1
- Committer: John Arbash Meinel
- Date: 2005-09-15 21:35:53 UTC
- mfrom: (907.1.57)
- mto: (1393.2.1)
- mto: This revision was merged to the branch mainline in revision 1396.
- Revision ID: john@arbash-meinel.com-20050915213552-a6c83a5ef1e20897
(broken) Transport work is merged in. Tests do not pass yet.
- files added:
- bzrlib/meta_store.py
- files removed:
- NEWS.developers
- files renamed:
- contrib/newinventory.py => bzrlib/newinventory.py
- bzrlib/upgrade.py => tools/history2weaves.py
- bzrlib/selftest/test_weave.py => tools/testweave.py
- files modified:
- .bzrignore
added
removed
tools/history2weaves.py
1
1
#! /usr/bin/python
2
#
2
3
3
# Copyright (C) 2005 Canonical Ltd
4
#
4
5
5
# This program is free software; you can redistribute it and/or modify
6
6
# it under the terms of the GNU General Public License as published by
7
7
# the Free Software Foundation; either version 2 of the License, or
8
8
# (at your option) any later version.
9
#
9
10
10
# This program is distributed in the hope that it will be useful,
11
11
# but WITHOUT ANY WARRANTY; without even the implied warranty of
12
12
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
13
# GNU General Public License for more details.
14
#
14
15
15
# You should have received a copy of the GNU General Public License
16
16
# along with this program; if not, write to the Free Software
17
17
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18
18
19
19
"""Experiment in converting existing bzr branches to weaves."""
20
20
21
# To make this properly useful
22
#
23
# 1. assign text version ids, and put those text versions into
24
# the inventory as they're converted.
25
#
26
# 2. keep track of the previous version of each file, rather than
27
# just using the last one imported
28
#
29
# 3. assign entry versions when files are added, renamed or moved.
30
#
31
# 4. when merged-in versions are observed, walk down through them
32
# to discover everything, then commit bottom-up
33
#
34
# 5. track ancestry as things are merged in, and commit that in each
35
# revision
36
#
37
# Perhaps it's best to first walk the whole graph and make a plan for
38
# what should be imported in what order? Need a kind of topological
39
# sort of all revisions. (Or do we, can we just before doing a revision
40
# see that all its parents have either been converted or abandoned?)
41
42
43
# Cannot import a revision until all its parents have been
44
# imported. in other words, we can only import revisions whose
45
# parents have all been imported. the first step must be to
46
# import a revision with no parents, of which there must be at
47
# least one. (So perhaps it's useful to store forward pointers
48
# from a list of parents to their children?)
49
#
50
# Another (equivalent?) approach is to build up the ordered
51
# ancestry list for the last revision, and walk through that. We
52
# are going to need that.
53
#
54
# We don't want to have to recurse all the way back down the list.
55
#
56
# Suppose we keep a queue of the revisions able to be processed at
57
# any point. This starts out with all the revisions having no
58
# parents.
59
#
60
# This seems like a generally useful algorithm...
61
#
62
# The current algorithm is dumb (O(n**2)?) but will do the job, and
63
# takes less than a second on the bzr.dev branch.
64
65
# This currently does a kind of lazy conversion of file texts, where a
66
# new text is written in every version. That's unnecessary but for
67
# the moment saves us having to worry about when files need new
68
# versions.
69
70
71
# TODO: Don't create a progress bar here, have it passed by the caller.
72
# At least do it from the UI factory.
73
74
if False:
75
try:
76
import psyco
77
psyco.full()
78
except ImportError:
79
pass
80
81
82
import os
83
import tempfile
84
import sys
21
try:
22
import psyco
23
psyco.full()
24
except ImportError:
25
pass
26
27
85
28
import logging
86
import shutil
87
29
88
from bzrlib.branch import Branch, find_branch, BZR_BRANCH_FORMAT_5
30
import bzrlib.branch
89
31
from bzrlib.revfile import Revfile
90
32
from bzrlib.weave import Weave
91
33
from bzrlib.weavefile import read_weave, write_weave
92
from bzrlib.ui import ui_factory
34
from bzrlib.progress import ProgressBar
93
35
from bzrlib.atomicfile import AtomicFile
94
from bzrlib.xml4 import serializer_v4
95
from bzrlib.xml5 import serializer_v5
96
from bzrlib.trace import mutter, note, warning, enable_default_logging
97
from bzrlib.osutils import sha_strings, sha_string
98
99
100
class Convert(object):
101
def __init__(self, base_dir):
102
self.base = base_dir
103
self.converted_revs = set()
104
self.absent_revisions = set()
105
self.text_count = 0
106
self.revisions = {}
107
self.convert()
108
109
110
def convert(self):
111
if not self._open_branch():
112
return
113
note('starting upgrade of %s', os.path.abspath(self.base))
114
self._backup_control_dir()
115
note('starting upgrade')
116
note('note: upgrade may be faster if all store files are ungzipped first')
117
self.pb = ui_factory.progress_bar()
118
if not os.path.isdir(self.base + '/.bzr/weaves'):
119
os.mkdir(self.base + '/.bzr/weaves')
120
self.inv_weave = Weave('inventory')
121
# holds in-memory weaves for all files
122
self.text_weaves = {}
123
os.remove(self.branch.controlfilename('branch-format'))
124
self._convert_working_inv()
125
rev_history = self.branch.revision_history()
126
# to_read is a stack holding the revisions we still need to process;
127
# appending to it adds new highest-priority revisions
128
self.known_revisions = set(rev_history)
129
self.to_read = [rev_history[-1]]
130
while self.to_read:
131
rev_id = self.to_read.pop()
132
if (rev_id not in self.revisions
133
and rev_id not in self.absent_revisions):
134
self._load_one_rev(rev_id)
135
self.pb.clear()
136
to_import = self._make_order()
137
for i, rev_id in enumerate(to_import):
138
self.pb.update('converting revision', i, len(to_import))
139
self._convert_one_rev(rev_id)
140
self.pb.clear()
141
note('upgraded to weaves:')
142
note(' %6d revisions and inventories' % len(self.revisions))
143
note(' %6d revisions not present' % len(self.absent_revisions))
144
note(' %6d texts' % self.text_count)
145
self._write_all_weaves()
146
self._write_all_revs()
147
self._set_new_format()
148
self._cleanup_spare_files()
149
150
151
def _open_branch(self):
152
self.branch = Branch.open_downlevel(self.base)
153
if self.branch._branch_format == 5:
154
note('this branch is already in the most current format')
155
return False
156
if self.branch._branch_format != 4:
157
raise BzrError("cannot upgrade from branch format %r" %
158
self.branch._branch_format)
159
return True
160
161
162
def _set_new_format(self):
163
self.branch.put_controlfile('branch-format', BZR_BRANCH_FORMAT_5)
164
165
166
def _cleanup_spare_files(self):
167
for n in 'merged-patches', 'pending-merged-patches':
168
p = self.branch.controlfilename(n)
169
if not os.path.exists(p):
170
continue
171
## assert os.path.getsize(p) == 0
172
os.remove(p)
173
shutil.rmtree(self.base + '/.bzr/inventory-store')
174
shutil.rmtree(self.base + '/.bzr/text-store')
175
176
177
def _backup_control_dir(self):
178
orig = self.base + '/.bzr'
179
backup = orig + '.backup'
180
note('making backup of tree history')
181
shutil.copytree(orig, backup)
182
note('%s has been backed up to %s', orig, backup)
183
note('if conversion fails, you can move this directory back to .bzr')
184
note('if it succeeds, you can remove this directory if you wish')
185
186
187
def _convert_working_inv(self):
188
branch = self.branch
189
inv = serializer_v4.read_inventory(branch.controlfile('inventory', 'rb'))
190
new_inv_xml = serializer_v5.write_inventory_to_string(inv)
191
branch.put_controlfile('inventory', new_inv_xml)
192
193
194
195
def _write_all_weaves(self):
196
write_a_weave(self.inv_weave, self.base + '/.bzr/inventory.weave')
197
i = 0
198
try:
199
for file_id, file_weave in self.text_weaves.items():
200
self.pb.update('writing weave', i, len(self.text_weaves))
201
write_a_weave(file_weave, self.base + '/.bzr/weaves/%s.weave' % file_id)
202
i += 1
203
finally:
204
self.pb.clear()
205
206
207
def _write_all_revs(self):
208
"""Write all revisions out in new form."""
209
shutil.rmtree(self.base + '/.bzr/revision-store')
210
os.mkdir(self.base + '/.bzr/revision-store')
211
try:
212
for i, rev_id in enumerate(self.converted_revs):
213
self.pb.update('write revision', i, len(self.converted_revs))
214
f = file(self.base + '/.bzr/revision-store/%s' % rev_id, 'wb')
215
try:
216
serializer_v5.write_revision(self.revisions[rev_id], f)
217
finally:
218
f.close()
219
finally:
220
self.pb.clear()
221
222
223
def _load_one_rev(self, rev_id):
224
"""Load a revision object into memory.
225
226
Any parents not either loaded or abandoned get queued to be
227
loaded."""
228
self.pb.update('loading revision',
229
len(self.revisions),
230
len(self.known_revisions))
231
if rev_id not in self.branch.revision_store:
232
self.pb.clear()
233
note('revision {%s} not present in branch; '
234
'will be converted as a ghost',
235
rev_id)
236
self.absent_revisions.add(rev_id)
237
else:
238
rev_xml = self.branch.revision_store[rev_id].read()
239
rev = serializer_v4.read_revision_from_string(rev_xml)
240
for parent_id in rev.parent_ids:
241
self.known_revisions.add(parent_id)
242
self.to_read.append(parent_id)
243
self.revisions[rev_id] = rev
244
245
246
def _load_old_inventory(self, rev_id):
247
assert rev_id not in self.converted_revs
248
old_inv_xml = self.branch.inventory_store[rev_id].read()
249
inv = serializer_v4.read_inventory_from_string(old_inv_xml)
250
rev = self.revisions[rev_id]
251
if rev.inventory_sha1:
252
assert rev.inventory_sha1 == sha_string(old_inv_xml), \
253
'inventory sha mismatch for {%s}' % rev_id
254
return inv
36
import bzrlib.trace
37
import tempfile
38
import hotshot, hotshot.stats
39
import sys
40
41
def convert():
42
bzrlib.trace.enable_default_logging()
43
44
pb = ProgressBar()
45
46
inv_weave = Weave()
47
48
last_text_sha = {}
49
50
# holds in-memory weaves for all files
51
text_weaves = {}
52
53
b = bzrlib.branch.find_branch('.')
54
55
revno = 1
56
rev_history = b.revision_history()
57
last_idx = None
58
inv_parents = []
59
text_count = 0
60
61
for rev_id in rev_history:
62
pb.update('converting revision', revno, len(rev_history))
255
63
256
257
def _load_updated_inventory(self, rev_id):
258
assert rev_id in self.converted_revs
259
inv_xml = self.inv_weave.get_text(rev_id)
260
inv = serializer_v5.read_inventory_from_string(inv_xml)
261
return inv
262
263
264
def _convert_one_rev(self, rev_id):
265
"""Convert revision and all referenced objects to new format."""
266
rev = self.revisions[rev_id]
267
inv = self._load_old_inventory(rev_id)
268
present_parents = [p for p in rev.parent_ids
269
if p not in self.absent_revisions]
270
self._convert_revision_contents(rev, inv, present_parents)
271
self._store_new_weave(rev, inv, present_parents)
272
self.converted_revs.add(rev_id)
273
274
275
def _store_new_weave(self, rev, inv, present_parents):
276
# the XML is now updated with text versions
277
if __debug__:
278
for file_id in inv:
279
ie = inv[file_id]
280
if ie.kind == 'root_directory':
281
continue
282
assert hasattr(ie, 'revision'), \
283
'no revision on {%s} in {%s}' % \
284
(file_id, rev.revision_id)
285
new_inv_xml = serializer_v5.write_inventory_to_string(inv)
286
new_inv_sha1 = sha_string(new_inv_xml)
287
self.inv_weave.add(rev.revision_id,
288
present_parents,
289
new_inv_xml.splitlines(True),
290
new_inv_sha1)
291
rev.inventory_sha1 = new_inv_sha1
292
293
def _convert_revision_contents(self, rev, inv, present_parents):
294
"""Convert all the files within a revision.
295
296
Also upgrade the inventory to refer to the text revision ids."""
297
rev_id = rev.revision_id
298
mutter('converting texts of revision {%s}',
299
rev_id)
300
parent_invs = map(self._load_updated_inventory, present_parents)
64
inv_xml = b.get_inventory_xml(rev_id).readlines()
65
66
new_idx = inv_weave.add(rev_id, inv_parents, inv_xml)
67
inv_parents = [new_idx]
68
69
tree = b.revision_tree(rev_id)
70
inv = tree.inventory
71
72
# for each file in the inventory, put it into its own revfile
301
73
for file_id in inv:
302
74
ie = inv[file_id]
303
self._convert_file_version(rev, ie, parent_invs)
304
305
def _convert_file_version(self, rev, ie, parent_invs):
306
"""Convert one version of one file.
307
308
The file needs to be added into the weave if it is a merge
309
of >=2 parents or if it's changed from its parent.
310
"""
311
if ie.kind == 'root_directory':
312
return
313
file_id = ie.file_id
314
rev_id = rev.revision_id
315
w = self.text_weaves.get(file_id)
316
if w is None:
317
w = Weave(file_id)
318
self.text_weaves[file_id] = w
319
text_changed = False
320
previous_entries = ie.find_previous_heads(parent_invs, w)
321
for old_revision in previous_entries:
322
# if this fails, its a ghost ?
323
assert old_revision in self.converted_revs
324
self.snapshot_ie(previous_entries, ie, w, rev_id)
325
del ie.text_id
326
assert getattr(ie, 'revision', None) is not None
327
328
def snapshot_ie(self, previous_revisions, ie, w, rev_id):
329
# TODO: convert this logic, which is ~= snapshot to
330
# a call to:. This needs the path figured out. rather than a work_tree
331
# a v4 revision_tree can be given, or something that looks enough like
332
# one to give the file content to the entry if it needs it.
333
# and we need something that looks like a weave store for snapshot to
334
# save against.
335
#ie.snapshot(rev, PATH, previous_revisions, REVISION_TREE, InMemoryWeaveStore(self.text_weaves))
336
if len(previous_revisions) == 1:
337
previous_ie = previous_revisions.values()[0]
338
if ie._unchanged(previous_ie):
339
ie.revision = previous_ie.revision
340
return
341
parent_indexes = map(w.lookup, previous_revisions)
342
if ie.has_text():
343
file_lines = self.branch.text_store[ie.text_id].readlines()
344
assert sha_strings(file_lines) == ie.text_sha1
345
assert sum(map(len, file_lines)) == ie.text_size
346
w.add(rev_id, parent_indexes, file_lines, ie.text_sha1)
347
self.text_count += 1
348
else:
349
w.add(rev_id, parent_indexes, [], None)
350
ie.revision = rev_id
351
##mutter('import text {%s} of {%s}',
352
## ie.text_id, file_id)
353
354
def _make_order(self):
355
"""Return a suitable order for importing revisions.
356
357
The order must be such that an revision is imported after all
358
its (present) parents.
359
"""
360
todo = set(self.revisions.keys())
361
done = self.absent_revisions.copy()
362
o = []
363
while todo:
364
# scan through looking for a revision whose parents
365
# are all done
366
for rev_id in sorted(list(todo)):
367
rev = self.revisions[rev_id]
368
parent_ids = set(rev.parent_ids)
369
if parent_ids.issubset(done):
370
# can take this one now
371
o.append(rev_id)
372
todo.remove(rev_id)
373
done.add(rev_id)
374
return o
375
376
377
def write_a_weave(weave, filename):
378
inv_wf = file(filename, 'wb')
75
if ie.kind != 'file':
76
continue
77
if last_text_sha.get(file_id) == ie.text_sha1:
78
# same as last time
79
continue
80
last_text_sha[file_id] = ie.text_sha1
81
82
# new text (though possibly already stored); need to store it
83
text_lines = tree.get_file(file_id).readlines()
84
85
# if the file's created for the first time in this
86
# revision then make a new weave; else find the old one
87
if file_id not in text_weaves:
88
text_weaves[file_id] = Weave()
89
90
w = text_weaves[file_id]
91
92
# base the new text version off whatever was last
93
# (actually it'd be better to track this, to allow for
94
# files that are deleted and then reappear)
95
last = len(w)
96
if last == 0:
97
parents = []
98
else:
99
parents = [last-1]
100
101
w.add(rev_id, parents, text_lines)
102
text_count += 1
103
104
revno += 1
105
106
pb.clear()
107
print '%6d revisions and inventories' % revno
108
print '%6d texts' % text_count
109
110
i = 0
111
# TODO: commit them all atomically at the end, not one by one
112
write_atomic_weave(inv_weave, 'weaves/inventory.weave')
113
for file_id, file_weave in text_weaves.items():
114
pb.update('writing weave', i, len(text_weaves))
115
write_atomic_weave(file_weave, 'weaves/%s.weave' % file_id)
116
i += 1
117
118
pb.clear()
119
120
121
def write_atomic_weave(weave, filename):
122
inv_wf = AtomicFile(filename)
379
123
try:
380
124
write_weave(weave, inv_wf)
125
inv_wf.commit()
381
126
finally:
382
127
inv_wf.close()
383
128
384
385
def upgrade(base_dir):
386
Convert(base_dir)
129
130
131
132
def profile_convert():
133
prof_f = tempfile.NamedTemporaryFile()
134
135
prof = hotshot.Profile(prof_f.name)
136
137
prof.runcall(convert)
138
prof.close()
139
140
stats = hotshot.stats.load(prof_f.name)
141
#stats.strip_dirs()
142
stats.sort_stats('time')
143
## XXX: Might like to write to stderr or the trace file instead but
144
## print_stats seems hardcoded to stdout
145
stats.print_stats(20)
146
147
148
if '-p' in sys.argv[1:]:
149
profile_convert()
150
else:
151
convert()
152
Loggerhead is a web-based interface for Breezy
Version: 2.0.1