1
# Copyright (C) 2005 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
"""Experiment in converting existing bzr branches to weaves."""
19
# To make this properly useful
21
# 1. assign text version ids, and put those text versions into
22
# the inventory as they're converted.
24
# 2. keep track of the previous version of each file, rather than
25
# just using the last one imported
27
# 3. assign entry versions when files are added, renamed or moved.
29
# 4. when merged-in versions are observed, walk down through them
30
# to discover everything, then commit bottom-up
32
# 5. track ancestry as things are merged in, and commit that in each
35
# Perhaps it's best to first walk the whole graph and make a plan for
36
# what should be imported in what order? Need a kind of topological
37
# sort of all revisions. (Or do we, can we just before doing a revision
38
# see that all its parents have either been converted or abandoned?)
41
# Cannot import a revision until all its parents have been
42
# imported. in other words, we can only import revisions whose
43
# parents have all been imported. the first step must be to
44
# import a revision with no parents, of which there must be at
45
# least one. (So perhaps it's useful to store forward pointers
46
# from a list of parents to their children?)
48
# Another (equivalent?) approach is to build up the ordered
49
# ancestry list for the last revision, and walk through that. We
50
# are going to need that.
52
# We don't want to have to recurse all the way back down the list.
54
# Suppose we keep a queue of the revisions able to be processed at
55
# any point. This starts out with all the revisions having no
58
# This seems like a generally useful algorithm...
60
# The current algorithm is dumb (O(n**2)?) but will do the job, and
61
# takes less than a second on the bzr.dev branch.
63
# This currently does a kind of lazy conversion of file texts, where a
64
# new text is written in every version. That's unnecessary but for
65
# the moment saves us having to worry about when files need new
68
from cStringIO import StringIO
74
from bzrlib.branch import Branch, find_branch
75
from bzrlib.branch import BZR_BRANCH_FORMAT_5, BZR_BRANCH_FORMAT_6
76
from bzrlib.branch import BzrBranchFormat, BzrBranchFormat4, BzrBranchFormat5, BzrBranchFormat6
77
from bzrlib.errors import NoSuchFile, UpgradeReadonly
78
import bzrlib.hashcache as hashcache
79
from bzrlib.osutils import sha_strings, sha_string, pathjoin, abspath
80
from bzrlib.ui import ui_factory
81
from bzrlib.store.text import TextStore
82
from bzrlib.store.weave import WeaveStore
83
from bzrlib.trace import mutter, note, warning
84
from bzrlib.transactions import PassThroughTransaction
85
from bzrlib.transport import get_transport
86
from bzrlib.transport.local import LocalTransport
87
from bzrlib.weave import Weave
88
from bzrlib.weavefile import read_weave, write_weave
89
from bzrlib.xml4 import serializer_v4
90
from bzrlib.xml5 import serializer_v5
93
class Convert(object):
95
def __init__(self, transport):
96
self.base = transport.base
97
self.converted_revs = set()
98
self.absent_revisions = set()
101
self.transport = transport
105
if self.transport.is_readonly():
106
raise UpgradeReadonly
107
if not self._open_branch():
109
note('starting upgrade of %s', self.base)
110
self._backup_control_dir()
111
self.pb = ui_factory.progress_bar()
112
if isinstance(self.old_format, BzrBranchFormat4):
113
note('starting upgrade from format 4 to 5')
114
self._convert_to_weaves()
115
if isinstance(self.old_format, BzrBranchFormat5):
116
note('starting upgrade from format 5 to 6')
117
self._convert_to_prefixed()
118
if isinstance(self.transport, LocalTransport):
119
cache = hashcache.HashCache(abspath(self.base))
124
def _convert_to_prefixed(self):
125
from bzrlib.store import hash_prefix
126
bzr_transport = self.transport.clone('.bzr')
127
bzr_transport.delete('branch-format')
128
for store_name in ["weaves", "revision-store"]:
129
note("adding prefixes to %s" % store_name)
130
store_transport = bzr_transport.clone(store_name)
131
for filename in store_transport.list_dir('.'):
132
if filename.endswith(".weave") or filename.endswith(".gz"):
133
file_id = os.path.splitext(filename)[0]
136
prefix_dir = hash_prefix(file_id)
137
# FIXME keep track of the dirs made RBC 20060121
139
store_transport.move(filename, prefix_dir + '/' + filename)
140
except NoSuchFile: # catches missing dirs strangely enough
141
store_transport.mkdir(prefix_dir)
142
store_transport.move(filename, prefix_dir + '/' + filename)
143
self._set_new_format(BZR_BRANCH_FORMAT_6)
144
self.branch = BzrBranchFormat6().open(self.transport)
145
self.old_format = self.branch._branch_format
147
def _convert_to_weaves(self):
148
note('note: upgrade may be faster if all store files are ungzipped first')
149
bzr_transport = self.transport.clone('.bzr')
152
stat = bzr_transport.stat('weaves')
153
if not S_ISDIR(stat.st_mode):
154
bzr_transport.delete('weaves')
155
bzr_transport.mkdir('weaves')
157
bzr_transport.mkdir('weaves')
158
self.inv_weave = Weave('inventory')
159
# holds in-memory weaves for all files
160
self.text_weaves = {}
161
bzr_transport.delete('branch-format')
162
self._convert_working_inv()
163
rev_history = self.branch.revision_history()
164
# to_read is a stack holding the revisions we still need to process;
165
# appending to it adds new highest-priority revisions
166
self.known_revisions = set(rev_history)
167
self.to_read = rev_history[-1:]
169
rev_id = self.to_read.pop()
170
if (rev_id not in self.revisions
171
and rev_id not in self.absent_revisions):
172
self._load_one_rev(rev_id)
174
to_import = self._make_order()
175
for i, rev_id in enumerate(to_import):
176
self.pb.update('converting revision', i, len(to_import))
177
self._convert_one_rev(rev_id)
179
self._write_all_weaves()
180
self._write_all_revs()
181
note('upgraded to weaves:')
182
note(' %6d revisions and inventories' % len(self.revisions))
183
note(' %6d revisions not present' % len(self.absent_revisions))
184
note(' %6d texts' % self.text_count)
185
self._cleanup_spare_files_after_format4()
186
self._set_new_format(BZR_BRANCH_FORMAT_5)
187
self.branch = BzrBranchFormat5().open(self.transport)
188
self.old_format = self.branch._branch_format
190
def _open_branch(self):
191
self.old_format = BzrBranchFormat.find_format(self.transport)
192
self.branch = self.old_format.open(self.transport)
193
if isinstance(self.old_format, BzrBranchFormat6):
194
note('this branch is in the most current format (%s)', self.old_format)
196
if (not isinstance(self.old_format, BzrBranchFormat4) and
197
not isinstance(self.old_format, BzrBranchFormat5)):
198
raise BzrError("cannot upgrade from branch format %s" %
199
self.branch._branch_format)
202
def _set_new_format(self, format):
203
self.branch.put_controlfile('branch-format', format)
205
def _cleanup_spare_files_after_format4(self):
206
transport = self.transport.clone('.bzr')
207
for n in 'merged-patches', 'pending-merged-patches':
209
## assert os.path.getsize(p) == 0
213
transport.delete_tree('inventory-store')
214
transport.delete_tree('text-store')
216
def _backup_control_dir(self):
217
note('making backup of tree history')
218
source = self.transport.clone('.bzr')
219
self.transport.mkdir('.bzr.backup')
220
backup = self.transport.clone('.bzr.backup')
224
dir = directories.pop()
227
for path in source.list_dir(dir):
228
path = dir + '/' + path
229
stat = source.stat(path)
230
if S_ISDIR(stat.st_mode):
231
directories.append(path)
234
source.copy_to(files, backup)
235
note('%s has been backed up to %s', source.base, backup.base)
236
note('if conversion fails, you can move this directory back to .bzr')
237
note('if it succeeds, you can remove this directory if you wish')
239
def _convert_working_inv(self):
241
inv = serializer_v4.read_inventory(branch.controlfile('inventory', 'rb'))
242
new_inv_xml = serializer_v5.write_inventory_to_string(inv)
243
branch.put_controlfile('inventory', new_inv_xml)
245
def _write_all_weaves(self):
246
bzr_transport = self.transport.clone('.bzr')
247
controlweaves = WeaveStore(bzr_transport, prefixed=False)
248
weave_transport = bzr_transport.clone('weaves')
249
weaves = WeaveStore(weave_transport, prefixed=False)
250
transaction = PassThroughTransaction()
252
controlweaves.put_weave('inventory', self.inv_weave, transaction)
255
for file_id, file_weave in self.text_weaves.items():
256
self.pb.update('writing weave', i, len(self.text_weaves))
257
weaves.put_weave(file_id, file_weave, transaction)
262
def _write_all_revs(self):
263
"""Write all revisions out in new form."""
264
transport = self.transport.clone('.bzr')
265
transport.delete_tree('revision-store')
266
transport.mkdir('revision-store')
267
revision_transport = transport.clone('revision-store')
269
revision_store = TextStore(revision_transport,
273
for i, rev_id in enumerate(self.converted_revs):
274
self.pb.update('write revision', i, len(self.converted_revs))
276
serializer_v5.write_revision(self.revisions[rev_id], rev_tmp)
278
revision_store.add(rev_tmp, rev_id)
283
def _load_one_rev(self, rev_id):
284
"""Load a revision object into memory.
286
Any parents not either loaded or abandoned get queued to be
288
self.pb.update('loading revision',
290
len(self.known_revisions))
291
if not self.branch.revision_store.has_id(rev_id):
293
note('revision {%s} not present in branch; '
294
'will be converted as a ghost',
296
self.absent_revisions.add(rev_id)
298
rev_xml = self.branch.revision_store.get(rev_id).read()
299
rev = serializer_v4.read_revision_from_string(rev_xml)
300
for parent_id in rev.parent_ids:
301
self.known_revisions.add(parent_id)
302
self.to_read.append(parent_id)
303
self.revisions[rev_id] = rev
306
def _load_old_inventory(self, rev_id):
307
assert rev_id not in self.converted_revs
308
old_inv_xml = self.branch.inventory_store.get(rev_id).read()
309
inv = serializer_v4.read_inventory_from_string(old_inv_xml)
310
rev = self.revisions[rev_id]
311
if rev.inventory_sha1:
312
assert rev.inventory_sha1 == sha_string(old_inv_xml), \
313
'inventory sha mismatch for {%s}' % rev_id
317
def _load_updated_inventory(self, rev_id):
318
assert rev_id in self.converted_revs
319
inv_xml = self.inv_weave.get_text(rev_id)
320
inv = serializer_v5.read_inventory_from_string(inv_xml)
324
def _convert_one_rev(self, rev_id):
325
"""Convert revision and all referenced objects to new format."""
326
rev = self.revisions[rev_id]
327
inv = self._load_old_inventory(rev_id)
328
present_parents = [p for p in rev.parent_ids
329
if p not in self.absent_revisions]
330
self._convert_revision_contents(rev, inv, present_parents)
331
self._store_new_weave(rev, inv, present_parents)
332
self.converted_revs.add(rev_id)
335
def _store_new_weave(self, rev, inv, present_parents):
336
# the XML is now updated with text versions
340
if ie.kind == 'root_directory':
342
assert hasattr(ie, 'revision'), \
343
'no revision on {%s} in {%s}' % \
344
(file_id, rev.revision_id)
345
new_inv_xml = serializer_v5.write_inventory_to_string(inv)
346
new_inv_sha1 = sha_string(new_inv_xml)
347
self.inv_weave.add(rev.revision_id,
349
new_inv_xml.splitlines(True),
351
rev.inventory_sha1 = new_inv_sha1
353
def _convert_revision_contents(self, rev, inv, present_parents):
354
"""Convert all the files within a revision.
356
Also upgrade the inventory to refer to the text revision ids."""
357
rev_id = rev.revision_id
358
mutter('converting texts of revision {%s}',
360
parent_invs = map(self._load_updated_inventory, present_parents)
363
self._convert_file_version(rev, ie, parent_invs)
365
def _convert_file_version(self, rev, ie, parent_invs):
366
"""Convert one version of one file.
368
The file needs to be added into the weave if it is a merge
369
of >=2 parents or if it's changed from its parent.
371
if ie.kind == 'root_directory':
374
rev_id = rev.revision_id
375
w = self.text_weaves.get(file_id)
378
self.text_weaves[file_id] = w
380
previous_entries = ie.find_previous_heads(parent_invs, w)
381
for old_revision in previous_entries:
382
# if this fails, its a ghost ?
383
assert old_revision in self.converted_revs
384
self.snapshot_ie(previous_entries, ie, w, rev_id)
386
assert getattr(ie, 'revision', None) is not None
388
def snapshot_ie(self, previous_revisions, ie, w, rev_id):
389
# TODO: convert this logic, which is ~= snapshot to
390
# a call to:. This needs the path figured out. rather than a work_tree
391
# a v4 revision_tree can be given, or something that looks enough like
392
# one to give the file content to the entry if it needs it.
393
# and we need something that looks like a weave store for snapshot to
395
#ie.snapshot(rev, PATH, previous_revisions, REVISION_TREE, InMemoryWeaveStore(self.text_weaves))
396
if len(previous_revisions) == 1:
397
previous_ie = previous_revisions.values()[0]
398
if ie._unchanged(previous_ie):
399
ie.revision = previous_ie.revision
401
parent_indexes = map(w.lookup, previous_revisions)
403
file_lines = self.branch.text_store.get(ie.text_id).readlines()
404
assert sha_strings(file_lines) == ie.text_sha1
405
assert sum(map(len, file_lines)) == ie.text_size
406
w.add(rev_id, parent_indexes, file_lines, ie.text_sha1)
409
w.add(rev_id, parent_indexes, [], None)
411
##mutter('import text {%s} of {%s}',
412
## ie.text_id, file_id)
414
def _make_order(self):
415
"""Return a suitable order for importing revisions.
417
The order must be such that an revision is imported after all
418
its (present) parents.
420
todo = set(self.revisions.keys())
421
done = self.absent_revisions.copy()
424
# scan through looking for a revision whose parents
426
for rev_id in sorted(list(todo)):
427
rev = self.revisions[rev_id]
428
parent_ids = set(rev.parent_ids)
429
if parent_ids.issubset(done):
430
# can take this one now
438
t = get_transport(url)
added
removed
bzrlib/upgrade.py
