67
68
# the moment saves us having to worry about when files need new
71
from cStringIO import StringIO
85
from bzrlib.branch import Branch, find_branch, BZR_BRANCH_FORMAT_5
86
from bzrlib.revfile import Revfile
78
from bzrlib.branch import Branch
79
import bzrlib.bzrdir as bzrdir
80
from bzrlib.bzrdir import BzrDirFormat, BzrDirFormat4, BzrDirFormat5, BzrDirFormat6
81
import bzrlib.errors as errors
82
from bzrlib.errors import NoSuchFile, UpgradeReadonly
83
import bzrlib.hashcache as hashcache
84
from bzrlib.lockable_files import LockableFiles
85
from bzrlib.osutils import sha_strings, sha_string, pathjoin, abspath
86
from bzrlib.ui import ui_factory
87
from bzrlib.store.text import TextStore
88
from bzrlib.store.weave import WeaveStore
89
from bzrlib.trace import mutter, note, warning
90
from bzrlib.transactions import PassThroughTransaction
91
from bzrlib.transport import get_transport
92
from bzrlib.transport.local import LocalTransport
87
93
from bzrlib.weave import Weave
88
94
from bzrlib.weavefile import read_weave, write_weave
89
from bzrlib.progress import ProgressBar
90
from bzrlib.atomicfile import AtomicFile
91
95
from bzrlib.xml4 import serializer_v4
92
96
from bzrlib.xml5 import serializer_v5
93
from bzrlib.trace import mutter, note, warning, enable_default_logging
94
from bzrlib.osutils import sha_strings, sha_string
95
from bzrlib.commit import merge_ancestry_lines
98
99
class Convert(object):
99
def __init__(self, base_dir):
101
def __init__(self, transport):
102
self.base = transport.base
101
103
self.converted_revs = set()
102
104
self.absent_revisions = set()
103
105
self.text_count = 0
104
106
self.revisions = {}
107
self.transport = transport
108
if self.transport.is_readonly():
109
raise UpgradeReadonly
110
self.control_files = LockableFiles(transport.clone(bzrlib.BZRDIR), 'branch-lock')
111
# Lock the branch (soon to be meta dir) to prevent anyone racing with us
112
# This is currently windows incompatible, it will deadlock. When the upgrade
113
# logic becomes format specific, then we can have the format know how to pass this
114
# on. Also note that we probably have an 'upgrade meta' which upgrades the constituent
116
print "FIXME: control files reuse"
117
self.control_files.lock_write()
121
self.control_files.unlock()
108
123
def convert(self):
109
124
if not self._open_branch():
111
note('starting upgrade of %s', os.path.abspath(self.base))
126
note('starting upgrade of %s', self.base)
112
127
self._backup_control_dir()
113
note('starting upgrade')
128
self.pb = ui_factory.progress_bar()
129
if isinstance(self.old_format, BzrDirFormat4):
130
note('starting upgrade from format 4 to 5')
131
if isinstance(self.transport, LocalTransport):
132
self.bzrdir.get_workingtree_transport(None).delete('stat-cache')
133
self._convert_to_weaves()
134
if isinstance(self.old_format, BzrDirFormat5):
135
note('starting upgrade from format 5 to 6')
136
self._convert_to_prefixed()
139
def _convert_to_prefixed(self):
140
from bzrlib.store import hash_prefix
141
bzr_transport = self.transport.clone('.bzr')
142
bzr_transport.delete('branch-format')
143
for store_name in ["weaves", "revision-store"]:
144
note("adding prefixes to %s" % store_name)
145
store_transport = bzr_transport.clone(store_name)
146
for filename in store_transport.list_dir('.'):
147
if (filename.endswith(".weave") or
148
filename.endswith(".gz") or
149
filename.endswith(".sig")):
150
file_id = os.path.splitext(filename)[0]
153
prefix_dir = hash_prefix(file_id)
154
# FIXME keep track of the dirs made RBC 20060121
156
store_transport.move(filename, prefix_dir + '/' + filename)
157
except NoSuchFile: # catches missing dirs strangely enough
158
store_transport.mkdir(prefix_dir)
159
store_transport.move(filename, prefix_dir + '/' + filename)
160
self.old_format = BzrDirFormat6()
161
self._set_new_format(self.old_format.get_format_string())
162
self.bzrdir = self.old_format.open(self.transport)
163
self.branch = self.bzrdir.open_branch()
165
def _convert_to_weaves(self):
114
166
note('note: upgrade may be faster if all store files are ungzipped first')
115
self.pb = ProgressBar()
116
if not os.path.isdir(self.base + '/.bzr/weaves'):
117
os.mkdir(self.base + '/.bzr/weaves')
167
bzr_transport = self.transport.clone('.bzr')
170
stat = bzr_transport.stat('weaves')
171
if not S_ISDIR(stat.st_mode):
172
bzr_transport.delete('weaves')
173
bzr_transport.mkdir('weaves')
175
bzr_transport.mkdir('weaves')
118
176
self.inv_weave = Weave('inventory')
119
self.anc_weave = Weave('ancestry')
121
177
# holds in-memory weaves for all files
122
178
self.text_weaves = {}
123
os.remove(self.branch.controlfilename('branch-format'))
179
bzr_transport.delete('branch-format')
124
180
self._convert_working_inv()
125
181
rev_history = self.branch.revision_history()
126
182
# to_read is a stack holding the revisions we still need to process;
127
183
# appending to it adds new highest-priority revisions
128
184
self.known_revisions = set(rev_history)
129
self.to_read = [rev_history[-1]]
185
self.to_read = rev_history[-1:]
130
186
while self.to_read:
131
187
rev_id = self.to_read.pop()
132
188
if (rev_id not in self.revisions
138
194
self.pb.update('converting revision', i, len(to_import))
139
195
self._convert_one_rev(rev_id)
197
self._write_all_weaves()
198
self._write_all_revs()
141
199
note('upgraded to weaves:')
142
200
note(' %6d revisions and inventories' % len(self.revisions))
143
note(' %6d absent revisions removed' % len(self.absent_revisions))
201
note(' %6d revisions not present' % len(self.absent_revisions))
144
202
note(' %6d texts' % self.text_count)
145
self._write_all_weaves()
146
self._write_all_revs()
147
self._set_new_format()
148
self._cleanup_spare_files()
203
self._cleanup_spare_files_after_format4()
204
self.old_format = BzrDirFormat5()
205
self._set_new_format(self.old_format.get_format_string())
206
self.bzrdir = self.old_format.open(self.transport)
207
self.branch = self.bzrdir.open_branch()
151
209
def _open_branch(self):
152
self.branch = Branch.open_downlevel(self.base)
153
if self.branch._branch_format == 5:
154
note('this branch is already in the most current format')
210
self.old_format = BzrDirFormat.find_format(self.transport)
211
self.bzrdir = self.old_format.open(self.transport)
212
self.branch = self.bzrdir.open_branch()
213
if isinstance(self.old_format, BzrDirFormat6):
214
note('this branch is in the most current format (%s)', self.old_format)
156
if self.branch._branch_format != 4:
157
raise BzrError("cannot upgrade from branch format %r" %
216
if (not isinstance(self.old_format, BzrDirFormat4) and
217
not isinstance(self.old_format, BzrDirFormat5) and
218
not isinstance(self.old_format, bzrdir.BzrDirMetaFormat1)):
219
raise errors.BzrError("cannot upgrade from branch format %s" %
158
220
self.branch._branch_format)
162
def _set_new_format(self):
163
self.branch.put_controlfile('branch-format', BZR_BRANCH_FORMAT_5)
166
def _cleanup_spare_files(self):
223
def _set_new_format(self, format):
224
self.branch.control_files.put_utf8('branch-format', format)
226
def _cleanup_spare_files_after_format4(self):
227
transport = self.transport.clone('.bzr')
228
print "FIXME working tree upgrade foo."
167
229
for n in 'merged-patches', 'pending-merged-patches':
168
p = self.branch.controlfilename(n)
169
if not os.path.exists(p):
171
## assert os.path.getsize(p) == 0
173
shutil.rmtree(self.base + '/.bzr/inventory-store')
174
shutil.rmtree(self.base + '/.bzr/text-store')
231
## assert os.path.getsize(p) == 0
235
transport.delete_tree('inventory-store')
236
transport.delete_tree('text-store')
177
238
def _backup_control_dir(self):
178
orig = self.base + '/.bzr'
179
backup = orig + '.backup'
180
239
note('making backup of tree history')
181
shutil.copytree(orig, backup)
182
note('%s has been backed up to %s', orig, backup)
240
self.transport.copy_tree('.bzr', '.bzr.backup')
241
note('%s.bzr has been backed up to %s.bzr.backup',
183
244
note('if conversion fails, you can move this directory back to .bzr')
184
245
note('if it succeeds, you can remove this directory if you wish')
187
247
def _convert_working_inv(self):
188
248
branch = self.branch
189
inv = serializer_v4.read_inventory(branch.controlfile('inventory', 'rb'))
249
inv = serializer_v4.read_inventory(branch.control_files.get('inventory'))
190
250
new_inv_xml = serializer_v5.write_inventory_to_string(inv)
191
branch.put_controlfile('inventory', new_inv_xml)
251
print "fixme inventory is a working tree change."
252
branch.control_files.put('inventory', new_inv_xml)
195
254
def _write_all_weaves(self):
196
write_a_weave(self.inv_weave, self.base + '/.bzr/inventory.weave')
197
write_a_weave(self.anc_weave, self.base + '/.bzr/ancestry.weave')
255
bzr_transport = self.transport.clone('.bzr')
256
controlweaves = WeaveStore(bzr_transport, prefixed=False)
257
weave_transport = bzr_transport.clone('weaves')
258
weaves = WeaveStore(weave_transport, prefixed=False)
259
transaction = PassThroughTransaction()
261
controlweaves.put_weave('inventory', self.inv_weave, transaction)
200
264
for file_id, file_weave in self.text_weaves.items():
201
265
self.pb.update('writing weave', i, len(self.text_weaves))
202
write_a_weave(file_weave, self.base + '/.bzr/weaves/%s.weave' % file_id)
266
weaves.put_weave(file_id, file_weave, transaction)
208
271
def _write_all_revs(self):
209
272
"""Write all revisions out in new form."""
210
shutil.rmtree(self.base + '/.bzr/revision-store')
211
os.mkdir(self.base + '/.bzr/revision-store')
273
transport = self.transport.clone('.bzr')
274
transport.delete_tree('revision-store')
275
transport.mkdir('revision-store')
276
revision_transport = transport.clone('revision-store')
278
revision_store = TextStore(revision_transport,
213
282
for i, rev_id in enumerate(self.converted_revs):
214
283
self.pb.update('write revision', i, len(self.converted_revs))
215
f = file(self.base + '/.bzr/revision-store/%s' % rev_id, 'wb')
217
serializer_v5.write_revision(self.revisions[rev_id], f)
285
serializer_v5.write_revision(self.revisions[rev_id], rev_tmp)
287
revision_store.add(rev_tmp, rev_id)
266
334
"""Convert revision and all referenced objects to new format."""
267
335
rev = self.revisions[rev_id]
268
336
inv = self._load_old_inventory(rev_id)
269
for parent_id in rev.parent_ids[:]:
270
if parent_id in self.absent_revisions:
271
rev.parent_ids.remove(parent_id)
273
note('remove {%s} as parent of {%s}', parent_id, rev_id)
274
self._convert_revision_contents(rev, inv)
275
self._store_new_weave(rev, inv)
276
self._make_rev_ancestry(rev)
337
present_parents = [p for p in rev.parent_ids
338
if p not in self.absent_revisions]
339
self._convert_revision_contents(rev, inv, present_parents)
340
self._store_new_weave(rev, inv, present_parents)
277
341
self.converted_revs.add(rev_id)
280
def _store_new_weave(self, rev, inv):
344
def _store_new_weave(self, rev, inv, present_parents):
281
345
# the XML is now updated with text versions
283
347
for file_id in inv:
287
351
assert hasattr(ie, 'revision'), \
288
352
'no revision on {%s} in {%s}' % \
289
353
(file_id, rev.revision_id)
291
354
new_inv_xml = serializer_v5.write_inventory_to_string(inv)
292
355
new_inv_sha1 = sha_string(new_inv_xml)
293
self.inv_weave.add(rev.revision_id, rev.parent_ids,
356
self.inv_weave.add(rev.revision_id,
294
358
new_inv_xml.splitlines(True),
296
360
rev.inventory_sha1 = new_inv_sha1
299
def _make_rev_ancestry(self, rev):
300
rev_id = rev.revision_id
301
for parent_id in rev.parent_ids:
302
assert parent_id in self.converted_revs
304
lines = list(self.anc_weave.mash_iter(rev.parent_ids))
307
lines.append(rev_id + '\n')
309
parent_ancestries = [self.ancestries[p] for p in rev.parent_ids]
310
new_lines = merge_ancestry_lines(rev_id, parent_ancestries)
311
assert set(lines) == set(new_lines)
312
self.ancestries[rev_id] = new_lines
313
self.anc_weave.add(rev_id, rev.parent_ids, lines)
316
def _convert_revision_contents(self, rev, inv):
362
def _convert_revision_contents(self, rev, inv, present_parents):
317
363
"""Convert all the files within a revision.
319
365
Also upgrade the inventory to refer to the text revision ids."""
320
366
rev_id = rev.revision_id
321
367
mutter('converting texts of revision {%s}',
323
parent_invs = map(self._load_updated_inventory, rev.parent_ids)
369
parent_invs = map(self._load_updated_inventory, present_parents)
324
370
for file_id in inv:
325
371
ie = inv[file_id]
326
self._set_revision(rev, ie, parent_invs)
327
if not ie.has_text():
329
372
self._convert_file_version(rev, ie, parent_invs)
332
def _set_revision(self, rev, ie, parent_invs):
333
"""Set name version for a file.
335
Done in a slightly lazy way: if the file is renamed or in a merge revision
336
it gets a new version, otherwise the same as before.
339
if ie.kind == 'root_directory':
341
if len(parent_invs) != 1:
342
ie.revision = rev.revision_id
344
old_inv = parent_invs[0]
345
if not old_inv.has_id(file_id):
346
ie.revision = rev.revision_id
348
old_ie = old_inv[file_id]
349
if (old_ie.parent_id != ie.parent_id
350
or old_ie.name != ie.name):
351
ie.revision = rev.revision_id
353
ie.revision = old_ie.revision
357
374
def _convert_file_version(self, rev, ie, parent_invs):
358
375
"""Convert one version of one file.
360
377
The file needs to be added into the weave if it is a merge
361
378
of >=2 parents or if it's changed from its parent.
380
if ie.kind == 'root_directory':
363
382
file_id = ie.file_id
364
383
rev_id = rev.revision_id
365
384
w = self.text_weaves.get(file_id)
367
386
w = Weave(file_id)
368
387
self.text_weaves[file_id] = w
370
388
text_changed = False
371
for parent_inv in parent_invs:
372
if parent_inv.has_id(file_id):
373
parent_ie = parent_inv[file_id]
374
old_revision = parent_ie.revision
389
previous_entries = ie.find_previous_heads(parent_invs, w)
390
for old_revision in previous_entries:
375
391
# if this fails, its a ghost ?
376
392
assert old_revision in self.converted_revs
377
if old_revision not in file_parents:
378
file_parents.append(old_revision)
379
if parent_ie.text_sha1 != ie.text_sha1:
381
if len(file_parents) != 1 or text_changed:
382
file_lines = self.branch.text_store[ie.text_id].readlines()
393
self.snapshot_ie(previous_entries, ie, w, rev_id)
395
assert getattr(ie, 'revision', None) is not None
397
def snapshot_ie(self, previous_revisions, ie, w, rev_id):
398
# TODO: convert this logic, which is ~= snapshot to
399
# a call to:. This needs the path figured out. rather than a work_tree
400
# a v4 revision_tree can be given, or something that looks enough like
401
# one to give the file content to the entry if it needs it.
402
# and we need something that looks like a weave store for snapshot to
404
#ie.snapshot(rev, PATH, previous_revisions, REVISION_TREE, InMemoryWeaveStore(self.text_weaves))
405
if len(previous_revisions) == 1:
406
previous_ie = previous_revisions.values()[0]
407
if ie._unchanged(previous_ie):
408
ie.revision = previous_ie.revision
410
parent_indexes = map(w.lookup, previous_revisions)
412
text = self.branch.repository.text_store.get(ie.text_id)
413
file_lines = text.readlines()
383
414
assert sha_strings(file_lines) == ie.text_sha1
384
415
assert sum(map(len, file_lines)) == ie.text_size
385
w.add(rev_id, file_parents, file_lines, ie.text_sha1)
416
w.add(rev_id, parent_indexes, file_lines, ie.text_sha1)
387
417
self.text_count += 1
388
##mutter('import text {%s} of {%s}',
389
## ie.text_id, file_id)
391
##mutter('text of {%s} unchanged from parent', file_id)
392
ie.revision = file_parents[0]
419
w.add(rev_id, parent_indexes, [], None)
421
##mutter('import text {%s} of {%s}',
422
## ie.text_id, file_id)
397
424
def _make_order(self):
398
425
"""Return a suitable order for importing revisions.
added
removed
bzrlib/upgrade.py
