13
13
# You should have received a copy of the GNU General Public License
14
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
17
"""bzr upgrade logic."""
20
from bzrlib.bzrdir import BzrDir, format_registry
21
import bzrlib.errors as errors
22
from bzrlib.remote import RemoteBzrDir
23
import bzrlib.ui as ui
19
# change upgrade from .bzr to create a '.bzr-new', then do a bait and switch.
22
# To make this properly useful
24
# 1. assign text version ids, and put those text versions into
25
# the inventory as they're converted.
27
# 2. keep track of the previous version of each file, rather than
28
# just using the last one imported
30
# 3. assign entry versions when files are added, renamed or moved.
32
# 4. when merged-in versions are observed, walk down through them
33
# to discover everything, then commit bottom-up
35
# 5. track ancestry as things are merged in, and commit that in each
38
# Perhaps it's best to first walk the whole graph and make a plan for
39
# what should be imported in what order? Need a kind of topological
40
# sort of all revisions. (Or do we, can we just before doing a revision
41
# see that all its parents have either been converted or abandoned?)
44
# Cannot import a revision until all its parents have been
45
# imported. in other words, we can only import revisions whose
46
# parents have all been imported. the first step must be to
47
# import a revision with no parents, of which there must be at
48
# least one. (So perhaps it's useful to store forward pointers
49
# from a list of parents to their children?)
51
# Another (equivalent?) approach is to build up the ordered
52
# ancestry list for the last revision, and walk through that. We
53
# are going to need that.
55
# We don't want to have to recurse all the way back down the list.
57
# Suppose we keep a queue of the revisions able to be processed at
58
# any point. This starts out with all the revisions having no
61
# This seems like a generally useful algorithm...
63
# The current algorithm is dumb (O(n**2)?) but will do the job, and
64
# takes less than a second on the bzr.dev branch.
66
# This currently does a kind of lazy conversion of file texts, where a
67
# new text is written in every version. That's unnecessary but for
68
# the moment saves us having to worry about when files need new
71
from cStringIO import StringIO
78
from bzrlib.branch import Branch
79
from bzrlib.branch import BZR_BRANCH_FORMAT_5, BZR_BRANCH_FORMAT_6
80
from bzrlib.branch import BzrBranchFormat, BzrBranchFormat4, BzrBranchFormat5, BzrBranchFormat6
81
from bzrlib.errors import NoSuchFile, UpgradeReadonly
82
import bzrlib.hashcache as hashcache
83
from bzrlib.lockable_files import LockableFiles
84
from bzrlib.osutils import sha_strings, sha_string, pathjoin, abspath
85
from bzrlib.ui import ui_factory
86
from bzrlib.store.text import TextStore
87
from bzrlib.store.weave import WeaveStore
88
from bzrlib.trace import mutter, note, warning
89
from bzrlib.transactions import PassThroughTransaction
90
from bzrlib.transport import get_transport
91
from bzrlib.transport.local import LocalTransport
92
from bzrlib.weave import Weave
93
from bzrlib.weavefile import read_weave, write_weave
94
from bzrlib.xml4 import serializer_v4
95
from bzrlib.xml5 import serializer_v5
26
98
class Convert(object):
28
def __init__(self, url, format=None):
30
self.bzrdir = BzrDir.open_unsupported(url)
31
# XXX: Change to cleanup
32
warning_id = 'cross_format_fetch'
33
saved_warning = warning_id in ui.ui_factory.suppressed_warnings
34
if isinstance(self.bzrdir, RemoteBzrDir):
35
self.bzrdir._ensure_real()
36
self.bzrdir = self.bzrdir._real_bzrdir
37
if self.bzrdir.root_transport.is_readonly():
38
raise errors.UpgradeReadonly
39
self.transport = self.bzrdir.root_transport
40
ui.ui_factory.suppressed_warnings.add(warning_id)
100
def __init__(self, transport):
101
self.base = transport.base
102
self.converted_revs = set()
103
self.absent_revisions = set()
106
self.transport = transport
107
if self.transport.is_readonly():
108
raise UpgradeReadonly
109
self.control_files = LockableFiles(transport.clone(bzrlib.BZRDIR), 'branch-lock')
110
# Lock the branch (soon to be meta dir) to prevent anyone racing with us
111
# This is currently windows incompatible, it will deadlock. When the upgrade
112
# logic becomes format specific, then we can have the format know how to pass this
113
# on. Also note that we probably have an 'upgrade meta' which upgrades the constituent
115
print "FIXME: control files reuse"
116
self.control_files.lock_write()
45
ui.ui_factory.suppressed_warnings.remove(warning_id)
120
self.control_files.unlock()
47
122
def convert(self):
123
if not self._open_branch():
125
note('starting upgrade of %s', self.base)
126
self._backup_control_dir()
127
self.pb = ui_factory.progress_bar()
128
if isinstance(self.old_format, BzrBranchFormat4):
129
note('starting upgrade from format 4 to 5')
130
self._convert_to_weaves()
131
if isinstance(self.old_format, BzrBranchFormat5):
132
note('starting upgrade from format 5 to 6')
133
self._convert_to_prefixed()
134
if isinstance(self.transport, LocalTransport):
135
cache = hashcache.HashCache(abspath(self.base))
140
def _convert_to_prefixed(self):
141
from bzrlib.store import hash_prefix
142
bzr_transport = self.transport.clone('.bzr')
143
bzr_transport.delete('branch-format')
144
for store_name in ["weaves", "revision-store"]:
145
note("adding prefixes to %s" % store_name)
146
store_transport = bzr_transport.clone(store_name)
147
for filename in store_transport.list_dir('.'):
148
if (filename.endswith(".weave") or
149
filename.endswith(".gz") or
150
filename.endswith(".sig")):
151
file_id = os.path.splitext(filename)[0]
154
prefix_dir = hash_prefix(file_id)
155
# FIXME keep track of the dirs made RBC 20060121
157
store_transport.move(filename, prefix_dir + '/' + filename)
158
except NoSuchFile: # catches missing dirs strangely enough
159
store_transport.mkdir(prefix_dir)
160
store_transport.move(filename, prefix_dir + '/' + filename)
161
self._set_new_format(BZR_BRANCH_FORMAT_6)
162
self.branch = BzrBranchFormat6().open(self.transport)
163
self.old_format = self.branch._branch_format
165
def _convert_to_weaves(self):
166
note('note: upgrade may be faster if all store files are ungzipped first')
167
bzr_transport = self.transport.clone('.bzr')
49
branch = self.bzrdir.open_branch()
50
if branch.user_url != self.bzrdir.user_url:
51
ui.ui_factory.note("This is a checkout. The branch (%s) needs to be "
52
"upgraded separately." %
55
except (errors.NotBranchError, errors.IncompatibleRepositories):
56
# might not be a format we can open without upgrading; see e.g.
57
# https://bugs.launchpad.net/bzr/+bug/253891
59
if self.format is None:
170
stat = bzr_transport.stat('weaves')
171
if not S_ISDIR(stat.st_mode):
172
bzr_transport.delete('weaves')
173
bzr_transport.mkdir('weaves')
175
bzr_transport.mkdir('weaves')
176
self.inv_weave = Weave('inventory')
177
# holds in-memory weaves for all files
178
self.text_weaves = {}
179
bzr_transport.delete('branch-format')
180
self._convert_working_inv()
181
rev_history = self.branch.revision_history()
182
# to_read is a stack holding the revisions we still need to process;
183
# appending to it adds new highest-priority revisions
184
self.known_revisions = set(rev_history)
185
self.to_read = rev_history[-1:]
187
rev_id = self.to_read.pop()
188
if (rev_id not in self.revisions
189
and rev_id not in self.absent_revisions):
190
self._load_one_rev(rev_id)
192
to_import = self._make_order()
193
for i, rev_id in enumerate(to_import):
194
self.pb.update('converting revision', i, len(to_import))
195
self._convert_one_rev(rev_id)
197
self._write_all_weaves()
198
self._write_all_revs()
199
note('upgraded to weaves:')
200
note(' %6d revisions and inventories' % len(self.revisions))
201
note(' %6d revisions not present' % len(self.absent_revisions))
202
note(' %6d texts' % self.text_count)
203
self._cleanup_spare_files_after_format4()
204
self._set_new_format(BZR_BRANCH_FORMAT_5)
205
self.branch = BzrBranchFormat5().open(self.transport)
206
self.old_format = self.branch._branch_format
208
def _open_branch(self):
209
self.old_format = BzrBranchFormat.find_format(self.transport)
210
self.branch = self.old_format.open(self.transport)
211
if isinstance(self.old_format, BzrBranchFormat6):
212
note('this branch is in the most current format (%s)', self.old_format)
214
if (not isinstance(self.old_format, BzrBranchFormat4) and
215
not isinstance(self.old_format, BzrBranchFormat5)):
216
raise BzrError("cannot upgrade from branch format %s" %
217
self.branch._branch_format)
220
def _set_new_format(self, format):
221
self.branch.control_files.put_utf8('branch-format', format)
223
def _cleanup_spare_files_after_format4(self):
224
transport = self.transport.clone('.bzr')
225
print "FIXME working tree upgrade foo."
226
for n in 'merged-patches', 'pending-merged-patches':
61
rich_root = self.bzrdir.find_repository()._format.rich_root_data
62
except errors.NoRepositoryPresent:
63
rich_root = False # assume no rich roots
65
format_name = "default-rich-root"
67
format_name = "default"
68
format = format_registry.make_bzrdir(format_name)
71
if not self.bzrdir.needs_format_conversion(format):
72
raise errors.UpToDateFormat(self.bzrdir._format)
73
if not self.bzrdir.can_convert_format():
74
raise errors.BzrError("cannot upgrade from bzrdir format %s" %
76
self.bzrdir.check_conversion_target(format)
77
ui.ui_factory.note('starting upgrade of %s' % self.transport.base)
79
self.bzrdir.backup_bzrdir()
80
while self.bzrdir.needs_format_conversion(format):
81
converter = self.bzrdir._format.get_converter(format)
82
self.bzrdir = converter.convert(self.bzrdir, None)
83
ui.ui_factory.note("finished")
86
def upgrade(url, format=None):
87
"""Upgrade to format, or the default bzrdir format if not supplied."""
228
## assert os.path.getsize(p) == 0
232
transport.delete_tree('inventory-store')
233
transport.delete_tree('text-store')
235
def _backup_control_dir(self):
236
note('making backup of tree history')
237
self.transport.copy_tree('.bzr', '.bzr.backup')
238
note('%s.bzr has been backed up to %s.bzr.backup',
241
note('if conversion fails, you can move this directory back to .bzr')
242
note('if it succeeds, you can remove this directory if you wish')
244
def _convert_working_inv(self):
246
inv = serializer_v4.read_inventory(branch.control_files.get('inventory'))
247
new_inv_xml = serializer_v5.write_inventory_to_string(inv)
248
print "fixme inventory is a working tree change."
249
branch.control_files.put('inventory', new_inv_xml)
251
def _write_all_weaves(self):
252
bzr_transport = self.transport.clone('.bzr')
253
controlweaves = WeaveStore(bzr_transport, prefixed=False)
254
weave_transport = bzr_transport.clone('weaves')
255
weaves = WeaveStore(weave_transport, prefixed=False)
256
transaction = PassThroughTransaction()
258
controlweaves.put_weave('inventory', self.inv_weave, transaction)
261
for file_id, file_weave in self.text_weaves.items():
262
self.pb.update('writing weave', i, len(self.text_weaves))
263
weaves.put_weave(file_id, file_weave, transaction)
268
def _write_all_revs(self):
269
"""Write all revisions out in new form."""
270
transport = self.transport.clone('.bzr')
271
transport.delete_tree('revision-store')
272
transport.mkdir('revision-store')
273
revision_transport = transport.clone('revision-store')
275
revision_store = TextStore(revision_transport,
279
for i, rev_id in enumerate(self.converted_revs):
280
self.pb.update('write revision', i, len(self.converted_revs))
282
serializer_v5.write_revision(self.revisions[rev_id], rev_tmp)
284
revision_store.add(rev_tmp, rev_id)
289
def _load_one_rev(self, rev_id):
290
"""Load a revision object into memory.
292
Any parents not either loaded or abandoned get queued to be
294
self.pb.update('loading revision',
296
len(self.known_revisions))
297
if not self.branch.repository.revision_store.has_id(rev_id):
299
note('revision {%s} not present in branch; '
300
'will be converted as a ghost',
302
self.absent_revisions.add(rev_id)
304
rev_xml = self.branch.repository.revision_store.get(rev_id).read()
305
rev = serializer_v4.read_revision_from_string(rev_xml)
306
for parent_id in rev.parent_ids:
307
self.known_revisions.add(parent_id)
308
self.to_read.append(parent_id)
309
self.revisions[rev_id] = rev
312
def _load_old_inventory(self, rev_id):
313
assert rev_id not in self.converted_revs
314
old_inv_xml = self.branch.repository.inventory_store.get(rev_id).read()
315
inv = serializer_v4.read_inventory_from_string(old_inv_xml)
316
rev = self.revisions[rev_id]
317
if rev.inventory_sha1:
318
assert rev.inventory_sha1 == sha_string(old_inv_xml), \
319
'inventory sha mismatch for {%s}' % rev_id
323
def _load_updated_inventory(self, rev_id):
324
assert rev_id in self.converted_revs
325
inv_xml = self.inv_weave.get_text(rev_id)
326
inv = serializer_v5.read_inventory_from_string(inv_xml)
330
def _convert_one_rev(self, rev_id):
331
"""Convert revision and all referenced objects to new format."""
332
rev = self.revisions[rev_id]
333
inv = self._load_old_inventory(rev_id)
334
present_parents = [p for p in rev.parent_ids
335
if p not in self.absent_revisions]
336
self._convert_revision_contents(rev, inv, present_parents)
337
self._store_new_weave(rev, inv, present_parents)
338
self.converted_revs.add(rev_id)
341
def _store_new_weave(self, rev, inv, present_parents):
342
# the XML is now updated with text versions
346
if ie.kind == 'root_directory':
348
assert hasattr(ie, 'revision'), \
349
'no revision on {%s} in {%s}' % \
350
(file_id, rev.revision_id)
351
new_inv_xml = serializer_v5.write_inventory_to_string(inv)
352
new_inv_sha1 = sha_string(new_inv_xml)
353
self.inv_weave.add(rev.revision_id,
355
new_inv_xml.splitlines(True),
357
rev.inventory_sha1 = new_inv_sha1
359
def _convert_revision_contents(self, rev, inv, present_parents):
360
"""Convert all the files within a revision.
362
Also upgrade the inventory to refer to the text revision ids."""
363
rev_id = rev.revision_id
364
mutter('converting texts of revision {%s}',
366
parent_invs = map(self._load_updated_inventory, present_parents)
369
self._convert_file_version(rev, ie, parent_invs)
371
def _convert_file_version(self, rev, ie, parent_invs):
372
"""Convert one version of one file.
374
The file needs to be added into the weave if it is a merge
375
of >=2 parents or if it's changed from its parent.
377
if ie.kind == 'root_directory':
380
rev_id = rev.revision_id
381
w = self.text_weaves.get(file_id)
384
self.text_weaves[file_id] = w
386
previous_entries = ie.find_previous_heads(parent_invs, w)
387
for old_revision in previous_entries:
388
# if this fails, its a ghost ?
389
assert old_revision in self.converted_revs
390
self.snapshot_ie(previous_entries, ie, w, rev_id)
392
assert getattr(ie, 'revision', None) is not None
394
def snapshot_ie(self, previous_revisions, ie, w, rev_id):
395
# TODO: convert this logic, which is ~= snapshot to
396
# a call to:. This needs the path figured out. rather than a work_tree
397
# a v4 revision_tree can be given, or something that looks enough like
398
# one to give the file content to the entry if it needs it.
399
# and we need something that looks like a weave store for snapshot to
401
#ie.snapshot(rev, PATH, previous_revisions, REVISION_TREE, InMemoryWeaveStore(self.text_weaves))
402
if len(previous_revisions) == 1:
403
previous_ie = previous_revisions.values()[0]
404
if ie._unchanged(previous_ie):
405
ie.revision = previous_ie.revision
407
parent_indexes = map(w.lookup, previous_revisions)
409
text = self.branch.repository.text_store.get(ie.text_id)
410
file_lines = text.readlines()
411
assert sha_strings(file_lines) == ie.text_sha1
412
assert sum(map(len, file_lines)) == ie.text_size
413
w.add(rev_id, parent_indexes, file_lines, ie.text_sha1)
416
w.add(rev_id, parent_indexes, [], None)
418
##mutter('import text {%s} of {%s}',
419
## ie.text_id, file_id)
421
def _make_order(self):
422
"""Return a suitable order for importing revisions.
424
The order must be such that an revision is imported after all
425
its (present) parents.
427
todo = set(self.revisions.keys())
428
done = self.absent_revisions.copy()
431
# scan through looking for a revision whose parents
433
for rev_id in sorted(list(todo)):
434
rev = self.revisions[rev_id]
435
parent_ids = set(rev.parent_ids)
436
if parent_ids.issubset(done):
437
# can take this one now
445
t = get_transport(url)
added
removed
bzrlib/upgrade.py
