13
15
# You should have received a copy of the GNU General Public License
14
16
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
"""bzr upgrade logic."""
25
from bzrlib.bzrdir import (
29
from bzrlib.remote import RemoteBzrDir
17
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19
"""Experiment in converting existing bzr branches to weaves."""
21
# To make this properly useful
23
# 1. assign text version ids, and put those text versions into
24
# the inventory as they're converted.
26
# 2. keep track of the previous version of each file, rather than
27
# just using the last one imported
29
# 3. assign entry versions when files are added, renamed or moved.
31
# 4. when merged-in versions are observed, walk down through them
32
# to discover everything, then commit bottom-up
34
# 5. track ancestry as things are merged in, and commit that in each
37
# Perhaps it's best to first walk the whole graph and make a plan for
38
# what should be imported in what order? Need a kind of topological
39
# sort of all revisions. (Or do we, can we just before doing a revision
40
# see that all its parents have either been converted or abandoned?)
43
# Cannot import a revision until all its parents have been
44
# imported. in other words, we can only import revisions whose
45
# parents have all been imported. the first step must be to
46
# import a revision with no parents, of which there must be at
47
# least one. (So perhaps it's useful to store forward pointers
48
# from a list of parents to their children?)
50
# Another (equivalent?) approach is to build up the ordered
51
# ancestry list for the last revision, and walk through that. We
52
# are going to need that.
54
# We don't want to have to recurse all the way back down the list.
56
# Suppose we keep a queue of the revisions able to be processed at
57
# any point. This starts out with all the revisions having no
60
# This seems like a generally useful algorithm...
62
# The current algorithm is dumb (O(n**2)?) but will do the job, and
63
# takes less than a second on the bzr.dev branch.
65
# This currently does a kind of lazy conversion of file texts, where a
66
# new text is written in every version. That's unnecessary but for
67
# the moment saves us having to worry about when files need new
85
from bzrlib.branch import Branch, find_branch, BZR_BRANCH_FORMAT_5
86
from bzrlib.revfile import Revfile
87
from bzrlib.weave import Weave
88
from bzrlib.weavefile import read_weave, write_weave
89
from bzrlib.progress import ProgressBar
90
from bzrlib.atomicfile import AtomicFile
91
from bzrlib.xml4 import serializer_v4
92
from bzrlib.xml5 import serializer_v5
93
from bzrlib.trace import mutter, note, warning, enable_default_logging
94
from bzrlib.osutils import sha_strings, sha_string
95
from bzrlib.commit import merge_ancestry_lines
32
98
class Convert(object):
34
def __init__(self, url=None, format=None, control_dir=None):
35
"""Convert a Bazaar control directory to a given format.
37
Either the url or control_dir parameter must be given.
39
:param url: the URL of the control directory or None if the
40
control_dir is explicitly given instead
41
:param format: the format to convert to or None for the default
42
:param control_dir: the control directory or None if it is
43
specified via the URL parameter instead
46
# XXX: Change to cleanup
47
warning_id = 'cross_format_fetch'
48
saved_warning = warning_id in ui.ui_factory.suppressed_warnings
49
if url is None and control_dir is None:
51
"either the url or control_dir parameter must be set.")
52
if control_dir is not None:
53
self.bzrdir = control_dir
55
self.bzrdir = BzrDir.open_unsupported(url)
56
if isinstance(self.bzrdir, RemoteBzrDir):
57
self.bzrdir._ensure_real()
58
self.bzrdir = self.bzrdir._real_bzrdir
59
if self.bzrdir.root_transport.is_readonly():
60
raise errors.UpgradeReadonly
61
self.transport = self.bzrdir.root_transport
62
ui.ui_factory.suppressed_warnings.add(warning_id)
67
ui.ui_factory.suppressed_warnings.remove(warning_id)
99
def __init__(self, base_dir):
101
self.converted_revs = set()
102
self.absent_revisions = set()
69
108
def convert(self):
71
branch = self.bzrdir.open_branch()
72
if branch.user_url != self.bzrdir.user_url:
74
'This is a checkout. The branch (%s) needs to be upgraded'
75
' separately.' % (branch.user_url,))
77
except (errors.NotBranchError, errors.IncompatibleRepositories):
78
# might not be a format we can open without upgrading; see e.g.
79
# https://bugs.launchpad.net/bzr/+bug/253891
81
if self.format is None:
83
rich_root = self.bzrdir.find_repository()._format.rich_root_data
84
except errors.NoRepositoryPresent:
85
rich_root = False # assume no rich roots
87
format_name = "default-rich-root"
109
if not self._open_branch():
111
note('starting upgrade of %s', os.path.abspath(self.base))
112
self._backup_control_dir()
113
note('starting upgrade')
114
note('note: upgrade may be faster if all store files are ungzipped first')
115
self.pb = ProgressBar()
116
if not os.path.isdir(self.base + '/.bzr/weaves'):
117
os.mkdir(self.base + '/.bzr/weaves')
118
self.inv_weave = Weave('inventory')
119
self.anc_weave = Weave('ancestry')
121
# holds in-memory weaves for all files
122
self.text_weaves = {}
123
os.remove(self.branch.controlfilename('branch-format'))
124
self._convert_working_inv()
125
rev_history = self.branch.revision_history()
126
# to_read is a stack holding the revisions we still need to process;
127
# appending to it adds new highest-priority revisions
128
self.known_revisions = set(rev_history)
129
self.to_read = [rev_history[-1]]
131
rev_id = self.to_read.pop()
132
if (rev_id not in self.revisions
133
and rev_id not in self.absent_revisions):
134
self._load_one_rev(rev_id)
136
to_import = self._make_order()
137
for i, rev_id in enumerate(to_import):
138
self.pb.update('converting revision', i, len(to_import))
139
self._convert_one_rev(rev_id)
141
note('upgraded to weaves:')
142
note(' %6d revisions and inventories' % len(self.revisions))
143
note(' %6d absent revisions removed' % len(self.absent_revisions))
144
note(' %6d texts' % self.text_count)
145
self._write_all_weaves()
146
self._write_all_revs()
147
self._set_new_format()
148
self._cleanup_spare_files()
151
def _open_branch(self):
152
self.branch = Branch.open_downlevel(self.base)
153
if self.branch._branch_format == 5:
154
note('this branch is already in the most current format')
156
if self.branch._branch_format != 4:
157
raise BzrError("cannot upgrade from branch format %r" %
158
self.branch._branch_format)
162
def _set_new_format(self):
163
f = self.branch.controlfile('branch-format', 'wb')
165
f.write(BZR_BRANCH_FORMAT_5)
170
def _cleanup_spare_files(self):
171
for n in 'merged-patches', 'pending-merged-patches':
172
p = self.branch.controlfilename(n)
173
if not os.path.exists(p):
175
## assert os.path.getsize(p) == 0
177
shutil.rmtree(self.base + '/.bzr/inventory-store')
178
shutil.rmtree(self.base + '/.bzr/text-store')
181
def _backup_control_dir(self):
182
orig = self.base + '/.bzr'
183
backup = orig + '.backup'
184
note('making backup of tree history')
185
shutil.copytree(orig, backup)
186
note('%s has been backed up to %s', orig, backup)
187
note('if conversion fails, you can move this directory back to .bzr')
188
note('if it succeeds, you can remove this directory if you wish')
191
def _convert_working_inv(self):
193
inv = serializer_v4.read_inventory(branch.controlfile('inventory', 'rb'))
194
serializer_v5.write_inventory(inv, branch.controlfile('inventory', 'wb'))
198
def _write_all_weaves(self):
199
write_a_weave(self.inv_weave, self.base + '/.bzr/inventory.weave')
200
write_a_weave(self.anc_weave, self.base + '/.bzr/ancestry.weave')
203
for file_id, file_weave in self.text_weaves.items():
204
self.pb.update('writing weave', i, len(self.text_weaves))
205
write_a_weave(file_weave, self.base + '/.bzr/weaves/%s.weave' % file_id)
211
def _write_all_revs(self):
212
"""Write all revisions out in new form."""
213
shutil.rmtree(self.base + '/.bzr/revision-store')
214
os.mkdir(self.base + '/.bzr/revision-store')
216
for i, rev_id in enumerate(self.converted_revs):
217
self.pb.update('write revision', i, len(self.converted_revs))
218
f = file(self.base + '/.bzr/revision-store/%s' % rev_id, 'wb')
220
serializer_v5.write_revision(self.revisions[rev_id], f)
227
def _load_one_rev(self, rev_id):
228
"""Load a revision object into memory.
230
Any parents not either loaded or abandoned get queued to be
232
self.pb.update('loading revision',
234
len(self.known_revisions))
235
if rev_id not in self.branch.revision_store:
237
note('revision {%s} not present in branch; '
238
'will not be converted',
240
self.absent_revisions.add(rev_id)
242
rev_xml = self.branch.revision_store[rev_id].read()
243
rev = serializer_v4.read_revision_from_string(rev_xml)
244
for parent_id in rev.parent_ids:
245
self.known_revisions.add(parent_id)
246
self.to_read.append(parent_id)
247
self.revisions[rev_id] = rev
250
def _load_old_inventory(self, rev_id):
251
assert rev_id not in self.converted_revs
252
old_inv_xml = self.branch.inventory_store[rev_id].read()
253
inv = serializer_v4.read_inventory_from_string(old_inv_xml)
254
rev = self.revisions[rev_id]
255
if rev.inventory_sha1:
256
assert rev.inventory_sha1 == sha_string(old_inv_xml), \
257
'inventory sha mismatch for {%s}' % rev_id
261
def _load_updated_inventory(self, rev_id):
262
assert rev_id in self.converted_revs
263
inv_xml = self.inv_weave.get_text(rev_id)
264
inv = serializer_v5.read_inventory_from_string(inv_xml)
268
def _convert_one_rev(self, rev_id):
269
"""Convert revision and all referenced objects to new format."""
270
rev = self.revisions[rev_id]
271
inv = self._load_old_inventory(rev_id)
272
for parent_id in rev.parent_ids[:]:
273
if parent_id in self.absent_revisions:
274
rev.parent_ids.remove(parent_id)
276
note('remove {%s} as parent of {%s}', parent_id, rev_id)
277
self._convert_revision_contents(rev, inv)
278
self._store_new_weave(rev, inv)
279
self._make_rev_ancestry(rev)
280
self.converted_revs.add(rev_id)
283
def _store_new_weave(self, rev, inv):
284
# the XML is now updated with text versions
288
if ie.kind == 'root_directory':
290
assert hasattr(ie, 'name_version'), \
291
'no name_version on {%s} in {%s}' % \
292
(file_id, rev.revision_id)
293
if ie.kind == 'file':
294
assert hasattr(ie, 'text_version')
296
new_inv_xml = serializer_v5.write_inventory_to_string(inv)
297
new_inv_sha1 = sha_string(new_inv_xml)
298
self.inv_weave.add(rev.revision_id, rev.parent_ids,
299
new_inv_xml.splitlines(True),
301
rev.inventory_sha1 = new_inv_sha1
304
def _make_rev_ancestry(self, rev):
305
rev_id = rev.revision_id
306
for parent_id in rev.parent_ids:
307
assert parent_id in self.converted_revs
309
lines = list(self.anc_weave.mash_iter(rev.parent_ids))
312
lines.append(rev_id + '\n')
314
parent_ancestries = [self.ancestries[p] for p in rev.parent_ids]
315
new_lines = merge_ancestry_lines(rev_id, parent_ancestries)
316
assert set(lines) == set(new_lines)
317
self.ancestries[rev_id] = new_lines
318
self.anc_weave.add(rev_id, rev.parent_ids, lines)
321
def _convert_revision_contents(self, rev, inv):
322
"""Convert all the files within a revision.
324
Also upgrade the inventory to refer to the text revision ids."""
325
rev_id = rev.revision_id
326
mutter('converting texts of revision {%s}',
328
parent_invs = map(self._load_updated_inventory, rev.parent_ids)
331
self._set_name_version(rev, ie, parent_invs)
332
if ie.kind != 'file':
334
self._convert_file_version(rev, ie, parent_invs)
337
def _set_name_version(self, rev, ie, parent_invs):
338
"""Set name version for a file.
340
Done in a slightly lazy way: if the file is renamed or in a merge revision
341
it gets a new version, otherwise the same as before.
344
if ie.kind == 'root_directory':
346
if len(parent_invs) != 1:
347
ie.name_version = rev.revision_id
349
old_inv = parent_invs[0]
350
if not old_inv.has_id(file_id):
351
ie.name_version = rev.revision_id
89
format_name = "default"
90
format = format_registry.make_bzrdir(format_name)
353
old_ie = old_inv[file_id]
354
if (old_ie.parent_id != ie.parent_id
355
or old_ie.name != ie.name):
356
ie.name_version = rev.revision_id
358
ie.name_version = old_ie.name_version
362
def _convert_file_version(self, rev, ie, parent_invs):
363
"""Convert one version of one file.
365
The file needs to be added into the weave if it is a merge
366
of >=2 parents or if it's changed from its parent.
369
rev_id = rev.revision_id
370
w = self.text_weaves.get(file_id)
373
self.text_weaves[file_id] = w
376
for parent_inv in parent_invs:
377
if parent_inv.has_id(file_id):
378
parent_ie = parent_inv[file_id]
379
old_text_version = parent_ie.text_version
380
assert old_text_version in self.converted_revs
381
if old_text_version not in file_parents:
382
file_parents.append(old_text_version)
383
if parent_ie.text_sha1 != ie.text_sha1:
385
if len(file_parents) != 1 or text_changed:
386
file_lines = self.branch.text_store[ie.text_id].readlines()
387
assert sha_strings(file_lines) == ie.text_sha1
388
assert sum(map(len, file_lines)) == ie.text_size
389
w.add(rev_id, file_parents, file_lines, ie.text_sha1)
390
ie.text_version = rev_id
392
##mutter('import text {%s} of {%s}',
393
## ie.text_id, file_id)
93
if not self.bzrdir.needs_format_conversion(format):
94
raise errors.UpToDateFormat(self.bzrdir._format)
95
if not self.bzrdir.can_convert_format():
96
raise errors.BzrError("cannot upgrade from bzrdir format %s" %
98
self.bzrdir.check_conversion_target(format)
99
ui.ui_factory.note('starting upgrade of %s' % self.transport.base)
101
self.backup_oldpath, self.backup_newpath = self.bzrdir.backup_bzrdir()
102
while self.bzrdir.needs_format_conversion(format):
103
converter = self.bzrdir._format.get_converter(format)
104
self.bzrdir = converter.convert(self.bzrdir, None)
105
ui.ui_factory.note('finished')
108
"""Clean-up after a conversion.
110
This removes the backup.bzr directory.
395
##mutter('text of {%s} unchanged from parent', file_id)
396
ie.text_version = file_parents[0]
401
def _make_order(self):
402
"""Return a suitable order for importing revisions.
404
The order must be such that an revision is imported after all
405
its (present) parents.
112
transport = self.transport
113
backup_relpath = transport.relpath(self.backup_newpath)
114
child_pb = ui.ui_factory.nested_progress_bar()
115
child_pb.update('Deleting backup.bzr')
117
transport.delete_tree(backup_relpath)
122
def upgrade(url, format=None, clean_up=False, dry_run=False):
123
"""Upgrade locations to format.
125
This routine wraps the smart_upgrade() routine with a nicer UI.
126
In particular, it ensures all URLs can be opened before starting
127
and reports a summary at the end if more than one upgrade was attempted.
128
This routine is useful for command line tools. Other bzrlib clients
129
probably ought to use smart_upgrade() instead.
131
:param url: a URL of the locations to upgrade.
132
:param format: the format to convert to or None for the best default
133
:param clean-up: if True, the backup.bzr directory is removed if the
134
upgrade succeeded for a given repo/branch/tree
135
:param dry_run: show what would happen but don't actually do any upgrades
136
:return: the list of exceptions encountered
138
control_dirs = [BzrDir.open_unsupported(url)]
139
attempted, succeeded, exceptions = smart_upgrade(control_dirs,
140
format, clean_up=clean_up, dry_run=dry_run)
141
if len(attempted) > 1:
142
attempted_count = len(attempted)
143
succeeded_count = len(succeeded)
144
failed_count = attempted_count - succeeded_count
146
'\nSUMMARY: %d upgrades attempted, %d succeeded, %d failed'
147
% (attempted_count, succeeded_count, failed_count))
151
def smart_upgrade(control_dirs, format, clean_up=False,
153
"""Convert control directories to a new format intelligently.
155
If the control directory is a shared repository, dependent branches
156
are also converted provided the repository converted successfully.
157
If the conversion of a branch fails, remaining branches are still tried.
159
:param control_dirs: the BzrDirs to upgrade
160
:param format: the format to convert to or None for the best default
161
:param clean_up: if True, the backup.bzr directory is removed if the
162
upgrade succeeded for a given repo/branch/tree
163
:param dry_run: show what would happen but don't actually do any upgrades
164
:return: attempted-control-dirs, succeeded-control-dirs, exceptions
169
for control_dir in control_dirs:
170
attempted, succeeded, exceptions = _smart_upgrade_one(control_dir,
171
format, clean_up=clean_up, dry_run=dry_run)
172
all_attempted.extend(attempted)
173
all_succeeded.extend(succeeded)
174
all_exceptions.extend(exceptions)
175
return all_attempted, all_succeeded, all_exceptions
178
def _smart_upgrade_one(control_dir, format, clean_up=False,
180
"""Convert a control directory to a new format intelligently.
182
See smart_upgrade for parameter details.
184
# If the URL is a shared repository, find the dependent branches
187
repo = control_dir.open_repository()
188
except errors.NoRepositoryPresent:
189
# A branch or checkout using a shared repository higher up
192
# The URL is a repository. If it successfully upgrades,
193
# then upgrade the dependent branches as well.
195
dependents = repo.find_branches(using=True)
198
attempted = [control_dir]
199
succeeded, exceptions = _convert_items([control_dir], format, clean_up,
201
if succeeded and dependents:
202
ui.ui_factory.note('Found %d dependent branches - upgrading ...'
203
% (len(dependents),))
204
# Convert dependent branches
205
branch_cdirs = [b.bzrdir for b in dependents]
206
successes, problems = _convert_items(branch_cdirs, format, clean_up,
207
dry_run, label="branch")
208
attempted.extend(branch_cdirs)
209
succeeded.extend(successes)
210
exceptions.extend(problems)
213
return attempted, succeeded, exceptions
215
# FIXME: There are several problems below:
216
# - RemoteRepository doesn't support _unsupported (really ?)
217
# - raising AssertionError is rude and may not be necessary
219
# - the only caller uses only the label
220
def _get_object_and_label(control_dir):
221
"""Return the primary object and type label for a control directory.
223
:return: object, label where:
224
* object is a Branch, Repository or WorkingTree and
227
* repository - a repository
228
* tree - a lightweight checkout
232
br = control_dir.open_branch(unsupported=True,
233
ignore_fallbacks=True)
234
except NotImplementedError:
235
# RemoteRepository doesn't support the unsupported parameter
236
br = control_dir.open_branch(ignore_fallbacks=True)
237
except errors.NotBranchError:
242
repo = control_dir.open_repository()
243
except errors.NoRepositoryPresent:
246
return repo, "repository"
248
wt = control_dir.open_workingtree()
249
except (errors.NoWorkingTree, errors.NotLocalUrl):
253
raise AssertionError("unknown type of control directory %s", control_dir)
256
def _convert_items(items, format, clean_up, dry_run, label=None):
257
"""Convert a sequence of control directories to the given format.
259
:param items: the control directories to upgrade
260
:param format: the format to convert to or None for the best default
261
:param clean-up: if True, the backup.bzr directory is removed if the
262
upgrade succeeded for a given repo/branch/tree
263
:param dry_run: show what would happen but don't actually do any upgrades
264
:param label: the label for these items or None to calculate one
265
:return: items successfully upgraded, exceptions
269
child_pb = ui.ui_factory.nested_progress_bar()
270
child_pb.update('Upgrading bzrdirs', 0, len(items))
271
for i, control_dir in enumerate(items):
273
location = control_dir.root_transport.base
274
bzr_object, bzr_label = _get_object_and_label(control_dir)
275
type_label = label or bzr_label
276
child_pb.update("Upgrading %s" % (type_label), i+1, len(items))
277
ui.ui_factory.note('Upgrading %s %s ...' % (type_label, location,))
280
cv = Convert(control_dir=control_dir, format=format)
281
except Exception, ex:
282
trace.warning('conversion error: %s' % ex)
283
exceptions.append(ex)
286
# Do any required post processing
287
succeeded.append(control_dir)
290
ui.ui_factory.note('Removing backup ...')
293
except Exception, ex:
294
trace.warning('failed to clean-up %s: %s' % (location, ex))
295
exceptions.append(ex)
300
return succeeded, exceptions
407
todo = set(self.revisions.keys())
408
done = self.absent_revisions.copy()
411
# scan through looking for a revision whose parents
413
for rev_id in sorted(list(todo)):
414
rev = self.revisions[rev_id]
415
parent_ids = set(rev.parent_ids)
416
if parent_ids.issubset(done):
417
# can take this one now
424
def write_a_weave(weave, filename):
425
inv_wf = file(filename, 'wb')
427
write_weave(weave, inv_wf)
432
def upgrade(base_dir):