15
13
# You should have received a copy of the GNU General Public License
16
14
# along with this program; if not, write to the Free Software
17
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19
"""Experiment in converting existing bzr branches to weaves."""
21
# To make this properly useful
23
# 1. assign text version ids, and put those text versions into
24
# the inventory as they're converted.
26
# 2. keep track of the previous version of each file, rather than
27
# just using the last one imported
29
# 3. assign entry versions when files are added, renamed or moved.
31
# 4. when merged-in versions are observed, walk down through them
32
# to discover everything, then commit bottom-up
34
# 5. track ancestry as things are merged in, and commit that in each
37
# Perhaps it's best to first walk the whole graph and make a plan for
38
# what should be imported in what order? Need a kind of topological
39
# sort of all revisions. (Or do we, can we just before doing a revision
40
# see that all its parents have either been converted or abandoned?)
43
# Cannot import a revision until all its parents have been
44
# imported. in other words, we can only import revisions whose
45
# parents have all been imported. the first step must be to
46
# import a revision with no parents, of which there must be at
47
# least one. (So perhaps it's useful to store forward pointers
48
# from a list of parents to their children?)
50
# Another (equivalent?) approach is to build up the ordered
51
# ancestry list for the last revision, and walk through that. We
52
# are going to need that.
54
# We don't want to have to recurse all the way back down the list.
56
# Suppose we keep a queue of the revisions able to be processed at
57
# any point. This starts out with all the revisions having no
60
# This seems like a generally useful algorithm...
62
# The current algorithm is dumb (O(n**2)?) but will do the job, and
63
# takes less than a second on the bzr.dev branch.
65
# This currently does a kind of lazy conversion of file texts, where a
66
# new text is written in every version. That's unnecessary but for
67
# the moment saves us having to worry about when files need new
71
# TODO: Don't create a progress bar here, have it passed by the caller.
72
# At least do it from the UI factory.
88
from bzrlib.branch import Branch, find_branch
89
from bzrlib.branch import BZR_BRANCH_FORMAT_5, BZR_BRANCH_FORMAT_6
90
import bzrlib.hashcache as hashcache
91
from bzrlib.revfile import Revfile
92
from bzrlib.weave import Weave
93
from bzrlib.weavefile import read_weave, write_weave
94
from bzrlib.ui import ui_factory
95
from bzrlib.atomicfile import AtomicFile
96
from bzrlib.xml4 import serializer_v4
97
from bzrlib.xml5 import serializer_v5
98
from bzrlib.trace import mutter, note, warning, enable_default_logging
99
from bzrlib.osutils import sha_strings, sha_string
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
"""bzr upgrade logic."""
19
from __future__ import absolute_import
27
from bzrlib.controldir import (
31
from bzrlib.i18n import gettext
32
from bzrlib.remote import RemoteBzrDir
102
35
class Convert(object):
103
def __init__(self, base_dir):
105
self.converted_revs = set()
106
self.absent_revisions = set()
37
def __init__(self, url=None, format=None, control_dir=None):
38
"""Convert a Bazaar control directory to a given format.
40
Either the url or control_dir parameter must be given.
42
:param url: the URL of the control directory or None if the
43
control_dir is explicitly given instead
44
:param format: the format to convert to or None for the default
45
:param control_dir: the control directory or None if it is
46
specified via the URL parameter instead
49
# XXX: Change to cleanup
50
warning_id = 'cross_format_fetch'
51
saved_warning = warning_id in ui.ui_factory.suppressed_warnings
52
if url is None and control_dir is None:
54
"either the url or control_dir parameter must be set.")
55
if control_dir is not None:
56
self.bzrdir = control_dir
58
self.bzrdir = ControlDir.open_unsupported(url)
59
if isinstance(self.bzrdir, RemoteBzrDir):
60
self.bzrdir._ensure_real()
61
self.bzrdir = self.bzrdir._real_bzrdir
62
if self.bzrdir.root_transport.is_readonly():
63
raise errors.UpgradeReadonly
64
self.transport = self.bzrdir.root_transport
65
ui.ui_factory.suppressed_warnings.add(warning_id)
70
ui.ui_factory.suppressed_warnings.remove(warning_id)
112
72
def convert(self):
113
if not self._open_branch():
115
note('starting upgrade of %s', os.path.abspath(self.base))
116
self._backup_control_dir()
117
self.pb = ui_factory.progress_bar()
118
if self.old_format == 4:
119
note('starting upgrade from format 4 to 5')
120
self._convert_to_weaves()
122
if self.old_format == 5:
123
note('starting upgrade from format 5 to 6')
124
self._convert_to_prefixed()
126
cache = hashcache.HashCache(os.path.abspath(self.base))
132
def _convert_to_prefixed(self):
133
from bzrlib.store import hash_prefix
134
for store_name in ["weaves", "revision-store"]:
135
note("adding prefixes to %s" % store_name)
136
store_dir = os.path.join(self.base, ".bzr", store_name)
137
for filename in os.listdir(store_dir):
138
if filename.endswith(".weave") or filename.endswith(".gz"):
139
file_id = os.path.splitext(filename)[0]
142
prefix_dir = os.path.join(store_dir, hash_prefix(file_id))
143
if not os.path.isdir(prefix_dir):
145
os.rename(os.path.join(store_dir, filename),
146
os.path.join(prefix_dir, filename))
147
self._set_new_format(BZR_BRANCH_FORMAT_6)
150
def _convert_to_weaves(self):
151
note('note: upgrade may be faster if all store files are ungzipped first')
152
if not os.path.isdir(self.base + '/.bzr/weaves'):
153
os.mkdir(self.base + '/.bzr/weaves')
154
self.inv_weave = Weave('inventory')
155
# holds in-memory weaves for all files
156
self.text_weaves = {}
157
os.remove(self.branch.controlfilename('branch-format'))
158
self._convert_working_inv()
159
rev_history = self.branch.revision_history()
160
# to_read is a stack holding the revisions we still need to process;
161
# appending to it adds new highest-priority revisions
162
self.known_revisions = set(rev_history)
163
self.to_read = [rev_history[-1]]
165
rev_id = self.to_read.pop()
166
if (rev_id not in self.revisions
167
and rev_id not in self.absent_revisions):
168
self._load_one_rev(rev_id)
170
to_import = self._make_order()
171
for i, rev_id in enumerate(to_import):
172
self.pb.update('converting revision', i, len(to_import))
173
self._convert_one_rev(rev_id)
175
note('upgraded to weaves:')
176
note(' %6d revisions and inventories' % len(self.revisions))
177
note(' %6d revisions not present' % len(self.absent_revisions))
178
note(' %6d texts' % self.text_count)
179
self._write_all_weaves()
180
self._write_all_revs()
181
self._cleanup_spare_files()
182
self._set_new_format(BZR_BRANCH_FORMAT_5)
185
def _open_branch(self):
186
self.branch = Branch.open_downlevel(self.base)
187
self.old_format = self.branch._branch_format
188
if self.old_format == 6:
189
note('this branch is in the most current format')
191
if self.old_format not in (4, 5):
192
raise BzrError("cannot upgrade from branch format %r" %
193
self.branch._branch_format)
197
def _set_new_format(self, format):
198
self.branch.put_controlfile('branch-format', format)
201
def _cleanup_spare_files(self):
202
for n in 'merged-patches', 'pending-merged-patches':
203
p = self.branch.controlfilename(n)
204
if not os.path.exists(p):
206
## assert os.path.getsize(p) == 0
208
shutil.rmtree(self.base + '/.bzr/inventory-store')
209
shutil.rmtree(self.base + '/.bzr/text-store')
212
def _backup_control_dir(self):
213
orig = self.base + '/.bzr'
214
backup = orig + '.backup'
215
note('making backup of tree history')
216
shutil.copytree(orig, backup)
217
note('%s has been backed up to %s', orig, backup)
218
note('if conversion fails, you can move this directory back to .bzr')
219
note('if it succeeds, you can remove this directory if you wish')
222
def _convert_working_inv(self):
224
inv = serializer_v4.read_inventory(branch.controlfile('inventory', 'rb'))
225
new_inv_xml = serializer_v5.write_inventory_to_string(inv)
226
branch.put_controlfile('inventory', new_inv_xml)
230
def _write_all_weaves(self):
231
write_a_weave(self.inv_weave, self.base + '/.bzr/inventory.weave')
234
for file_id, file_weave in self.text_weaves.items():
235
self.pb.update('writing weave', i, len(self.text_weaves))
236
write_a_weave(file_weave, self.base + '/.bzr/weaves/%s.weave' % file_id)
242
def _write_all_revs(self):
243
"""Write all revisions out in new form."""
244
shutil.rmtree(self.base + '/.bzr/revision-store')
245
os.mkdir(self.base + '/.bzr/revision-store')
247
for i, rev_id in enumerate(self.converted_revs):
248
self.pb.update('write revision', i, len(self.converted_revs))
249
f = file(self.base + '/.bzr/revision-store/%s' % rev_id, 'wb')
251
serializer_v5.write_revision(self.revisions[rev_id], f)
258
def _load_one_rev(self, rev_id):
259
"""Load a revision object into memory.
261
Any parents not either loaded or abandoned get queued to be
263
self.pb.update('loading revision',
265
len(self.known_revisions))
266
if not self.branch.revision_store.has_id(rev_id):
268
note('revision {%s} not present in branch; '
269
'will be converted as a ghost',
271
self.absent_revisions.add(rev_id)
273
rev_xml = self.branch.revision_store.get(rev_id).read()
274
rev = serializer_v4.read_revision_from_string(rev_xml)
275
for parent_id in rev.parent_ids:
276
self.known_revisions.add(parent_id)
277
self.to_read.append(parent_id)
278
self.revisions[rev_id] = rev
281
def _load_old_inventory(self, rev_id):
282
assert rev_id not in self.converted_revs
283
old_inv_xml = self.branch.inventory_store.get(rev_id).read()
284
inv = serializer_v4.read_inventory_from_string(old_inv_xml)
285
rev = self.revisions[rev_id]
286
if rev.inventory_sha1:
287
assert rev.inventory_sha1 == sha_string(old_inv_xml), \
288
'inventory sha mismatch for {%s}' % rev_id
292
def _load_updated_inventory(self, rev_id):
293
assert rev_id in self.converted_revs
294
inv_xml = self.inv_weave.get_text(rev_id)
295
inv = serializer_v5.read_inventory_from_string(inv_xml)
299
def _convert_one_rev(self, rev_id):
300
"""Convert revision and all referenced objects to new format."""
301
rev = self.revisions[rev_id]
302
inv = self._load_old_inventory(rev_id)
303
present_parents = [p for p in rev.parent_ids
304
if p not in self.absent_revisions]
305
self._convert_revision_contents(rev, inv, present_parents)
306
self._store_new_weave(rev, inv, present_parents)
307
self.converted_revs.add(rev_id)
310
def _store_new_weave(self, rev, inv, present_parents):
311
# the XML is now updated with text versions
315
if ie.kind == 'root_directory':
317
assert hasattr(ie, 'revision'), \
318
'no revision on {%s} in {%s}' % \
319
(file_id, rev.revision_id)
320
new_inv_xml = serializer_v5.write_inventory_to_string(inv)
321
new_inv_sha1 = sha_string(new_inv_xml)
322
self.inv_weave.add(rev.revision_id,
324
new_inv_xml.splitlines(True),
326
rev.inventory_sha1 = new_inv_sha1
328
def _convert_revision_contents(self, rev, inv, present_parents):
329
"""Convert all the files within a revision.
331
Also upgrade the inventory to refer to the text revision ids."""
332
rev_id = rev.revision_id
333
mutter('converting texts of revision {%s}',
335
parent_invs = map(self._load_updated_inventory, present_parents)
338
self._convert_file_version(rev, ie, parent_invs)
340
def _convert_file_version(self, rev, ie, parent_invs):
341
"""Convert one version of one file.
343
The file needs to be added into the weave if it is a merge
344
of >=2 parents or if it's changed from its parent.
346
if ie.kind == 'root_directory':
349
rev_id = rev.revision_id
350
w = self.text_weaves.get(file_id)
353
self.text_weaves[file_id] = w
355
previous_entries = ie.find_previous_heads(parent_invs, w)
356
for old_revision in previous_entries:
357
# if this fails, its a ghost ?
358
assert old_revision in self.converted_revs
359
self.snapshot_ie(previous_entries, ie, w, rev_id)
361
assert getattr(ie, 'revision', None) is not None
363
def snapshot_ie(self, previous_revisions, ie, w, rev_id):
364
# TODO: convert this logic, which is ~= snapshot to
365
# a call to:. This needs the path figured out. rather than a work_tree
366
# a v4 revision_tree can be given, or something that looks enough like
367
# one to give the file content to the entry if it needs it.
368
# and we need something that looks like a weave store for snapshot to
370
#ie.snapshot(rev, PATH, previous_revisions, REVISION_TREE, InMemoryWeaveStore(self.text_weaves))
371
if len(previous_revisions) == 1:
372
previous_ie = previous_revisions.values()[0]
373
if ie._unchanged(previous_ie):
374
ie.revision = previous_ie.revision
376
parent_indexes = map(w.lookup, previous_revisions)
378
file_lines = self.branch.text_store.get(ie.text_id).readlines()
379
assert sha_strings(file_lines) == ie.text_sha1
380
assert sum(map(len, file_lines)) == ie.text_size
381
w.add(rev_id, parent_indexes, file_lines, ie.text_sha1)
384
w.add(rev_id, parent_indexes, [], None)
386
##mutter('import text {%s} of {%s}',
387
## ie.text_id, file_id)
389
def _make_order(self):
390
"""Return a suitable order for importing revisions.
392
The order must be such that an revision is imported after all
393
its (present) parents.
395
todo = set(self.revisions.keys())
396
done = self.absent_revisions.copy()
399
# scan through looking for a revision whose parents
401
for rev_id in sorted(list(todo)):
402
rev = self.revisions[rev_id]
403
parent_ids = set(rev.parent_ids)
404
if parent_ids.issubset(done):
405
# can take this one now
412
def write_a_weave(weave, filename):
413
inv_wf = file(filename, 'wb')
415
write_weave(weave, inv_wf)
420
def upgrade(base_dir):
74
branch = self.bzrdir.open_branch()
75
if branch.user_url != self.bzrdir.user_url:
76
ui.ui_factory.note(gettext(
77
'This is a checkout. The branch (%s) needs to be upgraded'
78
' separately.') % (urlutils.unescape_for_display(
79
branch.user_url, 'utf-8')))
81
except (errors.NotBranchError, errors.IncompatibleRepositories):
82
# might not be a format we can open without upgrading; see e.g.
83
# https://bugs.launchpad.net/bzr/+bug/253891
85
if self.format is None:
87
rich_root = self.bzrdir.find_repository()._format.rich_root_data
88
except errors.NoRepositoryPresent:
89
rich_root = False # assume no rich roots
91
format_name = "default-rich-root"
93
format_name = "default"
94
format = format_registry.make_bzrdir(format_name)
97
if not self.bzrdir.needs_format_conversion(format):
98
raise errors.UpToDateFormat(self.bzrdir._format)
99
if not self.bzrdir.can_convert_format():
100
raise errors.BzrError(gettext("cannot upgrade from bzrdir format %s") %
102
self.bzrdir.check_conversion_target(format)
103
ui.ui_factory.note(gettext('starting upgrade of %s') %
104
urlutils.unescape_for_display(self.transport.base, 'utf-8'))
106
self.backup_oldpath, self.backup_newpath = self.bzrdir.backup_bzrdir()
107
while self.bzrdir.needs_format_conversion(format):
108
converter = self.bzrdir._format.get_converter(format)
109
self.bzrdir = converter.convert(self.bzrdir, None)
110
ui.ui_factory.note(gettext('finished'))
113
"""Clean-up after a conversion.
115
This removes the backup.bzr directory.
117
transport = self.transport
118
backup_relpath = transport.relpath(self.backup_newpath)
119
child_pb = ui.ui_factory.nested_progress_bar()
120
child_pb.update(gettext('Deleting backup.bzr'))
122
transport.delete_tree(backup_relpath)
127
def upgrade(url, format=None, clean_up=False, dry_run=False):
128
"""Upgrade locations to format.
130
This routine wraps the smart_upgrade() routine with a nicer UI.
131
In particular, it ensures all URLs can be opened before starting
132
and reports a summary at the end if more than one upgrade was attempted.
133
This routine is useful for command line tools. Other bzrlib clients
134
probably ought to use smart_upgrade() instead.
136
:param url: a URL of the locations to upgrade.
137
:param format: the format to convert to or None for the best default
138
:param clean-up: if True, the backup.bzr directory is removed if the
139
upgrade succeeded for a given repo/branch/tree
140
:param dry_run: show what would happen but don't actually do any upgrades
141
:return: the list of exceptions encountered
143
control_dirs = [ControlDir.open_unsupported(url)]
144
attempted, succeeded, exceptions = smart_upgrade(control_dirs,
145
format, clean_up=clean_up, dry_run=dry_run)
146
if len(attempted) > 1:
147
attempted_count = len(attempted)
148
succeeded_count = len(succeeded)
149
failed_count = attempted_count - succeeded_count
151
gettext('\nSUMMARY: {0} upgrades attempted, {1} succeeded,'\
152
' {2} failed').format(
153
attempted_count, succeeded_count, failed_count))
157
def smart_upgrade(control_dirs, format, clean_up=False,
159
"""Convert control directories to a new format intelligently.
161
If the control directory is a shared repository, dependent branches
162
are also converted provided the repository converted successfully.
163
If the conversion of a branch fails, remaining branches are still tried.
165
:param control_dirs: the BzrDirs to upgrade
166
:param format: the format to convert to or None for the best default
167
:param clean_up: if True, the backup.bzr directory is removed if the
168
upgrade succeeded for a given repo/branch/tree
169
:param dry_run: show what would happen but don't actually do any upgrades
170
:return: attempted-control-dirs, succeeded-control-dirs, exceptions
175
for control_dir in control_dirs:
176
attempted, succeeded, exceptions = _smart_upgrade_one(control_dir,
177
format, clean_up=clean_up, dry_run=dry_run)
178
all_attempted.extend(attempted)
179
all_succeeded.extend(succeeded)
180
all_exceptions.extend(exceptions)
181
return all_attempted, all_succeeded, all_exceptions
184
def _smart_upgrade_one(control_dir, format, clean_up=False,
186
"""Convert a control directory to a new format intelligently.
188
See smart_upgrade for parameter details.
190
# If the URL is a shared repository, find the dependent branches
193
repo = control_dir.open_repository()
194
except errors.NoRepositoryPresent:
195
# A branch or checkout using a shared repository higher up
198
# The URL is a repository. If it successfully upgrades,
199
# then upgrade the dependent branches as well.
201
dependents = repo.find_branches(using=True)
204
attempted = [control_dir]
205
succeeded, exceptions = _convert_items([control_dir], format, clean_up,
207
if succeeded and dependents:
208
ui.ui_factory.note(gettext('Found %d dependent branches - upgrading ...')
209
% (len(dependents),))
210
# Convert dependent branches
211
branch_cdirs = [b.bzrdir for b in dependents]
212
successes, problems = _convert_items(branch_cdirs, format, clean_up,
213
dry_run, label="branch")
214
attempted.extend(branch_cdirs)
215
succeeded.extend(successes)
216
exceptions.extend(problems)
219
return attempted, succeeded, exceptions
221
# FIXME: There are several problems below:
222
# - RemoteRepository doesn't support _unsupported (really ?)
223
# - raising AssertionError is rude and may not be necessary
225
# - the only caller uses only the label
226
def _get_object_and_label(control_dir):
227
"""Return the primary object and type label for a control directory.
229
:return: object, label where:
230
* object is a Branch, Repository or WorkingTree and
233
* repository - a repository
234
* tree - a lightweight checkout
238
br = control_dir.open_branch(unsupported=True,
239
ignore_fallbacks=True)
240
except NotImplementedError:
241
# RemoteRepository doesn't support the unsupported parameter
242
br = control_dir.open_branch(ignore_fallbacks=True)
243
except errors.NotBranchError:
248
repo = control_dir.open_repository()
249
except errors.NoRepositoryPresent:
252
return repo, "repository"
254
wt = control_dir.open_workingtree()
255
except (errors.NoWorkingTree, errors.NotLocalUrl):
259
raise AssertionError("unknown type of control directory %s", control_dir)
262
def _convert_items(items, format, clean_up, dry_run, label=None):
263
"""Convert a sequence of control directories to the given format.
265
:param items: the control directories to upgrade
266
:param format: the format to convert to or None for the best default
267
:param clean-up: if True, the backup.bzr directory is removed if the
268
upgrade succeeded for a given repo/branch/tree
269
:param dry_run: show what would happen but don't actually do any upgrades
270
:param label: the label for these items or None to calculate one
271
:return: items successfully upgraded, exceptions
275
child_pb = ui.ui_factory.nested_progress_bar()
276
child_pb.update(gettext('Upgrading bzrdirs'), 0, len(items))
277
for i, control_dir in enumerate(items):
279
location = control_dir.root_transport.base
280
bzr_object, bzr_label = _get_object_and_label(control_dir)
281
type_label = label or bzr_label
282
child_pb.update(gettext("Upgrading %s") % (type_label), i+1, len(items))
283
ui.ui_factory.note(gettext('Upgrading {0} {1} ...').format(type_label,
284
urlutils.unescape_for_display(location, 'utf-8'),))
287
cv = Convert(control_dir=control_dir, format=format)
288
except errors.UpToDateFormat, ex:
289
ui.ui_factory.note(str(ex))
290
succeeded.append(control_dir)
292
except Exception, ex:
293
trace.warning('conversion error: %s' % ex)
294
exceptions.append(ex)
297
# Do any required post processing
298
succeeded.append(control_dir)
301
ui.ui_factory.note(gettext('Removing backup ...'))
304
except Exception, ex:
305
trace.warning(gettext('failed to clean-up {0}: {1}') % (location, ex))
306
exceptions.append(ex)
311
return succeeded, exceptions
added
removed
bzrlib/upgrade.py
