← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/upgrade.py

[merge] update from bzr.dev

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/upgrade.py
65
65
# the moment saves us having to worry about when files need new
66
66
# versions.
67
67
 
68
 
 
 
68
from cStringIO import StringIO
69
69
import os
70
70
import tempfile
71
71
import sys
72
 
import shutil
 
72
from stat import *
73
73
 
74
 
from bzrlib.branch import Branch, find_branch
 
74
import bzrlib
 
75
from bzrlib.branch import Branch
75
76
from bzrlib.branch import BZR_BRANCH_FORMAT_5, BZR_BRANCH_FORMAT_6
 
77
from bzrlib.branch import BzrBranchFormat, BzrBranchFormat4, BzrBranchFormat5, BzrBranchFormat6
 
78
from bzrlib.errors import NoSuchFile, UpgradeReadonly
76
79
import bzrlib.hashcache as hashcache
 
80
from bzrlib.lockable_files import LockableFiles
 
81
from bzrlib.osutils import sha_strings, sha_string, pathjoin, abspath
 
82
from bzrlib.ui import ui_factory
 
83
from bzrlib.store.text import TextStore
 
84
from bzrlib.store.weave import WeaveStore
 
85
from bzrlib.trace import mutter, note, warning
 
86
from bzrlib.transactions import PassThroughTransaction
 
87
from bzrlib.transport import get_transport
 
88
from bzrlib.transport.local import LocalTransport
77
89
from bzrlib.weave import Weave
78
90
from bzrlib.weavefile import read_weave, write_weave
79
 
from bzrlib.ui import ui_factory
80
 
from bzrlib.atomicfile import AtomicFile
81
91
from bzrlib.xml4 import serializer_v4
82
92
from bzrlib.xml5 import serializer_v5
83
 
from bzrlib.trace import mutter, note, warning
84
 
from bzrlib.osutils import sha_strings, sha_string, pathjoin, abspath
85
93
 
86
94
 
87
95
class Convert(object):
88
 
    def __init__(self, base_dir):
89
 
        self.base = base_dir
 
96
 
 
97
    def __init__(self, transport):
 
98
        self.base = transport.base
90
99
        self.converted_revs = set()
91
100
        self.absent_revisions = set()
92
101
        self.text_count = 0
93
102
        self.revisions = {}
94
 
        self.convert()
95
 
 
 
103
        self.transport = transport
 
104
        if self.transport.is_readonly():
 
105
            raise UpgradeReadonly
 
106
        self.control_files = LockableFiles(transport.clone(bzrlib.BZRDIR), 'branch-lock')
 
107
        # Lock the branch (soon to be meta dir) to prevent anyone racing with us
 
108
        # This is currently windows incompatible, it will deadlock. When the upgrade
 
109
        # logic becomes format specific, then we can have the format know how to pass this
 
110
        # on. Also note that we probably have an 'upgrade meta' which upgrades the constituent
 
111
        # parts.
 
112
        print "FIXME: control files reuse" 
 
113
        self.control_files.lock_write()
 
114
        try:
 
115
            self.convert()
 
116
        finally:
 
117
            self.control_files.unlock()
96
118
 
97
119
    def convert(self):
98
120
        if not self._open_branch():
99
121
            return
100
 
        note('starting upgrade of %s', os.path.abspath(self.base))
 
122
        note('starting upgrade of %s', self.base)
101
123
        self._backup_control_dir()
102
124
        self.pb = ui_factory.progress_bar()
103
 
        if self.old_format == 4:
 
125
        if isinstance(self.old_format, BzrBranchFormat4):
104
126
            note('starting upgrade from format 4 to 5')
105
127
            self._convert_to_weaves()
106
 
            self._open_branch()
107
 
        if self.old_format == 5:
 
128
        if isinstance(self.old_format, BzrBranchFormat5):
108
129
            note('starting upgrade from format 5 to 6')
109
130
            self._convert_to_prefixed()
110
 
            self._open_branch()
111
 
        cache = hashcache.HashCache(abspath(self.base))
112
 
        cache.clear()
113
 
        cache.write()
 
131
        if isinstance(self.transport, LocalTransport):
 
132
            cache = hashcache.HashCache(abspath(self.base))
 
133
            cache.clear()
 
134
            cache.write()
114
135
        note("finished")
115
136
 
116
 
 
117
137
    def _convert_to_prefixed(self):
118
138
        from bzrlib.store import hash_prefix
 
139
        bzr_transport = self.transport.clone('.bzr')
 
140
        bzr_transport.delete('branch-format')
119
141
        for store_name in ["weaves", "revision-store"]:
120
142
            note("adding prefixes to %s" % store_name) 
121
 
            store_dir = pathjoin(self.base, ".bzr", store_name)
122
 
            for filename in os.listdir(store_dir):
123
 
                if filename.endswith(".weave") or filename.endswith(".gz"):
 
143
            store_transport = bzr_transport.clone(store_name)
 
144
            for filename in store_transport.list_dir('.'):
 
145
                if (filename.endswith(".weave") or
 
146
                    filename.endswith(".gz") or
 
147
                    filename.endswith(".sig")):
124
148
                    file_id = os.path.splitext(filename)[0]
125
149
                else:
126
150
                    file_id = filename
127
 
                prefix_dir = pathjoin(store_dir, hash_prefix(file_id))
128
 
                if not os.path.isdir(prefix_dir):
129
 
                    os.mkdir(prefix_dir)
130
 
                os.rename(pathjoin(store_dir, filename),
131
 
                          pathjoin(prefix_dir, filename))
 
151
                prefix_dir = hash_prefix(file_id)
 
152
                # FIXME keep track of the dirs made RBC 20060121
 
153
                try:
 
154
                    store_transport.move(filename, prefix_dir + '/' + filename)
 
155
                except NoSuchFile: # catches missing dirs strangely enough
 
156
                    store_transport.mkdir(prefix_dir)
 
157
                    store_transport.move(filename, prefix_dir + '/' + filename)
132
158
        self._set_new_format(BZR_BRANCH_FORMAT_6)
133
 
 
 
159
        self.branch = BzrBranchFormat6().open(self.transport)
 
160
        self.old_format = self.branch._branch_format
134
161
 
135
162
    def _convert_to_weaves(self):
136
163
        note('note: upgrade may be faster if all store files are ungzipped first')
137
 
        if not os.path.isdir(self.base + '/.bzr/weaves'):
138
 
            os.mkdir(self.base + '/.bzr/weaves')
 
164
        bzr_transport = self.transport.clone('.bzr')
 
165
        try:
 
166
            # TODO permissions
 
167
            stat = bzr_transport.stat('weaves')
 
168
            if not S_ISDIR(stat.st_mode):
 
169
                bzr_transport.delete('weaves')
 
170
                bzr_transport.mkdir('weaves')
 
171
        except NoSuchFile:
 
172
            bzr_transport.mkdir('weaves')
139
173
        self.inv_weave = Weave('inventory')
140
174
        # holds in-memory weaves for all files
141
175
        self.text_weaves = {}
142
 
        os.remove(self.branch.controlfilename('branch-format'))
 
176
        bzr_transport.delete('branch-format')
143
177
        self._convert_working_inv()
144
178
        rev_history = self.branch.revision_history()
145
179
        # to_read is a stack holding the revisions we still need to process;
157
191
            self.pb.update('converting revision', i, len(to_import))
158
192
            self._convert_one_rev(rev_id)
159
193
        self.pb.clear()
 
194
        self._write_all_weaves()
 
195
        self._write_all_revs()
160
196
        note('upgraded to weaves:')
161
197
        note('  %6d revisions and inventories' % len(self.revisions))
162
198
        note('  %6d revisions not present' % len(self.absent_revisions))
163
199
        note('  %6d texts' % self.text_count)
164
 
        self._write_all_weaves()
165
 
        self._write_all_revs()
166
 
        self._cleanup_spare_files()
 
200
        self._cleanup_spare_files_after_format4()
167
201
        self._set_new_format(BZR_BRANCH_FORMAT_5)
168
 
 
 
202
        self.branch = BzrBranchFormat5().open(self.transport)
 
203
        self.old_format = self.branch._branch_format
169
204
 
170
205
    def _open_branch(self):
171
 
        self.branch = Branch.open_downlevel(self.base)
172
 
        self.old_format = self.branch._branch_format
173
 
        if self.old_format == 6:
174
 
            note('this branch is in the most current format')
 
206
        self.old_format = BzrBranchFormat.find_format(self.transport)
 
207
        self.branch = self.old_format.open(self.transport)
 
208
        if isinstance(self.old_format, BzrBranchFormat6):
 
209
            note('this branch is in the most current format (%s)', self.old_format)
175
210
            return False
176
 
        if self.old_format not in (4, 5):
177
 
            raise BzrError("cannot upgrade from branch format %r" %
 
211
        if (not isinstance(self.old_format, BzrBranchFormat4) and
 
212
            not isinstance(self.old_format, BzrBranchFormat5)):
 
213
            raise BzrError("cannot upgrade from branch format %s" %
178
214
                           self.branch._branch_format)
179
215
        return True
180
216
 
181
 
 
182
217
    def _set_new_format(self, format):
183
 
        self.branch.put_controlfile('branch-format', format)
184
 
 
185
 
 
186
 
    def _cleanup_spare_files(self):
 
218
        self.branch.control_files.put_utf8('branch-format', format)
 
219
 
 
220
    def _cleanup_spare_files_after_format4(self):
 
221
        transport = self.transport.clone('.bzr')
 
222
        print "FIXME working tree upgrade foo."
187
223
        for n in 'merged-patches', 'pending-merged-patches':
188
 
            p = self.branch.controlfilename(n)
189
 
            if not os.path.exists(p):
190
 
                continue
191
 
            ## assert os.path.getsize(p) == 0
192
 
            os.remove(p)
193
 
        shutil.rmtree(self.base + '/.bzr/inventory-store')
194
 
        shutil.rmtree(self.base + '/.bzr/text-store')
195
 
 
 
224
            try:
 
225
                ## assert os.path.getsize(p) == 0
 
226
                transport.delete(n)
 
227
            except NoSuchFile:
 
228
                pass
 
229
        transport.delete_tree('inventory-store')
 
230
        transport.delete_tree('text-store')
196
231
 
197
232
    def _backup_control_dir(self):
198
 
        orig = self.base + '/.bzr'
199
 
        backup = orig + '.backup'
200
233
        note('making backup of tree history')
201
 
        shutil.copytree(orig, backup)
202
 
        note('%s has been backed up to %s', orig, backup)
 
234
        self.transport.copy_tree('.bzr', '.bzr.backup')
 
235
        note('%s.bzr has been backed up to %s.bzr.backup',
 
236
             self.transport.base,
 
237
             self.transport.base)
203
238
        note('if conversion fails, you can move this directory back to .bzr')
204
239
        note('if it succeeds, you can remove this directory if you wish')
205
240
 
206
 
 
207
241
    def _convert_working_inv(self):
208
242
        branch = self.branch
209
 
        inv = serializer_v4.read_inventory(branch.controlfile('inventory', 'rb'))
 
243
        inv = serializer_v4.read_inventory(branch.control_files.get('inventory'))
210
244
        new_inv_xml = serializer_v5.write_inventory_to_string(inv)
211
 
        branch.put_controlfile('inventory', new_inv_xml)
212
 
 
213
 
 
 
245
        print "fixme inventory is a working tree change."
 
246
        branch.control_files.put('inventory', new_inv_xml)
214
247
 
215
248
    def _write_all_weaves(self):
216
 
        write_a_weave(self.inv_weave, self.base + '/.bzr/inventory.weave')
 
249
        bzr_transport = self.transport.clone('.bzr')
 
250
        controlweaves = WeaveStore(bzr_transport, prefixed=False)
 
251
        weave_transport = bzr_transport.clone('weaves')
 
252
        weaves = WeaveStore(weave_transport, prefixed=False)
 
253
        transaction = PassThroughTransaction()
 
254
 
 
255
        controlweaves.put_weave('inventory', self.inv_weave, transaction)
217
256
        i = 0
218
257
        try:
219
258
            for file_id, file_weave in self.text_weaves.items():
220
259
                self.pb.update('writing weave', i, len(self.text_weaves))
221
 
                write_a_weave(file_weave, self.base + '/.bzr/weaves/%s.weave' % file_id)
 
260
                weaves.put_weave(file_id, file_weave, transaction)
222
261
                i += 1
223
262
        finally:
224
263
            self.pb.clear()
225
264
 
226
 
 
227
265
    def _write_all_revs(self):
228
266
        """Write all revisions out in new form."""
229
 
        shutil.rmtree(self.base + '/.bzr/revision-store')
230
 
        os.mkdir(self.base + '/.bzr/revision-store')
 
267
        transport = self.transport.clone('.bzr')
 
268
        transport.delete_tree('revision-store')
 
269
        transport.mkdir('revision-store')
 
270
        revision_transport = transport.clone('revision-store')
 
271
        # TODO permissions
 
272
        revision_store = TextStore(revision_transport,
 
273
                                   prefixed=False,
 
274
                                   compressed=True)
231
275
        try:
232
276
            for i, rev_id in enumerate(self.converted_revs):
233
277
                self.pb.update('write revision', i, len(self.converted_revs))
234
 
                f = file(self.base + '/.bzr/revision-store/%s' % rev_id, 'wb')
235
 
                try:
236
 
                    serializer_v5.write_revision(self.revisions[rev_id], f)
237
 
                finally:
238
 
                    f.close()
 
278
                rev_tmp = StringIO()
 
279
                serializer_v5.write_revision(self.revisions[rev_id], rev_tmp)
 
280
                rev_tmp.seek(0)
 
281
                revision_store.add(rev_tmp, rev_id)
239
282
        finally:
240
283
            self.pb.clear()
241
284
 
248
291
        self.pb.update('loading revision',
249
292
                       len(self.revisions),
250
293
                       len(self.known_revisions))
251
 
        if not self.branch.revision_store.has_id(rev_id):
 
294
        if not self.branch.repository.revision_store.has_id(rev_id):
252
295
            self.pb.clear()
253
296
            note('revision {%s} not present in branch; '
254
297
                 'will be converted as a ghost',
255
298
                 rev_id)
256
299
            self.absent_revisions.add(rev_id)
257
300
        else:
258
 
            rev_xml = self.branch.revision_store.get(rev_id).read()
 
301
            rev_xml = self.branch.repository.revision_store.get(rev_id).read()
259
302
            rev = serializer_v4.read_revision_from_string(rev_xml)
260
303
            for parent_id in rev.parent_ids:
261
304
                self.known_revisions.add(parent_id)
265
308
 
266
309
    def _load_old_inventory(self, rev_id):
267
310
        assert rev_id not in self.converted_revs
268
 
        old_inv_xml = self.branch.inventory_store.get(rev_id).read()
 
311
        old_inv_xml = self.branch.repository.inventory_store.get(rev_id).read()
269
312
        inv = serializer_v4.read_inventory_from_string(old_inv_xml)
270
313
        rev = self.revisions[rev_id]
271
314
        if rev.inventory_sha1:
360
403
                return
361
404
        parent_indexes = map(w.lookup, previous_revisions)
362
405
        if ie.has_text():
363
 
            file_lines = self.branch.text_store.get(ie.text_id).readlines()
 
406
            text = self.branch.repository.text_store.get(ie.text_id)
 
407
            file_lines = text.readlines()
364
408
            assert sha_strings(file_lines) == ie.text_sha1
365
409
            assert sum(map(len, file_lines)) == ie.text_size
366
410
            w.add(rev_id, parent_indexes, file_lines, ie.text_sha1)
394
438
        return o
395
439
 
396
440
 
397
 
def write_a_weave(weave, filename):
398
 
    inv_wf = file(filename, 'wb')
399
 
    try:
400
 
        write_weave(weave, inv_wf)
401
 
    finally:
402
 
        inv_wf.close()
403
 
 
404
 
 
405
 
def upgrade(base_dir):
406
 
    Convert(base_dir)
 
441
def upgrade(url):
 
442
    t = get_transport(url)
 
443
    Convert(t)