← Back to branch summary

~abentley/bzrtools/bzrtools.dev

« back to all changes in this revision

Viewing changes to upstream_import.py

  • Committer: Aaron Bentley
  • Date: 2012-01-20 02:07:15 UTC
  • Revision ID: aaron@aaronbentley.com-20120120020715-ar6jbqnrjcuebggz
Tags: release-2.5
Update for 2.5 release.

Show diffs side-by-side

added added

removed removed

Lines of Context:
upstream_import.py
1
1
"""Import upstream source into a branch"""
2
2
 
3
 
from bz2 import BZ2File
4
3
import errno
5
4
import os
6
 
from shutil import rmtree
 
5
import re
7
6
from StringIO import StringIO
 
7
import stat
8
8
import tarfile
9
 
from unittest import makeSuite
 
9
import zipfile
10
10
 
 
11
from bzrlib import generate_ids
11
12
from bzrlib.bzrdir import BzrDir
12
13
from bzrlib.errors import NoSuchFile, BzrCommandError, NotBranchError
13
 
from bzrlib.osutils import pathjoin, isdir, file_iterator
14
 
from bzrlib.tests import TestCaseInTempDir
 
14
from bzrlib.osutils import (pathjoin, isdir, file_iterator, basename,
 
15
                            file_kind, splitpath)
15
16
from bzrlib.trace import warning
16
17
from bzrlib.transform import TreeTransform, resolve_conflicts, cook_conflicts
17
18
from bzrlib.workingtree import WorkingTree
18
 
 
19
 
 
20
 
def top_directory(path):
 
19
from bzrlib.plugins.bzrtools.bzrtools import open_from_url
 
20
from bzrlib.plugins.bzrtools import errors
 
21
 
 
22
class ZipFileWrapper(object):
 
23
 
 
24
    def __init__(self, fileobj, mode):
 
25
        self.zipfile = zipfile.ZipFile(fileobj, mode)
 
26
 
 
27
    def getmembers(self):
 
28
        for info in self.zipfile.infolist():
 
29
            yield ZipInfoWrapper(self.zipfile, info)
 
30
 
 
31
    def extractfile(self, infowrapper):
 
32
        return StringIO(self.zipfile.read(infowrapper.name))
 
33
 
 
34
    def add(self, filename):
 
35
        if isdir(filename):
 
36
            self.zipfile.writestr(filename+'/', '')
 
37
        else:
 
38
            self.zipfile.write(filename)
 
39
 
 
40
    def close(self):
 
41
        self.zipfile.close()
 
42
 
 
43
 
 
44
class ZipInfoWrapper(object):
 
45
 
 
46
    def __init__(self, zipfile, info):
 
47
        self.info = info
 
48
        self.type = None
 
49
        self.name = info.filename
 
50
        self.zipfile = zipfile
 
51
        self.mode = 0666
 
52
 
 
53
    def isdir(self):
 
54
        # Really? Eeeew!
 
55
        return bool(self.name.endswith('/'))
 
56
 
 
57
    def isreg(self):
 
58
        # Really? Eeeew!
 
59
        return not self.isdir()
 
60
 
 
61
 
 
62
class DirWrapper(object):
 
63
    def __init__(self, fileobj, mode='r'):
 
64
        assert mode == 'r', mode
 
65
        self.root = os.path.realpath(fileobj.read())
 
66
 
 
67
    def __repr__(self):
 
68
        return 'DirWrapper(%r)' % self.root
 
69
 
 
70
    def getmembers(self, subdir=None):
 
71
        if subdir is not None:
 
72
            mydir = pathjoin(self.root, subdir)
 
73
        else:
 
74
            mydir = self.root
 
75
        for child in os.listdir(mydir):
 
76
            if subdir is not None:
 
77
                child = pathjoin(subdir, child)
 
78
            fi = FileInfo(self.root, child)
 
79
            yield fi
 
80
            if fi.isdir():
 
81
                for v in self.getmembers(child):
 
82
                    yield v
 
83
 
 
84
    def extractfile(self, member):
 
85
        return open(member.fullpath)
 
86
 
 
87
 
 
88
class FileInfo(object):
 
89
 
 
90
    def __init__(self, root, filepath):
 
91
        self.fullpath = pathjoin(root, filepath)
 
92
        self.root = root
 
93
        if filepath != '':
 
94
            self.name = pathjoin(basename(root), filepath)
 
95
        else:
 
96
            print 'root %r' % root
 
97
            self.name = basename(root)
 
98
        self.type = None
 
99
        stat = os.lstat(self.fullpath)
 
100
        self.mode = stat.st_mode
 
101
        if self.isdir():
 
102
            self.name += '/'
 
103
 
 
104
    def __repr__(self):
 
105
        return 'FileInfo(%r)' % self.name
 
106
 
 
107
    def isreg(self):
 
108
        return stat.S_ISREG(self.mode)
 
109
 
 
110
    def isdir(self):
 
111
        return stat.S_ISDIR(self.mode)
 
112
 
 
113
    def issym(self):
 
114
        if stat.S_ISLNK(self.mode):
 
115
            self.linkname = os.readlink(self.fullpath)
 
116
            return True
 
117
        else:
 
118
            return False
 
119
 
 
120
 
 
121
def top_path(path):
21
122
    """Return the top directory given in a path."""
22
 
    dirname = os.path.dirname(path)
23
 
    last_dirname = dirname
24
 
    while True:
25
 
        dirname = os.path.dirname(dirname)
26
 
        if dirname == '' or dirname == last_dirname:
27
 
            return last_dirname
28
 
        last_dirname = dirname
 
123
    components = splitpath(path)
 
124
    if len(components) > 0:
 
125
        return components[0]
 
126
    else:
 
127
        return ''
29
128
 
30
129
 
31
130
def common_directory(names):
32
131
    """Determine a single directory prefix from a list of names"""
33
132
    possible_prefix = None
34
133
    for name in names:
35
 
        name_top = top_directory(name)
 
134
        name_top = top_path(name)
 
135
        if name_top == '':
 
136
            return None
36
137
        if possible_prefix is None:
37
138
            possible_prefix = name_top
38
139
        else:
63
164
            yield member.name
64
165
 
65
166
 
 
167
def should_ignore(relative_path):
 
168
    return top_path(relative_path) == '.bzr'
 
169
 
 
170
 
66
171
def import_tar(tree, tar_input):
67
172
    """Replace the contents of a working directory with tarfile contents.
68
173
    The tarfile may be a gzipped stream.  File ids will be updated.
69
174
    """
70
175
    tar_file = tarfile.open('lala', 'r', tar_input)
71
 
    prefix = common_directory(names_of_files(tar_file))
 
176
    import_archive(tree, tar_file)
 
177
 
 
178
def import_zip(tree, zip_input):
 
179
    zip_file = ZipFileWrapper(zip_input, 'r')
 
180
    import_archive(tree, zip_file)
 
181
 
 
182
def import_dir(tree, dir_input):
 
183
    dir_file = DirWrapper(dir_input)
 
184
    import_archive(tree, dir_file)
 
185
 
 
186
 
 
187
def import_archive(tree, archive_file):
72
188
    tt = TreeTransform(tree)
73
 
 
 
189
    try:
 
190
        import_archive_to_transform(tree, archive_file, tt)
 
191
        tt.apply()
 
192
    finally:
 
193
        tt.finalize()
 
194
 
 
195
 
 
196
def import_archive_to_transform(tree, archive_file, tt):
 
197
    prefix = common_directory(names_of_files(archive_file))
74
198
    removed = set()
75
199
    for path, entry in tree.inventory.iter_entries():
76
200
        if entry.parent_id is None:
79
203
        tt.delete_contents(trans_id)
80
204
        removed.add(path)
81
205
 
82
 
    added = set() 
 
206
    added = set()
83
207
    implied_parents = set()
84
208
    seen = set()
85
 
    for member in tar_file.getmembers():
 
209
    for member in archive_file.getmembers():
86
210
        if member.type == 'g':
87
211
            # type 'g' is a header
88
212
            continue
89
 
        relative_path = member.name 
 
213
        # Inverse functionality in bzr uses utf-8.  We could also
 
214
        # interpret relative to fs encoding, which would match native
 
215
        # behaviour better.
 
216
        relative_path = member.name.decode('utf-8')
90
217
        if prefix is not None:
91
218
            relative_path = relative_path[len(prefix)+1:]
 
219
            relative_path = relative_path.rstrip('/')
92
220
        if relative_path == '':
93
221
            continue
 
222
        if should_ignore(relative_path):
 
223
            continue
94
224
        add_implied_parents(implied_parents, relative_path)
95
225
        trans_id = tt.trans_id_tree_path(relative_path)
96
226
        added.add(relative_path.rstrip('/'))
97
227
        path = tree.abspath(relative_path)
98
228
        if member.name in seen:
 
229
            if tt.final_kind(trans_id) == 'file':
 
230
                tt.set_executability(None, trans_id)
99
231
            tt.cancel_creation(trans_id)
100
232
        seen.add(member.name)
101
233
        if member.isreg():
102
 
            tt.create_file(file_iterator(tar_file.extractfile(member)), 
 
234
            tt.create_file(file_iterator(archive_file.extractfile(member)),
103
235
                           trans_id)
 
236
            executable = (member.mode & 0111) != 0
 
237
            tt.set_executability(executable, trans_id)
104
238
        elif member.isdir():
105
239
            do_directory(tt, trans_id, tree, relative_path, path)
106
240
        elif member.issym():
107
241
            tt.create_symlink(member.linkname, trans_id)
 
242
        else:
 
243
            continue
 
244
        if tt.tree_file_id(trans_id) is None:
 
245
            name = basename(member.name.rstrip('/'))
 
246
            file_id = generate_ids.gen_file_id(name)
 
247
            tt.version_file(file_id, trans_id)
108
248
 
109
249
    for relative_path in implied_parents.difference(added):
110
250
        if relative_path == "":
112
252
        trans_id = tt.trans_id_tree_path(relative_path)
113
253
        path = tree.abspath(relative_path)
114
254
        do_directory(tt, trans_id, tree, relative_path, path)
 
255
        if tt.tree_file_id(trans_id) is None:
 
256
            tt.version_file(trans_id, trans_id)
115
257
        added.add(relative_path)
116
258
 
 
259
    for path in removed.difference(added):
 
260
        tt.unversion_file(tt.trans_id_tree_path(path))
 
261
 
117
262
    for conflict in cook_conflicts(resolve_conflicts(tt), tt):
118
263
        warning(conflict)
119
 
    tt.apply()
120
 
    update_ids(tree, added, removed)
121
 
 
122
 
 
123
 
def update_ids(tree, added, removed):
124
 
    """Make sure that all present files files have file_ids.
125
 
    """
126
 
    # XXX detect renames
127
 
    new = added.difference(removed)
128
 
    deleted = removed.difference(added)
129
 
    tree.add(sorted(new))
130
 
    tree.remove(sorted(deleted, reverse=True))
131
264
 
132
265
 
133
266
def do_import(source, tree_directory=None):
147
280
        if tree.changes_from(tree.basis_tree()).has_changed():
148
281
            raise BzrCommandError("Working tree has uncommitted changes.")
149
282
 
150
 
        if (source.endswith('.tar') or source.endswith('.tar.gz') or 
151
 
            source.endswith('.tar.bz2')) or source.endswith('.tgz'):
152
 
            try:
153
 
                if source.endswith('.bz2'):
154
 
                    tar_input = BZ2File(source, 'r')
155
 
                    tar_input = StringIO(tar_input.read())
156
 
                else:
157
 
                    tar_input = file(source, 'rb')
158
 
            except IOError, e:
159
 
                if e.errno == errno.ENOENT:
160
 
                    raise NoSuchFile(source)
161
 
            try:
162
 
                import_tar(tree, tar_input)
163
 
            finally:
164
 
                tar_input.close()
 
283
        try:
 
284
            archive, external_compressor = get_archive_type(source)
 
285
        except errors.NotArchiveType:
 
286
            if file_kind(source) == 'directory':
 
287
                s = StringIO(source)
 
288
                s.seek(0)
 
289
                import_dir(tree, s)
 
290
            else:
 
291
                raise BzrCommandError('Unhandled import source')
 
292
        else:
 
293
            if archive == 'zip':
 
294
                import_zip(tree, open_from_url(source))
 
295
            elif archive == 'tar':
 
296
                try:
 
297
                    tar_input = open_from_url(source)
 
298
                    if external_compressor == 'bz2':
 
299
                        import bz2
 
300
                        tar_input = StringIO(bz2.decompress(tar_input.read()))
 
301
                    elif external_compressor == 'lzma':
 
302
                        import lzma
 
303
                        tar_input = StringIO(lzma.decompress(tar_input.read()))
 
304
                except IOError, e:
 
305
                    if e.errno == errno.ENOENT:
 
306
                        raise NoSuchFile(source)
 
307
                try:
 
308
                    import_tar(tree, tar_input)
 
309
                finally:
 
310
                    tar_input.close()
165
311
    finally:
166
312
        tree.unlock()
167
313
 
168
 
class TestImport(TestCaseInTempDir):
169
 
 
170
 
    def make_tar(self, mode='w'):
171
 
        result = StringIO()
172
 
        tar_file = tarfile.open('project-0.1.tar', mode, result)
173
 
        os.mkdir('project-0.1')
174
 
        tar_file.add('project-0.1')
175
 
        os.mkdir('project-0.1/junk')
176
 
        tar_file.add('project-0.1/junk')
177
 
        
178
 
        f = file('project-0.1/README', 'wb')
179
 
        f.write('What?')
180
 
        f.close()
181
 
        tar_file.add('project-0.1/README')
182
 
 
183
 
        f = file('project-0.1/FEEDME', 'wb')
184
 
        f.write('Hungry!!')
185
 
        f.close()
186
 
        tar_file.add('project-0.1/FEEDME')
187
 
 
188
 
        tar_file.close()
189
 
        rmtree('project-0.1')
190
 
        result.seek(0)
191
 
        return result
192
 
 
193
 
    def make_tar2(self):
194
 
        result = StringIO()
195
 
        tar_file = tarfile.open('project-0.2.tar', 'w', result)
196
 
        os.mkdir('project-0.2')
197
 
        tar_file.add('project-0.2')
198
 
        
199
 
        os.mkdir('project-0.2/junk')
200
 
        tar_file.add('project-0.2/junk')
201
 
 
202
 
        f = file('project-0.2/README', 'wb')
203
 
        f.write('Now?')
204
 
        f.close()
205
 
        tar_file.add('project-0.2/README')
206
 
        tar_file.close()
207
 
 
208
 
        tar_file = tarfile.open('project-0.2.tar', 'a', result)
209
 
        tar_file.add('project-0.2/README')
210
 
 
211
 
        rmtree('project-0.2')
212
 
        return result
213
 
 
214
 
    def make_messed_tar(self):
215
 
        result = StringIO()
216
 
        tar_file = tarfile.open('project-0.1.tar', 'w', result)
217
 
        os.mkdir('project-0.1')
218
 
        tar_file.add('project-0.1')
219
 
 
220
 
        os.mkdir('project-0.2')
221
 
        tar_file.add('project-0.2')
222
 
        
223
 
        f = file('project-0.1/README', 'wb')
224
 
        f.write('What?')
225
 
        f.close()
226
 
        tar_file.add('project-0.1/README')
227
 
        tar_file.close()
228
 
        rmtree('project-0.1')
229
 
        result.seek(0)
230
 
        return result
231
 
 
232
 
    def test_top_directory(self):
233
 
        self.assertEqual(top_directory('ab/b/c'), 'ab')
234
 
        self.assertEqual(top_directory('/etc'), '/')
235
 
 
236
 
    def test_common_directory(self):
237
 
        self.assertEqual(common_directory(['ab/c/d', 'ab/c/e']), 'ab')
238
 
        self.assertIs(common_directory(['ab/c/d', 'ac/c/e']), None)
239
 
 
240
 
    def test_untar(self):
241
 
        tar_file = self.make_tar()
242
 
        tree = BzrDir.create_standalone_workingtree('tree')
243
 
        import_tar(tree, tar_file)
244
 
        self.assertTrue(tree.path2id('README') is not None) 
245
 
        self.assertTrue(tree.path2id('FEEDME') is not None)
246
 
        self.assertTrue(os.path.isfile(tree.abspath('README')))
247
 
        self.assertEqual(tree.inventory[tree.path2id('README')].kind, 'file')
248
 
        self.assertEqual(tree.inventory[tree.path2id('FEEDME')].kind, 'file')
249
 
        
250
 
        f = file(tree.abspath('junk/food'), 'wb')
251
 
        f.write('I like food\n')
252
 
        f.close()
253
 
 
254
 
        tar_file = self.make_tar2()
255
 
        import_tar(tree, tar_file)
256
 
        self.assertTrue(tree.path2id('README') is not None) 
257
 
        self.assertTrue(not os.path.exists(tree.abspath('FEEDME')))
258
 
 
259
 
 
260
 
    def test_untar2(self):
261
 
        tar_file = self.make_messed_tar()
262
 
        tree = BzrDir.create_standalone_workingtree('tree')
263
 
        import_tar(tree, tar_file)
264
 
        self.assertTrue(tree.path2id('project-0.1/README') is not None) 
265
 
 
266
 
    def test_untar_gzip(self):
267
 
        tar_file = self.make_tar(mode='w:gz')
268
 
        tree = BzrDir.create_standalone_workingtree('tree')
269
 
        import_tar(tree, tar_file)
270
 
        self.assertTrue(tree.path2id('README') is not None) 
271
 
 
272
 
 
273
 
def test_suite():
274
 
    return makeSuite(TestImport)
 
314
 
 
315
def get_archive_type(path):
 
316
    """Return the type of archive and compressor indicated by path name.
 
317
 
 
318
    Only external compressors are returned, so zip files are only
 
319
    ('zip', None).  .tgz is treated as ('tar', 'gz') and '.tar.xz' is treated
 
320
    as ('tar', 'lzma').
 
321
    """
 
322
    matches = re.match(r'.*\.(zip|tgz|tar(.(gz|bz2|lzma|xz))?)$', path)
 
323
    if not matches:
 
324
        raise errors.NotArchiveType(path)
 
325
    external_compressor = None
 
326
    if matches.group(3) is not None:
 
327
        archive = 'tar'
 
328
        external_compressor = matches.group(3)
 
329
        if external_compressor == 'xz':
 
330
            external_compressor = 'lzma'
 
331
    elif matches.group(1) == 'tgz':
 
332
        return 'tar', 'gz'
 
333
    else:
 
334
        archive = matches.group(1)
 
335
    return archive, external_compressor