← Back to branch summary

~abentley/bzrtools/bzrtools.dev

« back to all changes in this revision

Viewing changes to upstream_import.py

  • Committer: Aaron Bentley
  • Date: 2013-08-20 03:02:43 UTC
  • Revision ID: aaron@aaronbentley.com-20130820030243-r8v1xfbcnd8f10p4
Fix zap command for 2.6/7

Show diffs side-by-side

added added

removed removed

Lines of Context:
upstream_import.py
1
1
"""Import upstream source into a branch"""
2
2
 
3
 
from bz2 import BZ2File
4
3
import errno
5
4
import os
6
 
from shutil import rmtree
 
5
import re
7
6
from StringIO import StringIO
 
7
import stat
8
8
import tarfile
9
 
from unittest import makeSuite
10
9
import zipfile
11
10
 
 
11
from bzrlib import generate_ids
12
12
from bzrlib.bzrdir import BzrDir
13
13
from bzrlib.errors import NoSuchFile, BzrCommandError, NotBranchError
14
 
from bzrlib.osutils import pathjoin, isdir, file_iterator
15
 
from bzrlib.tests import TestCaseInTempDir
 
14
from bzrlib.osutils import (pathjoin, isdir, file_iterator, basename,
 
15
                            file_kind, splitpath)
16
16
from bzrlib.trace import warning
17
17
from bzrlib.transform import TreeTransform, resolve_conflicts, cook_conflicts
18
18
from bzrlib.workingtree import WorkingTree
 
19
from bzrlib.plugins.bzrtools.bzrtools import open_from_url
 
20
from bzrlib.plugins.bzrtools import errors
19
21
 
20
22
class ZipFileWrapper(object):
21
23
 
22
 
    def __init__(self, zipfile):
23
 
        self.zipfile = zipfile
 
24
    def __init__(self, fileobj, mode):
 
25
        self.zipfile = zipfile.ZipFile(fileobj, mode)
24
26
 
25
27
    def getmembers(self):
26
28
        for info in self.zipfile.infolist():
35
37
        else:
36
38
            self.zipfile.write(filename)
37
39
 
38
 
 
39
40
    def close(self):
40
41
        self.zipfile.close()
41
42
 
42
43
 
43
44
class ZipInfoWrapper(object):
44
 
    
 
45
 
45
46
    def __init__(self, zipfile, info):
46
47
        self.info = info
47
48
        self.type = None
48
49
        self.name = info.filename
49
50
        self.zipfile = zipfile
 
51
        self.mode = 0666
50
52
 
51
53
    def isdir(self):
52
54
        # Really? Eeeew!
57
59
        return not self.isdir()
58
60
 
59
61
 
60
 
def top_directory(path):
 
62
class DirWrapper(object):
 
63
    def __init__(self, fileobj, mode='r'):
 
64
        assert mode == 'r', mode
 
65
        self.root = os.path.realpath(fileobj.read())
 
66
 
 
67
    def __repr__(self):
 
68
        return 'DirWrapper(%r)' % self.root
 
69
 
 
70
    def getmembers(self, subdir=None):
 
71
        if subdir is not None:
 
72
            mydir = pathjoin(self.root, subdir)
 
73
        else:
 
74
            mydir = self.root
 
75
        for child in os.listdir(mydir):
 
76
            if subdir is not None:
 
77
                child = pathjoin(subdir, child)
 
78
            fi = FileInfo(self.root, child)
 
79
            yield fi
 
80
            if fi.isdir():
 
81
                for v in self.getmembers(child):
 
82
                    yield v
 
83
 
 
84
    def extractfile(self, member):
 
85
        return open(member.fullpath)
 
86
 
 
87
 
 
88
class FileInfo(object):
 
89
 
 
90
    def __init__(self, root, filepath):
 
91
        self.fullpath = pathjoin(root, filepath)
 
92
        self.root = root
 
93
        if filepath != '':
 
94
            self.name = pathjoin(basename(root), filepath)
 
95
        else:
 
96
            print 'root %r' % root
 
97
            self.name = basename(root)
 
98
        self.type = None
 
99
        stat = os.lstat(self.fullpath)
 
100
        self.mode = stat.st_mode
 
101
        if self.isdir():
 
102
            self.name += '/'
 
103
 
 
104
    def __repr__(self):
 
105
        return 'FileInfo(%r)' % self.name
 
106
 
 
107
    def isreg(self):
 
108
        return stat.S_ISREG(self.mode)
 
109
 
 
110
    def isdir(self):
 
111
        return stat.S_ISDIR(self.mode)
 
112
 
 
113
    def issym(self):
 
114
        if stat.S_ISLNK(self.mode):
 
115
            self.linkname = os.readlink(self.fullpath)
 
116
            return True
 
117
        else:
 
118
            return False
 
119
 
 
120
 
 
121
def top_path(path):
61
122
    """Return the top directory given in a path."""
62
 
    dirname = os.path.dirname(path)
63
 
    last_dirname = dirname
64
 
    while True:
65
 
        dirname = os.path.dirname(dirname)
66
 
        if dirname == '' or dirname == last_dirname:
67
 
            return last_dirname
68
 
        last_dirname = dirname
 
123
    components = splitpath(path)
 
124
    if len(components) > 0:
 
125
        return components[0]
 
126
    else:
 
127
        return ''
69
128
 
70
129
 
71
130
def common_directory(names):
72
131
    """Determine a single directory prefix from a list of names"""
73
132
    possible_prefix = None
74
133
    for name in names:
75
 
        name_top = top_directory(name)
 
134
        name_top = top_path(name)
 
135
        if name_top == '':
 
136
            return None
76
137
        if possible_prefix is None:
77
138
            possible_prefix = name_top
78
139
        else:
103
164
            yield member.name
104
165
 
105
166
 
 
167
def should_ignore(relative_path):
 
168
    return top_path(relative_path) == '.bzr'
 
169
 
 
170
 
106
171
def import_tar(tree, tar_input):
107
172
    """Replace the contents of a working directory with tarfile contents.
108
173
    The tarfile may be a gzipped stream.  File ids will be updated.
111
176
    import_archive(tree, tar_file)
112
177
 
113
178
def import_zip(tree, zip_input):
114
 
    zip_file = ZipFileWrapper(zipfile.ZipFile(zip_input))
 
179
    zip_file = ZipFileWrapper(zip_input, 'r')
115
180
    import_archive(tree, zip_file)
116
181
 
 
182
def import_dir(tree, dir_input):
 
183
    dir_file = DirWrapper(dir_input)
 
184
    import_archive(tree, dir_file)
 
185
 
 
186
 
117
187
def import_archive(tree, archive_file):
 
188
    tt = TreeTransform(tree)
 
189
    try:
 
190
        import_archive_to_transform(tree, archive_file, tt)
 
191
        tt.apply()
 
192
    finally:
 
193
        tt.finalize()
 
194
 
 
195
 
 
196
def import_archive_to_transform(tree, archive_file, tt):
118
197
    prefix = common_directory(names_of_files(archive_file))
119
 
    tt = TreeTransform(tree)
120
 
 
121
198
    removed = set()
122
 
    for path, entry in tree.inventory.iter_entries():
 
199
    for path, entry in tree.iter_entries_by_dir():
123
200
        if entry.parent_id is None:
124
201
            continue
125
202
        trans_id = tt.trans_id_tree_path(path)
126
203
        tt.delete_contents(trans_id)
127
204
        removed.add(path)
128
205
 
129
 
    added = set() 
 
206
    added = set()
130
207
    implied_parents = set()
131
208
    seen = set()
132
209
    for member in archive_file.getmembers():
133
210
        if member.type == 'g':
134
211
            # type 'g' is a header
135
212
            continue
136
 
        relative_path = member.name 
 
213
        # Inverse functionality in bzr uses utf-8.  We could also
 
214
        # interpret relative to fs encoding, which would match native
 
215
        # behaviour better.
 
216
        relative_path = member.name.decode('utf-8')
137
217
        if prefix is not None:
138
218
            relative_path = relative_path[len(prefix)+1:]
 
219
            relative_path = relative_path.rstrip('/')
139
220
        if relative_path == '':
140
221
            continue
 
222
        if should_ignore(relative_path):
 
223
            continue
141
224
        add_implied_parents(implied_parents, relative_path)
142
225
        trans_id = tt.trans_id_tree_path(relative_path)
143
226
        added.add(relative_path.rstrip('/'))
144
227
        path = tree.abspath(relative_path)
145
228
        if member.name in seen:
 
229
            if tt.final_kind(trans_id) == 'file':
 
230
                tt.set_executability(None, trans_id)
146
231
            tt.cancel_creation(trans_id)
147
232
        seen.add(member.name)
148
233
        if member.isreg():
149
 
            tt.create_file(file_iterator(archive_file.extractfile(member)), 
 
234
            tt.create_file(file_iterator(archive_file.extractfile(member)),
150
235
                           trans_id)
 
236
            executable = (member.mode & 0111) != 0
 
237
            tt.set_executability(executable, trans_id)
151
238
        elif member.isdir():
152
239
            do_directory(tt, trans_id, tree, relative_path, path)
153
240
        elif member.issym():
154
241
            tt.create_symlink(member.linkname, trans_id)
 
242
        else:
 
243
            continue
 
244
        if tt.tree_file_id(trans_id) is None:
 
245
            name = basename(member.name.rstrip('/'))
 
246
            file_id = generate_ids.gen_file_id(name)
 
247
            tt.version_file(file_id, trans_id)
155
248
 
156
249
    for relative_path in implied_parents.difference(added):
157
250
        if relative_path == "":
159
252
        trans_id = tt.trans_id_tree_path(relative_path)
160
253
        path = tree.abspath(relative_path)
161
254
        do_directory(tt, trans_id, tree, relative_path, path)
 
255
        if tt.tree_file_id(trans_id) is None:
 
256
            tt.version_file(trans_id, trans_id)
162
257
        added.add(relative_path)
163
258
 
 
259
    for path in removed.difference(added):
 
260
        tt.unversion_file(tt.trans_id_tree_path(path))
 
261
 
164
262
    for conflict in cook_conflicts(resolve_conflicts(tt), tt):
165
263
        warning(conflict)
166
 
    tt.apply()
167
 
    update_ids(tree, added, removed)
168
 
 
169
 
 
170
 
def update_ids(tree, added, removed):
171
 
    """Make sure that all present files files have file_ids.
172
 
    """
173
 
    # XXX detect renames
174
 
    new = added.difference(removed)
175
 
    deleted = removed.difference(added)
176
 
    tree.add(sorted(new))
177
 
    tree.remove(sorted(deleted, reverse=True))
178
264
 
179
265
 
180
266
def do_import(source, tree_directory=None):
194
280
        if tree.changes_from(tree.basis_tree()).has_changed():
195
281
            raise BzrCommandError("Working tree has uncommitted changes.")
196
282
 
197
 
        if (source.endswith('.tar') or source.endswith('.tar.gz') or 
198
 
            source.endswith('.tar.bz2')) or source.endswith('.tgz'):
199
 
            try:
200
 
                if source.endswith('.bz2'):
201
 
                    tar_input = BZ2File(source, 'r')
202
 
                    tar_input = StringIO(tar_input.read())
203
 
                else:
204
 
                    tar_input = file(source, 'rb')
205
 
            except IOError, e:
206
 
                if e.errno == errno.ENOENT:
207
 
                    raise NoSuchFile(source)
208
 
            try:
209
 
                import_tar(tree, tar_input)
210
 
            finally:
211
 
                tar_input.close()
212
 
        elif source.endswith('.zip'):
213
 
            import_zip(tree, open(source, 'rb'))
 
283
        try:
 
284
            archive, external_compressor = get_archive_type(source)
 
285
        except errors.NotArchiveType:
 
286
            if file_kind(source) == 'directory':
 
287
                s = StringIO(source)
 
288
                s.seek(0)
 
289
                import_dir(tree, s)
 
290
            else:
 
291
                raise BzrCommandError('Unhandled import source')
214
292
        else:
215
 
            raise BzrCommandError('Unhandled import source')
 
293
            if archive == 'zip':
 
294
                import_zip(tree, open_from_url(source))
 
295
            elif archive == 'tar':
 
296
                try:
 
297
                    tar_input = open_from_url(source)
 
298
                    if external_compressor == 'bz2':
 
299
                        import bz2
 
300
                        tar_input = StringIO(bz2.decompress(tar_input.read()))
 
301
                    elif external_compressor == 'lzma':
 
302
                        import lzma
 
303
                        tar_input = StringIO(lzma.decompress(tar_input.read()))
 
304
                except IOError, e:
 
305
                    if e.errno == errno.ENOENT:
 
306
                        raise NoSuchFile(source)
 
307
                try:
 
308
                    import_tar(tree, tar_input)
 
309
                finally:
 
310
                    tar_input.close()
216
311
    finally:
217
312
        tree.unlock()
218
313
 
219
 
class TestImport(TestCaseInTempDir):
220
 
 
221
 
    def make_tar(self, mode='w'):
222
 
        def maker(fileobj):
223
 
            return tarfile.open('project-0.1.tar', mode, fileobj)
224
 
        return self.make_archive(maker)
225
 
 
226
 
    def make_archive(self, maker):
227
 
        result = StringIO()
228
 
        archive_file = maker(result)
229
 
        os.mkdir('project-0.1')
230
 
        archive_file.add('project-0.1')
231
 
        os.mkdir('project-0.1/junk')
232
 
        archive_file.add('project-0.1/junk')
233
 
        
234
 
        f = file('project-0.1/README', 'wb')
235
 
        f.write('What?')
236
 
        f.close()
237
 
        archive_file.add('project-0.1/README')
238
 
 
239
 
        f = file('project-0.1/FEEDME', 'wb')
240
 
        f.write('Hungry!!')
241
 
        f.close()
242
 
        archive_file.add('project-0.1/FEEDME')
243
 
 
244
 
        archive_file.close()
245
 
        rmtree('project-0.1')
246
 
        result.seek(0)
247
 
        return result
248
 
 
249
 
    def make_tar2(self):
250
 
        result = StringIO()
251
 
        tar_file = tarfile.open('project-0.2.tar', 'w', result)
252
 
        os.mkdir('project-0.2')
253
 
        tar_file.add('project-0.2')
254
 
        
255
 
        os.mkdir('project-0.2/junk')
256
 
        tar_file.add('project-0.2/junk')
257
 
 
258
 
        f = file('project-0.2/README', 'wb')
259
 
        f.write('Now?')
260
 
        f.close()
261
 
        tar_file.add('project-0.2/README')
262
 
        tar_file.close()
263
 
 
264
 
        tar_file = tarfile.open('project-0.2.tar', 'a', result)
265
 
        tar_file.add('project-0.2/README')
266
 
 
267
 
        rmtree('project-0.2')
268
 
        return result
269
 
 
270
 
    def make_messed_tar(self):
271
 
        result = StringIO()
272
 
        tar_file = tarfile.open('project-0.1.tar', 'w', result)
273
 
        os.mkdir('project-0.1')
274
 
        tar_file.add('project-0.1')
275
 
 
276
 
        os.mkdir('project-0.2')
277
 
        tar_file.add('project-0.2')
278
 
        
279
 
        f = file('project-0.1/README', 'wb')
280
 
        f.write('What?')
281
 
        f.close()
282
 
        tar_file.add('project-0.1/README')
283
 
        tar_file.close()
284
 
        rmtree('project-0.1')
285
 
        result.seek(0)
286
 
        return result
287
 
 
288
 
    def make_zip(self):
289
 
        def maker(fileobj):
290
 
            return ZipFileWrapper(zipfile.ZipFile(fileobj, 'w'))
291
 
        return self.make_archive(maker)
292
 
 
293
 
    def test_top_directory(self):
294
 
        self.assertEqual(top_directory('ab/b/c'), 'ab')
295
 
        self.assertEqual(top_directory('/etc'), '/')
296
 
 
297
 
    def test_common_directory(self):
298
 
        self.assertEqual(common_directory(['ab/c/d', 'ab/c/e']), 'ab')
299
 
        self.assertIs(common_directory(['ab/c/d', 'ac/c/e']), None)
300
 
 
301
 
    def test_untar(self):
302
 
        tar_file = self.make_tar()
303
 
        tree = BzrDir.create_standalone_workingtree('tree')
304
 
        import_tar(tree, tar_file)
305
 
        self.assertTrue(tree.path2id('README') is not None) 
306
 
        self.assertTrue(tree.path2id('FEEDME') is not None)
307
 
        self.assertTrue(os.path.isfile(tree.abspath('README')))
308
 
        self.assertEqual(tree.inventory[tree.path2id('README')].kind, 'file')
309
 
        self.assertEqual(tree.inventory[tree.path2id('FEEDME')].kind, 'file')
310
 
        
311
 
        f = file(tree.abspath('junk/food'), 'wb')
312
 
        f.write('I like food\n')
313
 
        f.close()
314
 
 
315
 
        tar_file = self.make_tar2()
316
 
        import_tar(tree, tar_file)
317
 
        self.assertTrue(tree.path2id('README') is not None) 
318
 
        self.assertTrue(not os.path.exists(tree.abspath('FEEDME')))
319
 
 
320
 
 
321
 
    def test_untar2(self):
322
 
        tar_file = self.make_messed_tar()
323
 
        tree = BzrDir.create_standalone_workingtree('tree')
324
 
        import_tar(tree, tar_file)
325
 
        self.assertTrue(tree.path2id('project-0.1/README') is not None) 
326
 
 
327
 
    def test_untar_gzip(self):
328
 
        tar_file = self.make_tar(mode='w:gz')
329
 
        tree = BzrDir.create_standalone_workingtree('tree')
330
 
        import_tar(tree, tar_file)
331
 
        self.assertTrue(tree.path2id('README') is not None) 
332
 
 
333
 
    def test_unzip(self):
334
 
        zip_file = self.make_zip()
335
 
        tree = BzrDir.create_standalone_workingtree('tree')
336
 
        import_zip(tree, zip_file)
337
 
        self.assertTrue(tree.path2id('README') is not None) 
338
 
        self.assertTrue(tree.path2id('FEEDME') is not None)
339
 
        self.assertTrue(os.path.isfile(tree.abspath('README')))
340
 
        self.assertEqual(tree.inventory[tree.path2id('README')].kind, 'file')
341
 
        self.assertEqual(tree.inventory[tree.path2id('FEEDME')].kind, 'file')
342
 
        
343
 
        f = file(tree.abspath('junk/food'), 'wb')
344
 
        f.write('I like food\n')
345
 
        f.close()
346
 
 
347
 
 
348
 
def test_suite():
349
 
    return makeSuite(TestImport)
 
314
 
 
315
def get_archive_type(path):
 
316
    """Return the type of archive and compressor indicated by path name.
 
317
 
 
318
    Only external compressors are returned, so zip files are only
 
319
    ('zip', None).  .tgz is treated as ('tar', 'gz') and '.tar.xz' is treated
 
320
    as ('tar', 'lzma').
 
321
    """
 
322
    matches = re.match(r'.*\.(zip|tgz|tar(.(gz|bz2|lzma|xz))?)$', path)
 
323
    if not matches:
 
324
        raise errors.NotArchiveType(path)
 
325
    external_compressor = None
 
326
    if matches.group(3) is not None:
 
327
        archive = 'tar'
 
328
        external_compressor = matches.group(3)
 
329
        if external_compressor == 'xz':
 
330
            external_compressor = 'lzma'
 
331
    elif matches.group(1) == 'tgz':
 
332
        return 'tar', 'gz'
 
333
    else:
 
334
        archive = matches.group(1)
 
335
    return archive, external_compressor