← Back to branch summary

~abentley/bzrtools/bzrtools.dev

« back to all changes in this revision

Viewing changes to upstream_import.py

  • Committer: Aaron Bentley
  • Date: 2011-01-26 01:11:59 UTC
  • Revision ID: aaron@aaronbentley.com-20110126011159-xsh6z4n8bgxy0k9k
Ignore the_kraken changelogs.

Show diffs side-by-side

added added

removed removed

Lines of Context:
upstream_import.py
3
3
from bz2 import BZ2File
4
4
import errno
5
5
import os
6
 
from shutil import rmtree
7
6
from StringIO import StringIO
 
7
import stat
8
8
import tarfile
9
 
from unittest import makeSuite
 
9
import zipfile
10
10
 
 
11
from bzrlib import generate_ids
11
12
from bzrlib.bzrdir import BzrDir
12
13
from bzrlib.errors import NoSuchFile, BzrCommandError, NotBranchError
13
 
from bzrlib.osutils import pathjoin, isdir, file_iterator
14
 
from bzrlib.tests import TestCaseInTempDir
 
14
from bzrlib.osutils import (pathjoin, isdir, file_iterator, basename,
 
15
                            file_kind, splitpath)
15
16
from bzrlib.trace import warning
16
17
from bzrlib.transform import TreeTransform, resolve_conflicts, cook_conflicts
17
18
from bzrlib.workingtree import WorkingTree
18
 
 
19
 
 
20
 
def top_directory(path):
 
19
from bzrlib.plugins.bzrtools.bzrtools import open_from_url
 
20
 
 
21
class ZipFileWrapper(object):
 
22
 
 
23
    def __init__(self, fileobj, mode):
 
24
        self.zipfile = zipfile.ZipFile(fileobj, mode)
 
25
 
 
26
    def getmembers(self):
 
27
        for info in self.zipfile.infolist():
 
28
            yield ZipInfoWrapper(self.zipfile, info)
 
29
 
 
30
    def extractfile(self, infowrapper):
 
31
        return StringIO(self.zipfile.read(infowrapper.name))
 
32
 
 
33
    def add(self, filename):
 
34
        if isdir(filename):
 
35
            self.zipfile.writestr(filename+'/', '')
 
36
        else:
 
37
            self.zipfile.write(filename)
 
38
 
 
39
    def close(self):
 
40
        self.zipfile.close()
 
41
 
 
42
 
 
43
class ZipInfoWrapper(object):
 
44
 
 
45
    def __init__(self, zipfile, info):
 
46
        self.info = info
 
47
        self.type = None
 
48
        self.name = info.filename
 
49
        self.zipfile = zipfile
 
50
        self.mode = 0666
 
51
 
 
52
    def isdir(self):
 
53
        # Really? Eeeew!
 
54
        return bool(self.name.endswith('/'))
 
55
 
 
56
    def isreg(self):
 
57
        # Really? Eeeew!
 
58
        return not self.isdir()
 
59
 
 
60
 
 
61
class DirWrapper(object):
 
62
    def __init__(self, fileobj, mode='r'):
 
63
        assert mode == 'r', mode
 
64
        self.root = os.path.realpath(fileobj.read())
 
65
 
 
66
    def __repr__(self):
 
67
        return 'DirWrapper(%r)' % self.root
 
68
 
 
69
    def getmembers(self, subdir=None):
 
70
        if subdir is not None:
 
71
            mydir = pathjoin(self.root, subdir)
 
72
        else:
 
73
            mydir = self.root
 
74
        for child in os.listdir(mydir):
 
75
            if subdir is not None:
 
76
                child = pathjoin(subdir, child)
 
77
            fi = FileInfo(self.root, child)
 
78
            yield fi
 
79
            if fi.isdir():
 
80
                for v in self.getmembers(child):
 
81
                    yield v
 
82
 
 
83
    def extractfile(self, member):
 
84
        return open(member.fullpath)
 
85
 
 
86
 
 
87
class FileInfo(object):
 
88
 
 
89
    def __init__(self, root, filepath):
 
90
        self.fullpath = pathjoin(root, filepath)
 
91
        self.root = root
 
92
        if filepath != '':
 
93
            self.name = pathjoin(basename(root), filepath)
 
94
        else:
 
95
            print 'root %r' % root
 
96
            self.name = basename(root)
 
97
        self.type = None
 
98
        stat = os.lstat(self.fullpath)
 
99
        self.mode = stat.st_mode
 
100
        if self.isdir():
 
101
            self.name += '/'
 
102
 
 
103
    def __repr__(self):
 
104
        return 'FileInfo(%r)' % self.name
 
105
 
 
106
    def isreg(self):
 
107
        return stat.S_ISREG(self.mode)
 
108
 
 
109
    def isdir(self):
 
110
        return stat.S_ISDIR(self.mode)
 
111
 
 
112
    def issym(self):
 
113
        if stat.S_ISLNK(self.mode):
 
114
            self.linkname = os.readlink(self.fullpath)
 
115
            return True
 
116
        else:
 
117
            return False
 
118
 
 
119
 
 
120
def top_path(path):
21
121
    """Return the top directory given in a path."""
22
 
    dirname = os.path.dirname(path)
23
 
    last_dirname = dirname
24
 
    while True:
25
 
        dirname = os.path.dirname(dirname)
26
 
        if dirname == '' or dirname == last_dirname:
27
 
            return last_dirname
28
 
        last_dirname = dirname
 
122
    components = splitpath(path)
 
123
    if len(components) > 0:
 
124
        return components[0]
 
125
    else:
 
126
        return ''
29
127
 
30
128
 
31
129
def common_directory(names):
32
130
    """Determine a single directory prefix from a list of names"""
33
131
    possible_prefix = None
34
132
    for name in names:
35
 
        name_top = top_directory(name)
 
133
        name_top = top_path(name)
 
134
        if name_top == '':
 
135
            return None
36
136
        if possible_prefix is None:
37
137
            possible_prefix = name_top
38
138
        else:
63
163
            yield member.name
64
164
 
65
165
 
 
166
def should_ignore(relative_path):
 
167
    return top_path(relative_path) == '.bzr'
 
168
 
 
169
 
66
170
def import_tar(tree, tar_input):
67
171
    """Replace the contents of a working directory with tarfile contents.
68
172
    The tarfile may be a gzipped stream.  File ids will be updated.
69
173
    """
70
174
    tar_file = tarfile.open('lala', 'r', tar_input)
71
 
    prefix = common_directory(names_of_files(tar_file))
 
175
    import_archive(tree, tar_file)
 
176
 
 
177
def import_zip(tree, zip_input):
 
178
    zip_file = ZipFileWrapper(zip_input, 'r')
 
179
    import_archive(tree, zip_file)
 
180
 
 
181
def import_dir(tree, dir_input):
 
182
    dir_file = DirWrapper(dir_input)
 
183
    import_archive(tree, dir_file)
 
184
 
 
185
def import_archive(tree, archive_file):
 
186
    prefix = common_directory(names_of_files(archive_file))
72
187
    tt = TreeTransform(tree)
73
188
 
74
189
    removed = set()
79
194
        tt.delete_contents(trans_id)
80
195
        removed.add(path)
81
196
 
82
 
    added = set() 
 
197
    added = set()
83
198
    implied_parents = set()
84
199
    seen = set()
85
 
    for member in tar_file.getmembers():
 
200
    for member in archive_file.getmembers():
86
201
        if member.type == 'g':
87
202
            # type 'g' is a header
88
203
            continue
89
 
        relative_path = member.name 
 
204
        relative_path = member.name
90
205
        if prefix is not None:
91
206
            relative_path = relative_path[len(prefix)+1:]
 
207
            relative_path = relative_path.rstrip('/')
92
208
        if relative_path == '':
93
209
            continue
 
210
        if should_ignore(relative_path):
 
211
            continue
94
212
        add_implied_parents(implied_parents, relative_path)
95
213
        trans_id = tt.trans_id_tree_path(relative_path)
96
214
        added.add(relative_path.rstrip('/'))
97
215
        path = tree.abspath(relative_path)
98
216
        if member.name in seen:
 
217
            if tt.final_kind(trans_id) == 'file':
 
218
                tt.set_executability(None, trans_id)
99
219
            tt.cancel_creation(trans_id)
100
220
        seen.add(member.name)
101
221
        if member.isreg():
102
 
            tt.create_file(file_iterator(tar_file.extractfile(member)), 
 
222
            tt.create_file(file_iterator(archive_file.extractfile(member)),
103
223
                           trans_id)
 
224
            executable = (member.mode & 0111) != 0
 
225
            tt.set_executability(executable, trans_id)
104
226
        elif member.isdir():
105
227
            do_directory(tt, trans_id, tree, relative_path, path)
106
228
        elif member.issym():
107
229
            tt.create_symlink(member.linkname, trans_id)
 
230
        else:
 
231
            continue
 
232
        if tt.tree_file_id(trans_id) is None:
 
233
            name = basename(member.name.rstrip('/'))
 
234
            file_id = generate_ids.gen_file_id(name)
 
235
            tt.version_file(file_id, trans_id)
108
236
 
109
237
    for relative_path in implied_parents.difference(added):
110
238
        if relative_path == "":
112
240
        trans_id = tt.trans_id_tree_path(relative_path)
113
241
        path = tree.abspath(relative_path)
114
242
        do_directory(tt, trans_id, tree, relative_path, path)
 
243
        if tt.tree_file_id(trans_id) is None:
 
244
            tt.version_file(trans_id, trans_id)
115
245
        added.add(relative_path)
116
246
 
 
247
    for path in removed.difference(added):
 
248
        tt.unversion_file(tt.trans_id_tree_path(path))
 
249
 
117
250
    for conflict in cook_conflicts(resolve_conflicts(tt), tt):
118
251
        warning(conflict)
119
252
    tt.apply()
120
 
    update_ids(tree, added, removed)
121
 
 
122
 
 
123
 
def update_ids(tree, added, removed):
124
 
    """Make sure that all present files files have file_ids.
125
 
    """
126
 
    # XXX detect renames
127
 
    new = added.difference(removed)
128
 
    deleted = removed.difference(added)
129
 
    tree.add(sorted(new))
130
 
    tree.remove(sorted(deleted, reverse=True))
131
253
 
132
254
 
133
255
def do_import(source, tree_directory=None):
147
269
        if tree.changes_from(tree.basis_tree()).has_changed():
148
270
            raise BzrCommandError("Working tree has uncommitted changes.")
149
271
 
150
 
        if (source.endswith('.tar') or source.endswith('.tar.gz') or 
 
272
        if (source.endswith('.tar') or source.endswith('.tar.gz') or
151
273
            source.endswith('.tar.bz2')) or source.endswith('.tgz'):
152
274
            try:
 
275
                tar_input = open_from_url(source)
153
276
                if source.endswith('.bz2'):
154
 
                    tar_input = BZ2File(source, 'r')
155
 
                    tar_input = StringIO(tar_input.read())
156
 
                else:
157
 
                    tar_input = file(source, 'rb')
 
277
                    tar_input = StringIO(tar_input.read().decode('bz2'))
158
278
            except IOError, e:
159
279
                if e.errno == errno.ENOENT:
160
280
                    raise NoSuchFile(source)
162
282
                import_tar(tree, tar_input)
163
283
            finally:
164
284
                tar_input.close()
 
285
        elif source.endswith('.zip'):
 
286
            import_zip(tree, open_from_url(source))
 
287
        elif file_kind(source) == 'directory':
 
288
            s = StringIO(source)
 
289
            s.seek(0)
 
290
            import_dir(tree, s)
 
291
        else:
 
292
            raise BzrCommandError('Unhandled import source')
165
293
    finally:
166
294
        tree.unlock()
167
 
 
168
 
class TestImport(TestCaseInTempDir):
169
 
 
170
 
    def make_tar(self, mode='w'):
171
 
        result = StringIO()
172
 
        tar_file = tarfile.open('project-0.1.tar', mode, result)
173
 
        os.mkdir('project-0.1')
174
 
        tar_file.add('project-0.1')
175
 
        os.mkdir('project-0.1/junk')
176
 
        tar_file.add('project-0.1/junk')
177
 
        
178
 
        f = file('project-0.1/README', 'wb')
179
 
        f.write('What?')
180
 
        f.close()
181
 
        tar_file.add('project-0.1/README')
182
 
 
183
 
        f = file('project-0.1/FEEDME', 'wb')
184
 
        f.write('Hungry!!')
185
 
        f.close()
186
 
        tar_file.add('project-0.1/FEEDME')
187
 
 
188
 
        tar_file.close()
189
 
        rmtree('project-0.1')
190
 
        result.seek(0)
191
 
        return result
192
 
 
193
 
    def make_tar2(self):
194
 
        result = StringIO()
195
 
        tar_file = tarfile.open('project-0.2.tar', 'w', result)
196
 
        os.mkdir('project-0.2')
197
 
        tar_file.add('project-0.2')
198
 
        
199
 
        os.mkdir('project-0.2/junk')
200
 
        tar_file.add('project-0.2/junk')
201
 
 
202
 
        f = file('project-0.2/README', 'wb')
203
 
        f.write('Now?')
204
 
        f.close()
205
 
        tar_file.add('project-0.2/README')
206
 
        tar_file.close()
207
 
 
208
 
        tar_file = tarfile.open('project-0.2.tar', 'a', result)
209
 
        tar_file.add('project-0.2/README')
210
 
 
211
 
        rmtree('project-0.2')
212
 
        return result
213
 
 
214
 
    def make_messed_tar(self):
215
 
        result = StringIO()
216
 
        tar_file = tarfile.open('project-0.1.tar', 'w', result)
217
 
        os.mkdir('project-0.1')
218
 
        tar_file.add('project-0.1')
219
 
 
220
 
        os.mkdir('project-0.2')
221
 
        tar_file.add('project-0.2')
222
 
        
223
 
        f = file('project-0.1/README', 'wb')
224
 
        f.write('What?')
225
 
        f.close()
226
 
        tar_file.add('project-0.1/README')
227
 
        tar_file.close()
228
 
        rmtree('project-0.1')
229
 
        result.seek(0)
230
 
        return result
231
 
 
232
 
    def test_top_directory(self):
233
 
        self.assertEqual(top_directory('ab/b/c'), 'ab')
234
 
        self.assertEqual(top_directory('/etc'), '/')
235
 
 
236
 
    def test_common_directory(self):
237
 
        self.assertEqual(common_directory(['ab/c/d', 'ab/c/e']), 'ab')
238
 
        self.assertIs(common_directory(['ab/c/d', 'ac/c/e']), None)
239
 
 
240
 
    def test_untar(self):
241
 
        tar_file = self.make_tar()
242
 
        tree = BzrDir.create_standalone_workingtree('tree')
243
 
        import_tar(tree, tar_file)
244
 
        self.assertTrue(tree.path2id('README') is not None) 
245
 
        self.assertTrue(tree.path2id('FEEDME') is not None)
246
 
        self.assertTrue(os.path.isfile(tree.abspath('README')))
247
 
        self.assertEqual(tree.inventory[tree.path2id('README')].kind, 'file')
248
 
        self.assertEqual(tree.inventory[tree.path2id('FEEDME')].kind, 'file')
249
 
        
250
 
        f = file(tree.abspath('junk/food'), 'wb')
251
 
        f.write('I like food\n')
252
 
        f.close()
253
 
 
254
 
        tar_file = self.make_tar2()
255
 
        import_tar(tree, tar_file)
256
 
        self.assertTrue(tree.path2id('README') is not None) 
257
 
        self.assertTrue(not os.path.exists(tree.abspath('FEEDME')))
258
 
 
259
 
 
260
 
    def test_untar2(self):
261
 
        tar_file = self.make_messed_tar()
262
 
        tree = BzrDir.create_standalone_workingtree('tree')
263
 
        import_tar(tree, tar_file)
264
 
        self.assertTrue(tree.path2id('project-0.1/README') is not None) 
265
 
 
266
 
    def test_untar_gzip(self):
267
 
        tar_file = self.make_tar(mode='w:gz')
268
 
        tree = BzrDir.create_standalone_workingtree('tree')
269
 
        import_tar(tree, tar_file)
270
 
        self.assertTrue(tree.path2id('README') is not None) 
271
 
 
272
 
 
273
 
def test_suite():
274
 
    return makeSuite(TestImport)