← Back to branch summary

~abentley/bzrtools/bzrtools.dev

« back to all changes in this revision

Viewing changes to upstream_import.py

  • Committer: Aaron Bentley
  • Date: 2006-08-08 02:29:34 UTC
  • mto: This revision was merged to the branch mainline in revision 425.
  • Revision ID: aaron.bentley@utoronto.ca-20060808022934-c5e50b01aefe43d2
More updates for 0.9

Show diffs side-by-side

added added

removed removed

Lines of Context:
upstream_import.py
1
1
"""Import upstream source into a branch"""
2
2
 
 
3
from bz2 import BZ2File
3
4
import errno
4
5
import os
 
6
from shutil import rmtree
5
7
from StringIO import StringIO
6
 
import stat
7
8
import tarfile
8
 
import zipfile
 
9
from unittest import makeSuite
9
10
 
10
 
from bzrlib import generate_ids
11
11
from bzrlib.bzrdir import BzrDir
12
12
from bzrlib.errors import NoSuchFile, BzrCommandError, NotBranchError
13
 
from bzrlib.osutils import (pathjoin, isdir, file_iterator, basename,
14
 
                            file_kind, splitpath)
 
13
from bzrlib.osutils import pathjoin, isdir, file_iterator
 
14
from bzrlib.tests import TestCaseInTempDir
15
15
from bzrlib.trace import warning
16
16
from bzrlib.transform import TreeTransform, resolve_conflicts, cook_conflicts
17
17
from bzrlib.workingtree import WorkingTree
18
 
from bzrlib.plugins.bzrtools.bzrtools import open_from_url
19
 
 
20
 
class ZipFileWrapper(object):
21
 
 
22
 
    def __init__(self, fileobj, mode):
23
 
        self.zipfile = zipfile.ZipFile(fileobj, mode)
24
 
 
25
 
    def getmembers(self):
26
 
        for info in self.zipfile.infolist():
27
 
            yield ZipInfoWrapper(self.zipfile, info)
28
 
 
29
 
    def extractfile(self, infowrapper):
30
 
        return StringIO(self.zipfile.read(infowrapper.name))
31
 
 
32
 
    def add(self, filename):
33
 
        if isdir(filename):
34
 
            self.zipfile.writestr(filename+'/', '')
35
 
        else:
36
 
            self.zipfile.write(filename)
37
 
 
38
 
    def close(self):
39
 
        self.zipfile.close()
40
 
 
41
 
 
42
 
class ZipInfoWrapper(object):
43
 
 
44
 
    def __init__(self, zipfile, info):
45
 
        self.info = info
46
 
        self.type = None
47
 
        self.name = info.filename
48
 
        self.zipfile = zipfile
49
 
        self.mode = 0666
50
 
 
51
 
    def isdir(self):
52
 
        # Really? Eeeew!
53
 
        return bool(self.name.endswith('/'))
54
 
 
55
 
    def isreg(self):
56
 
        # Really? Eeeew!
57
 
        return not self.isdir()
58
 
 
59
 
 
60
 
class DirWrapper(object):
61
 
    def __init__(self, fileobj, mode='r'):
62
 
        assert mode == 'r', mode
63
 
        self.root = os.path.realpath(fileobj.read())
64
 
 
65
 
    def __repr__(self):
66
 
        return 'DirWrapper(%r)' % self.root
67
 
 
68
 
    def getmembers(self, subdir=None):
69
 
        if subdir is not None:
70
 
            mydir = pathjoin(self.root, subdir)
71
 
        else:
72
 
            mydir = self.root
73
 
        for child in os.listdir(mydir):
74
 
            if subdir is not None:
75
 
                child = pathjoin(subdir, child)
76
 
            fi = FileInfo(self.root, child)
77
 
            yield fi
78
 
            if fi.isdir():
79
 
                for v in self.getmembers(child):
80
 
                    yield v
81
 
 
82
 
    def extractfile(self, member):
83
 
        return open(member.fullpath)
84
 
 
85
 
 
86
 
class FileInfo(object):
87
 
 
88
 
    def __init__(self, root, filepath):
89
 
        self.fullpath = pathjoin(root, filepath)
90
 
        self.root = root
91
 
        if filepath != '':
92
 
            self.name = pathjoin(basename(root), filepath)
93
 
        else:
94
 
            print 'root %r' % root
95
 
            self.name = basename(root)
96
 
        self.type = None
97
 
        stat = os.lstat(self.fullpath)
98
 
        self.mode = stat.st_mode
99
 
        if self.isdir():
100
 
            self.name += '/'
101
 
 
102
 
    def __repr__(self):
103
 
        return 'FileInfo(%r)' % self.name
104
 
 
105
 
    def isreg(self):
106
 
        return stat.S_ISREG(self.mode)
107
 
 
108
 
    def isdir(self):
109
 
        return stat.S_ISDIR(self.mode)
110
 
 
111
 
    def issym(self):
112
 
        if stat.S_ISLNK(self.mode):
113
 
            self.linkname = os.readlink(self.fullpath)
114
 
            return True
115
 
        else:
116
 
            return False
117
 
 
118
 
 
119
 
def top_path(path):
 
18
 
 
19
 
 
20
def top_directory(path):
120
21
    """Return the top directory given in a path."""
121
 
    components = splitpath(path)
122
 
    if len(components) > 0:
123
 
        return components[0]
124
 
    else:
125
 
        return ''
 
22
    dirname = os.path.dirname(path)
 
23
    last_dirname = dirname
 
24
    while True:
 
25
        dirname = os.path.dirname(dirname)
 
26
        if dirname == '' or dirname == last_dirname:
 
27
            return last_dirname
 
28
        last_dirname = dirname
126
29
 
127
30
 
128
31
def common_directory(names):
129
32
    """Determine a single directory prefix from a list of names"""
130
33
    possible_prefix = None
131
34
    for name in names:
132
 
        name_top = top_path(name)
133
 
        if name_top == '':
134
 
            return None
 
35
        name_top = top_directory(name)
135
36
        if possible_prefix is None:
136
37
            possible_prefix = name_top
137
38
        else:
162
63
            yield member.name
163
64
 
164
65
 
165
 
def should_ignore(relative_path):
166
 
    return top_path(relative_path) == '.bzr'
167
 
 
168
 
 
169
66
def import_tar(tree, tar_input):
170
67
    """Replace the contents of a working directory with tarfile contents.
171
68
    The tarfile may be a gzipped stream.  File ids will be updated.
172
69
    """
173
70
    tar_file = tarfile.open('lala', 'r', tar_input)
174
 
    import_archive(tree, tar_file)
175
 
 
176
 
def import_zip(tree, zip_input):
177
 
    zip_file = ZipFileWrapper(zip_input, 'r')
178
 
    import_archive(tree, zip_file)
179
 
 
180
 
def import_dir(tree, dir_input):
181
 
    dir_file = DirWrapper(dir_input)
182
 
    import_archive(tree, dir_file)
183
 
 
184
 
 
185
 
def import_archive(tree, archive_file):
 
71
    prefix = common_directory(names_of_files(tar_file))
186
72
    tt = TreeTransform(tree)
187
 
    try:
188
 
        import_archive_to_transform(tree, archive_file, tt)
189
 
        tt.apply()
190
 
    finally:
191
 
        tt.finalize()
192
 
 
193
 
 
194
 
def import_archive_to_transform(tree, archive_file, tt):
195
 
    prefix = common_directory(names_of_files(archive_file))
 
73
 
196
74
    removed = set()
197
75
    for path, entry in tree.inventory.iter_entries():
198
 
        if entry.parent_id is None:
199
 
            continue
200
76
        trans_id = tt.trans_id_tree_path(path)
201
77
        tt.delete_contents(trans_id)
202
78
        removed.add(path)
203
79
 
204
 
    added = set()
 
80
    added = set() 
205
81
    implied_parents = set()
206
82
    seen = set()
207
 
    for member in archive_file.getmembers():
 
83
    for member in tar_file.getmembers():
208
84
        if member.type == 'g':
209
85
            # type 'g' is a header
210
86
            continue
211
 
        # Inverse functionality in bzr uses utf-8.  We could also
212
 
        # interpret relative to fs encoding, which would match native
213
 
        # behaviour better.
214
 
        relative_path = member.name.decode('utf-8')
 
87
        relative_path = member.name 
215
88
        if prefix is not None:
216
89
            relative_path = relative_path[len(prefix)+1:]
217
 
            relative_path = relative_path.rstrip('/')
218
90
        if relative_path == '':
219
91
            continue
220
 
        if should_ignore(relative_path):
221
 
            continue
222
92
        add_implied_parents(implied_parents, relative_path)
223
93
        trans_id = tt.trans_id_tree_path(relative_path)
224
94
        added.add(relative_path.rstrip('/'))
225
95
        path = tree.abspath(relative_path)
226
96
        if member.name in seen:
227
 
            if tt.final_kind(trans_id) == 'file':
228
 
                tt.set_executability(None, trans_id)
229
97
            tt.cancel_creation(trans_id)
230
98
        seen.add(member.name)
231
99
        if member.isreg():
232
 
            tt.create_file(file_iterator(archive_file.extractfile(member)),
 
100
            tt.create_file(file_iterator(tar_file.extractfile(member)), 
233
101
                           trans_id)
234
 
            executable = (member.mode & 0111) != 0
235
 
            tt.set_executability(executable, trans_id)
236
102
        elif member.isdir():
237
103
            do_directory(tt, trans_id, tree, relative_path, path)
238
104
        elif member.issym():
239
105
            tt.create_symlink(member.linkname, trans_id)
240
 
        else:
241
 
            continue
242
 
        if tt.tree_file_id(trans_id) is None:
243
 
            name = basename(member.name.rstrip('/'))
244
 
            file_id = generate_ids.gen_file_id(name)
245
 
            tt.version_file(file_id, trans_id)
246
106
 
247
107
    for relative_path in implied_parents.difference(added):
248
108
        if relative_path == "":
250
110
        trans_id = tt.trans_id_tree_path(relative_path)
251
111
        path = tree.abspath(relative_path)
252
112
        do_directory(tt, trans_id, tree, relative_path, path)
253
 
        if tt.tree_file_id(trans_id) is None:
254
 
            tt.version_file(trans_id, trans_id)
255
113
        added.add(relative_path)
256
114
 
257
 
    for path in removed.difference(added):
258
 
        tt.unversion_file(tt.trans_id_tree_path(path))
259
 
 
260
115
    for conflict in cook_conflicts(resolve_conflicts(tt), tt):
261
116
        warning(conflict)
 
117
    tt.apply()
 
118
    update_ids(tree, added, removed)
 
119
 
 
120
 
 
121
def update_ids(tree, added, removed):
 
122
    """Make sure that all present files files have file_ids.
 
123
    """
 
124
    # XXX detect renames
 
125
    new = added.difference(removed)
 
126
    deleted = removed.difference(added)
 
127
    tree.add(sorted(new))
 
128
    tree.remove(sorted(deleted, reverse=True))
262
129
 
263
130
 
264
131
def do_import(source, tree_directory=None):
278
145
        if tree.changes_from(tree.basis_tree()).has_changed():
279
146
            raise BzrCommandError("Working tree has uncommitted changes.")
280
147
 
281
 
        if (source.endswith('.tar') or source.endswith('.tar.gz') or
282
 
            source.endswith('.tar.bz2') or source.endswith('.tgz') or
283
 
            source.endswith('.tar.lzma') or source.endswith('.tar.xz')):
 
148
        if (source.endswith('.tar') or source.endswith('.tar.gz') or 
 
149
            source.endswith('.tar.bz2')) or source.endswith('.tgz'):
284
150
            try:
285
 
                tar_input = open_from_url(source)
286
151
                if source.endswith('.bz2'):
287
 
                    import bz2
288
 
                    tar_input = StringIO(bz2.decompress(tar_input.read()))
289
 
                elif source.endswith('.xz') or source.endswith('.lzma'):
290
 
                    import lzma
291
 
                    tar_input = StringIO(lzma.decompress(tar_input.read()))
 
152
                    tar_input = BZ2File(source, 'r')
 
153
                    tar_input = StringIO(tar_input.read())
 
154
                else:
 
155
                    tar_input = file(source, 'rb')
292
156
            except IOError, e:
293
157
                if e.errno == errno.ENOENT:
294
158
                    raise NoSuchFile(source)
296
160
                import_tar(tree, tar_input)
297
161
            finally:
298
162
                tar_input.close()
299
 
        elif source.endswith('.zip'):
300
 
            import_zip(tree, open_from_url(source))
301
 
        elif file_kind(source) == 'directory':
302
 
            s = StringIO(source)
303
 
            s.seek(0)
304
 
            import_dir(tree, s)
305
 
        else:
306
 
            raise BzrCommandError('Unhandled import source')
307
163
    finally:
308
164
        tree.unlock()
 
165
 
 
166
class TestImport(TestCaseInTempDir):
 
167
 
 
168
    def make_tar(self, mode='w'):
 
169
        result = StringIO()
 
170
        tar_file = tarfile.open('project-0.1.tar', mode, result)
 
171
        os.mkdir('project-0.1')
 
172
        tar_file.add('project-0.1')
 
173
        os.mkdir('project-0.1/junk')
 
174
        tar_file.add('project-0.1/junk')
 
175
        
 
176
        f = file('project-0.1/README', 'wb')
 
177
        f.write('What?')
 
178
        f.close()
 
179
        tar_file.add('project-0.1/README')
 
180
 
 
181
        f = file('project-0.1/FEEDME', 'wb')
 
182
        f.write('Hungry!!')
 
183
        f.close()
 
184
        tar_file.add('project-0.1/FEEDME')
 
185
 
 
186
        tar_file.close()
 
187
        rmtree('project-0.1')
 
188
        result.seek(0)
 
189
        return result
 
190
 
 
191
    def make_tar2(self):
 
192
        result = StringIO()
 
193
        tar_file = tarfile.open('project-0.2.tar', 'w', result)
 
194
        os.mkdir('project-0.2')
 
195
        tar_file.add('project-0.2')
 
196
        
 
197
        os.mkdir('project-0.2/junk')
 
198
        tar_file.add('project-0.2/junk')
 
199
 
 
200
        f = file('project-0.2/README', 'wb')
 
201
        f.write('Now?')
 
202
        f.close()
 
203
        tar_file.add('project-0.2/README')
 
204
        tar_file.close()
 
205
 
 
206
        tar_file = tarfile.open('project-0.2.tar', 'a', result)
 
207
        tar_file.add('project-0.2/README')
 
208
 
 
209
        rmtree('project-0.2')
 
210
        return result
 
211
 
 
212
    def make_messed_tar(self):
 
213
        result = StringIO()
 
214
        tar_file = tarfile.open('project-0.1.tar', 'w', result)
 
215
        os.mkdir('project-0.1')
 
216
        tar_file.add('project-0.1')
 
217
 
 
218
        os.mkdir('project-0.2')
 
219
        tar_file.add('project-0.2')
 
220
        
 
221
        f = file('project-0.1/README', 'wb')
 
222
        f.write('What?')
 
223
        f.close()
 
224
        tar_file.add('project-0.1/README')
 
225
        tar_file.close()
 
226
        rmtree('project-0.1')
 
227
        result.seek(0)
 
228
        return result
 
229
 
 
230
    def test_top_directory(self):
 
231
        self.assertEqual(top_directory('ab/b/c'), 'ab')
 
232
        self.assertEqual(top_directory('/etc'), '/')
 
233
 
 
234
    def test_common_directory(self):
 
235
        self.assertEqual(common_directory(['ab/c/d', 'ab/c/e']), 'ab')
 
236
        self.assertIs(common_directory(['ab/c/d', 'ac/c/e']), None)
 
237
 
 
238
    def test_untar(self):
 
239
        tar_file = self.make_tar()
 
240
        tree = BzrDir.create_standalone_workingtree('tree')
 
241
        import_tar(tree, tar_file)
 
242
        self.assertTrue(tree.path2id('README') is not None) 
 
243
        self.assertTrue(tree.path2id('FEEDME') is not None)
 
244
        self.assertTrue(os.path.isfile(tree.abspath('README')))
 
245
        self.assertEqual(tree.inventory[tree.path2id('README')].kind, 'file')
 
246
        self.assertEqual(tree.inventory[tree.path2id('FEEDME')].kind, 'file')
 
247
        
 
248
        f = file(tree.abspath('junk/food'), 'wb')
 
249
        f.write('I like food\n')
 
250
        f.close()
 
251
 
 
252
        tar_file = self.make_tar2()
 
253
        import_tar(tree, tar_file)
 
254
        self.assertTrue(tree.path2id('README') is not None) 
 
255
        self.assertTrue(not os.path.exists(tree.abspath('FEEDME')))
 
256
 
 
257
 
 
258
    def test_untar2(self):
 
259
        tar_file = self.make_messed_tar()
 
260
        tree = BzrDir.create_standalone_workingtree('tree')
 
261
        import_tar(tree, tar_file)
 
262
        self.assertTrue(tree.path2id('project-0.1/README') is not None) 
 
263
 
 
264
    def test_untar_gzip(self):
 
265
        tar_file = self.make_tar(mode='w:gz')
 
266
        tree = BzrDir.create_standalone_workingtree('tree')
 
267
        import_tar(tree, tar_file)
 
268
        self.assertTrue(tree.path2id('README') is not None) 
 
269
 
 
270
 
 
271
def test_suite():
 
272
    return makeSuite(TestImport)