← Back to branch summary

~abentley/bzrtools/bzrtools.dev

« back to all changes in this revision

Viewing changes to upstream_import.py

  • Committer: Aaron Bentley
  • Date: 2011-06-27 22:03:53 UTC
  • mfrom: (770.1.1 trunk)
  • Revision ID: aaron@aaronbentley.com-20110627220353-c7ikthkaap2amfzm
Support importing .tar.xz and .tar.lzma files.  (Jelmer)

Show diffs side-by-side

added added

removed removed

Lines of Context:
upstream_import.py
 
1
"""Import upstream source into a branch"""
 
2
 
 
3
import errno
 
4
import os
 
5
from StringIO import StringIO
 
6
import stat
 
7
import tarfile
 
8
import zipfile
 
9
 
 
10
from bzrlib import generate_ids
 
11
from bzrlib.bzrdir import BzrDir
 
12
from bzrlib.errors import NoSuchFile, BzrCommandError, NotBranchError
 
13
from bzrlib.osutils import (pathjoin, isdir, file_iterator, basename,
 
14
                            file_kind, splitpath)
 
15
from bzrlib.trace import warning
 
16
from bzrlib.transform import TreeTransform, resolve_conflicts, cook_conflicts
 
17
from bzrlib.workingtree import WorkingTree
 
18
from bzrlib.plugins.bzrtools.bzrtools import open_from_url
 
19
 
 
20
class ZipFileWrapper(object):
 
21
 
 
22
    def __init__(self, fileobj, mode):
 
23
        self.zipfile = zipfile.ZipFile(fileobj, mode)
 
24
 
 
25
    def getmembers(self):
 
26
        for info in self.zipfile.infolist():
 
27
            yield ZipInfoWrapper(self.zipfile, info)
 
28
 
 
29
    def extractfile(self, infowrapper):
 
30
        return StringIO(self.zipfile.read(infowrapper.name))
 
31
 
 
32
    def add(self, filename):
 
33
        if isdir(filename):
 
34
            self.zipfile.writestr(filename+'/', '')
 
35
        else:
 
36
            self.zipfile.write(filename)
 
37
 
 
38
    def close(self):
 
39
        self.zipfile.close()
 
40
 
 
41
 
 
42
class ZipInfoWrapper(object):
 
43
 
 
44
    def __init__(self, zipfile, info):
 
45
        self.info = info
 
46
        self.type = None
 
47
        self.name = info.filename
 
48
        self.zipfile = zipfile
 
49
        self.mode = 0666
 
50
 
 
51
    def isdir(self):
 
52
        # Really? Eeeew!
 
53
        return bool(self.name.endswith('/'))
 
54
 
 
55
    def isreg(self):
 
56
        # Really? Eeeew!
 
57
        return not self.isdir()
 
58
 
 
59
 
 
60
class DirWrapper(object):
 
61
    def __init__(self, fileobj, mode='r'):
 
62
        assert mode == 'r', mode
 
63
        self.root = os.path.realpath(fileobj.read())
 
64
 
 
65
    def __repr__(self):
 
66
        return 'DirWrapper(%r)' % self.root
 
67
 
 
68
    def getmembers(self, subdir=None):
 
69
        if subdir is not None:
 
70
            mydir = pathjoin(self.root, subdir)
 
71
        else:
 
72
            mydir = self.root
 
73
        for child in os.listdir(mydir):
 
74
            if subdir is not None:
 
75
                child = pathjoin(subdir, child)
 
76
            fi = FileInfo(self.root, child)
 
77
            yield fi
 
78
            if fi.isdir():
 
79
                for v in self.getmembers(child):
 
80
                    yield v
 
81
 
 
82
    def extractfile(self, member):
 
83
        return open(member.fullpath)
 
84
 
 
85
 
 
86
class FileInfo(object):
 
87
 
 
88
    def __init__(self, root, filepath):
 
89
        self.fullpath = pathjoin(root, filepath)
 
90
        self.root = root
 
91
        if filepath != '':
 
92
            self.name = pathjoin(basename(root), filepath)
 
93
        else:
 
94
            print 'root %r' % root
 
95
            self.name = basename(root)
 
96
        self.type = None
 
97
        stat = os.lstat(self.fullpath)
 
98
        self.mode = stat.st_mode
 
99
        if self.isdir():
 
100
            self.name += '/'
 
101
 
 
102
    def __repr__(self):
 
103
        return 'FileInfo(%r)' % self.name
 
104
 
 
105
    def isreg(self):
 
106
        return stat.S_ISREG(self.mode)
 
107
 
 
108
    def isdir(self):
 
109
        return stat.S_ISDIR(self.mode)
 
110
 
 
111
    def issym(self):
 
112
        if stat.S_ISLNK(self.mode):
 
113
            self.linkname = os.readlink(self.fullpath)
 
114
            return True
 
115
        else:
 
116
            return False
 
117
 
 
118
 
 
119
def top_path(path):
 
120
    """Return the top directory given in a path."""
 
121
    components = splitpath(path)
 
122
    if len(components) > 0:
 
123
        return components[0]
 
124
    else:
 
125
        return ''
 
126
 
 
127
 
 
128
def common_directory(names):
 
129
    """Determine a single directory prefix from a list of names"""
 
130
    possible_prefix = None
 
131
    for name in names:
 
132
        name_top = top_path(name)
 
133
        if name_top == '':
 
134
            return None
 
135
        if possible_prefix is None:
 
136
            possible_prefix = name_top
 
137
        else:
 
138
            if name_top != possible_prefix:
 
139
                return None
 
140
    return possible_prefix
 
141
 
 
142
 
 
143
def do_directory(tt, trans_id, tree, relative_path, path):
 
144
    if isdir(path) and tree.path2id(relative_path) is not None:
 
145
        tt.cancel_deletion(trans_id)
 
146
    else:
 
147
        tt.create_directory(trans_id)
 
148
 
 
149
 
 
150
def add_implied_parents(implied_parents, path):
 
151
    """Update the set of implied parents from a path"""
 
152
    parent = os.path.dirname(path)
 
153
    if parent in implied_parents:
 
154
        return
 
155
    implied_parents.add(parent)
 
156
    add_implied_parents(implied_parents, parent)
 
157
 
 
158
 
 
159
def names_of_files(tar_file):
 
160
    for member in tar_file.getmembers():
 
161
        if member.type != "g":
 
162
            yield member.name
 
163
 
 
164
 
 
165
def should_ignore(relative_path):
 
166
    return top_path(relative_path) == '.bzr'
 
167
 
 
168
 
 
169
def import_tar(tree, tar_input):
 
170
    """Replace the contents of a working directory with tarfile contents.
 
171
    The tarfile may be a gzipped stream.  File ids will be updated.
 
172
    """
 
173
    tar_file = tarfile.open('lala', 'r', tar_input)
 
174
    import_archive(tree, tar_file)
 
175
 
 
176
def import_zip(tree, zip_input):
 
177
    zip_file = ZipFileWrapper(zip_input, 'r')
 
178
    import_archive(tree, zip_file)
 
179
 
 
180
def import_dir(tree, dir_input):
 
181
    dir_file = DirWrapper(dir_input)
 
182
    import_archive(tree, dir_file)
 
183
 
 
184
 
 
185
def import_archive(tree, archive_file):
 
186
    tt = TreeTransform(tree)
 
187
    try:
 
188
        import_archive_to_transform(tree, archive_file, tt)
 
189
        tt.apply()
 
190
    finally:
 
191
        tt.finalize()
 
192
 
 
193
 
 
194
def import_archive_to_transform(tree, archive_file, tt):
 
195
    prefix = common_directory(names_of_files(archive_file))
 
196
    removed = set()
 
197
    for path, entry in tree.inventory.iter_entries():
 
198
        if entry.parent_id is None:
 
199
            continue
 
200
        trans_id = tt.trans_id_tree_path(path)
 
201
        tt.delete_contents(trans_id)
 
202
        removed.add(path)
 
203
 
 
204
    added = set()
 
205
    implied_parents = set()
 
206
    seen = set()
 
207
    for member in archive_file.getmembers():
 
208
        if member.type == 'g':
 
209
            # type 'g' is a header
 
210
            continue
 
211
        # Inverse functionality in bzr uses utf-8.  We could also
 
212
        # interpret relative to fs encoding, which would match native
 
213
        # behaviour better.
 
214
        relative_path = member.name.decode('utf-8')
 
215
        if prefix is not None:
 
216
            relative_path = relative_path[len(prefix)+1:]
 
217
            relative_path = relative_path.rstrip('/')
 
218
        if relative_path == '':
 
219
            continue
 
220
        if should_ignore(relative_path):
 
221
            continue
 
222
        add_implied_parents(implied_parents, relative_path)
 
223
        trans_id = tt.trans_id_tree_path(relative_path)
 
224
        added.add(relative_path.rstrip('/'))
 
225
        path = tree.abspath(relative_path)
 
226
        if member.name in seen:
 
227
            if tt.final_kind(trans_id) == 'file':
 
228
                tt.set_executability(None, trans_id)
 
229
            tt.cancel_creation(trans_id)
 
230
        seen.add(member.name)
 
231
        if member.isreg():
 
232
            tt.create_file(file_iterator(archive_file.extractfile(member)),
 
233
                           trans_id)
 
234
            executable = (member.mode & 0111) != 0
 
235
            tt.set_executability(executable, trans_id)
 
236
        elif member.isdir():
 
237
            do_directory(tt, trans_id, tree, relative_path, path)
 
238
        elif member.issym():
 
239
            tt.create_symlink(member.linkname, trans_id)
 
240
        else:
 
241
            continue
 
242
        if tt.tree_file_id(trans_id) is None:
 
243
            name = basename(member.name.rstrip('/'))
 
244
            file_id = generate_ids.gen_file_id(name)
 
245
            tt.version_file(file_id, trans_id)
 
246
 
 
247
    for relative_path in implied_parents.difference(added):
 
248
        if relative_path == "":
 
249
            continue
 
250
        trans_id = tt.trans_id_tree_path(relative_path)
 
251
        path = tree.abspath(relative_path)
 
252
        do_directory(tt, trans_id, tree, relative_path, path)
 
253
        if tt.tree_file_id(trans_id) is None:
 
254
            tt.version_file(trans_id, trans_id)
 
255
        added.add(relative_path)
 
256
 
 
257
    for path in removed.difference(added):
 
258
        tt.unversion_file(tt.trans_id_tree_path(path))
 
259
 
 
260
    for conflict in cook_conflicts(resolve_conflicts(tt), tt):
 
261
        warning(conflict)
 
262
 
 
263
 
 
264
def do_import(source, tree_directory=None):
 
265
    """Implementation of import command.  Intended for UI only"""
 
266
    if tree_directory is not None:
 
267
        try:
 
268
            tree = WorkingTree.open(tree_directory)
 
269
        except NotBranchError:
 
270
            if not os.path.exists(tree_directory):
 
271
                os.mkdir(tree_directory)
 
272
            branch = BzrDir.create_branch_convenience(tree_directory)
 
273
            tree = branch.bzrdir.open_workingtree()
 
274
    else:
 
275
        tree = WorkingTree.open_containing('.')[0]
 
276
    tree.lock_write()
 
277
    try:
 
278
        if tree.changes_from(tree.basis_tree()).has_changed():
 
279
            raise BzrCommandError("Working tree has uncommitted changes.")
 
280
 
 
281
        if (source.endswith('.tar') or source.endswith('.tar.gz') or
 
282
            source.endswith('.tar.bz2') or source.endswith('.tgz') or
 
283
            source.endswith('.tar.lzma') or source.endswith('.tar.xz')):
 
284
            try:
 
285
                tar_input = open_from_url(source)
 
286
                if source.endswith('.bz2'):
 
287
                    import bz2
 
288
                    tar_input = StringIO(bz2.decompress(tar_input.read()))
 
289
                elif source.endswith('.xz') or source.endswith('.lzma'):
 
290
                    import lzma
 
291
                    tar_input = StringIO(lzma.decompress(tar_input.read()))
 
292
            except IOError, e:
 
293
                if e.errno == errno.ENOENT:
 
294
                    raise NoSuchFile(source)
 
295
            try:
 
296
                import_tar(tree, tar_input)
 
297
            finally:
 
298
                tar_input.close()
 
299
        elif source.endswith('.zip'):
 
300
            import_zip(tree, open_from_url(source))
 
301
        elif file_kind(source) == 'directory':
 
302
            s = StringIO(source)
 
303
            s.seek(0)
 
304
            import_dir(tree, s)
 
305
        else:
 
306
            raise BzrCommandError('Unhandled import source')
 
307
    finally:
 
308
        tree.unlock()