← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/transform.py

  • Committer: Canonical.com Patch Queue Manager
  • Date: 2009-08-27 02:27:19 UTC
  • mfrom: (4634.3.19 gc-batching)
  • Revision ID: pqm@pqm.ubuntu.com-20090827022719-bl2yoqhpj3fcfczu
(andrew) Fix #402657: 2a fetch over dumb transport reads one group at
        a time.

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/transform.py
1
 
# Copyright (C) 2006, 2007, 2008 Canonical Ltd
 
1
# Copyright (C) 2006, 2007, 2008, 2009 Canonical Ltd
2
2
#
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
12
12
#
13
13
# You should have received a copy of the GNU General Public License
14
14
# along with this program; if not, write to the Free Software
15
 
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16
16
 
17
17
import os
18
18
import errno
22
22
lazy_import(globals(), """
23
23
from bzrlib import (
24
24
    annotate,
 
25
    bencode,
25
26
    bzrdir,
26
27
    delta,
27
28
    errors,
28
29
    inventory,
 
30
    multiparent,
29
31
    osutils,
30
32
    revision as _mod_revision,
31
33
    )
34
36
                           ReusingTransform, NotVersionedError, CantMoveRoot,
35
37
                           ExistingLimbo, ImmortalLimbo, NoFinalPath,
36
38
                           UnableCreateSymlink)
 
39
from bzrlib.filters import filtered_output_bytes, ContentFilterContext
37
40
from bzrlib.inventory import InventoryEntry
38
41
from bzrlib.osutils import (
39
42
    delete_any,
41
44
    has_symlinks,
42
45
    lexists,
43
46
    pathjoin,
 
47
    sha_file,
44
48
    splitpath,
45
49
    supports_executable,
46
50
)
47
51
from bzrlib.progress import DummyProgress, ProgressPhase
48
52
from bzrlib.symbol_versioning import (
49
53
        deprecated_function,
 
54
        deprecated_in,
50
55
        )
51
56
from bzrlib.trace import mutter, warning
52
57
from bzrlib import tree
71
76
 
72
77
 
73
78
class TreeTransformBase(object):
74
 
    """The base class for TreeTransform and TreeTransformBase"""
 
79
    """The base class for TreeTransform and its kin."""
75
80
 
76
 
    def __init__(self, tree, limbodir, pb=DummyProgress(),
 
81
    def __init__(self, tree, pb=DummyProgress(),
77
82
                 case_sensitive=True):
78
83
        """Constructor.
79
84
 
80
85
        :param tree: The tree that will be transformed, but not necessarily
81
86
            the output tree.
82
 
        :param limbodir: A directory where new files can be stored until
83
 
            they are installed in their proper places
84
 
        :param pb: A ProgressBar indicating how much progress is being made
 
87
        :param pb: A ProgressTask indicating how much progress is being made
85
88
        :param case_sensitive: If True, the target of the transform is
86
89
            case sensitive, not just case preserving.
87
90
        """
88
91
        object.__init__(self)
89
92
        self._tree = tree
90
 
        self._limbodir = limbodir
91
 
        self._deletiondir = None
92
93
        self._id_number = 0
93
94
        # mapping of trans_id -> new basename
94
95
        self._new_name = {}
96
97
        self._new_parent = {}
97
98
        # mapping of trans_id with new contents -> new file_kind
98
99
        self._new_contents = {}
99
 
        # A mapping of transform ids to their limbo filename
100
 
        self._limbo_files = {}
101
 
        # A mapping of transform ids to a set of the transform ids of children
102
 
        # that their limbo directory has
103
 
        self._limbo_children = {}
104
 
        # Map transform ids to maps of child filename to child transform id
105
 
        self._limbo_children_names = {}
106
 
        # List of transform ids that need to be renamed from limbo into place
107
 
        self._needs_rename = set()
108
100
        # Set of trans_ids whose contents will be removed
109
101
        self._removed_contents = set()
110
102
        # Mapping of trans_id -> new execute-bit value
117
109
        self._non_present_ids = {}
118
110
        # Mapping of new file_id -> trans_id
119
111
        self._r_new_id = {}
120
 
        # Set of file_ids that will be removed
 
112
        # Set of trans_ids that will be removed
121
113
        self._removed_id = set()
122
114
        # Mapping of path in old tree -> trans_id
123
115
        self._tree_path_ids = {}
124
116
        # Mapping trans_id -> path in old tree
125
117
        self._tree_id_paths = {}
126
 
        # Cache of realpath results, to speed up canonical_path
127
 
        self._realpaths = {}
128
 
        # Cache of relpath results, to speed up canonical_path
129
 
        self._relpaths = {}
130
118
        # The trans_id that will be used as the tree root
131
119
        root_id = tree.get_root_id()
132
120
        if root_id is not None:
142
130
        # A counter of how many files have been renamed
143
131
        self.rename_count = 0
144
132
 
 
133
    def finalize(self):
 
134
        """Release the working tree lock, if held.
 
135
 
 
136
        This is required if apply has not been invoked, but can be invoked
 
137
        even after apply.
 
138
        """
 
139
        if self._tree is None:
 
140
            return
 
141
        self._tree.unlock()
 
142
        self._tree = None
 
143
 
145
144
    def __get_root(self):
146
145
        return self._new_root
147
146
 
148
147
    root = property(__get_root)
149
148
 
150
 
    def finalize(self):
151
 
        """Release the working tree lock, if held, clean up limbo dir.
152
 
 
153
 
        This is required if apply has not been invoked, but can be invoked
154
 
        even after apply.
155
 
        """
156
 
        if self._tree is None:
157
 
            return
158
 
        try:
159
 
            entries = [(self._limbo_name(t), t, k) for t, k in
160
 
                       self._new_contents.iteritems()]
161
 
            entries.sort(reverse=True)
162
 
            for path, trans_id, kind in entries:
163
 
                if kind == "directory":
164
 
                    os.rmdir(path)
165
 
                else:
166
 
                    os.unlink(path)
167
 
            try:
168
 
                os.rmdir(self._limbodir)
169
 
            except OSError:
170
 
                # We don't especially care *why* the dir is immortal.
171
 
                raise ImmortalLimbo(self._limbodir)
172
 
            try:
173
 
                if self._deletiondir is not None:
174
 
                    os.rmdir(self._deletiondir)
175
 
            except OSError:
176
 
                raise errors.ImmortalPendingDeletion(self._deletiondir)
177
 
        finally:
178
 
            self._tree.unlock()
179
 
            self._tree = None
180
 
 
181
149
    def _assign_id(self):
182
150
        """Produce a new tranform id"""
183
151
        new_id = "new-%s" % self._id_number
195
163
        """Change the path that is assigned to a transaction id."""
196
164
        if trans_id == self._new_root:
197
165
            raise CantMoveRoot
198
 
        previous_parent = self._new_parent.get(trans_id)
199
 
        previous_name = self._new_name.get(trans_id)
200
166
        self._new_name[trans_id] = name
201
167
        self._new_parent[trans_id] = parent
202
168
        if parent == ROOT_PARENT:
203
169
            if self._new_root is not None:
204
170
                raise ValueError("Cannot have multiple roots.")
205
171
            self._new_root = trans_id
206
 
        if (trans_id in self._limbo_files and
207
 
            trans_id not in self._needs_rename):
208
 
            self._rename_in_limbo([trans_id])
209
 
            self._limbo_children[previous_parent].remove(trans_id)
210
 
            del self._limbo_children_names[previous_parent][previous_name]
211
 
 
212
 
    def _rename_in_limbo(self, trans_ids):
213
 
        """Fix limbo names so that the right final path is produced.
214
 
 
215
 
        This means we outsmarted ourselves-- we tried to avoid renaming
216
 
        these files later by creating them with their final names in their
217
 
        final parents.  But now the previous name or parent is no longer
218
 
        suitable, so we have to rename them.
219
 
 
220
 
        Even for trans_ids that have no new contents, we must remove their
221
 
        entries from _limbo_files, because they are now stale.
222
 
        """
223
 
        for trans_id in trans_ids:
224
 
            old_path = self._limbo_files.pop(trans_id)
225
 
            if trans_id not in self._new_contents:
226
 
                continue
227
 
            new_path = self._limbo_name(trans_id)
228
 
            os.rename(old_path, new_path)
229
172
 
230
173
    def adjust_root_path(self, name, parent):
231
174
        """Emulate moving the root by moving all children, instead.
232
 
        
 
175
 
233
176
        We do this by undoing the association of root's transaction id with the
234
177
        current tree.  This allows us to create a new directory with that
235
 
        transaction id.  We unversion the root directory and version the 
 
178
        transaction id.  We unversion the root directory and version the
236
179
        physically new directory, and hope someone versions the tree root
237
180
        later.
238
181
        """
241
184
        # force moving all children of root
242
185
        for child_id in self.iter_tree_children(old_root):
243
186
            if child_id != parent:
244
 
                self.adjust_path(self.final_name(child_id), 
 
187
                self.adjust_path(self.final_name(child_id),
245
188
                                 self.final_parent(child_id), child_id)
246
189
            file_id = self.final_file_id(child_id)
247
190
            if file_id is not None:
248
191
                self.unversion_file(child_id)
249
192
            self.version_file(file_id, child_id)
250
 
        
 
193
 
251
194
        # the physical root needs a new transaction id
252
195
        self._tree_path_ids.pop("")
253
196
        self._tree_id_paths.pop(old_root)
261
204
 
262
205
    def trans_id_tree_file_id(self, inventory_id):
263
206
        """Determine the transaction id of a working tree file.
264
 
        
 
207
 
265
208
        This reflects only files that already exist, not ones that will be
266
209
        added by transactions.
267
210
        """
280
223
            raise ValueError('None is not a valid file id')
281
224
        if file_id in self._r_new_id and self._r_new_id[file_id] is not None:
282
225
            return self._r_new_id[file_id]
283
 
        elif file_id in self._tree.inventory:
284
 
            return self.trans_id_tree_file_id(file_id)
285
 
        elif file_id in self._non_present_ids:
286
 
            return self._non_present_ids[file_id]
287
 
        else:
288
 
            trans_id = self._assign_id()
289
 
            self._non_present_ids[file_id] = trans_id
290
 
            return trans_id
291
 
 
292
 
    def canonical_path(self, path):
293
 
        """Get the canonical tree-relative path"""
294
 
        # don't follow final symlinks
295
 
        abs = self._tree.abspath(path)
296
 
        if abs in self._relpaths:
297
 
            return self._relpaths[abs]
298
 
        dirname, basename = os.path.split(abs)
299
 
        if dirname not in self._realpaths:
300
 
            self._realpaths[dirname] = os.path.realpath(dirname)
301
 
        dirname = self._realpaths[dirname]
302
 
        abs = pathjoin(dirname, basename)
303
 
        if dirname in self._relpaths:
304
 
            relpath = pathjoin(self._relpaths[dirname], basename)
305
 
            relpath = relpath.rstrip('/\\')
306
 
        else:
307
 
            relpath = self._tree.relpath(abs)
308
 
        self._relpaths[abs] = relpath
309
 
        return relpath
 
226
        else:
 
227
            try:
 
228
                self._tree.iter_entries_by_dir([file_id]).next()
 
229
            except StopIteration:
 
230
                if file_id in self._non_present_ids:
 
231
                    return self._non_present_ids[file_id]
 
232
                else:
 
233
                    trans_id = self._assign_id()
 
234
                    self._non_present_ids[file_id] = trans_id
 
235
                    return trans_id
 
236
            else:
 
237
                return self.trans_id_tree_file_id(file_id)
310
238
 
311
239
    def trans_id_tree_path(self, path):
312
240
        """Determine (and maybe set) the transaction ID for a tree path."""
323
251
            return ROOT_PARENT
324
252
        return self.trans_id_tree_path(os.path.dirname(path))
325
253
 
326
 
    def create_file(self, contents, trans_id, mode_id=None):
327
 
        """Schedule creation of a new file.
328
 
 
329
 
        See also new_file.
330
 
        
331
 
        Contents is an iterator of strings, all of which will be written
332
 
        to the target destination.
333
 
 
334
 
        New file takes the permissions of any existing file with that id,
335
 
        unless mode_id is specified.
336
 
        """
337
 
        name = self._limbo_name(trans_id)
338
 
        f = open(name, 'wb')
339
 
        try:
340
 
            try:
341
 
                unique_add(self._new_contents, trans_id, 'file')
342
 
            except:
343
 
                # Clean up the file, it never got registered so
344
 
                # TreeTransform.finalize() won't clean it up.
345
 
                f.close()
346
 
                os.unlink(name)
347
 
                raise
348
 
 
349
 
            f.writelines(contents)
350
 
        finally:
351
 
            f.close()
352
 
        self._set_mode(trans_id, mode_id, S_ISREG)
353
 
 
354
 
    def _set_mode(self, trans_id, mode_id, typefunc):
355
 
        """Set the mode of new file contents.
356
 
        The mode_id is the existing file to get the mode from (often the same
357
 
        as trans_id).  The operation is only performed if there's a mode match
358
 
        according to typefunc.
359
 
        """
360
 
        if mode_id is None:
361
 
            mode_id = trans_id
362
 
        try:
363
 
            old_path = self._tree_id_paths[mode_id]
364
 
        except KeyError:
365
 
            return
366
 
        try:
367
 
            mode = os.stat(self._tree.abspath(old_path)).st_mode
368
 
        except OSError, e:
369
 
            if e.errno in (errno.ENOENT, errno.ENOTDIR):
370
 
                # Either old_path doesn't exist, or the parent of the
371
 
                # target is not a directory (but will be one eventually)
372
 
                # Either way, we know it doesn't exist *right now*
373
 
                # See also bug #248448
374
 
                return
375
 
            else:
376
 
                raise
377
 
        if typefunc(mode):
378
 
            os.chmod(self._limbo_name(trans_id), mode)
379
 
 
380
 
    def create_hardlink(self, path, trans_id):
381
 
        """Schedule creation of a hard link"""
382
 
        name = self._limbo_name(trans_id)
383
 
        try:
384
 
            os.link(path, name)
385
 
        except OSError, e:
386
 
            if e.errno != errno.EPERM:
387
 
                raise
388
 
            raise errors.HardLinkNotSupported(path)
389
 
        try:
390
 
            unique_add(self._new_contents, trans_id, 'file')
391
 
        except:
392
 
            # Clean up the file, it never got registered so
393
 
            # TreeTransform.finalize() won't clean it up.
394
 
            os.unlink(name)
395
 
            raise
396
 
 
397
 
    def create_directory(self, trans_id):
398
 
        """Schedule creation of a new directory.
399
 
        
400
 
        See also new_directory.
401
 
        """
402
 
        os.mkdir(self._limbo_name(trans_id))
403
 
        unique_add(self._new_contents, trans_id, 'directory')
404
 
 
405
 
    def create_symlink(self, target, trans_id):
406
 
        """Schedule creation of a new symbolic link.
407
 
 
408
 
        target is a bytestring.
409
 
        See also new_symlink.
410
 
        """
411
 
        if has_symlinks():
412
 
            os.symlink(target, self._limbo_name(trans_id))
413
 
            unique_add(self._new_contents, trans_id, 'symlink')
414
 
        else:
415
 
            try:
416
 
                path = FinalPaths(self).get_path(trans_id)
417
 
            except KeyError:
418
 
                path = None
419
 
            raise UnableCreateSymlink(path=path)
420
 
 
421
 
    def cancel_creation(self, trans_id):
422
 
        """Cancel the creation of new file contents."""
423
 
        del self._new_contents[trans_id]
424
 
        children = self._limbo_children.get(trans_id)
425
 
        # if this is a limbo directory with children, move them before removing
426
 
        # the directory
427
 
        if children is not None:
428
 
            self._rename_in_limbo(children)
429
 
            del self._limbo_children[trans_id]
430
 
            del self._limbo_children_names[trans_id]
431
 
        delete_any(self._limbo_name(trans_id))
432
 
 
433
254
    def delete_contents(self, trans_id):
434
255
        """Schedule the contents of a path entry for deletion"""
435
256
        self.tree_kind(trans_id)
509
330
        new_ids.update(changed_kind)
510
331
        return sorted(FinalPaths(self).get_paths(new_ids))
511
332
 
512
 
    def tree_kind(self, trans_id):
513
 
        """Determine the file kind in the working tree.
514
 
 
515
 
        Raises NoSuchFile if the file does not exist
516
 
        """
517
 
        path = self._tree_id_paths.get(trans_id)
518
 
        if path is None:
519
 
            raise NoSuchFile(None)
520
 
        try:
521
 
            return file_kind(self._tree.abspath(path))
522
 
        except OSError, e:
523
 
            if e.errno != errno.ENOENT:
524
 
                raise
525
 
            else:
526
 
                raise NoSuchFile(path)
527
 
 
528
333
    def final_kind(self, trans_id):
529
334
        """Determine the final file kind, after any changes applied.
530
 
        
 
335
 
531
336
        Raises NoSuchFile if the file does not exist/has no contents.
532
337
        (It is conceivable that a path would be created without the
533
338
        corresponding contents insertion command)
549
354
        # the file is old; the old id is still valid
550
355
        if self._new_root == trans_id:
551
356
            return self._tree.get_root_id()
552
 
        return self._tree.inventory.path2id(path)
 
357
        return self._tree.path2id(path)
553
358
 
554
359
    def final_file_id(self, trans_id):
555
360
        """Determine the file id after any changes are applied, or None.
556
 
        
 
361
 
557
362
        None indicates that the file will not be versioned after changes are
558
363
        applied.
559
364
        """
598
403
 
599
404
    def by_parent(self):
600
405
        """Return a map of parent: children for known parents.
601
 
        
 
406
 
602
407
        Only new paths and parents of tree files with assigned ids are used.
603
408
        """
604
409
        by_parent = {}
605
410
        items = list(self._new_parent.iteritems())
606
 
        items.extend((t, self.final_parent(t)) for t in 
 
411
        items.extend((t, self.final_parent(t)) for t in
607
412
                      self._tree_id_paths.keys())
608
413
        for trans_id, parent_id in items:
609
414
            if parent_id not in by_parent:
637
442
        conflicts.extend(self._overwrite_conflicts())
638
443
        return conflicts
639
444
 
 
445
    def _check_malformed(self):
 
446
        conflicts = self.find_conflicts()
 
447
        if len(conflicts) != 0:
 
448
            raise MalformedTransform(conflicts=conflicts)
 
449
 
640
450
    def _add_tree_children(self):
641
451
        """Add all the children of all active parents to the known paths.
642
452
 
644
454
        removed.  This is a necessary first step in detecting conflicts.
645
455
        """
646
456
        parents = self.by_parent().keys()
647
 
        parents.extend([t for t in self._removed_contents if 
 
457
        parents.extend([t for t in self._removed_contents if
648
458
                        self.tree_kind(t) == 'directory'])
649
459
        for trans_id in self._removed_id:
650
460
            file_id = self.tree_file_id(trans_id)
658
468
            # ensure that all children are registered with the transaction
659
469
            list(self.iter_tree_children(parent_id))
660
470
 
661
 
    def iter_tree_children(self, parent_id):
662
 
        """Iterate through the entry's tree children, if any"""
663
 
        try:
664
 
            path = self._tree_id_paths[parent_id]
665
 
        except KeyError:
666
 
            return
667
 
        try:
668
 
            children = os.listdir(self._tree.abspath(path))
669
 
        except OSError, e:
670
 
            if not (osutils._is_error_enotdir(e)
671
 
                    or e.errno in (errno.ENOENT, errno.ESRCH)):
672
 
                raise
673
 
            return
674
 
 
675
 
        for child in children:
676
 
            childpath = joinpath(path, child)
677
 
            if self._tree.is_control_filename(childpath):
678
 
                continue
679
 
            yield self.trans_id_tree_path(childpath)
680
 
 
681
471
    def has_named_child(self, by_parent, parent_id, name):
682
472
        try:
683
473
            children = by_parent[parent_id]
737
527
 
738
528
    def _improper_versioning(self):
739
529
        """Cannot version a file with no contents, or a bad type.
740
 
        
 
530
 
741
531
        However, existing entries with no contents are okay.
742
532
        """
743
533
        conflicts = []
753
543
 
754
544
    def _executability_conflicts(self):
755
545
        """Check for bad executability changes.
756
 
        
 
546
 
757
547
        Only versioned files may have their executability set, because
758
548
        1. only versioned entries can have executability under windows
759
549
        2. only files can be executable.  (The execute bit on a directory
858
648
            return True
859
649
        return False
860
650
 
861
 
    def _limbo_name(self, trans_id):
862
 
        """Generate the limbo name of a file"""
863
 
        limbo_name = self._limbo_files.get(trans_id)
864
 
        if limbo_name is not None:
865
 
            return limbo_name
866
 
        parent = self._new_parent.get(trans_id)
867
 
        # if the parent directory is already in limbo (e.g. when building a
868
 
        # tree), choose a limbo name inside the parent, to reduce further
869
 
        # renames.
870
 
        use_direct_path = False
871
 
        if self._new_contents.get(parent) == 'directory':
872
 
            filename = self._new_name.get(trans_id)
873
 
            if filename is not None:
874
 
                if parent not in self._limbo_children:
875
 
                    self._limbo_children[parent] = set()
876
 
                    self._limbo_children_names[parent] = {}
877
 
                    use_direct_path = True
878
 
                # the direct path can only be used if no other file has
879
 
                # already taken this pathname, i.e. if the name is unused, or
880
 
                # if it is already associated with this trans_id.
881
 
                elif self._case_sensitive_target:
882
 
                    if (self._limbo_children_names[parent].get(filename)
883
 
                        in (trans_id, None)):
884
 
                        use_direct_path = True
885
 
                else:
886
 
                    for l_filename, l_trans_id in\
887
 
                        self._limbo_children_names[parent].iteritems():
888
 
                        if l_trans_id == trans_id:
889
 
                            continue
890
 
                        if l_filename.lower() == filename.lower():
891
 
                            break
892
 
                    else:
893
 
                        use_direct_path = True
894
 
 
895
 
        if use_direct_path:
896
 
            limbo_name = pathjoin(self._limbo_files[parent], filename)
897
 
            self._limbo_children[parent].add(trans_id)
898
 
            self._limbo_children_names[parent][filename] = trans_id
899
 
        else:
900
 
            limbo_name = pathjoin(self._limbodir, trans_id)
901
 
            self._needs_rename.add(trans_id)
902
 
        self._limbo_files[trans_id] = limbo_name
903
 
        return limbo_name
904
 
 
905
651
    def _set_executability(self, path, trans_id):
906
652
        """Set the executability of versioned files """
907
653
        if supports_executable():
928
674
            self.version_file(file_id, trans_id)
929
675
        return trans_id
930
676
 
931
 
    def new_file(self, name, parent_id, contents, file_id=None, 
 
677
    def new_file(self, name, parent_id, contents, file_id=None,
932
678
                 executable=None):
933
679
        """Convenience method to create files.
934
 
        
 
680
 
935
681
        name is the name of the file to create.
936
682
        parent_id is the transaction id of the parent directory of the file.
937
683
        contents is an iterator of bytestrings, which will be used to produce
957
703
        """
958
704
        trans_id = self._new_entry(name, parent_id, file_id)
959
705
        self.create_directory(trans_id)
960
 
        return trans_id 
 
706
        return trans_id
961
707
 
962
708
    def new_symlink(self, name, parent_id, target, file_id=None):
963
709
        """Convenience method to create symbolic link.
964
 
        
 
710
 
965
711
        name is the name of the symlink to create.
966
712
        parent_id is the transaction id of the parent directory of the symlink.
967
713
        target is a bytestring of the target of the symlink.
1006
752
        from_path = self._tree_id_paths.get(from_trans_id)
1007
753
        if from_versioned:
1008
754
            # get data from working tree if versioned
1009
 
            from_entry = self._tree.inventory[file_id]
 
755
            from_entry = self._tree.iter_entries_by_dir([file_id]).next()[1]
1010
756
            from_name = from_entry.name
1011
757
            from_parent = from_entry.parent_id
1012
758
        else:
1118
864
        """
1119
865
        return _PreviewTree(self)
1120
866
 
1121
 
 
1122
 
class TreeTransform(TreeTransformBase):
 
867
    def commit(self, branch, message, merge_parents=None, strict=False):
 
868
        """Commit the result of this TreeTransform to a branch.
 
869
 
 
870
        :param branch: The branch to commit to.
 
871
        :param message: The message to attach to the commit.
 
872
        :param merge_parents: Additional parents specified by pending merges.
 
873
        :return: The revision_id of the revision committed.
 
874
        """
 
875
        self._check_malformed()
 
876
        if strict:
 
877
            unversioned = set(self._new_contents).difference(set(self._new_id))
 
878
            for trans_id in unversioned:
 
879
                if self.final_file_id(trans_id) is None:
 
880
                    raise errors.StrictCommitFailed()
 
881
 
 
882
        revno, last_rev_id = branch.last_revision_info()
 
883
        if last_rev_id == _mod_revision.NULL_REVISION:
 
884
            if merge_parents is not None:
 
885
                raise ValueError('Cannot supply merge parents for first'
 
886
                                 ' commit.')
 
887
            parent_ids = []
 
888
        else:
 
889
            parent_ids = [last_rev_id]
 
890
            if merge_parents is not None:
 
891
                parent_ids.extend(merge_parents)
 
892
        if self._tree.get_revision_id() != last_rev_id:
 
893
            raise ValueError('TreeTransform not based on branch basis: %s' %
 
894
                             self._tree.get_revision_id())
 
895
        builder = branch.get_commit_builder(parent_ids)
 
896
        preview = self.get_preview_tree()
 
897
        list(builder.record_iter_changes(preview, last_rev_id,
 
898
                                         self.iter_changes()))
 
899
        builder.finish_inventory()
 
900
        revision_id = builder.commit(message)
 
901
        branch.set_last_revision_info(revno + 1, revision_id)
 
902
        return revision_id
 
903
 
 
904
    def _text_parent(self, trans_id):
 
905
        file_id = self.tree_file_id(trans_id)
 
906
        try:
 
907
            if file_id is None or self._tree.kind(file_id) != 'file':
 
908
                return None
 
909
        except errors.NoSuchFile:
 
910
            return None
 
911
        return file_id
 
912
 
 
913
    def _get_parents_texts(self, trans_id):
 
914
        """Get texts for compression parents of this file."""
 
915
        file_id = self._text_parent(trans_id)
 
916
        if file_id is None:
 
917
            return ()
 
918
        return (self._tree.get_file_text(file_id),)
 
919
 
 
920
    def _get_parents_lines(self, trans_id):
 
921
        """Get lines for compression parents of this file."""
 
922
        file_id = self._text_parent(trans_id)
 
923
        if file_id is None:
 
924
            return ()
 
925
        return (self._tree.get_file_lines(file_id),)
 
926
 
 
927
    def serialize(self, serializer):
 
928
        """Serialize this TreeTransform.
 
929
 
 
930
        :param serializer: A Serialiser like pack.ContainerSerializer.
 
931
        """
 
932
        new_name = dict((k, v.encode('utf-8')) for k, v in
 
933
                        self._new_name.items())
 
934
        new_executability = dict((k, int(v)) for k, v in
 
935
                                 self._new_executability.items())
 
936
        tree_path_ids = dict((k.encode('utf-8'), v)
 
937
                             for k, v in self._tree_path_ids.items())
 
938
        attribs = {
 
939
            '_id_number': self._id_number,
 
940
            '_new_name': new_name,
 
941
            '_new_parent': self._new_parent,
 
942
            '_new_executability': new_executability,
 
943
            '_new_id': self._new_id,
 
944
            '_tree_path_ids': tree_path_ids,
 
945
            '_removed_id': list(self._removed_id),
 
946
            '_removed_contents': list(self._removed_contents),
 
947
            '_non_present_ids': self._non_present_ids,
 
948
            }
 
949
        yield serializer.bytes_record(bencode.bencode(attribs),
 
950
                                      (('attribs',),))
 
951
        for trans_id, kind in self._new_contents.items():
 
952
            if kind == 'file':
 
953
                lines = osutils.chunks_to_lines(
 
954
                    self._read_file_chunks(trans_id))
 
955
                parents = self._get_parents_lines(trans_id)
 
956
                mpdiff = multiparent.MultiParent.from_lines(lines, parents)
 
957
                content = ''.join(mpdiff.to_patch())
 
958
            if kind == 'directory':
 
959
                content = ''
 
960
            if kind == 'symlink':
 
961
                content = self._read_symlink_target(trans_id)
 
962
            yield serializer.bytes_record(content, ((trans_id, kind),))
 
963
 
 
964
    def deserialize(self, records):
 
965
        """Deserialize a stored TreeTransform.
 
966
 
 
967
        :param records: An iterable of (names, content) tuples, as per
 
968
            pack.ContainerPushParser.
 
969
        """
 
970
        names, content = records.next()
 
971
        attribs = bencode.bdecode(content)
 
972
        self._id_number = attribs['_id_number']
 
973
        self._new_name = dict((k, v.decode('utf-8'))
 
974
                            for k, v in attribs['_new_name'].items())
 
975
        self._new_parent = attribs['_new_parent']
 
976
        self._new_executability = dict((k, bool(v)) for k, v in
 
977
            attribs['_new_executability'].items())
 
978
        self._new_id = attribs['_new_id']
 
979
        self._r_new_id = dict((v, k) for k, v in self._new_id.items())
 
980
        self._tree_path_ids = {}
 
981
        self._tree_id_paths = {}
 
982
        for bytepath, trans_id in attribs['_tree_path_ids'].items():
 
983
            path = bytepath.decode('utf-8')
 
984
            self._tree_path_ids[path] = trans_id
 
985
            self._tree_id_paths[trans_id] = path
 
986
        self._removed_id = set(attribs['_removed_id'])
 
987
        self._removed_contents = set(attribs['_removed_contents'])
 
988
        self._non_present_ids = attribs['_non_present_ids']
 
989
        for ((trans_id, kind),), content in records:
 
990
            if kind == 'file':
 
991
                mpdiff = multiparent.MultiParent.from_patch(content)
 
992
                lines = mpdiff.to_lines(self._get_parents_texts(trans_id))
 
993
                self.create_file(lines, trans_id)
 
994
            if kind == 'directory':
 
995
                self.create_directory(trans_id)
 
996
            if kind == 'symlink':
 
997
                self.create_symlink(content.decode('utf-8'), trans_id)
 
998
 
 
999
 
 
1000
class DiskTreeTransform(TreeTransformBase):
 
1001
    """Tree transform storing its contents on disk."""
 
1002
 
 
1003
    def __init__(self, tree, limbodir, pb=DummyProgress(),
 
1004
                 case_sensitive=True):
 
1005
        """Constructor.
 
1006
        :param tree: The tree that will be transformed, but not necessarily
 
1007
            the output tree.
 
1008
        :param limbodir: A directory where new files can be stored until
 
1009
            they are installed in their proper places
 
1010
        :param pb: A ProgressBar indicating how much progress is being made
 
1011
        :param case_sensitive: If True, the target of the transform is
 
1012
            case sensitive, not just case preserving.
 
1013
        """
 
1014
        TreeTransformBase.__init__(self, tree, pb, case_sensitive)
 
1015
        self._limbodir = limbodir
 
1016
        self._deletiondir = None
 
1017
        # A mapping of transform ids to their limbo filename
 
1018
        self._limbo_files = {}
 
1019
        # A mapping of transform ids to a set of the transform ids of children
 
1020
        # that their limbo directory has
 
1021
        self._limbo_children = {}
 
1022
        # Map transform ids to maps of child filename to child transform id
 
1023
        self._limbo_children_names = {}
 
1024
        # List of transform ids that need to be renamed from limbo into place
 
1025
        self._needs_rename = set()
 
1026
 
 
1027
    def finalize(self):
 
1028
        """Release the working tree lock, if held, clean up limbo dir.
 
1029
 
 
1030
        This is required if apply has not been invoked, but can be invoked
 
1031
        even after apply.
 
1032
        """
 
1033
        if self._tree is None:
 
1034
            return
 
1035
        try:
 
1036
            entries = [(self._limbo_name(t), t, k) for t, k in
 
1037
                       self._new_contents.iteritems()]
 
1038
            entries.sort(reverse=True)
 
1039
            for path, trans_id, kind in entries:
 
1040
                delete_any(path)
 
1041
            try:
 
1042
                delete_any(self._limbodir)
 
1043
            except OSError:
 
1044
                # We don't especially care *why* the dir is immortal.
 
1045
                raise ImmortalLimbo(self._limbodir)
 
1046
            try:
 
1047
                if self._deletiondir is not None:
 
1048
                    delete_any(self._deletiondir)
 
1049
            except OSError:
 
1050
                raise errors.ImmortalPendingDeletion(self._deletiondir)
 
1051
        finally:
 
1052
            TreeTransformBase.finalize(self)
 
1053
 
 
1054
    def _limbo_name(self, trans_id):
 
1055
        """Generate the limbo name of a file"""
 
1056
        limbo_name = self._limbo_files.get(trans_id)
 
1057
        if limbo_name is not None:
 
1058
            return limbo_name
 
1059
        parent = self._new_parent.get(trans_id)
 
1060
        # if the parent directory is already in limbo (e.g. when building a
 
1061
        # tree), choose a limbo name inside the parent, to reduce further
 
1062
        # renames.
 
1063
        use_direct_path = False
 
1064
        if self._new_contents.get(parent) == 'directory':
 
1065
            filename = self._new_name.get(trans_id)
 
1066
            if filename is not None:
 
1067
                if parent not in self._limbo_children:
 
1068
                    self._limbo_children[parent] = set()
 
1069
                    self._limbo_children_names[parent] = {}
 
1070
                    use_direct_path = True
 
1071
                # the direct path can only be used if no other file has
 
1072
                # already taken this pathname, i.e. if the name is unused, or
 
1073
                # if it is already associated with this trans_id.
 
1074
                elif self._case_sensitive_target:
 
1075
                    if (self._limbo_children_names[parent].get(filename)
 
1076
                        in (trans_id, None)):
 
1077
                        use_direct_path = True
 
1078
                else:
 
1079
                    for l_filename, l_trans_id in\
 
1080
                        self._limbo_children_names[parent].iteritems():
 
1081
                        if l_trans_id == trans_id:
 
1082
                            continue
 
1083
                        if l_filename.lower() == filename.lower():
 
1084
                            break
 
1085
                    else:
 
1086
                        use_direct_path = True
 
1087
 
 
1088
        if use_direct_path:
 
1089
            limbo_name = pathjoin(self._limbo_files[parent], filename)
 
1090
            self._limbo_children[parent].add(trans_id)
 
1091
            self._limbo_children_names[parent][filename] = trans_id
 
1092
        else:
 
1093
            limbo_name = pathjoin(self._limbodir, trans_id)
 
1094
            self._needs_rename.add(trans_id)
 
1095
        self._limbo_files[trans_id] = limbo_name
 
1096
        return limbo_name
 
1097
 
 
1098
    def adjust_path(self, name, parent, trans_id):
 
1099
        previous_parent = self._new_parent.get(trans_id)
 
1100
        previous_name = self._new_name.get(trans_id)
 
1101
        TreeTransformBase.adjust_path(self, name, parent, trans_id)
 
1102
        if (trans_id in self._limbo_files and
 
1103
            trans_id not in self._needs_rename):
 
1104
            self._rename_in_limbo([trans_id])
 
1105
            self._limbo_children[previous_parent].remove(trans_id)
 
1106
            del self._limbo_children_names[previous_parent][previous_name]
 
1107
 
 
1108
    def _rename_in_limbo(self, trans_ids):
 
1109
        """Fix limbo names so that the right final path is produced.
 
1110
 
 
1111
        This means we outsmarted ourselves-- we tried to avoid renaming
 
1112
        these files later by creating them with their final names in their
 
1113
        final parents.  But now the previous name or parent is no longer
 
1114
        suitable, so we have to rename them.
 
1115
 
 
1116
        Even for trans_ids that have no new contents, we must remove their
 
1117
        entries from _limbo_files, because they are now stale.
 
1118
        """
 
1119
        for trans_id in trans_ids:
 
1120
            old_path = self._limbo_files.pop(trans_id)
 
1121
            if trans_id not in self._new_contents:
 
1122
                continue
 
1123
            new_path = self._limbo_name(trans_id)
 
1124
            os.rename(old_path, new_path)
 
1125
 
 
1126
    def create_file(self, contents, trans_id, mode_id=None):
 
1127
        """Schedule creation of a new file.
 
1128
 
 
1129
        See also new_file.
 
1130
 
 
1131
        Contents is an iterator of strings, all of which will be written
 
1132
        to the target destination.
 
1133
 
 
1134
        New file takes the permissions of any existing file with that id,
 
1135
        unless mode_id is specified.
 
1136
        """
 
1137
        name = self._limbo_name(trans_id)
 
1138
        f = open(name, 'wb')
 
1139
        try:
 
1140
            try:
 
1141
                unique_add(self._new_contents, trans_id, 'file')
 
1142
            except:
 
1143
                # Clean up the file, it never got registered so
 
1144
                # TreeTransform.finalize() won't clean it up.
 
1145
                f.close()
 
1146
                os.unlink(name)
 
1147
                raise
 
1148
 
 
1149
            f.writelines(contents)
 
1150
        finally:
 
1151
            f.close()
 
1152
        self._set_mode(trans_id, mode_id, S_ISREG)
 
1153
 
 
1154
    def _read_file_chunks(self, trans_id):
 
1155
        cur_file = open(self._limbo_name(trans_id), 'rb')
 
1156
        try:
 
1157
            return cur_file.readlines()
 
1158
        finally:
 
1159
            cur_file.close()
 
1160
 
 
1161
    def _read_symlink_target(self, trans_id):
 
1162
        return os.readlink(self._limbo_name(trans_id))
 
1163
 
 
1164
    def create_hardlink(self, path, trans_id):
 
1165
        """Schedule creation of a hard link"""
 
1166
        name = self._limbo_name(trans_id)
 
1167
        try:
 
1168
            os.link(path, name)
 
1169
        except OSError, e:
 
1170
            if e.errno != errno.EPERM:
 
1171
                raise
 
1172
            raise errors.HardLinkNotSupported(path)
 
1173
        try:
 
1174
            unique_add(self._new_contents, trans_id, 'file')
 
1175
        except:
 
1176
            # Clean up the file, it never got registered so
 
1177
            # TreeTransform.finalize() won't clean it up.
 
1178
            os.unlink(name)
 
1179
            raise
 
1180
 
 
1181
    def create_directory(self, trans_id):
 
1182
        """Schedule creation of a new directory.
 
1183
 
 
1184
        See also new_directory.
 
1185
        """
 
1186
        os.mkdir(self._limbo_name(trans_id))
 
1187
        unique_add(self._new_contents, trans_id, 'directory')
 
1188
 
 
1189
    def create_symlink(self, target, trans_id):
 
1190
        """Schedule creation of a new symbolic link.
 
1191
 
 
1192
        target is a bytestring.
 
1193
        See also new_symlink.
 
1194
        """
 
1195
        if has_symlinks():
 
1196
            os.symlink(target, self._limbo_name(trans_id))
 
1197
            unique_add(self._new_contents, trans_id, 'symlink')
 
1198
        else:
 
1199
            try:
 
1200
                path = FinalPaths(self).get_path(trans_id)
 
1201
            except KeyError:
 
1202
                path = None
 
1203
            raise UnableCreateSymlink(path=path)
 
1204
 
 
1205
    def cancel_creation(self, trans_id):
 
1206
        """Cancel the creation of new file contents."""
 
1207
        del self._new_contents[trans_id]
 
1208
        children = self._limbo_children.get(trans_id)
 
1209
        # if this is a limbo directory with children, move them before removing
 
1210
        # the directory
 
1211
        if children is not None:
 
1212
            self._rename_in_limbo(children)
 
1213
            del self._limbo_children[trans_id]
 
1214
            del self._limbo_children_names[trans_id]
 
1215
        delete_any(self._limbo_name(trans_id))
 
1216
 
 
1217
 
 
1218
class TreeTransform(DiskTreeTransform):
1123
1219
    """Represent a tree transformation.
1124
1220
 
1125
1221
    This object is designed to support incremental generation of the transform,
1211
1307
            tree.unlock()
1212
1308
            raise
1213
1309
 
1214
 
        TreeTransformBase.__init__(self, tree, limbodir, pb,
 
1310
        # Cache of realpath results, to speed up canonical_path
 
1311
        self._realpaths = {}
 
1312
        # Cache of relpath results, to speed up canonical_path
 
1313
        self._relpaths = {}
 
1314
        DiskTreeTransform.__init__(self, tree, limbodir, pb,
1215
1315
                                   tree.case_sensitive)
1216
1316
        self._deletiondir = deletiondir
1217
1317
 
 
1318
    def canonical_path(self, path):
 
1319
        """Get the canonical tree-relative path"""
 
1320
        # don't follow final symlinks
 
1321
        abs = self._tree.abspath(path)
 
1322
        if abs in self._relpaths:
 
1323
            return self._relpaths[abs]
 
1324
        dirname, basename = os.path.split(abs)
 
1325
        if dirname not in self._realpaths:
 
1326
            self._realpaths[dirname] = os.path.realpath(dirname)
 
1327
        dirname = self._realpaths[dirname]
 
1328
        abs = pathjoin(dirname, basename)
 
1329
        if dirname in self._relpaths:
 
1330
            relpath = pathjoin(self._relpaths[dirname], basename)
 
1331
            relpath = relpath.rstrip('/\\')
 
1332
        else:
 
1333
            relpath = self._tree.relpath(abs)
 
1334
        self._relpaths[abs] = relpath
 
1335
        return relpath
 
1336
 
 
1337
    def tree_kind(self, trans_id):
 
1338
        """Determine the file kind in the working tree.
 
1339
 
 
1340
        Raises NoSuchFile if the file does not exist
 
1341
        """
 
1342
        path = self._tree_id_paths.get(trans_id)
 
1343
        if path is None:
 
1344
            raise NoSuchFile(None)
 
1345
        try:
 
1346
            return file_kind(self._tree.abspath(path))
 
1347
        except OSError, e:
 
1348
            if e.errno != errno.ENOENT:
 
1349
                raise
 
1350
            else:
 
1351
                raise NoSuchFile(path)
 
1352
 
 
1353
    def _set_mode(self, trans_id, mode_id, typefunc):
 
1354
        """Set the mode of new file contents.
 
1355
        The mode_id is the existing file to get the mode from (often the same
 
1356
        as trans_id).  The operation is only performed if there's a mode match
 
1357
        according to typefunc.
 
1358
        """
 
1359
        if mode_id is None:
 
1360
            mode_id = trans_id
 
1361
        try:
 
1362
            old_path = self._tree_id_paths[mode_id]
 
1363
        except KeyError:
 
1364
            return
 
1365
        try:
 
1366
            mode = os.stat(self._tree.abspath(old_path)).st_mode
 
1367
        except OSError, e:
 
1368
            if e.errno in (errno.ENOENT, errno.ENOTDIR):
 
1369
                # Either old_path doesn't exist, or the parent of the
 
1370
                # target is not a directory (but will be one eventually)
 
1371
                # Either way, we know it doesn't exist *right now*
 
1372
                # See also bug #248448
 
1373
                return
 
1374
            else:
 
1375
                raise
 
1376
        if typefunc(mode):
 
1377
            os.chmod(self._limbo_name(trans_id), mode)
 
1378
 
 
1379
    def iter_tree_children(self, parent_id):
 
1380
        """Iterate through the entry's tree children, if any"""
 
1381
        try:
 
1382
            path = self._tree_id_paths[parent_id]
 
1383
        except KeyError:
 
1384
            return
 
1385
        try:
 
1386
            children = os.listdir(self._tree.abspath(path))
 
1387
        except OSError, e:
 
1388
            if not (osutils._is_error_enotdir(e)
 
1389
                    or e.errno in (errno.ENOENT, errno.ESRCH)):
 
1390
                raise
 
1391
            return
 
1392
 
 
1393
        for child in children:
 
1394
            childpath = joinpath(path, child)
 
1395
            if self._tree.is_control_filename(childpath):
 
1396
                continue
 
1397
            yield self.trans_id_tree_path(childpath)
 
1398
 
1218
1399
    def apply(self, no_conflicts=False, precomputed_delta=None, _mover=None):
1219
1400
        """Apply all changes to the inventory and filesystem.
1220
1401
 
1230
1411
        :param _mover: Supply an alternate FileMover, for testing
1231
1412
        """
1232
1413
        if not no_conflicts:
1233
 
            conflicts = self.find_conflicts()
1234
 
            if len(conflicts) != 0:
1235
 
                raise MalformedTransform(conflicts=conflicts)
 
1414
            self._check_malformed()
1236
1415
        child_pb = bzrlib.ui.ui_factory.nested_progress_bar()
1237
1416
        try:
1238
1417
            if precomputed_delta is None:
1397
1576
        return modified_paths
1398
1577
 
1399
1578
 
1400
 
class TransformPreview(TreeTransformBase):
 
1579
class TransformPreview(DiskTreeTransform):
1401
1580
    """A TreeTransform for generating preview trees.
1402
1581
 
1403
1582
    Unlike TreeTransform, this version works when the input tree is a
1408
1587
    def __init__(self, tree, pb=DummyProgress(), case_sensitive=True):
1409
1588
        tree.lock_read()
1410
1589
        limbodir = osutils.mkdtemp(prefix='bzr-limbo-')
1411
 
        TreeTransformBase.__init__(self, tree, limbodir, pb, case_sensitive)
 
1590
        DiskTreeTransform.__init__(self, tree, limbodir, pb, case_sensitive)
1412
1591
 
1413
1592
    def canonical_path(self, path):
1414
1593
        return path
1438
1617
        file_id = self.tree_file_id(parent_id)
1439
1618
        if file_id is None:
1440
1619
            return
1441
 
        children = getattr(self._tree.inventory[file_id], 'children', {})
 
1620
        entry = self._tree.iter_entries_by_dir([file_id]).next()[1]
 
1621
        children = getattr(entry, 'children', {})
1442
1622
        for child in children:
1443
1623
            childpath = joinpath(path, child)
1444
1624
            yield self.trans_id_tree_path(childpath)
1452
1632
        self._final_paths = FinalPaths(transform)
1453
1633
        self.__by_parent = None
1454
1634
        self._parent_ids = []
 
1635
        self._all_children_cache = {}
 
1636
        self._path2trans_id_cache = {}
 
1637
        self._final_name_cache = {}
1455
1638
 
1456
1639
    def _changes(self, file_id):
1457
1640
        for changes in self._transform.iter_changes():
1500
1683
            self.__by_parent = self._transform.by_parent()
1501
1684
        return self.__by_parent
1502
1685
 
 
1686
    def _comparison_data(self, entry, path):
 
1687
        kind, size, executable, link_or_sha1 = self.path_content_summary(path)
 
1688
        if kind == 'missing':
 
1689
            kind = None
 
1690
            executable = False
 
1691
        else:
 
1692
            file_id = self._transform.final_file_id(self._path2trans_id(path))
 
1693
            executable = self.is_executable(file_id, path)
 
1694
        return kind, executable, None
 
1695
 
1503
1696
    def lock_read(self):
1504
1697
        # Perhaps in theory, this should lock the TreeTransform?
1505
1698
        pass
1525
1718
    def __iter__(self):
1526
1719
        return iter(self.all_file_ids())
1527
1720
 
1528
 
    def paths2ids(self, specific_files, trees=None, require_versioned=False):
1529
 
        """See Tree.paths2ids"""
1530
 
        to_find = set(specific_files)
1531
 
        result = set()
1532
 
        for (file_id, paths, changed, versioned, parent, name, kind,
1533
 
             executable) in self._transform.iter_changes():
1534
 
            if paths[1] in to_find:
1535
 
                result.add(file_id)
1536
 
                to_find.remove(paths[1])
1537
 
        result.update(self._transform._tree.paths2ids(to_find,
1538
 
                      trees=[], require_versioned=require_versioned))
1539
 
        return result
 
1721
    def _has_id(self, file_id, fallback_check):
 
1722
        if file_id in self._transform._r_new_id:
 
1723
            return True
 
1724
        elif file_id in set([self._transform.tree_file_id(trans_id) for
 
1725
            trans_id in self._transform._removed_id]):
 
1726
            return False
 
1727
        else:
 
1728
            return fallback_check(file_id)
 
1729
 
 
1730
    def has_id(self, file_id):
 
1731
        return self._has_id(file_id, self._transform._tree.has_id)
 
1732
 
 
1733
    def has_or_had_id(self, file_id):
 
1734
        return self._has_id(file_id, self._transform._tree.has_or_had_id)
1540
1735
 
1541
1736
    def _path2trans_id(self, path):
 
1737
        # We must not use None here, because that is a valid value to store.
 
1738
        trans_id = self._path2trans_id_cache.get(path, object)
 
1739
        if trans_id is not object:
 
1740
            return trans_id
1542
1741
        segments = splitpath(path)
1543
1742
        cur_parent = self._transform.root
1544
1743
        for cur_segment in segments:
1545
1744
            for child in self._all_children(cur_parent):
1546
 
                if self._transform.final_name(child) == cur_segment:
 
1745
                final_name = self._final_name_cache.get(child)
 
1746
                if final_name is None:
 
1747
                    final_name = self._transform.final_name(child)
 
1748
                    self._final_name_cache[child] = final_name
 
1749
                if final_name == cur_segment:
1547
1750
                    cur_parent = child
1548
1751
                    break
1549
1752
            else:
 
1753
                self._path2trans_id_cache[path] = None
1550
1754
                return None
 
1755
        self._path2trans_id_cache[path] = cur_parent
1551
1756
        return cur_parent
1552
1757
 
1553
1758
    def path2id(self, path):
1561
1766
            raise errors.NoSuchId(self, file_id)
1562
1767
 
1563
1768
    def _all_children(self, trans_id):
 
1769
        children = self._all_children_cache.get(trans_id)
 
1770
        if children is not None:
 
1771
            return children
1564
1772
        children = set(self._transform.iter_tree_children(trans_id))
1565
1773
        # children in the _new_parent set are provided by _by_parent.
1566
1774
        children.difference_update(self._transform._new_parent.keys())
1567
1775
        children.update(self._by_parent.get(trans_id, []))
 
1776
        self._all_children_cache[trans_id] = children
1568
1777
        return children
1569
1778
 
1570
 
    def _make_inv_entries(self, ordered_entries, specific_file_ids):
 
1779
    def iter_children(self, file_id):
 
1780
        trans_id = self._transform.trans_id_file_id(file_id)
 
1781
        for child_trans_id in self._all_children(trans_id):
 
1782
            yield self._transform.final_file_id(child_trans_id)
 
1783
 
 
1784
    def extras(self):
 
1785
        possible_extras = set(self._transform.trans_id_tree_path(p) for p
 
1786
                              in self._transform._tree.extras())
 
1787
        possible_extras.update(self._transform._new_contents)
 
1788
        possible_extras.update(self._transform._removed_id)
 
1789
        for trans_id in possible_extras:
 
1790
            if self._transform.final_file_id(trans_id) is None:
 
1791
                yield self._final_paths._determine_path(trans_id)
 
1792
 
 
1793
    def _make_inv_entries(self, ordered_entries, specific_file_ids=None,
 
1794
        yield_parents=False):
1571
1795
        for trans_id, parent_file_id in ordered_entries:
1572
1796
            file_id = self._transform.final_file_id(trans_id)
1573
1797
            if file_id is None:
1599
1823
                ordered_ids.append((trans_id, parent_file_id))
1600
1824
        return ordered_ids
1601
1825
 
1602
 
    def iter_entries_by_dir(self, specific_file_ids=None):
 
1826
    def iter_entries_by_dir(self, specific_file_ids=None, yield_parents=False):
1603
1827
        # This may not be a maximally efficient implementation, but it is
1604
1828
        # reasonably straightforward.  An implementation that grafts the
1605
1829
        # TreeTransform changes onto the tree's iter_entries_by_dir results
1607
1831
        # position.
1608
1832
        ordered_ids = self._list_files_by_dir()
1609
1833
        for entry, trans_id in self._make_inv_entries(ordered_ids,
1610
 
                                                      specific_file_ids):
1611
 
            yield unicode(self._final_paths.get_path(trans_id)), entry
1612
 
 
1613
 
    def list_files(self, include_root=False):
1614
 
        """See Tree.list_files."""
 
1834
            specific_file_ids, yield_parents=yield_parents):
 
1835
            yield unicode(self._final_paths.get_path(trans_id)), entry
 
1836
 
 
1837
    def _iter_entries_for_dir(self, dir_path):
 
1838
        """Return path, entry for items in a directory without recursing down."""
 
1839
        dir_file_id = self.path2id(dir_path)
 
1840
        ordered_ids = []
 
1841
        for file_id in self.iter_children(dir_file_id):
 
1842
            trans_id = self._transform.trans_id_file_id(file_id)
 
1843
            ordered_ids.append((trans_id, file_id))
 
1844
        for entry, trans_id in self._make_inv_entries(ordered_ids):
 
1845
            yield unicode(self._final_paths.get_path(trans_id)), entry
 
1846
 
 
1847
    def list_files(self, include_root=False, from_dir=None, recursive=True):
 
1848
        """See WorkingTree.list_files."""
1615
1849
        # XXX This should behave like WorkingTree.list_files, but is really
1616
1850
        # more like RevisionTree.list_files.
1617
 
        for path, entry in self.iter_entries_by_dir():
1618
 
            if entry.name == '' and not include_root:
1619
 
                continue
1620
 
            yield path, 'V', entry.kind, entry.file_id, entry
 
1851
        if recursive:
 
1852
            prefix = None
 
1853
            if from_dir:
 
1854
                prefix = from_dir + '/'
 
1855
            entries = self.iter_entries_by_dir()
 
1856
            for path, entry in entries:
 
1857
                if entry.name == '' and not include_root:
 
1858
                    continue
 
1859
                if prefix:
 
1860
                    if not path.startswith(prefix):
 
1861
                        continue
 
1862
                    path = path[len(prefix):]
 
1863
                yield path, 'V', entry.kind, entry.file_id, entry
 
1864
        else:
 
1865
            if from_dir is None and include_root is True:
 
1866
                root_entry = inventory.make_entry('directory', '',
 
1867
                    ROOT_PARENT, self.get_root_id())
 
1868
                yield '', 'V', 'directory', root_entry.file_id, root_entry
 
1869
            entries = self._iter_entries_for_dir(from_dir or '')
 
1870
            for path, entry in entries:
 
1871
                yield path, 'V', entry.kind, entry.file_id, entry
1621
1872
 
1622
1873
    def kind(self, file_id):
1623
1874
        trans_id = self._transform.trans_id_file_id(file_id)
1636
1887
            return self._transform._tree.get_file_mtime(file_id, path)
1637
1888
        return self._stat_limbo_file(file_id).st_mtime
1638
1889
 
 
1890
    def _file_size(self, entry, stat_value):
 
1891
        return self.get_file_size(entry.file_id)
 
1892
 
1639
1893
    def get_file_size(self, file_id):
1640
1894
        """See Tree.get_file_size"""
1641
1895
        if self.kind(file_id) == 'file':
1644
1898
            return None
1645
1899
 
1646
1900
    def get_file_sha1(self, file_id, path=None, stat_value=None):
1647
 
        return self._transform._tree.get_file_sha1(file_id)
 
1901
        trans_id = self._transform.trans_id_file_id(file_id)
 
1902
        kind = self._transform._new_contents.get(trans_id)
 
1903
        if kind is None:
 
1904
            return self._transform._tree.get_file_sha1(file_id)
 
1905
        if kind == 'file':
 
1906
            fileobj = self.get_file(file_id)
 
1907
            try:
 
1908
                return sha_file(fileobj)
 
1909
            finally:
 
1910
                fileobj.close()
1648
1911
 
1649
1912
    def is_executable(self, file_id, path=None):
 
1913
        if file_id is None:
 
1914
            return False
1650
1915
        trans_id = self._transform.trans_id_file_id(file_id)
1651
1916
        try:
1652
1917
            return self._transform._new_executability[trans_id]
1653
1918
        except KeyError:
1654
 
            return self._transform._tree.is_executable(file_id, path)
 
1919
            try:
 
1920
                return self._transform._tree.is_executable(file_id, path)
 
1921
            except OSError, e:
 
1922
                if e.errno == errno.ENOENT:
 
1923
                    return False
 
1924
                raise
 
1925
            except errors.NoSuchId:
 
1926
                return False
1655
1927
 
1656
1928
    def path_content_summary(self, path):
1657
1929
        trans_id = self._path2trans_id(path)
1679
1951
                size = None
1680
1952
                executable = None
1681
1953
            if kind == 'symlink':
1682
 
                link_or_sha1 = os.readlink(limbo_name)
 
1954
                link_or_sha1 = os.readlink(limbo_name).decode(osutils._fs_enc)
1683
1955
        if supports_executable():
1684
1956
            executable = tt._new_executability.get(trans_id, executable)
1685
1957
        return kind, size, executable, link_or_sha1
1689
1961
                      require_versioned=True, want_unversioned=False):
1690
1962
        """See InterTree.iter_changes.
1691
1963
 
1692
 
        This implementation does not support include_unchanged, specific_files,
1693
 
        or want_unversioned.  extra_trees, require_versioned, and pb are
1694
 
        ignored.
 
1964
        This has a fast path that is only used when the from_tree matches
 
1965
        the transform tree, and no fancy options are supplied.
1695
1966
        """
1696
 
        if from_tree is not self._transform._tree:
1697
 
            raise ValueError('from_tree must be transform source tree.')
1698
 
        if include_unchanged:
1699
 
            raise ValueError('include_unchanged is not supported')
1700
 
        if specific_files is not None:
1701
 
            raise ValueError('specific_files is not supported')
 
1967
        if (from_tree is not self._transform._tree or include_unchanged or
 
1968
            specific_files or want_unversioned):
 
1969
            return tree.InterTree(from_tree, self).iter_changes(
 
1970
                include_unchanged=include_unchanged,
 
1971
                specific_files=specific_files,
 
1972
                pb=pb,
 
1973
                extra_trees=extra_trees,
 
1974
                require_versioned=require_versioned,
 
1975
                want_unversioned=want_unversioned)
1702
1976
        if want_unversioned:
1703
1977
            raise ValueError('want_unversioned is not supported')
1704
1978
        return self._transform.iter_changes()
1711
1985
        name = self._transform._limbo_name(trans_id)
1712
1986
        return open(name, 'rb')
1713
1987
 
1714
 
    def get_file_text(self, file_id):
1715
 
        text_file = self.get_file(file_id)
1716
 
        try:
1717
 
            return text_file.read()
1718
 
        finally:
1719
 
            text_file.close()
 
1988
    def get_file_with_stat(self, file_id, path=None):
 
1989
        return self.get_file(file_id, path), None
1720
1990
 
1721
1991
    def annotate_iter(self, file_id,
1722
1992
                      default_revision=_mod_revision.CURRENT_REVISION):
1738
2008
            return old_annotation
1739
2009
        if not changed_content:
1740
2010
            return old_annotation
 
2011
        # TODO: This is doing something similar to what WT.annotate_iter is
 
2012
        #       doing, however it fails slightly because it doesn't know what
 
2013
        #       the *other* revision_id is, so it doesn't know how to give the
 
2014
        #       other as the origin for some lines, they all get
 
2015
        #       'default_revision'
 
2016
        #       It would be nice to be able to use the new Annotator based
 
2017
        #       approach, as well.
1741
2018
        return annotate.reannotate([old_annotation],
1742
2019
                                   self.get_file(file_id).readlines(),
1743
2020
                                   default_revision)
1748
2025
            return self._transform._tree.get_symlink_target(file_id)
1749
2026
        trans_id = self._transform.trans_id_file_id(file_id)
1750
2027
        name = self._transform._limbo_name(trans_id)
1751
 
        return os.readlink(name)
 
2028
        return osutils.readlink(name)
1752
2029
 
1753
2030
    def walkdirs(self, prefix=''):
1754
2031
        pending = [self._transform.root]
1809
2086
        self.transform = transform
1810
2087
 
1811
2088
    def _determine_path(self, trans_id):
1812
 
        if trans_id == self.transform.root:
 
2089
        if (trans_id == self.transform.root or trans_id == ROOT_PARENT):
1813
2090
            return ""
1814
2091
        name = self.transform.final_name(trans_id)
1815
2092
        parent_id = self.transform.final_parent(trans_id)
1839
2116
def build_tree(tree, wt, accelerator_tree=None, hardlink=False,
1840
2117
               delta_from_tree=False):
1841
2118
    """Create working tree for a branch, using a TreeTransform.
1842
 
    
 
2119
 
1843
2120
    This function should be used on empty trees, having a tree root at most.
1844
2121
    (see merge and revert functionality for working with existing trees)
1845
2122
 
1846
2123
    Existing files are handled like so:
1847
 
    
 
2124
 
1848
2125
    - Existing bzrdirs take precedence over creating new items.  They are
1849
2126
      created as '%s.diverted' % name.
1850
2127
    - Otherwise, if the content on disk matches the content we are building,
1952
2229
                    executable = tree.is_executable(file_id, tree_path)
1953
2230
                    if executable:
1954
2231
                        tt.set_executability(executable, trans_id)
1955
 
                    deferred_contents.append((file_id, trans_id))
 
2232
                    trans_data = (trans_id, tree_path)
 
2233
                    deferred_contents.append((file_id, trans_data))
1956
2234
                else:
1957
2235
                    file_trans_id[file_id] = new_by_entry(tt, entry, parent_id,
1958
2236
                                                          tree)
1989
2267
def _create_files(tt, tree, desired_files, pb, offset, accelerator_tree,
1990
2268
                  hardlink):
1991
2269
    total = len(desired_files) + offset
 
2270
    wt = tt._tree
1992
2271
    if accelerator_tree is None:
1993
2272
        new_desired_files = desired_files
1994
2273
    else:
1997
2276
                         in iter if not (c or e[0] != e[1]))
1998
2277
        new_desired_files = []
1999
2278
        count = 0
2000
 
        for file_id, trans_id in desired_files:
 
2279
        for file_id, (trans_id, tree_path) in desired_files:
2001
2280
            accelerator_path = unchanged.get(file_id)
2002
2281
            if accelerator_path is None:
2003
 
                new_desired_files.append((file_id, trans_id))
 
2282
                new_desired_files.append((file_id, (trans_id, tree_path)))
2004
2283
                continue
2005
2284
            pb.update('Adding file contents', count + offset, total)
2006
2285
            if hardlink:
2008
2287
                                   trans_id)
2009
2288
            else:
2010
2289
                contents = accelerator_tree.get_file(file_id, accelerator_path)
 
2290
                if wt.supports_content_filtering():
 
2291
                    filters = wt._content_filter_stack(tree_path)
 
2292
                    contents = filtered_output_bytes(contents, filters,
 
2293
                        ContentFilterContext(tree_path, tree))
2011
2294
                try:
2012
2295
                    tt.create_file(contents, trans_id)
2013
2296
                finally:
2014
 
                    contents.close()
 
2297
                    try:
 
2298
                        contents.close()
 
2299
                    except AttributeError:
 
2300
                        # after filtering, contents may no longer be file-like
 
2301
                        pass
2015
2302
            count += 1
2016
2303
        offset += count
2017
 
    for count, (trans_id, contents) in enumerate(tree.iter_files_bytes(
2018
 
                                                 new_desired_files)):
 
2304
    for count, ((trans_id, tree_path), contents) in enumerate(
 
2305
            tree.iter_files_bytes(new_desired_files)):
 
2306
        if wt.supports_content_filtering():
 
2307
            filters = wt._content_filter_stack(tree_path)
 
2308
            contents = filtered_output_bytes(contents, filters,
 
2309
                ContentFilterContext(tree_path, tree))
2019
2310
        tt.create_file(contents, trans_id)
2020
2311
        pb.update('Adding file contents', count + offset, total)
2021
2312
 
2081
2372
    if kind == 'file':
2082
2373
        contents = tree.get_file(entry.file_id).readlines()
2083
2374
        executable = tree.is_executable(entry.file_id)
2084
 
        return tt.new_file(name, parent_id, contents, entry.file_id, 
 
2375
        return tt.new_file(name, parent_id, contents, entry.file_id,
2085
2376
                           executable)
2086
2377
    elif kind in ('directory', 'tree-reference'):
2087
2378
        trans_id = tt.new_directory(name, parent_id, entry.file_id)
2088
2379
        if kind == 'tree-reference':
2089
2380
            tt.set_tree_reference(entry.reference_revision, trans_id)
2090
 
        return trans_id 
 
2381
        return trans_id
2091
2382
    elif kind == 'symlink':
2092
2383
        target = tree.get_symlink_target(entry.file_id)
2093
2384
        return tt.new_symlink(name, parent_id, target, entry.file_id)
2095
2386
        raise errors.BadFileKindError(name, kind)
2096
2387
 
2097
2388
 
 
2389
@deprecated_function(deprecated_in((1, 9, 0)))
2098
2390
def create_by_entry(tt, entry, tree, trans_id, lines=None, mode_id=None):
2099
 
    """Create new file contents according to an inventory entry."""
 
2391
    """Create new file contents according to an inventory entry.
 
2392
 
 
2393
    DEPRECATED.  Use create_from_tree instead.
 
2394
    """
2100
2395
    if entry.kind == "file":
2101
2396
        if lines is None:
2102
2397
            lines = tree.get_file(entry.file_id).readlines()
2107
2402
        tt.create_directory(trans_id)
2108
2403
 
2109
2404
 
 
2405
def create_from_tree(tt, trans_id, tree, file_id, bytes=None):
 
2406
    """Create new file contents according to tree contents."""
 
2407
    kind = tree.kind(file_id)
 
2408
    if kind == 'directory':
 
2409
        tt.create_directory(trans_id)
 
2410
    elif kind == "file":
 
2411
        if bytes is None:
 
2412
            tree_file = tree.get_file(file_id)
 
2413
            try:
 
2414
                bytes = tree_file.readlines()
 
2415
            finally:
 
2416
                tree_file.close()
 
2417
        tt.create_file(bytes, trans_id)
 
2418
    elif kind == "symlink":
 
2419
        tt.create_symlink(tree.get_symlink_target(file_id), trans_id)
 
2420
    else:
 
2421
        raise AssertionError('Unknown kind %r' % kind)
 
2422
 
 
2423
 
2110
2424
def create_entry_executability(tt, entry, trans_id):
2111
2425
    """Set the executability of a trans_id according to an inventory entry"""
2112
2426
    if entry.kind == "file":
2149
2463
        if entry.kind != working_kind:
2150
2464
            contents_mod, meta_mod = True, False
2151
2465
        else:
2152
 
            cur_entry._read_tree_state(working_tree.id2path(file_id), 
 
2466
            cur_entry._read_tree_state(working_tree.id2path(file_id),
2153
2467
                                       working_tree)
2154
2468
            contents_mod, meta_mod = entry.detect_changes(cur_entry)
2155
2469
            cur_entry._forget_tree_state()
2344
2658
                existing_file, new_file = conflict[1], conflict[2]
2345
2659
            new_name = tt.final_name(existing_file)+'.moved'
2346
2660
            tt.adjust_path(new_name, final_parent, existing_file)
2347
 
            new_conflicts.add((c_type, 'Moved existing file to', 
 
2661
            new_conflicts.add((c_type, 'Moved existing file to',
2348
2662
                               existing_file, new_file))
2349
2663
        elif c_type == 'parent loop':
2350
2664
            # break the loop by undoing one of the ops that caused the loop
2354
2668
            new_conflicts.add((c_type, 'Cancelled move', cur,
2355
2669
                               tt.final_parent(cur),))
2356
2670
            tt.adjust_path(tt.final_name(cur), tt.get_tree_parent(cur), cur)
2357
 
            
 
2671
 
2358
2672
        elif c_type == 'missing parent':
2359
2673
            trans_id = conflict[1]
2360
2674
            try:
2361
2675
                tt.cancel_deletion(trans_id)
2362
 
                new_conflicts.add(('deleting parent', 'Not deleting', 
 
2676
                new_conflicts.add(('deleting parent', 'Not deleting',
2363
2677
                                   trans_id))
2364
2678
            except KeyError:
2365
2679
                create = True
2430
2744
        if len(conflict) == 3:
2431
2745
            yield Conflict.factory(c_type, action=action, path=modified_path,
2432
2746
                                     file_id=modified_id)
2433
 
             
 
2747
 
2434
2748
        else:
2435
2749
            conflicting_path = fp.get_path(conflict[3])
2436
2750
            conflicting_id = tt.final_file_id(conflict[3])
2437
2751
            yield Conflict.factory(c_type, action=action, path=modified_path,
2438
 
                                   file_id=modified_id, 
 
2752
                                   file_id=modified_id,
2439
2753
                                   conflict_path=conflicting_path,
2440
2754
                                   conflict_file_id=conflicting_id)
2441
2755