~bzr-pqm/bzr/bzr.dev

0.1.1 by Martin Pool
Check in old existing knit code.
1
#! /usr/bin/python
2
3
# Copyright (C) 2005 Canonical Ltd
4
0.1.33 by Martin Pool
add gpl text
5
# This program is free software; you can redistribute it and/or modify
6
# it under the terms of the GNU General Public License as published by
7
# the Free Software Foundation; either version 2 of the License, or
8
# (at your option) any later version.
9
10
# This program is distributed in the hope that it will be useful,
11
# but WITHOUT ANY WARRANTY; without even the implied warranty of
12
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13
# GNU General Public License for more details.
14
15
# You should have received a copy of the GNU General Public License
16
# along with this program; if not, write to the Free Software
17
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
0.1.1 by Martin Pool
Check in old existing knit code.
18
19
# Author: Martin Pool <mbp@canonical.com>
20
21
0.1.38 by Martin Pool
Rename knit to weave. (I don't think there's an existing module called weave.)
22
"""Weave - storage of related text file versions"""
0.1.2 by Martin Pool
Import testsweet module adapted from bzr.
23
938 by Martin Pool
- various optimizations to weave add code
24
0.1.58 by Martin Pool
doc
25
# XXX: If we do weaves this way, will a merge still behave the same
26
# way if it's done in a different order?  That's a pretty desirable
27
# property.
28
0.1.62 by Martin Pool
Lame command-line client for reading and writing weaves.
29
# TODO: Nothing here so far assumes the lines are really \n newlines,
1759.2.1 by Jelmer Vernooij
Fix some types (found using aspell).
30
# rather than being split up in some other way.  We could accommodate
0.1.62 by Martin Pool
Lame command-line client for reading and writing weaves.
31
# binaries, perhaps by naively splitting on \n or perhaps using
32
# something like a rolling checksum.
33
0.1.85 by Martin Pool
doc
34
# TODO: End marker for each version so we can stop reading?
0.1.69 by Martin Pool
Simple text-based format for storing weaves, cleaner than
35
36
# TODO: Check that no insertion occurs inside a deletion that was
37
# active in the version of the insertion.
38
912 by Martin Pool
- update todos for weave
39
# TODO: In addition to the SHA-1 check, perhaps have some code that
40
# checks structural constraints of the weave: ie that insertions are
41
# properly nested, that there is no text outside of an insertion, that
42
# insertions or deletions are not repeated, etc.
0.1.85 by Martin Pool
doc
43
918 by Martin Pool
- start doing new weave-merge algorithm
44
# TODO: Parallel-extract that passes back each line along with a
45
# description of which revisions include it.  Nice for checking all
1393.1.25 by Martin Pool
- comments in weave code
46
# shas or calculating stats in parallel.
918 by Martin Pool
- start doing new weave-merge algorithm
47
1082 by Martin Pool
- lift imports
48
# TODO: Using a single _extract routine and then processing the output
49
# is probably inefficient.  It's simple enough that we can afford to
50
# have slight specializations for different ways its used: annotate,
51
# basis for add, get, etc.
52
1185.16.13 by Martin Pool
todo
53
# TODO: Probably the API should work only in names to hide the integer
54
# indexes from the user.
1083 by Martin Pool
- add space to store revision-id in weave files
55
1328 by Martin Pool
- slight refactoring of weave code
56
# TODO: Is there any potential performance win by having an add()
57
# variant that is passed a pre-cooked version of the single basis
58
# version?
59
1185.16.13 by Martin Pool
todo
60
# TODO: Reweave can possibly be made faster by remembering diffs
61
# where the basis and destination are unchanged.
62
63
# FIXME: Sometimes we will be given a parents list for a revision
64
# that includes some redundant parents (i.e. already a parent of 
65
# something in the list.)  We should eliminate them.  This can 
66
# be done fairly efficiently because the sequence numbers constrain
67
# the possible relationships.
1083 by Martin Pool
- add space to store revision-id in weave files
68
1616.1.18 by Martin Pool
(weave-merge) don't treat killed-both lines as points of agreement;
69
# FIXME: the conflict markers should be *7* characters
1082 by Martin Pool
- lift imports
70
1594.2.11 by Robert Collins
Setup fast-code paths for copying into empty weaves and weave->empty knit.
71
from copy import copy
1563.2.9 by Robert Collins
Update versionedfile api tests to ensure that data is available after every operation.
72
from cStringIO import StringIO
1185.50.28 by John Arbash Meinel
Lots of updates for 'bzr check'
73
import os
1082 by Martin Pool
- lift imports
74
import sha
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
75
import time
1773.4.1 by Martin Pool
Add pyflakes makefile target; fix many warnings
76
import warnings
1237 by Martin Pool
- allow the same version to be repeatedly added to a weave
77
1393.1.68 by Martin Pool
- add reweave that joins with ghosts
78
from bzrlib.trace import mutter
1185.50.23 by John Arbash Meinel
Adding sha1 check when weave extracts a text.
79
from bzrlib.errors import (WeaveError, WeaveFormatError, WeaveParentMismatch,
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
80
        RevisionAlreadyPresent,
81
        RevisionNotPresent,
82
        WeaveRevisionAlreadyPresent,
83
        WeaveRevisionNotPresent,
84
        )
1185.50.23 by John Arbash Meinel
Adding sha1 check when weave extracts a text.
85
import bzrlib.errors as errors
1563.2.6 by Robert Collins
Start check tests for knits (pending), and remove dead code.
86
from bzrlib.osutils import sha_strings
1711.2.24 by John Arbash Meinel
Late bind to PatienceSequenceMatcher to allow plugin to override.
87
import bzrlib.patiencediff
1773.4.1 by Martin Pool
Add pyflakes makefile target; fix many warnings
88
from bzrlib.symbol_versioning import (deprecated_method,
89
        deprecated_function,
90
        zero_eight,
91
        )
1185.16.117 by mbp at sourcefrog
Reweave in topological-sort order.
92
from bzrlib.tsort import topo_sort
1563.2.13 by Robert Collins
InterVersionedFile implemented.
93
from bzrlib.versionedfile import VersionedFile, InterVersionedFile
1563.2.9 by Robert Collins
Update versionedfile api tests to ensure that data is available after every operation.
94
from bzrlib.weavefile import _read_weave_v5, write_weave_v5
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
95
96
97
class Weave(VersionedFile):
0.1.38 by Martin Pool
Rename knit to weave. (I don't think there's an existing module called weave.)
98
    """weave - versioned text file storage.
0.1.2 by Martin Pool
Import testsweet module adapted from bzr.
99
    
0.1.72 by Martin Pool
Go back to weave lines normally having newlines at the end.
100
    A Weave manages versions of line-based text files, keeping track
101
    of the originating version for each line.
102
103
    To clients the "lines" of the file are represented as a list of strings.
104
    These strings  will typically have terminal newline characters, but
105
    this is not required.  In particular files commonly do not have a newline
106
    at the end of the file.
0.1.2 by Martin Pool
Import testsweet module adapted from bzr.
107
0.1.4 by Martin Pool
Start indexing knits by both integer and version string.
108
    Texts can be identified in either of two ways:
109
110
    * a nonnegative index number.
111
1324 by Martin Pool
- doc
112
    * a version-id string.
0.1.4 by Martin Pool
Start indexing knits by both integer and version string.
113
0.1.38 by Martin Pool
Rename knit to weave. (I don't think there's an existing module called weave.)
114
    Typically the index number will be valid only inside this weave and
0.1.4 by Martin Pool
Start indexing knits by both integer and version string.
115
    the version-id is used to reference it in the larger world.
0.1.2 by Martin Pool
Import testsweet module adapted from bzr.
116
0.1.39 by Martin Pool
Change to a more realistic weave structure which can represent insertions and
117
    The weave is represented as a list mixing edit instructions and
944 by Martin Pool
- refactor member names in Weave code
118
    literal text.  Each entry in _weave can be either a string (or
0.1.39 by Martin Pool
Change to a more realistic weave structure which can represent insertions and
119
    unicode), or a tuple.  If a string, it means that the given line
120
    should be output in the currently active revisions.
121
122
    If a tuple, it gives a processing instruction saying in which
123
    revisions the enclosed lines are active.  The tuple has the form
124
    (instruction, version).
125
126
    The instruction can be '{' or '}' for an insertion block, and '['
127
    and ']' for a deletion block respectively.  The version is the
0.1.45 by Martin Pool
doc
128
    integer version index.  There is no replace operator, only deletes
1075 by Martin Pool
- don't store redundant version number at end of insert blocks
129
    and inserts.  For '}', the end of an insertion, there is no
130
    version parameter because it always closes the most recently
131
    opened insertion.
0.1.39 by Martin Pool
Change to a more realistic weave structure which can represent insertions and
132
0.1.41 by Martin Pool
Doc
133
    Constraints/notes:
0.1.39 by Martin Pool
Change to a more realistic weave structure which can represent insertions and
134
135
    * A later version can delete lines that were introduced by any
136
      number of ancestor versions; this implies that deletion
137
      instructions can span insertion blocks without regard to the
138
      insertion block's nesting.
139
0.1.41 by Martin Pool
Doc
140
    * Similarly, deletions need not be properly nested with regard to
141
      each other, because they might have been generated by
142
      independent revisions.
143
0.1.45 by Martin Pool
doc
144
    * Insertions are always made by inserting a new bracketed block
145
      into a single point in the previous weave.  This implies they
146
      can nest but not overlap, and the nesting must always have later
147
      insertions on the inside.
148
0.1.41 by Martin Pool
Doc
149
    * It doesn't seem very useful to have an active insertion
150
      inside an inactive insertion, but it might happen.
0.1.45 by Martin Pool
doc
151
      
0.1.41 by Martin Pool
Doc
152
    * Therefore, all instructions are always"considered"; that
153
      is passed onto and off the stack.  An outer inactive block
154
      doesn't disable an inner block.
155
156
    * Lines are enabled if the most recent enclosing insertion is
157
      active and none of the enclosing deletions are active.
0.1.39 by Martin Pool
Change to a more realistic weave structure which can represent insertions and
158
0.1.49 by Martin Pool
Add another constraint: revisions should not delete text that they
159
    * There is no point having a deletion directly inside its own
160
      insertion; you might as well just not write it.  And there
161
      should be no way to get an earlier version deleting a later
162
      version.
163
944 by Martin Pool
- refactor member names in Weave code
164
    _weave
165
        Text of the weave; list of control instruction tuples and strings.
0.1.4 by Martin Pool
Start indexing knits by both integer and version string.
166
944 by Martin Pool
- refactor member names in Weave code
167
    _parents
892 by Martin Pool
- weave stores only direct parents, and calculates and memoizes expansion as needed
168
        List of parents, indexed by version number.
169
        It is only necessary to store the minimal set of parents for
170
        each version; the parent's parents are implied.
0.1.13 by Martin Pool
Knit structure now allows for versions to include the lines present in other
171
0.1.89 by Martin Pool
Store SHA1 in weave file for later verification
172
    _sha1s
1083 by Martin Pool
- add space to store revision-id in weave files
173
        List of hex SHA-1 of each version.
174
175
    _names
176
        List of symbolic names for each version.  Each should be unique.
177
178
    _name_map
179
        For each name, the version number.
1209 by Martin Pool
- Add Weave._weave_name for debugging purposes
180
181
    _weave_name
182
        Descriptive name of this weave; typically the filename if known.
183
        Set by read_weave.
0.1.2 by Martin Pool
Import testsweet module adapted from bzr.
184
    """
938 by Martin Pool
- various optimizations to weave add code
185
1209 by Martin Pool
- Add Weave._weave_name for debugging purposes
186
    __slots__ = ['_weave', '_parents', '_sha1s', '_names', '_name_map',
1185.81.6 by John Arbash Meinel
Adding an optional matcher class, which allows to use cdvdifflib matcher.
187
                 '_weave_name', '_matcher']
938 by Martin Pool
- various optimizations to weave add code
188
    
1185.81.17 by Aaron Bentley
Merged latest bzr.dev
189
    def __init__(self, weave_name=None, access_mode='w', matcher=None):
1594.2.23 by Robert Collins
Test versioned file storage handling of clean/dirty status for accessed versioned files.
190
        super(Weave, self).__init__(access_mode)
944 by Martin Pool
- refactor member names in Weave code
191
        self._weave = []
192
        self._parents = []
0.1.89 by Martin Pool
Store SHA1 in weave file for later verification
193
        self._sha1s = []
1083 by Martin Pool
- add space to store revision-id in weave files
194
        self._names = []
195
        self._name_map = {}
1209 by Martin Pool
- Add Weave._weave_name for debugging purposes
196
        self._weave_name = weave_name
1185.81.6 by John Arbash Meinel
Adding an optional matcher class, which allows to use cdvdifflib matcher.
197
        if matcher is None:
1711.2.24 by John Arbash Meinel
Late bind to PatienceSequenceMatcher to allow plugin to override.
198
            self._matcher = bzrlib.patiencediff.PatienceSequenceMatcher
1185.81.6 by John Arbash Meinel
Adding an optional matcher class, which allows to use cdvdifflib matcher.
199
        else:
200
            self._matcher = matcher
0.1.60 by Martin Pool
Weave eq and ne methods
201
1185.35.41 by Aaron Bentley
Added repr for Weave
202
    def __repr__(self):
203
        return "Weave(%r)" % self._weave_name
204
1393.1.51 by Martin Pool
- new Weave.copy()
205
    def copy(self):
206
        """Return a deep copy of self.
207
        
208
        The copy can be modified without affecting the original weave."""
209
        other = Weave()
210
        other._weave = self._weave[:]
211
        other._parents = self._parents[:]
212
        other._sha1s = self._sha1s[:]
213
        other._names = self._names[:]
214
        other._name_map = self._name_map.copy()
215
        other._weave_name = self._weave_name
216
        return other
217
0.1.60 by Martin Pool
Weave eq and ne methods
218
    def __eq__(self, other):
219
        if not isinstance(other, Weave):
220
            return False
944 by Martin Pool
- refactor member names in Weave code
221
        return self._parents == other._parents \
1083 by Martin Pool
- add space to store revision-id in weave files
222
               and self._weave == other._weave \
223
               and self._sha1s == other._sha1s 
0.1.60 by Martin Pool
Weave eq and ne methods
224
    
225
    def __ne__(self, other):
226
        return not self.__eq__(other)
227
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
228
    @deprecated_method(zero_eight)
229
    def idx_to_name(self, index):
230
        """Old public interface, the public interface is all names now."""
231
        return index
232
233
    def _idx_to_name(self, version):
234
        return self._names[version]
235
236
    @deprecated_method(zero_eight)
1083 by Martin Pool
- add space to store revision-id in weave files
237
    def lookup(self, name):
1759.2.1 by Jelmer Vernooij
Fix some types (found using aspell).
238
        """Backwards compatibility thunk:
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
239
240
        Return name, as name is valid in the api now, and spew deprecation
241
        warnings everywhere.
242
        """
243
        return name
244
245
    def _lookup(self, name):
1269 by Martin Pool
- some weave operations automatically look up symbolic names if supplied
246
        """Convert symbolic version name to index."""
1083 by Martin Pool
- add space to store revision-id in weave files
247
        try:
248
            return self._name_map[name]
249
        except KeyError:
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
250
            raise RevisionNotPresent(name, self._weave_name)
251
252
    @deprecated_method(zero_eight)
253
    def iter_names(self):
254
        """Deprecated convenience function, please see VersionedFile.names()."""
255
        return iter(self.names())
256
257
    @deprecated_method(zero_eight)
1185.16.104 by mbp at sourcefrog
Add Weave.names()
258
    def names(self):
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
259
        """See Weave.versions for the current api."""
260
        return self.versions()
261
262
    def versions(self):
263
        """See VersionedFile.versions."""
1185.16.105 by mbp at sourcefrog
- fix Weave.names()
264
        return self._names[:]
1229 by Martin Pool
- new Weave.idx_to_name and .parents methods
265
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
266
    def has_version(self, version_id):
267
        """See VersionedFile.has_version."""
268
        return self._name_map.has_key(version_id)
269
270
    __contains__ = has_version
271
1596.2.36 by Robert Collins
add a get_delta api to versioned_file.
272
    def get_delta(self, version_id):
273
        """See VersionedFile.get_delta."""
1596.2.38 by Robert Collins
rollback from using deltas to using fulltexts - deltas need more work to be ready.
274
        return self.get_deltas([version_id])[version_id]
275
276
    def get_deltas(self, version_ids):
277
        """See VersionedFile.get_deltas."""
278
        version_ids = self.get_ancestry(version_ids)
279
        for version_id in version_ids:
280
            if not self.has_version(version_id):
281
                raise RevisionNotPresent(version_id, self)
282
        # try extracting all versions; parallel extraction is used
283
        nv = self.num_versions()
284
        sha1s = {}
285
        deltas = {}
286
        texts = {}
287
        inclusions = {}
288
        noeols = {}
289
        last_parent_lines = {}
290
        parents = {}
291
        parent_inclusions = {}
292
        parent_linenums = {}
293
        parent_noeols = {}
294
        current_hunks = {}
295
        diff_hunks = {}
296
        # its simplest to generate a full set of prepared variables.
297
        for i in range(nv):
298
            name = self._names[i]
299
            sha1s[name] = self.get_sha1(name)
300
            parents_list = self.get_parents(name)
301
            try:
302
                parent = parents_list[0]
303
                parents[name] = parent
304
                parent_inclusions[name] = inclusions[parent]
305
            except IndexError:
306
                parents[name] = None
307
                parent_inclusions[name] = set()
308
            # we want to emit start, finish, replacement_length, replacement_lines tuples.
309
            diff_hunks[name] = []
310
            current_hunks[name] = [0, 0, 0, []] # #start, finish, repl_length, repl_tuples
311
            parent_linenums[name] = 0
312
            noeols[name] = False
313
            parent_noeols[name] = False
314
            last_parent_lines[name] = None
315
            new_inc = set([name])
316
            for p in self._parents[i]:
317
                new_inc.update(inclusions[self._idx_to_name(p)])
318
            # debug only, known good so far.
319
            #assert set(new_inc) == set(self.get_ancestry(name)), \
320
            #    'failed %s != %s' % (set(new_inc), set(self.get_ancestry(name)))
321
            inclusions[name] = new_inc
322
323
        nlines = len(self._weave)
324
325
        for lineno, inserted, deletes, line in self._walk_internal():
326
            # a line is active in a version if:
327
            # insert is in the versions inclusions
328
            # and
329
            # deleteset & the versions inclusions is an empty set.
330
            # so - if we have a included by mapping - version is included by
331
            # children, we get a list of children to examine for deletes affect
332
            # ing them, which is less than the entire set of children.
333
            for version_id in version_ids:  
334
                # The active inclusion must be an ancestor,
335
                # and no ancestors must have deleted this line,
336
                # because we don't support resurrection.
337
                parent_inclusion = parent_inclusions[version_id]
338
                inclusion = inclusions[version_id]
339
                parent_active = inserted in parent_inclusion and not (deletes & parent_inclusion)
340
                version_active = inserted in inclusion and not (deletes & inclusion)
341
                if not parent_active and not version_active:
342
                    # unrelated line of ancestry
343
                    continue
344
                elif parent_active and version_active:
345
                    # shared line
346
                    parent_linenum = parent_linenums[version_id]
347
                    if current_hunks[version_id] != [parent_linenum, parent_linenum, 0, []]:
348
                        diff_hunks[version_id].append(tuple(current_hunks[version_id]))
349
                    parent_linenum += 1
350
                    current_hunks[version_id] = [parent_linenum, parent_linenum, 0, []]
351
                    parent_linenums[version_id] = parent_linenum
352
                    try:
353
                        if line[-1] != '\n':
354
                            noeols[version_id] = True
355
                    except IndexError:
356
                        pass
357
                elif parent_active and not version_active:
358
                    # deleted line
359
                    current_hunks[version_id][1] += 1
360
                    parent_linenums[version_id] += 1
361
                    last_parent_lines[version_id] = line
362
                elif not parent_active and version_active:
363
                    # replacement line
364
                    # noeol only occurs at the end of a file because we 
365
                    # diff linewise. We want to show noeol changes as a
366
                    # empty diff unless the actual eol-less content changed.
367
                    theline = line
368
                    try:
369
                        if last_parent_lines[version_id][-1] != '\n':
370
                            parent_noeols[version_id] = True
371
                    except (TypeError, IndexError):
372
                        pass
373
                    try:
374
                        if theline[-1] != '\n':
375
                            noeols[version_id] = True
376
                    except IndexError:
377
                        pass
378
                    new_line = False
379
                    parent_should_go = False
380
381
                    if parent_noeols[version_id] == noeols[version_id]:
382
                        # no noeol toggle, so trust the weaves statement
383
                        # that this line is changed.
384
                        new_line = True
385
                        if parent_noeols[version_id]:
386
                            theline = theline + '\n'
387
                    elif parent_noeols[version_id]:
388
                        # parent has no eol, we do:
389
                        # our line is new, report as such..
390
                        new_line = True
391
                    elif noeols[version_id]:
392
                        # append a eol so that it looks like
393
                        # a normalised delta
394
                        theline = theline + '\n'
395
                        if parents[version_id] is not None:
396
                        #if last_parent_lines[version_id] is not None:
397
                            parent_should_go = True
398
                        if last_parent_lines[version_id] != theline:
399
                            # but changed anyway
400
                            new_line = True
401
                            #parent_should_go = False
402
                    if new_line:
403
                        current_hunks[version_id][2] += 1
404
                        current_hunks[version_id][3].append((inserted, theline))
405
                    if parent_should_go:
406
                        # last hunk last parent line is not eaten
407
                        current_hunks[version_id][1] -= 1
408
                    if current_hunks[version_id][1] < 0:
409
                        current_hunks[version_id][1] = 0
410
                        # import pdb;pdb.set_trace()
411
                    # assert current_hunks[version_id][1] >= 0
412
1596.2.36 by Robert Collins
add a get_delta api to versioned_file.
413
        # flush last hunk
1596.2.38 by Robert Collins
rollback from using deltas to using fulltexts - deltas need more work to be ready.
414
        for i in range(nv):
415
            version = self._idx_to_name(i)
416
            if current_hunks[version] != [0, 0, 0, []]:
417
                diff_hunks[version].append(tuple(current_hunks[version]))
418
        result = {}
419
        for version_id in version_ids:
420
            result[version_id] = (
421
                                  parents[version_id],
422
                                  sha1s[version_id],
423
                                  noeols[version_id],
424
                                  diff_hunks[version_id],
425
                                  )
426
        return result
427
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
428
    def get_parents(self, version_id):
429
        """See VersionedFile.get_parent."""
430
        return map(self._idx_to_name, self._parents[self._lookup(version_id)])
1229 by Martin Pool
- new Weave.idx_to_name and .parents methods
431
1323 by Martin Pool
- caller can pass SHA-1 to Weave.add for efficiency
432
    def _check_repeated_add(self, name, parents, text, sha1):
1237 by Martin Pool
- allow the same version to be repeatedly added to a weave
433
        """Check that a duplicated add is OK.
434
435
        If it is, return the (old) index; otherwise raise an exception.
436
        """
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
437
        idx = self._lookup(name)
1185.16.116 by mbp at sourcefrog
Move weave errors into bzrlib.errors and make consistent with new system.
438
        if sorted(self._parents[idx]) != sorted(parents) \
439
            or sha1 != self._sha1s[idx]:
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
440
            raise RevisionAlreadyPresent(name, self._weave_name)
1237 by Martin Pool
- allow the same version to be repeatedly added to a weave
441
        return idx
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
442
443
    @deprecated_method(zero_eight)
444
    def add_identical(self, old_rev_id, new_rev_id, parents):
445
        """Please use Weave.clone_text now."""
1563.2.5 by Robert Collins
Remove unused transaction references from knit.py and the versionedfile interface.
446
        return self.clone_text(new_rev_id, old_rev_id, parents)
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
447
1596.2.32 by Robert Collins
Reduce re-extraction of texts during weave to knit joins by providing a memoisation facility.
448
    def _add_lines(self, version_id, parents, lines, parent_texts):
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
449
        """See VersionedFile.add_lines."""
450
        return self._add(version_id, lines, map(self._lookup, parents))
451
452
    @deprecated_method(zero_eight)
1323 by Martin Pool
- caller can pass SHA-1 to Weave.add for efficiency
453
    def add(self, name, parents, text, sha1=None):
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
454
        """See VersionedFile.add_lines for the non deprecated api."""
455
        return self._add(name, text, map(self._maybe_lookup, parents), sha1)
456
457
    def _add(self, version_id, lines, parents, sha1=None):
0.1.4 by Martin Pool
Start indexing knits by both integer and version string.
458
        """Add a single text on top of the weave.
0.1.36 by Martin Pool
doc
459
  
0.1.26 by Martin Pool
Refactor parameters to add command
460
        Returns the index number of the newly added version.
461
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
462
        version_id
1083 by Martin Pool
- add space to store revision-id in weave files
463
            Symbolic name for this version.
464
            (Typically the revision-id of the revision that added it.)
465
0.1.26 by Martin Pool
Refactor parameters to add command
466
        parents
892 by Martin Pool
- weave stores only direct parents, and calculates and memoizes expansion as needed
467
            List or set of direct parent version numbers.
468
            
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
469
        lines
1323 by Martin Pool
- caller can pass SHA-1 to Weave.add for efficiency
470
            Sequence of lines to be added in the new version.
471
        """
938 by Martin Pool
- various optimizations to weave add code
472
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
473
        assert isinstance(version_id, basestring)
1666.1.6 by Robert Collins
Make knit the default format.
474
        self._check_lines_not_unicode(lines)
475
        self._check_lines_are_lines(lines)
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
476
        if not sha1:
477
            sha1 = sha_strings(lines)
478
        if version_id in self._name_map:
479
            return self._check_repeated_add(version_id, parents, lines, sha1)
1269 by Martin Pool
- some weave operations automatically look up symbolic names if supplied
480
938 by Martin Pool
- various optimizations to weave add code
481
        self._check_versions(parents)
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
482
        ## self._check_lines(lines)
944 by Martin Pool
- refactor member names in Weave code
483
        new_version = len(self._parents)
0.1.5 by Martin Pool
Add test for storing two text versions.
484
1083 by Martin Pool
- add space to store revision-id in weave files
485
        # if we abort after here the (in-memory) weave will be corrupt because only
486
        # some fields are updated
1570.1.12 by Robert Collins
Add a FIXME for weave.add and remove some extraneous whitespace.
487
        # XXX: FIXME implement a succeed-or-fail of the rest of this routine.
488
        #      - Robert Collins 20060226
1083 by Martin Pool
- add space to store revision-id in weave files
489
        self._parents.append(parents[:])
0.1.89 by Martin Pool
Store SHA1 in weave file for later verification
490
        self._sha1s.append(sha1)
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
491
        self._names.append(version_id)
492
        self._name_map[version_id] = new_version
938 by Martin Pool
- various optimizations to weave add code
493
494
            
495
        if not parents:
496
            # special case; adding with no parents revision; can do
497
            # this more quickly by just appending unconditionally.
498
            # even more specially, if we're adding an empty text we
499
            # need do nothing at all.
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
500
            if lines:
944 by Martin Pool
- refactor member names in Weave code
501
                self._weave.append(('{', new_version))
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
502
                self._weave.extend(lines)
1075 by Martin Pool
- don't store redundant version number at end of insert blocks
503
                self._weave.append(('}', None))
938 by Martin Pool
- various optimizations to weave add code
504
            return new_version
505
941 by Martin Pool
- allow for parents specified to Weave.add to be a set
506
        if len(parents) == 1:
507
            pv = list(parents)[0]
508
            if sha1 == self._sha1s[pv]:
509
                # special case: same as the single parent
510
                return new_version
938 by Martin Pool
- various optimizations to weave add code
511
            
512
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
513
        ancestors = self._inclusions(parents)
938 by Martin Pool
- various optimizations to weave add code
514
944 by Martin Pool
- refactor member names in Weave code
515
        l = self._weave
938 by Martin Pool
- various optimizations to weave add code
516
517
        # basis a list of (origin, lineno, line)
518
        basis_lineno = []
519
        basis_lines = []
520
        for origin, lineno, line in self._extract(ancestors):
521
            basis_lineno.append(lineno)
522
            basis_lines.append(line)
523
974.1.26 by aaron.bentley at utoronto
merged mbp@sourcefrog.net-20050817233101-0939da1cf91f2472
524
        # another small special case: a merge, producing the same text
525
        # as auto-merge
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
526
        if lines == basis_lines:
938 by Martin Pool
- various optimizations to weave add code
527
            return new_version            
528
1759.2.1 by Jelmer Vernooij
Fix some types (found using aspell).
529
        # add a sentinel, because we can also match against the final line
944 by Martin Pool
- refactor member names in Weave code
530
        basis_lineno.append(len(self._weave))
938 by Martin Pool
- various optimizations to weave add code
531
532
        # XXX: which line of the weave should we really consider
533
        # matches the end of the file?  the current code says it's the
534
        # last line of the weave?
535
536
        #print 'basis_lines:', basis_lines
537
        #print 'new_lines:  ', lines
538
1185.81.17 by Aaron Bentley
Merged latest bzr.dev
539
        s = self._matcher(None, basis_lines, lines)
938 by Martin Pool
- various optimizations to weave add code
540
541
        # offset gives the number of lines that have been inserted
542
        # into the weave up to the current point; if the original edit instruction
543
        # says to change line A then we actually change (A+offset)
544
        offset = 0
545
546
        for tag, i1, i2, j1, j2 in s.get_opcodes():
547
            # i1,i2 are given in offsets within basis_lines; we need to map them
548
            # back to offsets within the entire weave
549
            #print 'raw match', tag, i1, i2, j1, j2
550
            if tag == 'equal':
551
                continue
552
553
            i1 = basis_lineno[i1]
554
            i2 = basis_lineno[i2]
555
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
556
            assert 0 <= j1 <= j2 <= len(lines)
938 by Martin Pool
- various optimizations to weave add code
557
558
            #print tag, i1, i2, j1, j2
559
560
            # the deletion and insertion are handled separately.
561
            # first delete the region.
562
            if i1 != i2:
944 by Martin Pool
- refactor member names in Weave code
563
                self._weave.insert(i1+offset, ('[', new_version))
564
                self._weave.insert(i2+offset+1, (']', new_version))
938 by Martin Pool
- various optimizations to weave add code
565
                offset += 2
566
567
            if j1 != j2:
568
                # there may have been a deletion spanning up to
569
                # i2; we want to insert after this region to make sure
570
                # we don't destroy ourselves
571
                i = i2 + offset
944 by Martin Pool
- refactor member names in Weave code
572
                self._weave[i:i] = ([('{', new_version)] 
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
573
                                    + lines[j1:j2] 
1075 by Martin Pool
- don't store redundant version number at end of insert blocks
574
                                    + [('}', None)])
938 by Martin Pool
- various optimizations to weave add code
575
                offset += 2 + (j2 - j1)
576
        return new_version
0.1.2 by Martin Pool
Import testsweet module adapted from bzr.
577
1594.2.24 by Robert Collins
Make use of the transaction finalisation warning support to implement in-knit caching.
578
    def _clone_text(self, new_version_id, old_version_id, parents):
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
579
        """See VersionedFile.clone_text."""
580
        old_lines = self.get_text(old_version_id)
581
        self.add_lines(new_version_id, parents, old_lines)
0.1.27 by Martin Pool
Check that version numbers passed in are reasonable
582
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
583
    def _inclusions(self, versions):
893 by Martin Pool
- Refactor weave calculation of inclusions
584
        """Return set of all ancestors of given version(s)."""
1563.2.29 by Robert Collins
Remove all but fetch references to repository.revision_store.
585
        if not len(versions):
586
            return []
928 by Martin Pool
- go back to using plain builtin set()
587
        i = set(versions)
1328 by Martin Pool
- slight refactoring of weave code
588
        for v in xrange(max(versions), 0, -1):
589
            if v in i:
590
                # include all its parents
591
                i.update(self._parents[v])
592
        return i
593
        ## except IndexError:
594
        ##     raise ValueError("version %d not present in weave" % v)
0.1.77 by Martin Pool
New Weave.get_included() does transitive expansion
595
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
596
    @deprecated_method(zero_eight)
597
    def inclusions(self, version_ids):
598
        """Deprecated - see VersionedFile.get_ancestry for the replacement."""
599
        if not version_ids:
890 by Martin Pool
- weave info should show minimal expression of parents
600
            return []
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
601
        if isinstance(version_ids[0], int):
602
            return [self._idx_to_name(v) for v in self._inclusions(version_ids)]
603
        else:
604
            return self.get_ancestry(version_ids)
605
606
    def get_ancestry(self, version_ids):
607
        """See VersionedFile.get_ancestry."""
608
        if isinstance(version_ids, basestring):
609
            version_ids = [version_ids]
610
        i = self._inclusions([self._lookup(v) for v in version_ids])
611
        return [self._idx_to_name(v) for v in i]
0.1.75 by Martin Pool
Remove VerInfo class; just store sets directly in the list of
612
0.1.39 by Martin Pool
Change to a more realistic weave structure which can represent insertions and
613
    def _check_lines(self, text):
614
        if not isinstance(text, list):
615
            raise ValueError("text should be a list, not %s" % type(text))
616
617
        for l in text:
618
            if not isinstance(l, basestring):
869 by Martin Pool
- more weave.py command line options
619
                raise ValueError("text line should be a string or unicode, not %s"
620
                                 % type(l))
0.1.39 by Martin Pool
Change to a more realistic weave structure which can represent insertions and
621
        
622
623
0.1.27 by Martin Pool
Check that version numbers passed in are reasonable
624
    def _check_versions(self, indexes):
625
        """Check everything in the sequence of indexes is valid"""
626
        for i in indexes:
627
            try:
944 by Martin Pool
- refactor member names in Weave code
628
                self._parents[i]
0.1.27 by Martin Pool
Check that version numbers passed in are reasonable
629
            except IndexError:
630
                raise IndexError("invalid version number %r" % i)
631
1563.2.24 by Robert Collins
Make join cheaper for compatibly inconsistent parents.
632
    def _compatible_parents(self, my_parents, other_parents):
633
        """During join check that other_parents are joinable with my_parents.
634
635
        Joinable is defined as 'is a subset of' - supersets may require 
636
        regeneration of diffs, but subsets do not.
637
        """
638
        return len(other_parents.difference(my_parents)) == 0
639
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
640
    def annotate(self, version_id):
641
        if isinstance(version_id, int):
1773.4.1 by Martin Pool
Add pyflakes makefile target; fix many warnings
642
            warnings.warn('Weave.annotate(int) is deprecated. Please use version names'
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
643
                 ' in all circumstances as of 0.8',
644
                 DeprecationWarning,
645
                 stacklevel=2
646
                 )
647
            result = []
648
            for origin, lineno, text in self._extract([version_id]):
649
                result.append((origin, text))
650
            return result
651
        else:
652
            return super(Weave, self).annotate(version_id)
0.1.2 by Martin Pool
Import testsweet module adapted from bzr.
653
    
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
654
    def annotate_iter(self, version_id):
655
        """Yield list of (version-id, line) pairs for the specified version.
0.1.7 by Martin Pool
Add trivial annotate text
656
657
        The index indicates when the line originated in the weave."""
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
658
        incls = [self._lookup(version_id)]
1269 by Martin Pool
- some weave operations automatically look up symbolic names if supplied
659
        for origin, lineno, text in self._extract(incls):
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
660
            yield self._idx_to_name(origin), text
0.1.22 by Martin Pool
Calculate delta for new versions relative to a set of parent versions.
661
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
662
    @deprecated_method(zero_eight)
918 by Martin Pool
- start doing new weave-merge algorithm
663
    def _walk(self):
1594.2.6 by Robert Collins
Introduce a api specifically for looking at lines in some versions of the inventory, for fileid_involved.
664
        """_walk has become visit, a supported api."""
665
        return self._walk_internal()
666
667
    def iter_lines_added_or_present_in_versions(self, version_ids=None):
668
        """See VersionedFile.iter_lines_added_or_present_in_versions()."""
669
        if version_ids is None:
670
            version_ids = self.versions()
671
        version_ids = set(version_ids)
672
        for lineno, inserted, deletes, line in self._walk_internal(version_ids):
673
            # if inserted not in version_ids then it was inserted before the
674
            # versions we care about, but because weaves cannot represent ghosts
675
            # properly, we do not filter down to that
676
            # if inserted not in version_ids: continue
677
            if line[-1] != '\n':
678
                yield line + '\n'
679
            else:
680
                yield line
681
682
    #@deprecated_method(zero_eight)
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
683
    def walk(self, version_ids=None):
684
        """See VersionedFile.walk."""
1594.2.6 by Robert Collins
Introduce a api specifically for looking at lines in some versions of the inventory, for fileid_involved.
685
        return self._walk_internal(version_ids)
686
687
    def _walk_internal(self, version_ids=None):
688
        """Helper method for weave actions."""
918 by Martin Pool
- start doing new weave-merge algorithm
689
        
690
        istack = []
928 by Martin Pool
- go back to using plain builtin set()
691
        dset = set()
918 by Martin Pool
- start doing new weave-merge algorithm
692
693
        lineno = 0         # line of weave, 0-based
694
944 by Martin Pool
- refactor member names in Weave code
695
        for l in self._weave:
1596.2.38 by Robert Collins
rollback from using deltas to using fulltexts - deltas need more work to be ready.
696
            if l.__class__ == tuple:
918 by Martin Pool
- start doing new weave-merge algorithm
697
                c, v = l
698
                isactive = None
699
                if c == '{':
1596.2.38 by Robert Collins
rollback from using deltas to using fulltexts - deltas need more work to be ready.
700
                    istack.append(self._names[v])
918 by Martin Pool
- start doing new weave-merge algorithm
701
                elif c == '}':
1075 by Martin Pool
- don't store redundant version number at end of insert blocks
702
                    istack.pop()
918 by Martin Pool
- start doing new weave-merge algorithm
703
                elif c == '[':
1596.2.38 by Robert Collins
rollback from using deltas to using fulltexts - deltas need more work to be ready.
704
                    assert self._names[v] not in dset
705
                    dset.add(self._names[v])
918 by Martin Pool
- start doing new weave-merge algorithm
706
                elif c == ']':
1596.2.38 by Robert Collins
rollback from using deltas to using fulltexts - deltas need more work to be ready.
707
                    dset.remove(self._names[v])
918 by Martin Pool
- start doing new weave-merge algorithm
708
                else:
1185.16.116 by mbp at sourcefrog
Move weave errors into bzrlib.errors and make consistent with new system.
709
                    raise WeaveFormatError('unexpected instruction %r' % v)
918 by Martin Pool
- start doing new weave-merge algorithm
710
            else:
1596.2.38 by Robert Collins
rollback from using deltas to using fulltexts - deltas need more work to be ready.
711
                assert l.__class__ in (str, unicode)
918 by Martin Pool
- start doing new weave-merge algorithm
712
                assert istack
1596.2.38 by Robert Collins
rollback from using deltas to using fulltexts - deltas need more work to be ready.
713
                yield lineno, istack[-1], frozenset(dset), l
918 by Martin Pool
- start doing new weave-merge algorithm
714
            lineno += 1
715
1185.50.27 by John Arbash Meinel
Cleaned up the check code by using self._walk() instead of manually writing the loop.
716
        if istack:
717
            raise WeaveFormatError("unclosed insertion blocks "
718
                    "at end of weave: %s" % istack)
719
        if dset:
720
            raise WeaveFormatError("unclosed deletion blocks at end of weave: %s"
721
                                   % dset)
918 by Martin Pool
- start doing new weave-merge algorithm
722
1551.6.15 by Aaron Bentley
Moved plan_merge into Weave
723
    def plan_merge(self, ver_a, ver_b):
724
        """Return pseudo-annotation indicating how the two versions merge.
725
726
        This is computed between versions a and b and their common
727
        base.
728
729
        Weave lines present in none of them are skipped entirely.
730
        """
731
        inc_a = set(self.get_ancestry([ver_a]))
732
        inc_b = set(self.get_ancestry([ver_b]))
733
        inc_c = inc_a & inc_b
734
735
        for lineno, insert, deleteset, line in\
736
            self.walk([ver_a, ver_b]):
737
            if deleteset & inc_c:
738
                # killed in parent; can't be in either a or b
739
                # not relevant to our work
740
                yield 'killed-base', line
741
            elif insert in inc_c:
742
                # was inserted in base
743
                killed_a = bool(deleteset & inc_a)
744
                killed_b = bool(deleteset & inc_b)
745
                if killed_a and killed_b:
746
                    yield 'killed-both', line
747
                elif killed_a:
748
                    yield 'killed-a', line
749
                elif killed_b:
750
                    yield 'killed-b', line
751
                else:
752
                    yield 'unchanged', line
753
            elif insert in inc_a:
754
                if deleteset & inc_a:
755
                    yield 'ghost-a', line
756
                else:
757
                    # new in A; not in B
758
                    yield 'new-a', line
759
            elif insert in inc_b:
760
                if deleteset & inc_b:
761
                    yield 'ghost-b', line
762
                else:
763
                    yield 'new-b', line
764
            else:
765
                # not in either revision
766
                yield 'irrelevant', line
767
768
        yield 'unchanged', ''           # terminator
769
893 by Martin Pool
- Refactor weave calculation of inclusions
770
    def _extract(self, versions):
0.1.20 by Martin Pool
Factor out Knit.extract() method
771
        """Yield annotation of lines in included set.
772
0.1.39 by Martin Pool
Change to a more realistic weave structure which can represent insertions and
773
        Yields a sequence of tuples (origin, lineno, text), where
774
        origin is the origin version, lineno the index in the weave,
775
        and text the text of the line.
776
0.1.20 by Martin Pool
Factor out Knit.extract() method
777
        The set typically but not necessarily corresponds to a version.
778
        """
1196 by Martin Pool
- [WIP] retrieve historical texts from weaves
779
        for i in versions:
780
            if not isinstance(i, int):
781
                raise ValueError(i)
782
            
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
783
        included = self._inclusions(versions)
881 by Martin Pool
- faster weave extraction
784
785
        istack = []
1596.2.25 by Robert Collins
Weave._extract should be faster
786
        iset = set()
928 by Martin Pool
- go back to using plain builtin set()
787
        dset = set()
0.1.48 by Martin Pool
Basic parsing of delete instructions.
788
789
        lineno = 0         # line of weave, 0-based
891 by Martin Pool
- fix up refactoring of weave
790
894 by Martin Pool
- small optimization for weave extract
791
        isactive = None
0.1.85 by Martin Pool
doc
792
931 by Martin Pool
- experiment with making Weave._extract() return a list, not a generator - slightly faster
793
        result = []
794
0.1.63 by Martin Pool
Abbreviate WeaveFormatError in some code
795
        WFE = WeaveFormatError
0.1.95 by Martin Pool
- preliminary merge conflict detection
796
1596.2.25 by Robert Collins
Weave._extract should be faster
797
        # wow. 
798
        #  449       0   4474.6820   2356.5590   bzrlib.weave:556(_extract)
799
        #  +285282   0   1676.8040   1676.8040   +<isinstance>
800
        # 1.6 seconds in 'isinstance'.
801
        # changing the first isinstance:
802
        #  449       0   2814.2660   1577.1760   bzrlib.weave:556(_extract)
803
        #  +140414   0    762.8050    762.8050   +<isinstance>
804
        # note that the inline time actually dropped (less function calls)
805
        # and total processing time was halved.
806
        # we're still spending ~1/4 of the method in isinstance though.
807
        # so lets hard code the acceptable string classes we expect:
808
        #  449       0   1202.9420    786.2930   bzrlib.weave:556(_extract)
809
        # +71352     0    377.5560    377.5560   +<method 'append' of 'list' 
810
        #                                          objects>
811
        # yay, down to ~1/4 the initial extract time, and our inline time
812
        # has shrunk again, with isinstance no longer dominating.
813
        # tweaking the stack inclusion test to use a set gives:
814
        #  449       0   1122.8030    713.0080   bzrlib.weave:556(_extract)
815
        # +71352     0    354.9980    354.9980   +<method 'append' of 'list' 
816
        #                                          objects>
817
        # - a 5% win, or possibly just noise. However with large istacks that
818
        # 'in' test could dominate, so I'm leaving this change in place -
819
        # when its fast enough to consider profiling big datasets we can review.
820
821
              
822
             
823
944 by Martin Pool
- refactor member names in Weave code
824
        for l in self._weave:
1596.2.25 by Robert Collins
Weave._extract should be faster
825
            if l.__class__ == tuple:
0.1.39 by Martin Pool
Change to a more realistic weave structure which can represent insertions and
826
                c, v = l
894 by Martin Pool
- small optimization for weave extract
827
                isactive = None
891 by Martin Pool
- fix up refactoring of weave
828
                if c == '{':
1596.2.25 by Robert Collins
Weave._extract should be faster
829
                    assert v not in iset
891 by Martin Pool
- fix up refactoring of weave
830
                    istack.append(v)
1596.2.25 by Robert Collins
Weave._extract should be faster
831
                    iset.add(v)
891 by Martin Pool
- fix up refactoring of weave
832
                elif c == '}':
1596.2.25 by Robert Collins
Weave._extract should be faster
833
                    iset.remove(istack.pop())
891 by Martin Pool
- fix up refactoring of weave
834
                elif c == '[':
835
                    if v in included:
881 by Martin Pool
- faster weave extraction
836
                        assert v not in dset
0.1.48 by Martin Pool
Basic parsing of delete instructions.
837
                        dset.add(v)
891 by Martin Pool
- fix up refactoring of weave
838
                else:
839
                    assert c == ']'
840
                    if v in included:
881 by Martin Pool
- faster weave extraction
841
                        assert v in dset
0.1.48 by Martin Pool
Basic parsing of delete instructions.
842
                        dset.remove(v)
0.1.39 by Martin Pool
Change to a more realistic weave structure which can represent insertions and
843
            else:
1596.2.25 by Robert Collins
Weave._extract should be faster
844
                assert l.__class__ in (str, unicode)
894 by Martin Pool
- small optimization for weave extract
845
                if isactive is None:
846
                    isactive = (not dset) and istack and (istack[-1] in included)
0.1.39 by Martin Pool
Change to a more realistic weave structure which can represent insertions and
847
                if isactive:
931 by Martin Pool
- experiment with making Weave._extract() return a list, not a generator - slightly faster
848
                    result.append((istack[-1], lineno, l))
0.1.39 by Martin Pool
Change to a more realistic weave structure which can represent insertions and
849
            lineno += 1
0.1.46 by Martin Pool
More constraints on structure of weave, and checks that they work
850
        if istack:
1185.16.116 by mbp at sourcefrog
Move weave errors into bzrlib.errors and make consistent with new system.
851
            raise WeaveFormatError("unclosed insertion blocks "
852
                    "at end of weave: %s" % istack)
0.1.48 by Martin Pool
Basic parsing of delete instructions.
853
        if dset:
1185.16.116 by mbp at sourcefrog
Move weave errors into bzrlib.errors and make consistent with new system.
854
            raise WeaveFormatError("unclosed deletion blocks at end of weave: %s"
855
                                   % dset)
931 by Martin Pool
- experiment with making Weave._extract() return a list, not a generator - slightly faster
856
        return result
857
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
858
    @deprecated_method(zero_eight)
1269 by Martin Pool
- some weave operations automatically look up symbolic names if supplied
859
    def get_iter(self, name_or_index):
1563.2.13 by Robert Collins
InterVersionedFile implemented.
860
        """Deprecated, please do not use. Lookups are not not needed.
861
        
862
        Please use get_lines now.
863
        """
1596.2.29 by Robert Collins
Make weave.get_lines faster.
864
        return iter(self.get_lines(self._maybe_lookup(name_or_index)))
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
865
866
    @deprecated_method(zero_eight)
867
    def maybe_lookup(self, name_or_index):
868
        """Deprecated, please do not use. Lookups are not not needed."""
869
        return self._maybe_lookup(name_or_index)
870
871
    def _maybe_lookup(self, name_or_index):
872
        """Convert possible symbolic name to index, or pass through indexes.
873
        
874
        NOT FOR PUBLIC USE.
875
        """
876
        if isinstance(name_or_index, (int, long)):
877
            return name_or_index
878
        else:
879
            return self._lookup(name_or_index)
880
881
    @deprecated_method(zero_eight)
882
    def get(self, version_id):
883
        """Please use either Weave.get_text or Weave.get_lines as desired."""
884
        return self.get_lines(version_id)
885
886
    def get_lines(self, version_id):
887
        """See VersionedFile.get_lines()."""
1596.2.30 by Robert Collins
Unbreak backwards compatability on weave.get_lines.
888
        int_index = self._maybe_lookup(version_id)
1596.2.29 by Robert Collins
Make weave.get_lines faster.
889
        result = [line for (origin, lineno, line) in self._extract([int_index])]
890
        expected_sha1 = self._sha1s[int_index]
891
        measured_sha1 = sha_strings(result)
892
        if measured_sha1 != expected_sha1:
893
            raise errors.WeaveInvalidChecksum(
894
                    'file %s, revision %s, expected: %s, measured %s' 
895
                    % (self._weave_name, version_id,
896
                       expected_sha1, measured_sha1))
897
        return result
1369 by Martin Pool
- try to avoid redundant conversion of strings when retrieving from weaves
898
1666.1.6 by Robert Collins
Make knit the default format.
899
    def get_sha1(self, version_id):
900
        """See VersionedFile.get_sha1()."""
901
        return self._sha1s[self._lookup(version_id)]
0.1.91 by Martin Pool
Update Weave.check
902
1563.2.18 by Robert Collins
get knit repositories really using knits for text storage.
903
    @deprecated_method(zero_eight)
0.1.91 by Martin Pool
Update Weave.check
904
    def numversions(self):
1563.2.18 by Robert Collins
get knit repositories really using knits for text storage.
905
        """How many versions are in this weave?
906
907
        Deprecated in favour of num_versions.
908
        """
909
        return self.num_versions()
910
911
    def num_versions(self):
912
        """How many versions are in this weave?"""
944 by Martin Pool
- refactor member names in Weave code
913
        l = len(self._parents)
0.1.91 by Martin Pool
Update Weave.check
914
        assert l == len(self._sha1s)
915
        return l
916
1563.2.18 by Robert Collins
get knit repositories really using knits for text storage.
917
    __len__ = num_versions
946 by Martin Pool
- weave info only shows the weave headers, doesn't extract every version:
918
894 by Martin Pool
- small optimization for weave extract
919
    def check(self, progress_bar=None):
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
920
        # TODO evaluate performance hit of using string sets in this routine.
1616.1.5 by Martin Pool
Cleanup and document some check code
921
        # TODO: check no circular inclusions
922
        # TODO: create a nested progress bar
1563.2.18 by Robert Collins
get knit repositories really using knits for text storage.
923
        for version in range(self.num_versions()):
944 by Martin Pool
- refactor member names in Weave code
924
            inclusions = list(self._parents[version])
0.1.91 by Martin Pool
Update Weave.check
925
            if inclusions:
926
                inclusions.sort()
927
                if inclusions[-1] >= version:
0.1.47 by Martin Pool
New WeaveError and WeaveFormatError rather than assertions.
928
                    raise WeaveFormatError("invalid included version %d for index %d"
0.1.91 by Martin Pool
Update Weave.check
929
                                           % (inclusions[-1], version))
930
1185.50.26 by John Arbash Meinel
Change Weave.check() so that it checks all revisions in parallel.
931
        # try extracting all versions; parallel extraction is used
1563.2.18 by Robert Collins
get knit repositories really using knits for text storage.
932
        nv = self.num_versions()
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
933
        sha1s = {}
934
        texts = {}
935
        inclusions = {}
1185.50.26 by John Arbash Meinel
Change Weave.check() so that it checks all revisions in parallel.
936
        for i in range(nv):
937
            # For creating the ancestry, IntSet is much faster (3.7s vs 0.17s)
938
            # The problem is that set membership is much more expensive
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
939
            name = self._idx_to_name(i)
940
            sha1s[name] = sha.new()
941
            texts[name] = []
942
            new_inc = set([name])
1185.50.26 by John Arbash Meinel
Change Weave.check() so that it checks all revisions in parallel.
943
            for p in self._parents[i]:
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
944
                new_inc.update(inclusions[self._idx_to_name(p)])
1185.50.26 by John Arbash Meinel
Change Weave.check() so that it checks all revisions in parallel.
945
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
946
            assert set(new_inc) == set(self.get_ancestry(name)), \
947
                'failed %s != %s' % (set(new_inc), set(self.get_ancestry(name)))
948
            inclusions[name] = new_inc
1185.50.26 by John Arbash Meinel
Change Weave.check() so that it checks all revisions in parallel.
949
950
        nlines = len(self._weave)
1185.50.28 by John Arbash Meinel
Lots of updates for 'bzr check'
951
952
        update_text = 'checking weave'
953
        if self._weave_name:
954
            short_name = os.path.basename(self._weave_name)
955
            update_text = 'checking %s' % (short_name,)
956
            update_text = update_text[:25]
957
1594.2.6 by Robert Collins
Introduce a api specifically for looking at lines in some versions of the inventory, for fileid_involved.
958
        for lineno, insert, deleteset, line in self._walk_internal():
1185.50.26 by John Arbash Meinel
Change Weave.check() so that it checks all revisions in parallel.
959
            if progress_bar:
1185.50.28 by John Arbash Meinel
Lots of updates for 'bzr check'
960
                progress_bar.update(update_text, lineno, nlines)
1185.50.27 by John Arbash Meinel
Cleaned up the check code by using self._walk() instead of manually writing the loop.
961
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
962
            for name, name_inclusions in inclusions.items():
1185.50.27 by John Arbash Meinel
Cleaned up the check code by using self._walk() instead of manually writing the loop.
963
                # The active inclusion must be an ancestor,
964
                # and no ancestors must have deleted this line,
965
                # because we don't support resurrection.
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
966
                if (insert in name_inclusions) and not (deleteset & name_inclusions):
967
                    sha1s[name].update(line)
1185.50.26 by John Arbash Meinel
Change Weave.check() so that it checks all revisions in parallel.
968
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
969
        for i in range(nv):
970
            version = self._idx_to_name(i)
1185.50.26 by John Arbash Meinel
Change Weave.check() so that it checks all revisions in parallel.
971
            hd = sha1s[version].hexdigest()
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
972
            expected = self._sha1s[i]
0.1.91 by Martin Pool
Update Weave.check
973
            if hd != expected:
1185.50.26 by John Arbash Meinel
Change Weave.check() so that it checks all revisions in parallel.
974
                raise errors.WeaveInvalidChecksum(
975
                        "mismatched sha1 for version %s: "
976
                        "got %s, expected %s"
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
977
                        % (version, hd, expected))
0.1.18 by Martin Pool
Better Knit.dump method
978
881 by Martin Pool
- faster weave extraction
979
        # TODO: check insertions are properly nested, that there are
980
        # no lines outside of insertion blocks, that deletions are
981
        # properly paired, etc.
982
1563.2.31 by Robert Collins
Convert Knit repositories to use knits.
983
    def _join(self, other, pb, msg, version_ids, ignore_missing):
1563.2.11 by Robert Collins
Consolidate reweave and join as we have no separate usage, make reweave tests apply to all versionedfile implementations and deprecate the old reweave apis.
984
        """Worker routine for join()."""
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
985
        if not other.versions():
1393.1.48 by Martin Pool
- Add stub Weave.join() method
986
            return          # nothing to update, easy
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
987
1684.3.2 by Robert Collins
Factor out version_ids-to-join selection in InterVersionedfile.
988
        if not version_ids:
989
            # versions is never none, InterWeave checks this.
990
            return 0
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
991
1185.13.4 by Robert Collins
make reweave visible as a weave method, and quickly integrate into fetch
992
        # two loops so that we do not change ourselves before verifying it
993
        # will be ok
1393.1.50 by Martin Pool
- more development of Weave.join()
994
        # work through in index order to make sure we get all dependencies
1185.77.1 by John Arbash Meinel
Simplifying what bzr displays when merging
995
        names_to_join = []
996
        processed = 0
1563.2.40 by Robert Collins
Extra test for joining of version-limited sets.
997
        # get the selected versions only that are in other.versions.
1563.2.31 by Robert Collins
Convert Knit repositories to use knits.
998
        version_ids = set(other.versions()).intersection(set(version_ids))
1563.2.40 by Robert Collins
Extra test for joining of version-limited sets.
999
        # pull in the referenced graph.
1000
        version_ids = other.get_ancestry(version_ids)
1563.2.31 by Robert Collins
Convert Knit repositories to use knits.
1001
        pending_graph = [(version, other.get_parents(version)) for
1002
                         version in version_ids]
1003
        for name in topo_sort(pending_graph):
1004
            other_idx = other._name_map[name]
1607.1.1 by Robert Collins
Unbreak weave joins when the source weave has less parent information than the target.
1005
            # returns True if we have it, False if we need it.
1006
            if not self._check_version_consistent(other, other_idx, name):
1007
                names_to_join.append((other_idx, name))
1185.77.1 by John Arbash Meinel
Simplifying what bzr displays when merging
1008
            processed += 1
1009
1185.33.93 by Martin Pool
[patch] Speed improvement for Weave.join() (Goffredo Baroncelli)
1010
1551.3.11 by Aaron Bentley
Merge from Robert
1011
        if pb and not msg:
1012
            msg = 'weave join'
1013
1185.33.93 by Martin Pool
[patch] Speed improvement for Weave.join() (Goffredo Baroncelli)
1014
        merged = 0
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
1015
        time0 = time.time()
1185.77.1 by John Arbash Meinel
Simplifying what bzr displays when merging
1016
        for other_idx, name in names_to_join:
1185.33.93 by Martin Pool
[patch] Speed improvement for Weave.join() (Goffredo Baroncelli)
1017
            # TODO: If all the parents of the other version are already
1393.1.50 by Martin Pool
- more development of Weave.join()
1018
            # present then we can avoid some work by just taking the delta
1019
            # and adjusting the offsets.
1393.1.51 by Martin Pool
- new Weave.copy()
1020
            new_parents = self._imported_parents(other, other_idx)
1185.33.93 by Martin Pool
[patch] Speed improvement for Weave.join() (Goffredo Baroncelli)
1021
            sha1 = other._sha1s[other_idx]
1022
1185.77.1 by John Arbash Meinel
Simplifying what bzr displays when merging
1023
            merged += 1
1185.33.93 by Martin Pool
[patch] Speed improvement for Weave.join() (Goffredo Baroncelli)
1024
1551.3.11 by Aaron Bentley
Merge from Robert
1025
            if pb:
1185.77.1 by John Arbash Meinel
Simplifying what bzr displays when merging
1026
                pb.update(msg, merged, len(names_to_join))
1551.3.11 by Aaron Bentley
Merge from Robert
1027
           
1393.1.50 by Martin Pool
- more development of Weave.join()
1028
            lines = other.get_lines(other_idx)
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
1029
            self._add(name, lines, new_parents, sha1)
1393.1.50 by Martin Pool
- more development of Weave.join()
1030
1185.33.93 by Martin Pool
[patch] Speed improvement for Weave.join() (Goffredo Baroncelli)
1031
        mutter("merged = %d, processed = %d, file_id=%s; deltat=%d"%(
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
1032
                merged, processed, self._weave_name, time.time()-time0))
1185.33.93 by Martin Pool
[patch] Speed improvement for Weave.join() (Goffredo Baroncelli)
1033
 
1393.1.51 by Martin Pool
- new Weave.copy()
1034
    def _imported_parents(self, other, other_idx):
1035
        """Return list of parents in self corresponding to indexes in other."""
1036
        new_parents = []
1037
        for parent_idx in other._parents[other_idx]:
1038
            parent_name = other._names[parent_idx]
1596.2.31 by Robert Collins
Remove a list __contains__ usage in weave operations.
1039
            if parent_name not in self._name_map:
1393.1.51 by Martin Pool
- new Weave.copy()
1040
                # should not be possible
1041
                raise WeaveError("missing parent {%s} of {%s} in %r" 
1042
                                 % (parent_name, other._name_map[other_idx], self))
1043
            new_parents.append(self._name_map[parent_name])
1044
        return new_parents
1045
1393.1.50 by Martin Pool
- more development of Weave.join()
1046
    def _check_version_consistent(self, other, other_idx, name):
1047
        """Check if a version in consistent in this and other.
1393.1.66 by Martin Pool
- fix join of weaves where parents occur at different offsets
1048
1049
        To be consistent it must have:
1050
1051
         * the same text
1052
         * the same direct parents (by name, not index, and disregarding
1053
           order)
1393.1.50 by Martin Pool
- more development of Weave.join()
1054
        
1055
        If present & correct return True;
1056
        if not present in self return False; 
1057
        if inconsistent raise error."""
1058
        this_idx = self._name_map.get(name, -1)
1059
        if this_idx != -1:
1060
            if self._sha1s[this_idx] != other._sha1s[other_idx]:
1563.2.11 by Robert Collins
Consolidate reweave and join as we have no separate usage, make reweave tests apply to all versionedfile implementations and deprecate the old reweave apis.
1061
                raise errors.WeaveTextDiffers(name, self, other)
1393.1.66 by Martin Pool
- fix join of weaves where parents occur at different offsets
1062
            self_parents = self._parents[this_idx]
1063
            other_parents = other._parents[other_idx]
1551.3.11 by Aaron Bentley
Merge from Robert
1064
            n1 = set([self._names[i] for i in self_parents])
1065
            n2 = set([other._names[i] for i in other_parents])
1563.2.24 by Robert Collins
Make join cheaper for compatibly inconsistent parents.
1066
            if not self._compatible_parents(n1, n2):
1185.13.4 by Robert Collins
make reweave visible as a weave method, and quickly integrate into fetch
1067
                raise WeaveParentMismatch("inconsistent parents "
1068
                    "for version {%s}: %s vs %s" % (name, n1, n2))
1393.1.50 by Martin Pool
- more development of Weave.join()
1069
            else:
1070
                return True         # ok!
1071
        else:
1072
            return False
918 by Martin Pool
- start doing new weave-merge algorithm
1073
1563.2.11 by Robert Collins
Consolidate reweave and join as we have no separate usage, make reweave tests apply to all versionedfile implementations and deprecate the old reweave apis.
1074
    @deprecated_method(zero_eight)
1551.3.11 by Aaron Bentley
Merge from Robert
1075
    def reweave(self, other, pb=None, msg=None):
1759.2.1 by Jelmer Vernooij
Fix some types (found using aspell).
1076
        """reweave has been superseded by plain use of join."""
1563.2.11 by Robert Collins
Consolidate reweave and join as we have no separate usage, make reweave tests apply to all versionedfile implementations and deprecate the old reweave apis.
1077
        return self.join(other, pb, msg)
1078
1079
    def _reweave(self, other, pb, msg):
1080
        """Reweave self with other - internal helper for join().
1551.3.11 by Aaron Bentley
Merge from Robert
1081
1082
        :param other: The other weave to merge
1083
        :param pb: An optional progress bar, indicating how far done we are
1084
        :param msg: An optional message for the progress
1085
        """
1563.2.11 by Robert Collins
Consolidate reweave and join as we have no separate usage, make reweave tests apply to all versionedfile implementations and deprecate the old reweave apis.
1086
        new_weave = _reweave(self, other, pb=pb, msg=msg)
1594.2.11 by Robert Collins
Setup fast-code paths for copying into empty weaves and weave->empty knit.
1087
        self._copy_weave_content(new_weave)
1088
1089
    def _copy_weave_content(self, otherweave):
1090
        """adsorb the content from otherweave."""
1185.13.4 by Robert Collins
make reweave visible as a weave method, and quickly integrate into fetch
1091
        for attr in self.__slots__:
1563.2.11 by Robert Collins
Consolidate reweave and join as we have no separate usage, make reweave tests apply to all versionedfile implementations and deprecate the old reweave apis.
1092
            if attr != '_weave_name':
1594.2.11 by Robert Collins
Setup fast-code paths for copying into empty weaves and weave->empty knit.
1093
                setattr(self, attr, copy(getattr(otherweave, attr)))
1185.13.4 by Robert Collins
make reweave visible as a weave method, and quickly integrate into fetch
1094
0.1.62 by Martin Pool
Lame command-line client for reading and writing weaves.
1095
1563.2.9 by Robert Collins
Update versionedfile api tests to ensure that data is available after every operation.
1096
class WeaveFile(Weave):
1097
    """A WeaveFile represents a Weave on disk and writes on change."""
1098
1563.2.15 by Robert Collins
remove the weavestore assumptions about the number and nature of files it manages.
1099
    WEAVE_SUFFIX = '.weave'
1100
    
1594.2.23 by Robert Collins
Test versioned file storage handling of clean/dirty status for accessed versioned files.
1101
    def __init__(self, name, transport, filemode=None, create=False, access_mode='w'):
1563.2.25 by Robert Collins
Merge in upstream.
1102
        """Create a WeaveFile.
1103
        
1104
        :param create: If not True, only open an existing knit.
1105
        """
1594.2.23 by Robert Collins
Test versioned file storage handling of clean/dirty status for accessed versioned files.
1106
        super(WeaveFile, self).__init__(name, access_mode)
1563.2.9 by Robert Collins
Update versionedfile api tests to ensure that data is available after every operation.
1107
        self._transport = transport
1594.2.23 by Robert Collins
Test versioned file storage handling of clean/dirty status for accessed versioned files.
1108
        self._filemode = filemode
1563.2.9 by Robert Collins
Update versionedfile api tests to ensure that data is available after every operation.
1109
        try:
1563.2.15 by Robert Collins
remove the weavestore assumptions about the number and nature of files it manages.
1110
            _read_weave_v5(self._transport.get(name + WeaveFile.WEAVE_SUFFIX), self)
1563.2.9 by Robert Collins
Update versionedfile api tests to ensure that data is available after every operation.
1111
        except errors.NoSuchFile:
1563.2.25 by Robert Collins
Merge in upstream.
1112
            if not create:
1113
                raise
1563.2.14 by Robert Collins
Prepare weave store to delegate copy details to the versioned file.
1114
            # new file, save it
1115
            self._save()
1563.2.9 by Robert Collins
Update versionedfile api tests to ensure that data is available after every operation.
1116
1596.2.32 by Robert Collins
Reduce re-extraction of texts during weave to knit joins by providing a memoisation facility.
1117
    def _add_lines(self, version_id, parents, lines, parent_texts):
1563.2.9 by Robert Collins
Update versionedfile api tests to ensure that data is available after every operation.
1118
        """Add a version and save the weave."""
1596.2.32 by Robert Collins
Reduce re-extraction of texts during weave to knit joins by providing a memoisation facility.
1119
        result = super(WeaveFile, self)._add_lines(version_id, parents, lines,
1120
                                                   parent_texts)
1563.2.9 by Robert Collins
Update versionedfile api tests to ensure that data is available after every operation.
1121
        self._save()
1596.2.32 by Robert Collins
Reduce re-extraction of texts during weave to knit joins by providing a memoisation facility.
1122
        return result
1563.2.9 by Robert Collins
Update versionedfile api tests to ensure that data is available after every operation.
1123
1594.2.24 by Robert Collins
Make use of the transaction finalisation warning support to implement in-knit caching.
1124
    def _clone_text(self, new_version_id, old_version_id, parents):
1125
        """See VersionedFile.clone_text."""
1126
        super(WeaveFile, self)._clone_text(new_version_id, old_version_id, parents)
1127
        self._save
1128
1563.2.15 by Robert Collins
remove the weavestore assumptions about the number and nature of files it manages.
1129
    def copy_to(self, name, transport):
1130
        """See VersionedFile.copy_to()."""
1131
        # as we are all in memory always, just serialise to the new place.
1132
        sio = StringIO()
1133
        write_weave_v5(self, sio)
1134
        sio.seek(0)
1594.2.23 by Robert Collins
Test versioned file storage handling of clean/dirty status for accessed versioned files.
1135
        transport.put(name + WeaveFile.WEAVE_SUFFIX, sio, self._filemode)
1563.2.15 by Robert Collins
remove the weavestore assumptions about the number and nature of files it manages.
1136
1594.2.23 by Robert Collins
Test versioned file storage handling of clean/dirty status for accessed versioned files.
1137
    def create_empty(self, name, transport, filemode=None):
1138
        return WeaveFile(name, transport, filemode, create=True)
1563.2.13 by Robert Collins
InterVersionedFile implemented.
1139
1563.2.9 by Robert Collins
Update versionedfile api tests to ensure that data is available after every operation.
1140
    def _save(self):
1141
        """Save the weave."""
1594.2.23 by Robert Collins
Test versioned file storage handling of clean/dirty status for accessed versioned files.
1142
        self._check_write_ok()
1563.2.9 by Robert Collins
Update versionedfile api tests to ensure that data is available after every operation.
1143
        sio = StringIO()
1144
        write_weave_v5(self, sio)
1145
        sio.seek(0)
1563.2.15 by Robert Collins
remove the weavestore assumptions about the number and nature of files it manages.
1146
        self._transport.put(self._weave_name + WeaveFile.WEAVE_SUFFIX,
1147
                            sio,
1594.2.23 by Robert Collins
Test versioned file storage handling of clean/dirty status for accessed versioned files.
1148
                            self._filemode)
1563.2.15 by Robert Collins
remove the weavestore assumptions about the number and nature of files it manages.
1149
1150
    @staticmethod
1151
    def get_suffixes():
1152
        """See VersionedFile.get_suffixes()."""
1153
        return [WeaveFile.WEAVE_SUFFIX]
1563.2.9 by Robert Collins
Update versionedfile api tests to ensure that data is available after every operation.
1154
1563.2.31 by Robert Collins
Convert Knit repositories to use knits.
1155
    def join(self, other, pb=None, msg=None, version_ids=None,
1156
             ignore_missing=False):
1563.2.9 by Robert Collins
Update versionedfile api tests to ensure that data is available after every operation.
1157
        """Join other into self and save."""
1563.2.31 by Robert Collins
Convert Knit repositories to use knits.
1158
        super(WeaveFile, self).join(other, pb, msg, version_ids, ignore_missing)
1563.2.9 by Robert Collins
Update versionedfile api tests to ensure that data is available after every operation.
1159
        self._save()
1160
1161
1563.2.11 by Robert Collins
Consolidate reweave and join as we have no separate usage, make reweave tests apply to all versionedfile implementations and deprecate the old reweave apis.
1162
@deprecated_function(zero_eight)
1551.3.11 by Aaron Bentley
Merge from Robert
1163
def reweave(wa, wb, pb=None, msg=None):
1563.2.11 by Robert Collins
Consolidate reweave and join as we have no separate usage, make reweave tests apply to all versionedfile implementations and deprecate the old reweave apis.
1164
    """reweaving is deprecation, please just use weave.join()."""
1165
    _reweave(wa, wb, pb, msg)
1166
1167
def _reweave(wa, wb, pb=None, msg=None):
1393.1.68 by Martin Pool
- add reweave that joins with ghosts
1168
    """Combine two weaves and return the result.
1169
1170
    This works even if a revision R has different parents in 
1171
    wa and wb.  In the resulting weave all the parents are given.
1172
1173
    This is done by just building up a new weave, maintaining ordering 
1174
    of the versions in the two inputs.  More efficient approaches
1175
    might be possible but it should only be necessary to do 
1176
    this operation rarely, when a new previously ghost version is 
1177
    inserted.
1551.3.11 by Aaron Bentley
Merge from Robert
1178
1179
    :param pb: An optional progress bar, indicating how far done we are
1180
    :param msg: An optional message for the progress
1393.1.68 by Martin Pool
- add reweave that joins with ghosts
1181
    """
1182
    wr = Weave()
1183
    ia = ib = 0
1563.2.18 by Robert Collins
get knit repositories really using knits for text storage.
1184
    queue_a = range(wa.num_versions())
1185
    queue_b = range(wb.num_versions())
1393.1.68 by Martin Pool
- add reweave that joins with ghosts
1186
    # first determine combined parents of all versions
1187
    # map from version name -> all parent names
1188
    combined_parents = _reweave_parent_graphs(wa, wb)
1189
    mutter("combined parents: %r", combined_parents)
1185.16.117 by mbp at sourcefrog
Reweave in topological-sort order.
1190
    order = topo_sort(combined_parents.iteritems())
1393.1.68 by Martin Pool
- add reweave that joins with ghosts
1191
    mutter("order to reweave: %r", order)
1551.3.11 by Aaron Bentley
Merge from Robert
1192
1193
    if pb and not msg:
1194
        msg = 'reweave'
1195
1196
    for idx, name in enumerate(order):
1197
        if pb:
1198
            pb.update(msg, idx, len(order))
1393.1.68 by Martin Pool
- add reweave that joins with ghosts
1199
        if name in wa._name_map:
1200
            lines = wa.get_lines(name)
1201
            if name in wb._name_map:
1551.3.11 by Aaron Bentley
Merge from Robert
1202
                lines_b = wb.get_lines(name)
1203
                if lines != lines_b:
1204
                    mutter('Weaves differ on content. rev_id {%s}', name)
1205
                    mutter('weaves: %s, %s', wa._weave_name, wb._weave_name)
1206
                    import difflib
1207
                    lines = list(difflib.unified_diff(lines, lines_b,
1208
                            wa._weave_name, wb._weave_name))
1209
                    mutter('lines:\n%s', ''.join(lines))
1210
                    raise errors.WeaveTextDiffers(name, wa, wb)
1393.1.68 by Martin Pool
- add reweave that joins with ghosts
1211
        else:
1212
            lines = wb.get_lines(name)
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
1213
        wr._add(name, lines, [wr._lookup(i) for i in combined_parents[name]])
1393.1.68 by Martin Pool
- add reweave that joins with ghosts
1214
    return wr
1215
1216
def _reweave_parent_graphs(wa, wb):
1217
    """Return combined parent ancestry for two weaves.
1218
    
1219
    Returned as a list of (version_name, set(parent_names))"""
1220
    combined = {}
1221
    for weave in [wa, wb]:
1222
        for idx, name in enumerate(weave._names):
1223
            p = combined.setdefault(name, set())
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
1224
            p.update(map(weave._idx_to_name, weave._parents[idx]))
1393.1.68 by Martin Pool
- add reweave that joins with ghosts
1225
    return combined
1226
1227
1081 by Martin Pool
- if weave tool is invoked with no arguments, show help
1228
def weave_toc(w):
1229
    """Show the weave's table-of-contents"""
1083 by Martin Pool
- add space to store revision-id in weave files
1230
    print '%6s %50s %10s %10s' % ('ver', 'name', 'sha1', 'parents')
1231
    for i in (6, 50, 10, 10):
870 by Martin Pool
- better weave info display
1232
        print '-' * i,
1233
    print
1563.2.18 by Robert Collins
get knit repositories really using knits for text storage.
1234
    for i in range(w.num_versions()):
0.1.91 by Martin Pool
Update Weave.check
1235
        sha1 = w._sha1s[i]
1083 by Martin Pool
- add space to store revision-id in weave files
1236
        name = w._names[i]
1237
        parent_str = ' '.join(map(str, w._parents[i]))
1238
        print '%6d %-50.50s %10.10s %s' % (i, name, sha1, parent_str)
0.1.88 by Martin Pool
Add weave info command.
1239
869 by Martin Pool
- more weave.py command line options
1240
1241
1393.1.4 by Martin Pool
- weave tool now finds bzrlib if it's not on the default path
1242
def weave_stats(weave_file, pb):
947 by Martin Pool
- new 'weave stats' command
1243
    from bzrlib.weavefile import read_weave
1244
1245
    wf = file(weave_file, 'rb')
1773.4.1 by Martin Pool
Add pyflakes makefile target; fix many warnings
1246
    w = read_weave(wf)
947 by Martin Pool
- new 'weave stats' command
1247
    # FIXME: doesn't work on pipes
1248
    weave_size = wf.tell()
1249
1250
    total = 0
1251
    vers = len(w)
1252
    for i in range(vers):
1253
        pb.update('checking sizes', i, vers)
1393.1.24 by Martin Pool
- slight speedup for weave stats
1254
        for origin, lineno, line in w._extract([i]):
947 by Martin Pool
- new 'weave stats' command
1255
            total += len(line)
1256
1257
    pb.clear()
1258
1259
    print 'versions          %9d' % vers
1260
    print 'weave file        %9d bytes' % weave_size
1261
    print 'total contents    %9d bytes' % total
1262
    print 'compression ratio %9.2fx' % (float(total) / float(weave_size))
974.1.26 by aaron.bentley at utoronto
merged mbp@sourcefrog.net-20050817233101-0939da1cf91f2472
1263
    if vers:
1264
        avg = total/vers
1265
        print 'average size      %9d bytes' % avg
1266
        print 'relative size     %9.2fx' % (float(weave_size) / float(avg))
947 by Martin Pool
- new 'weave stats' command
1267
1268
869 by Martin Pool
- more weave.py command line options
1269
def usage():
871 by Martin Pool
- add command for merge-based weave
1270
    print """bzr weave tool
1271
1272
Experimental tool for weave algorithm.
1273
869 by Martin Pool
- more weave.py command line options
1274
usage:
1275
    weave init WEAVEFILE
1276
        Create an empty weave file
1277
    weave get WEAVEFILE VERSION
1278
        Write out specified version.
1279
    weave check WEAVEFILE
1280
        Check consistency of all versions.
1081 by Martin Pool
- if weave tool is invoked with no arguments, show help
1281
    weave toc WEAVEFILE
869 by Martin Pool
- more weave.py command line options
1282
        Display table of contents.
1084 by Martin Pool
- weave add command needs to take a symbolic name too
1283
    weave add WEAVEFILE NAME [BASE...] < NEWTEXT
869 by Martin Pool
- more weave.py command line options
1284
        Add NEWTEXT, with specified parent versions.
1285
    weave annotate WEAVEFILE VERSION
1286
        Display origin of each line.
1287
    weave merge WEAVEFILE VERSION1 VERSION2 > OUT
1288
        Auto-merge two versions and display conflicts.
1329 by Martin Pool
- add diff command to weave utility
1289
    weave diff WEAVEFILE VERSION1 VERSION2 
1290
        Show differences between two versions.
871 by Martin Pool
- add command for merge-based weave
1291
1292
example:
1293
1294
    % weave init foo.weave
1295
    % vi foo.txt
1084 by Martin Pool
- weave add command needs to take a symbolic name too
1296
    % weave add foo.weave ver0 < foo.txt
871 by Martin Pool
- add command for merge-based weave
1297
    added version 0
1298
1299
    (create updated version)
1300
    % vi foo.txt
1301
    % weave get foo.weave 0 | diff -u - foo.txt
1084 by Martin Pool
- weave add command needs to take a symbolic name too
1302
    % weave add foo.weave ver1 0 < foo.txt
871 by Martin Pool
- add command for merge-based weave
1303
    added version 1
1304
1305
    % weave get foo.weave 0 > foo.txt       (create forked version)
1306
    % vi foo.txt
1084 by Martin Pool
- weave add command needs to take a symbolic name too
1307
    % weave add foo.weave ver2 0 < foo.txt
871 by Martin Pool
- add command for merge-based weave
1308
    added version 2
1309
1310
    % weave merge foo.weave 1 2 > foo.txt   (merge them)
1311
    % vi foo.txt                            (resolve conflicts)
1084 by Martin Pool
- weave add command needs to take a symbolic name too
1312
    % weave add foo.weave merged 1 2 < foo.txt     (commit merged version)     
871 by Martin Pool
- add command for merge-based weave
1313
    
869 by Martin Pool
- more weave.py command line options
1314
"""
0.1.88 by Martin Pool
Add weave info command.
1315
    
1316
0.1.62 by Martin Pool
Lame command-line client for reading and writing weaves.
1317
1318
def main(argv):
1319
    import sys
1320
    import os
1393.1.4 by Martin Pool
- weave tool now finds bzrlib if it's not on the default path
1321
    try:
1322
        import bzrlib
1323
    except ImportError:
1324
        # in case we're run directly from the subdirectory
1325
        sys.path.append('..')
1326
        import bzrlib
1336 by Martin Pool
- tidy up import
1327
    from bzrlib.weavefile import write_weave, read_weave
894 by Martin Pool
- small optimization for weave extract
1328
    from bzrlib.progress import ProgressBar
1329
1078 by Martin Pool
- use psyco for weave if possible
1330
    try:
1331
        import psyco
1332
        psyco.full()
1333
    except ImportError:
1334
        pass
894 by Martin Pool
- small optimization for weave extract
1335
1081 by Martin Pool
- if weave tool is invoked with no arguments, show help
1336
    if len(argv) < 2:
1337
        usage()
1338
        return 0
1339
0.1.62 by Martin Pool
Lame command-line client for reading and writing weaves.
1340
    cmd = argv[1]
869 by Martin Pool
- more weave.py command line options
1341
1342
    def readit():
1343
        return read_weave(file(argv[2], 'rb'))
1344
    
1345
    if cmd == 'help':
1346
        usage()
1347
    elif cmd == 'add':
1348
        w = readit()
0.1.62 by Martin Pool
Lame command-line client for reading and writing weaves.
1349
        # at the moment, based on everything in the file
1084 by Martin Pool
- weave add command needs to take a symbolic name too
1350
        name = argv[3]
1351
        parents = map(int, argv[4:])
0.1.72 by Martin Pool
Go back to weave lines normally having newlines at the end.
1352
        lines = sys.stdin.readlines()
1084 by Martin Pool
- weave add command needs to take a symbolic name too
1353
        ver = w.add(name, parents, lines)
869 by Martin Pool
- more weave.py command line options
1354
        write_weave(w, file(argv[2], 'wb'))
1084 by Martin Pool
- weave add command needs to take a symbolic name too
1355
        print 'added version %r %d' % (name, ver)
0.1.62 by Martin Pool
Lame command-line client for reading and writing weaves.
1356
    elif cmd == 'init':
1357
        fn = argv[2]
1358
        if os.path.exists(fn):
1359
            raise IOError("file exists")
1360
        w = Weave()
869 by Martin Pool
- more weave.py command line options
1361
        write_weave(w, file(fn, 'wb'))
1362
    elif cmd == 'get': # get one version
1363
        w = readit()
0.1.94 by Martin Pool
Fix get_iter call
1364
        sys.stdout.writelines(w.get_iter(int(argv[3])))
869 by Martin Pool
- more weave.py command line options
1365
        
1329 by Martin Pool
- add diff command to weave utility
1366
    elif cmd == 'diff':
1367
        w = readit()
1368
        fn = argv[2]
1369
        v1, v2 = map(int, argv[3:5])
1370
        lines1 = w.get(v1)
1371
        lines2 = w.get(v2)
1711.2.24 by John Arbash Meinel
Late bind to PatienceSequenceMatcher to allow plugin to override.
1372
        diff_gen = bzrlib.patiencediff.unified_diff(lines1, lines2,
1329 by Martin Pool
- add diff command to weave utility
1373
                                '%s version %d' % (fn, v1),
1374
                                '%s version %d' % (fn, v2))
1375
        sys.stdout.writelines(diff_gen)
1376
            
0.1.62 by Martin Pool
Lame command-line client for reading and writing weaves.
1377
    elif cmd == 'annotate':
869 by Martin Pool
- more weave.py command line options
1378
        w = readit()
0.1.72 by Martin Pool
Go back to weave lines normally having newlines at the end.
1379
        # newline is added to all lines regardless; too hard to get
1380
        # reasonable formatting otherwise
0.1.62 by Martin Pool
Lame command-line client for reading and writing weaves.
1381
        lasto = None
1382
        for origin, text in w.annotate(int(argv[3])):
0.1.72 by Martin Pool
Go back to weave lines normally having newlines at the end.
1383
            text = text.rstrip('\r\n')
0.1.62 by Martin Pool
Lame command-line client for reading and writing weaves.
1384
            if origin == lasto:
1385
                print '      | %s' % (text)
1386
            else:
1387
                print '%5d | %s' % (origin, text)
1388
                lasto = origin
871 by Martin Pool
- add command for merge-based weave
1389
                
1081 by Martin Pool
- if weave tool is invoked with no arguments, show help
1390
    elif cmd == 'toc':
1391
        weave_toc(readit())
947 by Martin Pool
- new 'weave stats' command
1392
1393
    elif cmd == 'stats':
1393.1.4 by Martin Pool
- weave tool now finds bzrlib if it's not on the default path
1394
        weave_stats(argv[2], ProgressBar())
871 by Martin Pool
- add command for merge-based weave
1395
        
0.1.91 by Martin Pool
Update Weave.check
1396
    elif cmd == 'check':
869 by Martin Pool
- more weave.py command line options
1397
        w = readit()
894 by Martin Pool
- small optimization for weave extract
1398
        pb = ProgressBar()
1399
        w.check(pb)
1400
        pb.clear()
1563.2.18 by Robert Collins
get knit repositories really using knits for text storage.
1401
        print '%d versions ok' % w.num_versions()
871 by Martin Pool
- add command for merge-based weave
1402
892 by Martin Pool
- weave stores only direct parents, and calculates and memoizes expansion as needed
1403
    elif cmd == 'inclusions':
1404
        w = readit()
1405
        print ' '.join(map(str, w.inclusions([int(argv[3])])))
1406
1407
    elif cmd == 'parents':
1408
        w = readit()
944 by Martin Pool
- refactor member names in Weave code
1409
        print ' '.join(map(str, w._parents[int(argv[3])]))
892 by Martin Pool
- weave stores only direct parents, and calculates and memoizes expansion as needed
1410
918 by Martin Pool
- start doing new weave-merge algorithm
1411
    elif cmd == 'plan-merge':
1616.1.17 by Martin Pool
New 'weave-plan-merge' and 'weave-merge-text' commands lifted from weave.py
1412
        # replaced by 'bzr weave-plan-merge'
918 by Martin Pool
- start doing new weave-merge algorithm
1413
        w = readit()
1414
        for state, line in w.plan_merge(int(argv[3]), int(argv[4])):
919 by Martin Pool
- more development of weave-merge
1415
            if line:
1416
                print '%14s | %s' % (state, line),
871 by Martin Pool
- add command for merge-based weave
1417
    elif cmd == 'merge':
1616.1.17 by Martin Pool
New 'weave-plan-merge' and 'weave-merge-text' commands lifted from weave.py
1418
        # replaced by 'bzr weave-merge-text'
919 by Martin Pool
- more development of weave-merge
1419
        w = readit()
1420
        p = w.plan_merge(int(argv[3]), int(argv[4]))
1421
        sys.stdout.writelines(w.weave_merge(p))
0.1.62 by Martin Pool
Lame command-line client for reading and writing weaves.
1422
    else:
1423
        raise ValueError('unknown command %r' % cmd)
1424
    
1425
1426
if __name__ == '__main__':
1427
    import sys
1773.4.1 by Martin Pool
Add pyflakes makefile target; fix many warnings
1428
    sys.exit(main(sys.argv))
1076 by Martin Pool
- add code to run weave utility under profiler
1429
1563.2.13 by Robert Collins
InterVersionedFile implemented.
1430
1431
class InterWeave(InterVersionedFile):
1432
    """Optimised code paths for weave to weave operations."""
1433
    
1684.3.3 by Robert Collins
Add a special cased weaves to knit converter.
1434
    _matching_file_from_factory = staticmethod(WeaveFile)
1435
    _matching_file_to_factory = staticmethod(WeaveFile)
1563.2.13 by Robert Collins
InterVersionedFile implemented.
1436
    
1437
    @staticmethod
1438
    def is_compatible(source, target):
1439
        """Be compatible with weaves."""
1440
        try:
1441
            return (isinstance(source, Weave) and
1442
                    isinstance(target, Weave))
1443
        except AttributeError:
1444
            return False
1445
1563.2.31 by Robert Collins
Convert Knit repositories to use knits.
1446
    def join(self, pb=None, msg=None, version_ids=None, ignore_missing=False):
1563.2.13 by Robert Collins
InterVersionedFile implemented.
1447
        """See InterVersionedFile.join."""
1684.3.2 by Robert Collins
Factor out version_ids-to-join selection in InterVersionedfile.
1448
        version_ids = self._get_source_version_ids(version_ids, ignore_missing)
1684.3.1 by Robert Collins
Fix versioned file joins with empty targets.
1449
        if self.target.versions() == [] and version_ids is None:
1594.2.11 by Robert Collins
Setup fast-code paths for copying into empty weaves and weave->empty knit.
1450
            self.target._copy_weave_content(self.source)
1451
            return
1563.2.13 by Robert Collins
InterVersionedFile implemented.
1452
        try:
1563.2.31 by Robert Collins
Convert Knit repositories to use knits.
1453
            self.target._join(self.source, pb, msg, version_ids, ignore_missing)
1563.2.13 by Robert Collins
InterVersionedFile implemented.
1454
        except errors.WeaveParentMismatch:
1455
            self.target._reweave(self.source, pb, msg)
1456
1457
1458
InterVersionedFile.register_optimiser(InterWeave)