« back to all changes in this revision
Viewing changes to bzrlib/knit.py
- browse files at revision 1959.4.3
- compare with another revision
- download diff
- download tarball
- view history from revision 1959.4.3
- Committer: Aaron Bentley
- Date: 2007-02-06 14:52:16 UTC
- mfrom: (2266 +trunk)
- mto: This revision was merged to the branch mainline in revision 2268.
- Revision ID: abentley@panoramicfeedback.com-20070206145216-fcpi8o3ufvuzwbp9
Merge bzr.dev
- files added:
- NEWS.developers
- bzrlib/bundle/old
- bzrlib/tests/interversionedfile_implementations
- bzrlib/tests/revisionstore_implementations
- files removed:
- bzrlib/_dirstate_helpers_c.h
- bzrlib/help_topics
- bzrlib/help_topics/en
- bzrlib/smart
- bzrlib/tests/commands
- bzrlib/tests/inventory_implementations
- bzrlib/tests/per_lock
- bzrlib/tests/per_repository_reference
- bzrlib/util/tests
- doc/developers
- doc/en
- doc/en/admin-guide
- doc/en/developer-guide
- doc/en/mini-tutorial
- doc/en/quick-reference
- doc/en/release-notes
- doc/en/tutorials
- doc/en/user-guide
- doc/en/user-guide/images
- doc/en/user-reference
- doc/es
- doc/es/guia-desarrollador
- doc/es/guia-usuario
- doc/es/mini-tutorial
- doc/es/notas-version
- doc/es/referencia
- doc/es/referencia-rapida
- man1
- files renamed:
- doc/developers/HACKING.txt => HACKING
- bzrlib/deprecated_graph.py => bzrlib/graph.py
- bzrlib/help_topics/__init__.py => bzrlib/help_topics.py
- bzrlib/_patiencediff_py.py => bzrlib/patiencediff.py
- bzrlib/tests/http_utils.py => bzrlib/tests/HTTPTestUtil.py
- bzrlib/tests/http_server.py => bzrlib/tests/HttpServer.py
- bzrlib/tests/blackbox/test_send.py => bzrlib/tests/blackbox/test_bundle.py
- bzrlib/tests/test_deprecated_graph.py => bzrlib/tests/test_graph.py
- bzrlib/tests/test_revisionspec.py => bzrlib/tests/test_revisionnamespaces.py
- bzrlib/transport/remote.py => bzrlib/transport/smart.py
- bzrlib/xml8.py => bzrlib/xml5.py
- doc/en/tutorials/centralized_workflow.txt => doc/centralized_workflow.txt
- bzrlib/help_topics/en/configuration.txt => doc/configuration.txt
- doc/en/user-guide/http_smart_server.txt => doc/http_smart_server.txt
- doc/en/user-guide/index.txt => doc/index.txt
- doc/en/user-guide/plugins.txt => doc/plugins.txt
- doc/en/user-guide/server.txt => doc/server.txt
- doc/en/user-guide/setting_up_email.txt => doc/setting_up_email.txt
- doc/en/user-guide/specifying_revisions.txt => doc/specifying_revisions.txt
- doc/en/tutorials/tutorial.txt => doc/tutorial.txt
- doc/en/user-guide/using_aliases.txt => doc/using_aliases.txt
- doc/en/user-guide/version_info.txt => doc/version_info.txt
- files modified:
- .bzrignore
- setup.py *
added
removed
bzrlib/knit.py
1
# Copyright (C) 2005, 2006, 2007 Canonical Ltd
1
# Copyright (C) 2005, 2006 Canonical Ltd
2
2
#
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
60
60
# record content length ?
61
61
62
62
63
from copy import copy
63
64
from cStringIO import StringIO
65
import difflib
64
66
from itertools import izip, chain
65
67
import operator
66
68
import os
69
import sys
70
import warnings
67
71
68
from bzrlib.lazy_import import lazy_import
69
lazy_import(globals(), """
70
from bzrlib import (
71
annotate,
72
debug,
73
diff,
74
graph as _mod_graph,
75
index as _mod_index,
76
lru_cache,
77
pack,
78
progress,
79
trace,
80
tsort,
81
tuned_gzip,
82
)
83
""")
84
from bzrlib import (
72
import bzrlib
73
from bzrlib import (
74
cache_utf8,
85
75
errors,
86
osutils,
87
76
patiencediff,
77
progress,
78
ui,
88
79
)
89
80
from bzrlib.errors import (
90
81
FileExists,
96
87
RevisionNotPresent,
97
88
RevisionAlreadyPresent,
98
89
)
90
from bzrlib.tuned_gzip import GzipFile
91
from bzrlib.trace import mutter
99
92
from bzrlib.osutils import (
100
93
contains_whitespace,
101
94
contains_linebreaks,
102
sha_string,
103
95
sha_strings,
104
split_lines,
105
)
106
from bzrlib.versionedfile import (
107
AbsentContentFactory,
108
adapter_registry,
109
ConstantMapper,
110
ContentFactory,
111
FulltextContentFactory,
112
VersionedFile,
113
VersionedFiles,
114
)
96
)
97
from bzrlib.symbol_versioning import DEPRECATED_PARAMETER, deprecated_passed
98
from bzrlib.tsort import topo_sort
99
import bzrlib.ui
100
import bzrlib.weave
101
from bzrlib.versionedfile import VersionedFile, InterVersionedFile
115
102
116
103
117
104
# TODO: Split out code specific to this format into an associated object.
131
118
INDEX_SUFFIX = '.kndx'
132
119
133
120
134
class KnitAdapter(object):
135
"""Base class for knit record adaption."""
136
137
def __init__(self, basis_vf):
138
"""Create an adapter which accesses full texts from basis_vf.
139
140
:param basis_vf: A versioned file to access basis texts of deltas from.
141
May be None for adapters that do not need to access basis texts.
142
"""
143
self._data = KnitVersionedFiles(None, None)
144
self._annotate_factory = KnitAnnotateFactory()
145
self._plain_factory = KnitPlainFactory()
146
self._basis_vf = basis_vf
147
148
149
class FTAnnotatedToUnannotated(KnitAdapter):
150
"""An adapter from FT annotated knits to unannotated ones."""
151
152
def get_bytes(self, factory, annotated_compressed_bytes):
153
rec, contents = \
154
self._data._parse_record_unchecked(annotated_compressed_bytes)
155
content = self._annotate_factory.parse_fulltext(contents, rec[1])
156
size, bytes = self._data._record_to_data((rec[1],), rec[3], content.text())
157
return bytes
158
159
160
class DeltaAnnotatedToUnannotated(KnitAdapter):
161
"""An adapter for deltas from annotated to unannotated."""
162
163
def get_bytes(self, factory, annotated_compressed_bytes):
164
rec, contents = \
165
self._data._parse_record_unchecked(annotated_compressed_bytes)
166
delta = self._annotate_factory.parse_line_delta(contents, rec[1],
167
plain=True)
168
contents = self._plain_factory.lower_line_delta(delta)
169
size, bytes = self._data._record_to_data((rec[1],), rec[3], contents)
170
return bytes
171
172
173
class FTAnnotatedToFullText(KnitAdapter):
174
"""An adapter from FT annotated knits to unannotated ones."""
175
176
def get_bytes(self, factory, annotated_compressed_bytes):
177
rec, contents = \
178
self._data._parse_record_unchecked(annotated_compressed_bytes)
179
content, delta = self._annotate_factory.parse_record(factory.key[-1],
180
contents, factory._build_details, None)
181
return ''.join(content.text())
182
183
184
class DeltaAnnotatedToFullText(KnitAdapter):
185
"""An adapter for deltas from annotated to unannotated."""
186
187
def get_bytes(self, factory, annotated_compressed_bytes):
188
rec, contents = \
189
self._data._parse_record_unchecked(annotated_compressed_bytes)
190
delta = self._annotate_factory.parse_line_delta(contents, rec[1],
191
plain=True)
192
compression_parent = factory.parents[0]
193
basis_entry = self._basis_vf.get_record_stream(
194
[compression_parent], 'unordered', True).next()
195
if basis_entry.storage_kind == 'absent':
196
raise errors.RevisionNotPresent(compression_parent, self._basis_vf)
197
basis_lines = split_lines(basis_entry.get_bytes_as('fulltext'))
198
# Manually apply the delta because we have one annotated content and
199
# one plain.
200
basis_content = PlainKnitContent(basis_lines, compression_parent)
201
basis_content.apply_delta(delta, rec[1])
202
basis_content._should_strip_eol = factory._build_details[1]
203
return ''.join(basis_content.text())
204
205
206
class FTPlainToFullText(KnitAdapter):
207
"""An adapter from FT plain knits to unannotated ones."""
208
209
def get_bytes(self, factory, compressed_bytes):
210
rec, contents = \
211
self._data._parse_record_unchecked(compressed_bytes)
212
content, delta = self._plain_factory.parse_record(factory.key[-1],
213
contents, factory._build_details, None)
214
return ''.join(content.text())
215
216
217
class DeltaPlainToFullText(KnitAdapter):
218
"""An adapter for deltas from annotated to unannotated."""
219
220
def get_bytes(self, factory, compressed_bytes):
221
rec, contents = \
222
self._data._parse_record_unchecked(compressed_bytes)
223
delta = self._plain_factory.parse_line_delta(contents, rec[1])
224
compression_parent = factory.parents[0]
225
# XXX: string splitting overhead.
226
basis_entry = self._basis_vf.get_record_stream(
227
[compression_parent], 'unordered', True).next()
228
if basis_entry.storage_kind == 'absent':
229
raise errors.RevisionNotPresent(compression_parent, self._basis_vf)
230
basis_lines = split_lines(basis_entry.get_bytes_as('fulltext'))
231
basis_content = PlainKnitContent(basis_lines, compression_parent)
232
# Manually apply the delta because we have one annotated content and
233
# one plain.
234
content, _ = self._plain_factory.parse_record(rec[1], contents,
235
factory._build_details, basis_content)
236
return ''.join(content.text())
237
238
239
class KnitContentFactory(ContentFactory):
240
"""Content factory for streaming from knits.
241
242
:seealso ContentFactory:
243
"""
244
245
def __init__(self, key, parents, build_details, sha1, raw_record,
246
annotated, knit=None):
247
"""Create a KnitContentFactory for key.
248
249
:param key: The key.
250
:param parents: The parents.
251
:param build_details: The build details as returned from
252
get_build_details.
253
:param sha1: The sha1 expected from the full text of this object.
254
:param raw_record: The bytes of the knit data from disk.
255
:param annotated: True if the raw data is annotated.
256
"""
257
ContentFactory.__init__(self)
258
self.sha1 = sha1
259
self.key = key
260
self.parents = parents
261
if build_details[0] == 'line-delta':
262
kind = 'delta'
263
else:
264
kind = 'ft'
265
if annotated:
266
annotated_kind = 'annotated-'
267
else:
268
annotated_kind = ''
269
self.storage_kind = 'knit-%s%s-gz' % (annotated_kind, kind)
270
self._raw_record = raw_record
271
self._build_details = build_details
272
self._knit = knit
273
274
def get_bytes_as(self, storage_kind):
275
if storage_kind == self.storage_kind:
276
return self._raw_record
277
if storage_kind == 'fulltext' and self._knit is not None:
278
return self._knit.get_text(self.key[0])
279
else:
280
raise errors.UnavailableRepresentation(self.key, storage_kind,
281
self.storage_kind)
282
283
284
121
class KnitContent(object):
285
"""Content of a knit version to which deltas can be applied.
286
287
This is always stored in memory as a list of lines with \n at the end,
288
plus a flag saying if the final ending is really there or not, because that
289
corresponds to the on-disk knit representation.
290
"""
291
292
def __init__(self):
293
self._should_strip_eol = False
294
295
def apply_delta(self, delta, new_version_id):
296
"""Apply delta to this object to become new_version_id."""
297
raise NotImplementedError(self.apply_delta)
122
"""Content of a knit version to which deltas can be applied."""
123
124
def __init__(self, lines):
125
self._lines = lines
126
127
def annotate_iter(self):
128
"""Yield tuples of (origin, text) for each content line."""
129
return iter(self._lines)
130
131
def annotate(self):
132
"""Return a list of (origin, text) tuples."""
133
return list(self.annotate_iter())
298
134
299
135
def line_delta_iter(self, new_lines):
300
136
"""Generate line-based delta from this content to new_lines."""
301
137
new_texts = new_lines.text()
302
138
old_texts = self.text()
303
s = patiencediff.PatienceSequenceMatcher(None, old_texts, new_texts)
139
s = KnitSequenceMatcher(None, old_texts, new_texts)
304
140
for tag, i1, i2, j1, j2 in s.get_opcodes():
305
141
if tag == 'equal':
306
142
continue
310
146
def line_delta(self, new_lines):
311
147
return list(self.line_delta_iter(new_lines))
312
148
313
@staticmethod
314
def get_line_delta_blocks(knit_delta, source, target):
315
"""Extract SequenceMatcher.get_matching_blocks() from a knit delta"""
316
target_len = len(target)
317
s_pos = 0
318
t_pos = 0
319
for s_begin, s_end, t_len, new_text in knit_delta:
320
true_n = s_begin - s_pos
321
n = true_n
322
if n > 0:
323
# knit deltas do not provide reliable info about whether the
324
# last line of a file matches, due to eol handling.
325
if source[s_pos + n -1] != target[t_pos + n -1]:
326
n-=1
327
if n > 0:
328
yield s_pos, t_pos, n
329
t_pos += t_len + true_n
330
s_pos = s_end
331
n = target_len - t_pos
332
if n > 0:
333
if source[s_pos + n -1] != target[t_pos + n -1]:
334
n-=1
335
if n > 0:
336
yield s_pos, t_pos, n
337
yield s_pos + (target_len - t_pos), target_len, 0
338
339
340
class AnnotatedKnitContent(KnitContent):
341
"""Annotated content."""
342
343
def __init__(self, lines):
344
KnitContent.__init__(self)
345
self._lines = lines
346
347
def annotate(self):
348
"""Return a list of (origin, text) for each content line."""
349
lines = self._lines[:]
350
if self._should_strip_eol:
351
origin, last_line = lines[-1]
352
lines[-1] = (origin, last_line.rstrip('\n'))
353
return lines
354
355
def apply_delta(self, delta, new_version_id):
356
"""Apply delta to this object to become new_version_id."""
357
offset = 0
358
lines = self._lines
359
for start, end, count, delta_lines in delta:
360
lines[offset+start:offset+end] = delta_lines
361
offset = offset + (start - end) + count
362
363
def text(self):
364
try:
365
lines = [text for origin, text in self._lines]
366
except ValueError, e:
367
# most commonly (only?) caused by the internal form of the knit
368
# missing annotation information because of a bug - see thread
369
# around 20071015
370
raise KnitCorrupt(self,
371
"line in annotated knit missing annotation information: %s"
372
% (e,))
373
if self._should_strip_eol:
374
lines[-1] = lines[-1].rstrip('\n')
375
return lines
376
377
def copy(self):
378
return AnnotatedKnitContent(self._lines[:])
379
380
381
class PlainKnitContent(KnitContent):
382
"""Unannotated content.
383
384
When annotate[_iter] is called on this content, the same version is reported
385
for all lines. Generally, annotate[_iter] is not useful on PlainKnitContent
386
objects.
387
"""
388
389
def __init__(self, lines, version_id):
390
KnitContent.__init__(self)
391
self._lines = lines
392
self._version_id = version_id
393
394
def annotate(self):
395
"""Return a list of (origin, text) for each content line."""
396
return [(self._version_id, line) for line in self._lines]
397
398
def apply_delta(self, delta, new_version_id):
399
"""Apply delta to this object to become new_version_id."""
400
offset = 0
401
lines = self._lines
402
for start, end, count, delta_lines in delta:
403
lines[offset+start:offset+end] = delta_lines
404
offset = offset + (start - end) + count
405
self._version_id = new_version_id
406
407
def copy(self):
408
return PlainKnitContent(self._lines[:], self._version_id)
409
410
def text(self):
411
lines = self._lines
412
if self._should_strip_eol:
413
lines = lines[:]
414
lines[-1] = lines[-1].rstrip('\n')
415
return lines
149
def text(self):
150
return [text for origin, text in self._lines]
151
152
def copy(self):
153
return KnitContent(self._lines[:])
416
154
417
155
418
156
class _KnitFactory(object):
419
"""Base class for common Factory functions."""
420
421
def parse_record(self, version_id, record, record_details,
422
base_content, copy_base_content=True):
423
"""Parse a record into a full content object.
424
425
:param version_id: The official version id for this content
426
:param record: The data returned by read_records_iter()
427
:param record_details: Details about the record returned by
428
get_build_details
429
:param base_content: If get_build_details returns a compression_parent,
430
you must return a base_content here, else use None
431
:param copy_base_content: When building from the base_content, decide
432
you can either copy it and return a new object, or modify it in
433
place.
434
:return: (content, delta) A Content object and possibly a line-delta,
435
delta may be None
436
"""
437
method, noeol = record_details
438
if method == 'line-delta':
439
if copy_base_content:
440
content = base_content.copy()
441
else:
442
content = base_content
443
delta = self.parse_line_delta(record, version_id)
444
content.apply_delta(delta, version_id)
445
else:
446
content = self.parse_fulltext(record, version_id)
447
delta = None
448
content._should_strip_eol = noeol
449
return (content, delta)
157
"""Base factory for creating content objects."""
158
159
def make(self, lines, version):
160
num_lines = len(lines)
161
return KnitContent(zip([version] * num_lines, lines))
450
162
451
163
452
164
class KnitAnnotateFactory(_KnitFactory):
454
166
455
167
annotated = True
456
168
457
def make(self, lines, version_id):
458
num_lines = len(lines)
459
return AnnotatedKnitContent(zip([version_id] * num_lines, lines))
460
461
def parse_fulltext(self, content, version_id):
169
def parse_fulltext(self, content, version):
462
170
"""Convert fulltext to internal representation
463
171
464
172
fulltext content is of the format
466
174
internal representation is of the format:
467
175
(revid, plaintext)
468
176
"""
469
# TODO: jam 20070209 The tests expect this to be returned as tuples,
470
# but the code itself doesn't really depend on that.
471
# Figure out a way to not require the overhead of turning the
472
# list back into tuples.
473
lines = [tuple(line.split(' ', 1)) for line in content]
474
return AnnotatedKnitContent(lines)
177
decode_utf8 = cache_utf8.decode
178
lines = []
179
for line in content:
180
origin, text = line.split(' ', 1)
181
lines.append((decode_utf8(origin), text))
182
return KnitContent(lines)
475
183
476
184
def parse_line_delta_iter(self, lines):
477
185
return iter(self.parse_line_delta(lines))
478
186
479
def parse_line_delta(self, lines, version_id, plain=False):
187
def parse_line_delta(self, lines, version):
480
188
"""Convert a line based delta into internal representation.
481
189
482
190
line delta is in the form of:
485
193
revid(utf8) newline\n
486
194
internal representation is
487
195
(start, end, count, [1..count tuples (revid, newline)])
488
489
:param plain: If True, the lines are returned as a plain
490
list without annotations, not as a list of (origin, content) tuples, i.e.
491
(start, end, count, [1..count newline])
492
196
"""
197
decode_utf8 = cache_utf8.decode
493
198
result = []
494
199
lines = iter(lines)
495
200
next = lines.next
496
497
cache = {}
498
def cache_and_return(line):
499
origin, text = line.split(' ', 1)
500
return cache.setdefault(origin, origin), text
501
502
201
# walk through the lines parsing.
503
# Note that the plain test is explicitly pulled out of the
504
# loop to minimise any performance impact
505
if plain:
506
for header in lines:
507
start, end, count = [int(n) for n in header.split(',')]
508
contents = [next().split(' ', 1)[1] for i in xrange(count)]
509
result.append((start, end, count, contents))
510
else:
511
for header in lines:
512
start, end, count = [int(n) for n in header.split(',')]
513
contents = [tuple(next().split(' ', 1)) for i in xrange(count)]
514
result.append((start, end, count, contents))
202
for header in lines:
203
start, end, count = [int(n) for n in header.split(',')]
204
contents = []
205
remaining = count
206
while remaining:
207
origin, text = next().split(' ', 1)
208
remaining -= 1
209
contents.append((decode_utf8(origin), text))
210
result.append((start, end, count, contents))
515
211
return result
516
212
517
213
def get_fulltext_content(self, lines):
538
234
539
235
see parse_fulltext which this inverts.
540
236
"""
541
# TODO: jam 20070209 We only do the caching thing to make sure that
542
# the origin is a valid utf-8 line, eventually we could remove it
543
return ['%s %s' % (o, t) for o, t in content._lines]
237
encode_utf8 = cache_utf8.encode
238
return ['%s %s' % (encode_utf8(o), t) for o, t in content._lines]
544
239
545
240
def lower_line_delta(self, delta):
546
241
"""convert a delta into a serializable form.
547
242
548
243
See parse_line_delta which this inverts.
549
244
"""
550
# TODO: jam 20070209 We only do the caching thing to make sure that
551
# the origin is a valid utf-8 line, eventually we could remove it
245
encode_utf8 = cache_utf8.encode
552
246
out = []
553
247
for start, end, c, lines in delta:
554
248
out.append('%d,%d,%d\n' % (start, end, c))
555
out.extend(origin + ' ' + text
249
out.extend(encode_utf8(origin) + ' ' + text
556
250
for origin, text in lines)
557
251
return out
558
252
559
def annotate(self, knit, key):
560
content = knit._get_content(key)
561
# adjust for the fact that serialised annotations are only key suffixes
562
# for this factory.
563
if type(key) == tuple:
564
prefix = key[:-1]
565
origins = content.annotate()
566
result = []
567
for origin, line in origins:
568
result.append((prefix + (origin,), line))
569
return result
570
else:
571
# XXX: This smells a bit. Why would key ever be a non-tuple here?
572
# Aren't keys defined to be tuples? -- spiv 20080618
573
return content.annotate()
574
575
253
576
254
class KnitPlainFactory(_KnitFactory):
577
255
"""Factory for creating plain Content objects."""
578
256
579
257
annotated = False
580
258
581
def make(self, lines, version_id):
582
return PlainKnitContent(lines, version_id)
583
584
def parse_fulltext(self, content, version_id):
259
def parse_fulltext(self, content, version):
585
260
"""This parses an unannotated fulltext.
586
261
587
262
Note that this is not a noop - the internal representation
588
263
has (versionid, line) - its just a constant versionid.
589
264
"""
590
return self.make(content, version_id)
265
return self.make(content, version)
591
266
592
def parse_line_delta_iter(self, lines, version_id):
267
def parse_line_delta_iter(self, lines, version):
593
268
cur = 0
594
269
num_lines = len(lines)
595
270
while cur < num_lines:
596
271
header = lines[cur]
597
272
cur += 1
598
273
start, end, c = [int(n) for n in header.split(',')]
599
yield start, end, c, lines[cur:cur+c]
274
yield start, end, c, zip([version] * c, lines[cur:cur+c])
600
275
cur += c
601
276
602
def parse_line_delta(self, lines, version_id):
603
return list(self.parse_line_delta_iter(lines, version_id))
277
def parse_line_delta(self, lines, version):
278
return list(self.parse_line_delta_iter(lines, version))
604
279
605
280
def get_fulltext_content(self, lines):
606
281
"""Extract just the content lines from a fulltext."""
627
302
out = []
628
303
for start, end, c, lines in delta:
629
304
out.append('%d,%d,%d\n' % (start, end, c))
630
out.extend(lines)
305
out.extend([text for origin, text in lines])
631
306
return out
632
307
633
def annotate(self, knit, key):
634
annotator = _KnitAnnotator(knit)
635
return annotator.annotate(key)
636
637
638
639
def make_file_factory(annotated, mapper):
640
"""Create a factory for creating a file based KnitVersionedFiles.
641
642
This is only functional enough to run interface tests, it doesn't try to
643
provide a full pack environment.
644
645
:param annotated: knit annotations are wanted.
646
:param mapper: The mapper from keys to paths.
647
"""
648
def factory(transport):
649
index = _KndxIndex(transport, mapper, lambda:None, lambda:True, lambda:True)
650
access = _KnitKeyAccess(transport, mapper)
651
return KnitVersionedFiles(index, access, annotated=annotated)
652
return factory
653
654
655
def make_pack_factory(graph, delta, keylength):
656
"""Create a factory for creating a pack based VersionedFiles.
657
658
This is only functional enough to run interface tests, it doesn't try to
659
provide a full pack environment.
660
661
:param graph: Store a graph.
662
:param delta: Delta compress contents.
663
:param keylength: How long should keys be.
664
"""
665
def factory(transport):
666
parents = graph or delta
667
ref_length = 0
668
if graph:
669
ref_length += 1
670
if delta:
671
ref_length += 1
672
max_delta_chain = 200
673
else:
674
max_delta_chain = 0
675
graph_index = _mod_index.InMemoryGraphIndex(reference_lists=ref_length,
676
key_elements=keylength)
677
stream = transport.open_write_stream('newpack')
678
writer = pack.ContainerWriter(stream.write)
679
writer.begin()
680
index = _KnitGraphIndex(graph_index, lambda:True, parents=parents,
681
deltas=delta, add_callback=graph_index.add_nodes)
682
access = _DirectPackAccess({})
683
access.set_writer(writer, graph_index, (transport, 'newpack'))
684
result = KnitVersionedFiles(index, access,
685
max_delta_chain=max_delta_chain)
686
result.stream = stream
687
result.writer = writer
688
return result
689
return factory
690
691
692
def cleanup_pack_knit(versioned_files):
693
versioned_files.stream.close()
694
versioned_files.writer.end()
695
696
697
class KnitVersionedFiles(VersionedFiles):
698
"""Storage for many versioned files using knit compression.
699
700
Backend storage is managed by indices and data objects.
701
"""
702
703
def __init__(self, index, data_access, max_delta_chain=200,
704
annotated=False):
705
"""Create a KnitVersionedFiles with index and data_access.
706
707
:param index: The index for the knit data.
708
:param data_access: The access object to store and retrieve knit
709
records.
710
:param max_delta_chain: The maximum number of deltas to permit during
711
insertion. Set to 0 to prohibit the use of deltas.
712
:param annotated: Set to True to cause annotations to be calculated and
713
stored during insertion.
714
"""
715
self._index = index
716
self._access = data_access
717
self._max_delta_chain = max_delta_chain
718
if annotated:
719
self._factory = KnitAnnotateFactory()
720
else:
721
self._factory = KnitPlainFactory()
722
self._fallback_vfs = []
723
724
def add_fallback_versioned_files(self, a_versioned_files):
725
"""Add a source of texts for texts not present in this knit.
726
727
:param a_versioned_files: A VersionedFiles object.
728
"""
729
self._fallback_vfs.append(a_versioned_files)
730
731
def add_lines(self, key, parents, lines, parent_texts=None,
732
left_matching_blocks=None, nostore_sha=None, random_id=False,
733
check_content=True):
734
"""See VersionedFiles.add_lines()."""
735
self._index._check_write_ok()
736
self._check_add(key, lines, random_id, check_content)
737
if parents is None:
738
# The caller might pass None if there is no graph data, but kndx
739
# indexes can't directly store that, so we give them
740
# an empty tuple instead.
741
parents = ()
742
return self._add(key, lines, parents,
743
parent_texts, left_matching_blocks, nostore_sha, random_id)
744
745
def _add(self, key, lines, parents, parent_texts,
746
left_matching_blocks, nostore_sha, random_id):
747
"""Add a set of lines on top of version specified by parents.
748
749
Any versions not present will be converted into ghosts.
750
"""
751
# first thing, if the content is something we don't need to store, find
752
# that out.
753
line_bytes = ''.join(lines)
754
digest = sha_string(line_bytes)
755
if nostore_sha == digest:
756
raise errors.ExistingContent
757
758
present_parents = []
759
if parent_texts is None:
760
parent_texts = {}
761
# Do a single query to ascertain parent presence.
762
present_parent_map = self.get_parent_map(parents)
763
for parent in parents:
764
if parent in present_parent_map:
765
present_parents.append(parent)
766
767
# Currently we can only compress against the left most present parent.
768
if (len(present_parents) == 0 or
769
present_parents[0] != parents[0]):
770
delta = False
771
else:
772
# To speed the extract of texts the delta chain is limited
773
# to a fixed number of deltas. This should minimize both
774
# I/O and the time spend applying deltas.
775
delta = self._check_should_delta(present_parents[0])
776
777
text_length = len(line_bytes)
778
options = []
779
if lines:
780
if lines[-1][-1] != '\n':
781
# copy the contents of lines.
782
lines = lines[:]
783
options.append('no-eol')
784
lines[-1] = lines[-1] + '\n'
785
line_bytes += '\n'
786
787
for element in key:
788
if type(element) != str:
789
raise TypeError("key contains non-strings: %r" % (key,))
790
# Knit hunks are still last-element only
791
version_id = key[-1]
792
content = self._factory.make(lines, version_id)
793
if 'no-eol' in options:
794
# Hint to the content object that its text() call should strip the
795
# EOL.
796
content._should_strip_eol = True
797
if delta or (self._factory.annotated and len(present_parents) > 0):
798
# Merge annotations from parent texts if needed.
799
delta_hunks = self._merge_annotations(content, present_parents,
800
parent_texts, delta, self._factory.annotated,
801
left_matching_blocks)
802
803
if delta:
804
options.append('line-delta')
805
store_lines = self._factory.lower_line_delta(delta_hunks)
806
size, bytes = self._record_to_data(key, digest,
807
store_lines)
808
else:
809
options.append('fulltext')
810
# isinstance is slower and we have no hierarchy.
811
if self._factory.__class__ == KnitPlainFactory:
812
# Use the already joined bytes saving iteration time in
813
# _record_to_data.
814
size, bytes = self._record_to_data(key, digest,
815
lines, [line_bytes])
816
else:
817
# get mixed annotation + content and feed it into the
818
# serialiser.
819
store_lines = self._factory.lower_fulltext(content)
820
size, bytes = self._record_to_data(key, digest,
821
store_lines)
822
823
access_memo = self._access.add_raw_records([(key, size)], bytes)[0]
824
self._index.add_records(
825
((key, options, access_memo, parents),),
826
random_id=random_id)
827
return digest, text_length, content
828
829
def annotate(self, key):
830
"""See VersionedFiles.annotate."""
831
return self._factory.annotate(self, key)
832
833
def check(self, progress_bar=None):
834
"""See VersionedFiles.check()."""
835
# This doesn't actually test extraction of everything, but that will
836
# impact 'bzr check' substantially, and needs to be integrated with
837
# care. However, it does check for the obvious problem of a delta with
838
# no basis.
839
keys = self._index.keys()
840
parent_map = self.get_parent_map(keys)
841
for key in keys:
842
if self._index.get_method(key) != 'fulltext':
843
compression_parent = parent_map[key][0]
844
if compression_parent not in parent_map:
845
raise errors.KnitCorrupt(self,
846
"Missing basis parent %s for %s" % (
847
compression_parent, key))
848
for fallback_vfs in self._fallback_vfs:
849
fallback_vfs.check()
850
851
def _check_add(self, key, lines, random_id, check_content):
852
"""check that version_id and lines are safe to add."""
853
version_id = key[-1]
854
if contains_whitespace(version_id):
855
raise InvalidRevisionId(version_id, self)
856
self.check_not_reserved_id(version_id)
857
# TODO: If random_id==False and the key is already present, we should
858
# probably check that the existing content is identical to what is
859
# being inserted, and otherwise raise an exception. This would make
860
# the bundle code simpler.
861
if check_content:
862
self._check_lines_not_unicode(lines)
863
self._check_lines_are_lines(lines)
864
865
def _check_header(self, key, line):
866
rec = self._split_header(line)
867
self._check_header_version(rec, key[-1])
868
return rec
869
870
def _check_header_version(self, rec, version_id):
871
"""Checks the header version on original format knit records.
308
309
def make_empty_knit(transport, relpath):
310
"""Construct a empty knit at the specified location."""
311
k = KnitVersionedFile(transport, relpath, 'w', KnitPlainFactory)
312
k._data._open_file()
313
314
315
class KnitVersionedFile(VersionedFile):
316
"""Weave-like structure with faster random access.
317
318
A knit stores a number of texts and a summary of the relationships
319
between them. Texts are identified by a string version-id. Texts
320
are normally stored and retrieved as a series of lines, but can
321
also be passed as single strings.
322
323
Lines are stored with the trailing newline (if any) included, to
324
avoid special cases for files with no final newline. Lines are
325
composed of 8-bit characters, not unicode. The combination of
326
these approaches should mean any 'binary' file can be safely
327
stored and retrieved.
328
"""
329
330
def __init__(self, relpath, transport, file_mode=None, access_mode=None,
331
factory=None, basis_knit=DEPRECATED_PARAMETER, delta=True,
332
create=False, create_parent_dir=False, delay_create=False,
333
dir_mode=None):
334
"""Construct a knit at location specified by relpath.
872
335
873
These have the last component of the key embedded in the record.
336
:param create: If not True, only open an existing knit.
337
:param create_parent_dir: If True, create the parent directory if
338
creating the file fails. (This is used for stores with
339
hash-prefixes that may not exist yet)
340
:param delay_create: The calling code is aware that the knit won't
341
actually be created until the first data is stored.
874
342
"""
875
if rec[1] != version_id:
876
raise KnitCorrupt(self,
877
'unexpected version, wanted %r, got %r' % (version_id, rec[1]))
878
879
def _check_should_delta(self, parent):
343
if deprecated_passed(basis_knit):
344
warnings.warn("KnitVersionedFile.__(): The basis_knit parameter is"
345
" deprecated as of bzr 0.9.",
346
DeprecationWarning, stacklevel=2)
347
if access_mode is None:
348
access_mode = 'w'
349
super(KnitVersionedFile, self).__init__(access_mode)
350
assert access_mode in ('r', 'w'), "invalid mode specified %r" % access_mode
351
self.transport = transport
352
self.filename = relpath
353
self.factory = factory or KnitAnnotateFactory()
354
self.writable = (access_mode == 'w')
355
self.delta = delta
356
357
self._max_delta_chain = 200
358
359
self._index = _KnitIndex(transport, relpath + INDEX_SUFFIX,
360
access_mode, create=create, file_mode=file_mode,
361
create_parent_dir=create_parent_dir, delay_create=delay_create,
362
dir_mode=dir_mode)
363
self._data = _KnitData(transport, relpath + DATA_SUFFIX,
364
access_mode, create=create and not len(self), file_mode=file_mode,
365
create_parent_dir=create_parent_dir, delay_create=delay_create,
366
dir_mode=dir_mode)
367
368
def __repr__(self):
369
return '%s(%s)' % (self.__class__.__name__,
370
self.transport.abspath(self.filename))
371
372
def _check_should_delta(self, first_parents):
880
373
"""Iterate back through the parent listing, looking for a fulltext.
881
374
882
375
This is used when we want to decide whether to add a delta or a new
889
382
"""
890
383
delta_size = 0
891
384
fulltext_size = None
385
delta_parents = first_parents
892
386
for count in xrange(self._max_delta_chain):
893
# XXX: Collapse these two queries:
894
try:
895
method = self._index.get_method(parent)
896
except RevisionNotPresent:
897
# Some basis is not locally present: always delta
898
return False
899
index, pos, size = self._index.get_position(parent)
387
parent = delta_parents[0]
388
method = self._index.get_method(parent)
389
pos, size = self._index.get_position(parent)
900
390
if method == 'fulltext':
901
391
fulltext_size = size
902
392
break
903
393
delta_size += size
904
# We don't explicitly check for presence because this is in an
905
# inner loop, and if it's missing it'll fail anyhow.
906
# TODO: This should be asking for compression parent, not graph
907
# parent.
908
parent = self._index.get_parent_map([parent])[parent][0]
394
delta_parents = self._index.get_parents(parent)
909
395
else:
910
396
# We couldn't find a fulltext, so we must create a new one
911
397
return False
912
# Simple heuristic - if the total I/O wold be greater as a delta than
913
# the originally installed fulltext, we create a new fulltext.
398
914
399
return fulltext_size > delta_size
915
400
916
def _build_details_to_components(self, build_details):
917
"""Convert a build_details tuple to a position tuple."""
918
# record_details, access_memo, compression_parent
919
return build_details[3], build_details[0], build_details[1]
920
921
def _get_components_positions(self, keys, allow_missing=False):
922
"""Produce a map of position data for the components of keys.
401
def _add_delta(self, version_id, parents, delta_parent, sha1, noeol, delta):
402
"""See VersionedFile._add_delta()."""
403
self._check_add(version_id, []) # should we check the lines ?
404
self._check_versions_present(parents)
405
present_parents = []
406
ghosts = []
407
parent_texts = {}
408
for parent in parents:
409
if not self.has_version(parent):
410
ghosts.append(parent)
411
else:
412
present_parents.append(parent)
413
414
if delta_parent is None:
415
# reconstitute as full text.
416
assert len(delta) == 1 or len(delta) == 0
417
if len(delta):
418
assert delta[0][0] == 0
419
assert delta[0][1] == 0, delta[0][1]
420
return super(KnitVersionedFile, self)._add_delta(version_id,
421
parents,
422
delta_parent,
423
sha1,
424
noeol,
425
delta)
426
427
digest = sha1
428
429
options = []
430
if noeol:
431
options.append('no-eol')
432
433
if delta_parent is not None:
434
# determine the current delta chain length.
435
# To speed the extract of texts the delta chain is limited
436
# to a fixed number of deltas. This should minimize both
437
# I/O and the time spend applying deltas.
438
# The window was changed to a maximum of 200 deltas, but also added
439
# was a check that the total compressed size of the deltas is
440
# smaller than the compressed size of the fulltext.
441
if not self._check_should_delta([delta_parent]):
442
# We don't want a delta here, just do a normal insertion.
443
return super(KnitVersionedFile, self)._add_delta(version_id,
444
parents,
445
delta_parent,
446
sha1,
447
noeol,
448
delta)
449
450
options.append('line-delta')
451
store_lines = self.factory.lower_line_delta(delta)
452
453
where, size = self._data.add_record(version_id, digest, store_lines)
454
self._index.add_version(version_id, options, where, size, parents)
455
456
def _add_raw_records(self, records, data):
457
"""Add all the records 'records' with data pre-joined in 'data'.
458
459
:param records: A list of tuples(version_id, options, parents, size).
460
:param data: The data for the records. When it is written, the records
461
are adjusted to have pos pointing into data by the sum of
462
the preceding records sizes.
463
"""
464
# write all the data
465
pos = self._data.add_raw_record(data)
466
offset = 0
467
index_entries = []
468
for (version_id, options, parents, size) in records:
469
index_entries.append((version_id, options, pos+offset,
470
size, parents))
471
if self._data._do_cache:
472
self._data._cache[version_id] = data[offset:offset+size]
473
offset += size
474
self._index.add_versions(index_entries)
475
476
def enable_cache(self):
477
"""Start caching data for this knit"""
478
self._data.enable_cache()
479
480
def clear_cache(self):
481
"""Clear the data cache only."""
482
self._data.clear_cache()
483
484
def copy_to(self, name, transport):
485
"""See VersionedFile.copy_to()."""
486
# copy the current index to a temp index to avoid racing with local
487
# writes
488
transport.put_file_non_atomic(name + INDEX_SUFFIX + '.tmp',
489
self.transport.get(self._index._filename))
490
# copy the data file
491
f = self._data._open_file()
492
try:
493
transport.put_file(name + DATA_SUFFIX, f)
494
finally:
495
f.close()
496
# move the copied index into place
497
transport.move(name + INDEX_SUFFIX + '.tmp', name + INDEX_SUFFIX)
498
499
def create_empty(self, name, transport, mode=None):
500
return KnitVersionedFile(name, transport, factory=self.factory,
501
delta=self.delta, create=True)
502
503
def _fix_parents(self, version, new_parents):
504
"""Fix the parents list for version.
505
506
This is done by appending a new version to the index
507
with identical data except for the parents list.
508
the parents list must be a superset of the current
509
list.
510
"""
511
current_values = self._index._cache[version]
512
assert set(current_values[4]).difference(set(new_parents)) == set()
513
self._index.add_version(version,
514
current_values[1],
515
current_values[2],
516
current_values[3],
517
new_parents)
518
519
def get_delta(self, version_id):
520
"""Get a delta for constructing version from some other version."""
521
self.check_not_reserved_id(version_id)
522
if not self.has_version(version_id):
523
raise RevisionNotPresent(version_id, self.filename)
524
525
parents = self.get_parents(version_id)
526
if len(parents):
527
parent = parents[0]
528
else:
529
parent = None
530
data_pos, data_size = self._index.get_position(version_id)
531
data, sha1 = self._data.read_records(((version_id, data_pos, data_size),))[version_id]
532
version_idx = self._index.lookup(version_id)
533
noeol = 'no-eol' in self._index.get_options(version_id)
534
if 'fulltext' == self._index.get_method(version_id):
535
new_content = self.factory.parse_fulltext(data, version_idx)
536
if parent is not None:
537
reference_content = self._get_content(parent)
538
old_texts = reference_content.text()
539
else:
540
old_texts = []
541
new_texts = new_content.text()
542
delta_seq = KnitSequenceMatcher(None, old_texts, new_texts)
543
return parent, sha1, noeol, self._make_line_delta(delta_seq, new_content)
544
else:
545
delta = self.factory.parse_line_delta(data, version_idx)
546
return parent, sha1, noeol, delta
547
548
def get_graph_with_ghosts(self):
549
"""See VersionedFile.get_graph_with_ghosts()."""
550
graph_items = self._index.get_graph()
551
return dict(graph_items)
552
553
def get_sha1(self, version_id):
554
"""See VersionedFile.get_sha1()."""
555
record_map = self._get_record_map([version_id])
556
method, content, digest, next = record_map[version_id]
557
return digest
558
559
@staticmethod
560
def get_suffixes():
561
"""See VersionedFile.get_suffixes()."""
562
return [DATA_SUFFIX, INDEX_SUFFIX]
563
564
def has_ghost(self, version_id):
565
"""True if there is a ghost reference in the file to version_id."""
566
# maybe we have it
567
if self.has_version(version_id):
568
return False
569
# optimisable if needed by memoising the _ghosts set.
570
items = self._index.get_graph()
571
for node, parents in items:
572
for parent in parents:
573
if parent not in self._index._cache:
574
if parent == version_id:
575
return True
576
return False
577
578
def versions(self):
579
"""See VersionedFile.versions."""
580
return self._index.get_versions()
581
582
def has_version(self, version_id):
583
"""See VersionedFile.has_version."""
584
return self._index.has_version(version_id)
585
586
__contains__ = has_version
587
588
def _merge_annotations(self, content, parents, parent_texts={},
589
delta=None, annotated=None):
590
"""Merge annotations for content. This is done by comparing
591
the annotations based on changed to the text.
592
"""
593
if annotated:
594
delta_seq = None
595
for parent_id in parents:
596
merge_content = self._get_content(parent_id, parent_texts)
597
seq = patiencediff.PatienceSequenceMatcher(
598
None, merge_content.text(), content.text())
599
if delta_seq is None:
600
# setup a delta seq to reuse.
601
delta_seq = seq
602
for i, j, n in seq.get_matching_blocks():
603
if n == 0:
604
continue
605
# this appears to copy (origin, text) pairs across to the new
606
# content for any line that matches the last-checked parent.
607
# FIXME: save the sequence control data for delta compression
608
# against the most relevant parent rather than rediffing.
609
content._lines[j:j+n] = merge_content._lines[i:i+n]
610
if delta:
611
if not annotated:
612
reference_content = self._get_content(parents[0], parent_texts)
613
new_texts = content.text()
614
old_texts = reference_content.text()
615
delta_seq = patiencediff.PatienceSequenceMatcher(
616
None, old_texts, new_texts)
617
return self._make_line_delta(delta_seq, content)
618
619
def _make_line_delta(self, delta_seq, new_content):
620
"""Generate a line delta from delta_seq and new_content."""
621
diff_hunks = []
622
for op in delta_seq.get_opcodes():
623
if op[0] == 'equal':
624
continue
625
diff_hunks.append((op[1], op[2], op[4]-op[3], new_content._lines[op[3]:op[4]]))
626
return diff_hunks
627
628
def _get_components_positions(self, version_ids):
629
"""Produce a map of position data for the components of versions.
923
630
924
631
This data is intended to be used for retrieving the knit records.
925
632
926
A dict of key to (record_details, index_memo, next, parents) is
633
A dict of version_id to (method, data_pos, data_size, next) is
927
634
returned.
928
635
method is the way referenced data should be applied.
929
index_memo is the handle to pass to the data access to actually get the
930
data
636
data_pos is the position of the data in the knit.
637
data_size is the size of the data in the knit.
931
638
next is the build-parent of the version, or None for fulltexts.
932
parents is the version_ids of the parents of this version
933
934
:param allow_missing: If True do not raise an error on a missing component,
935
just ignore it.
936
639
"""
937
640
component_data = {}
938
pending_components = keys
939
while pending_components:
940
build_details = self._index.get_build_details(pending_components)
941
current_components = set(pending_components)
942
pending_components = set()
943
for key, details in build_details.iteritems():
944
(index_memo, compression_parent, parents,
945
record_details) = details
946
method = record_details[0]
947
if compression_parent is not None:
948
pending_components.add(compression_parent)
949
component_data[key] = self._build_details_to_components(details)
950
missing = current_components.difference(build_details)
951
if missing and not allow_missing:
952
raise errors.RevisionNotPresent(missing.pop(), self)
641
for version_id in version_ids:
642
cursor = version_id
643
644
while cursor is not None and cursor not in component_data:
645
method = self._index.get_method(cursor)
646
if method == 'fulltext':
647
next = None
648
else:
649
next = self.get_parents(cursor)[0]
650
data_pos, data_size = self._index.get_position(cursor)
651
component_data[cursor] = (method, data_pos, data_size, next)
652
cursor = next
953
653
return component_data
954
654
955
def _get_content(self, key, parent_texts={}):
655
def _get_content(self, version_id, parent_texts={}):
956
656
"""Returns a content object that makes up the specified
957
657
version."""
958
cached_version = parent_texts.get(key, None)
658
if not self.has_version(version_id):
659
raise RevisionNotPresent(version_id, self.filename)
660
661
cached_version = parent_texts.get(version_id, None)
959
662
if cached_version is not None:
960
# Ensure the cache dict is valid.
961
if not self.get_parent_map([key]):
962
raise RevisionNotPresent(key, self)
963
663
return cached_version
964
text_map, contents_map = self._get_content_maps([key])
965
return contents_map[key]
966
967
def _get_content_maps(self, keys, nonlocal_keys=None):
664
665
text_map, contents_map = self._get_content_maps([version_id])
666
return contents_map[version_id]
667
668
def _check_versions_present(self, version_ids):
669
"""Check that all specified versions are present."""
670
self._index.check_versions_present(version_ids)
671
672
def _add_lines_with_ghosts(self, version_id, parents, lines, parent_texts):
673
"""See VersionedFile.add_lines_with_ghosts()."""
674
self._check_add(version_id, lines)
675
return self._add(version_id, lines[:], parents, self.delta, parent_texts)
676
677
def _add_lines(self, version_id, parents, lines, parent_texts):
678
"""See VersionedFile.add_lines."""
679
self._check_add(version_id, lines)
680
self._check_versions_present(parents)
681
return self._add(version_id, lines[:], parents, self.delta, parent_texts)
682
683
def _check_add(self, version_id, lines):
684
"""check that version_id and lines are safe to add."""
685
assert self.writable, "knit is not opened for write"
686
### FIXME escape. RBC 20060228
687
if contains_whitespace(version_id):
688
raise InvalidRevisionId(version_id, self.filename)
689
self.check_not_reserved_id(version_id)
690
if self.has_version(version_id):
691
raise RevisionAlreadyPresent(version_id, self.filename)
692
self._check_lines_not_unicode(lines)
693
self._check_lines_are_lines(lines)
694
695
def _add(self, version_id, lines, parents, delta, parent_texts):
696
"""Add a set of lines on top of version specified by parents.
697
698
If delta is true, compress the text as a line-delta against
699
the first parent.
700
701
Any versions not present will be converted into ghosts.
702
"""
703
# 461 0 6546.0390 43.9100 bzrlib.knit:489(_add)
704
# +400 0 889.4890 418.9790 +bzrlib.knit:192(lower_fulltext)
705
# +461 0 1364.8070 108.8030 +bzrlib.knit:996(add_record)
706
# +461 0 193.3940 41.5720 +bzrlib.knit:898(add_version)
707
# +461 0 134.0590 18.3810 +bzrlib.osutils:361(sha_strings)
708
# +461 0 36.3420 15.4540 +bzrlib.knit:146(make)
709
# +1383 0 8.0370 8.0370 +<len>
710
# +61 0 13.5770 7.9190 +bzrlib.knit:199(lower_line_delta)
711
# +61 0 963.3470 7.8740 +bzrlib.knit:427(_get_content)
712
# +61 0 973.9950 5.2950 +bzrlib.knit:136(line_delta)
713
# +61 0 1918.1800 5.2640 +bzrlib.knit:359(_merge_annotations)
714
715
present_parents = []
716
ghosts = []
717
if parent_texts is None:
718
parent_texts = {}
719
for parent in parents:
720
if not self.has_version(parent):
721
ghosts.append(parent)
722
else:
723
present_parents.append(parent)
724
725
if delta and not len(present_parents):
726
delta = False
727
728
digest = sha_strings(lines)
729
options = []
730
if lines:
731
if lines[-1][-1] != '\n':
732
options.append('no-eol')
733
lines[-1] = lines[-1] + '\n'
734
735
if len(present_parents) and delta:
736
# To speed the extract of texts the delta chain is limited
737
# to a fixed number of deltas. This should minimize both
738
# I/O and the time spend applying deltas.
739
delta = self._check_should_delta(present_parents)
740
741
lines = self.factory.make(lines, version_id)
742
if delta or (self.factory.annotated and len(present_parents) > 0):
743
# Merge annotations from parent texts if so is needed.
744
delta_hunks = self._merge_annotations(lines, present_parents, parent_texts,
745
delta, self.factory.annotated)
746
747
if delta:
748
options.append('line-delta')
749
store_lines = self.factory.lower_line_delta(delta_hunks)
750
else:
751
options.append('fulltext')
752
store_lines = self.factory.lower_fulltext(lines)
753
754
where, size = self._data.add_record(version_id, digest, store_lines)
755
self._index.add_version(version_id, options, where, size, parents)
756
return lines
757
758
def check(self, progress_bar=None):
759
"""See VersionedFile.check()."""
760
761
def _clone_text(self, new_version_id, old_version_id, parents):
762
"""See VersionedFile.clone_text()."""
763
# FIXME RBC 20060228 make fast by only inserting an index with null
764
# delta.
765
self.add_lines(new_version_id, parents, self.get_lines(old_version_id))
766
767
def get_lines(self, version_id):
768
"""See VersionedFile.get_lines()."""
769
return self.get_line_list([version_id])[0]
770
771
def _get_record_map(self, version_ids):
772
"""Produce a dictionary of knit records.
773
774
The keys are version_ids, the values are tuples of (method, content,
775
digest, next).
776
method is the way the content should be applied.
777
content is a KnitContent object.
778
digest is the SHA1 digest of this version id after all steps are done
779
next is the build-parent of the version, i.e. the leftmost ancestor.
780
If the method is fulltext, next will be None.
781
"""
782
position_map = self._get_components_positions(version_ids)
783
# c = component_id, m = method, p = position, s = size, n = next
784
records = [(c, p, s) for c, (m, p, s, n) in position_map.iteritems()]
785
record_map = {}
786
for component_id, content, digest in \
787
self._data.read_records_iter(records):
788
method, position, size, next = position_map[component_id]
789
record_map[component_id] = method, content, digest, next
790
791
return record_map
792
793
def get_text(self, version_id):
794
"""See VersionedFile.get_text"""
795
return self.get_texts([version_id])[0]
796
797
def get_texts(self, version_ids):
798
return [''.join(l) for l in self.get_line_list(version_ids)]
799
800
def get_line_list(self, version_ids):
801
"""Return the texts of listed versions as a list of strings."""
802
for version_id in version_ids:
803
self.check_not_reserved_id(version_id)
804
text_map, content_map = self._get_content_maps(version_ids)
805
return [text_map[v] for v in version_ids]
806
807
def _get_content_maps(self, version_ids):
968
808
"""Produce maps of text and KnitContents
969
809
970
:param keys: The keys to produce content maps for.
971
:param nonlocal_keys: An iterable of keys(possibly intersecting keys)
972
which are known to not be in this knit, but rather in one of the
973
fallback knits.
974
810
:return: (text_map, content_map) where text_map contains the texts for
975
the requested versions and content_map contains the KnitContents.
811
the requested versions and content_map contains the KnitContents.
812
Both dicts take version_ids as their keys.
976
813
"""
977
# FUTURE: This function could be improved for the 'extract many' case
978
# by tracking each component and only doing the copy when the number of
979
# children than need to apply delta's to it is > 1 or it is part of the
980
# final output.
981
keys = list(keys)
982
multiple_versions = len(keys) != 1
983
record_map = self._get_record_map(keys, allow_missing=True)
814
for version_id in version_ids:
815
if not self.has_version(version_id):
816
raise RevisionNotPresent(version_id, self.filename)
817
record_map = self._get_record_map(version_ids)
984
818
985
819
text_map = {}
986
820
content_map = {}
987
821
final_content = {}
988
if nonlocal_keys is None:
989
nonlocal_keys = set()
990
else:
991
nonlocal_keys = frozenset(nonlocal_keys)
992
missing_keys = set(nonlocal_keys)
993
for source in self._fallback_vfs:
994
if not missing_keys:
995
break
996
for record in source.get_record_stream(missing_keys,
997
'unordered', True):
998
if record.storage_kind == 'absent':
999
continue
1000
missing_keys.remove(record.key)
1001
lines = split_lines(record.get_bytes_as('fulltext'))
1002
text_map[record.key] = lines
1003
content_map[record.key] = PlainKnitContent(lines, record.key)
1004
if record.key in keys:
1005
final_content[record.key] = content_map[record.key]
1006
for key in keys:
1007
if key in nonlocal_keys:
1008
# already handled
1009
continue
822
for version_id in version_ids:
1010
823
components = []
1011
cursor = key
824
cursor = version_id
1012
825
while cursor is not None:
1013
try:
1014
record, record_details, digest, next = record_map[cursor]
1015
except KeyError:
1016
raise RevisionNotPresent(cursor, self)
1017
components.append((cursor, record, record_details, digest))
1018
cursor = next
826
method, data, digest, next = record_map[cursor]
827
components.append((cursor, method, data, digest))
1019
828
if cursor in content_map:
1020
# no need to plan further back
1021
components.append((cursor, None, None, None))
1022
829
break
830
cursor = next
1023
831
1024
832
content = None
1025
for (component_id, record, record_details,
1026
digest) in reversed(components):
833
for component_id, method, data, digest in reversed(components):
1027
834
if component_id in content_map:
1028
835
content = content_map[component_id]
1029
836
else:
1030
content, delta = self._factory.parse_record(key[-1],
1031
record, record_details, content,
1032
copy_base_content=multiple_versions)
1033
if multiple_versions:
1034
content_map[component_id] = content
837
version_idx = self._index.lookup(component_id)
838
if method == 'fulltext':
839
assert content is None
840
content = self.factory.parse_fulltext(data, version_idx)
841
elif method == 'line-delta':
842
delta = self.factory.parse_line_delta(data, version_idx)
843
content = content.copy()
844
content._lines = self._apply_delta(content._lines,
845
delta)
846
content_map[component_id] = content
1035
847
1036
final_content[key] = content
848
if 'no-eol' in self._index.get_options(version_id):
849
content = content.copy()
850
line = content._lines[-1][1].rstrip('\n')
851
content._lines[-1] = (content._lines[-1][0], line)
852
final_content[version_id] = content
1037
853
1038
854
# digest here is the digest from the last applied component.
1039
855
text = content.text()
1040
actual_sha = sha_strings(text)
1041
if actual_sha != digest:
1042
raise KnitCorrupt(self,
1043
'\n sha-1 %s'
1044
'\n of reconstructed text does not match'
1045
'\n expected %s'
1046
'\n for version %s' %
1047
(actual_sha, digest, key))
1048
text_map[key] = text
1049
return text_map, final_content
1050
1051
def get_parent_map(self, keys):
1052
"""Get a map of the graph parents of keys.
1053
1054
:param keys: The keys to look up parents for.
1055
:return: A mapping from keys to parents. Absent keys are absent from
1056
the mapping.
1057
"""
1058
return self._get_parent_map_with_sources(keys)[0]
1059
1060
def _get_parent_map_with_sources(self, keys):
1061
"""Get a map of the parents of keys.
1062
1063
:param keys: The keys to look up parents for.
1064
:return: A tuple. The first element is a mapping from keys to parents.
1065
Absent keys are absent from the mapping. The second element is a
1066
list with the locations each key was found in. The first element
1067
is the in-this-knit parents, the second the first fallback source,
1068
and so on.
1069
"""
1070
result = {}
1071
sources = [self._index] + self._fallback_vfs
1072
source_results = []
1073
missing = set(keys)
1074
for source in sources:
1075
if not missing:
1076
break
1077
new_result = source.get_parent_map(missing)
1078
source_results.append(new_result)
1079
result.update(new_result)
1080
missing.difference_update(set(new_result))
1081
return result, source_results
1082
1083
def _get_record_map(self, keys, allow_missing=False):
1084
"""Produce a dictionary of knit records.
1085
1086
:return: {key:(record, record_details, digest, next)}
1087
record
1088
data returned from read_records
1089
record_details
1090
opaque information to pass to parse_record
1091
digest
1092
SHA1 digest of the full text after all steps are done
1093
next
1094
build-parent of the version, i.e. the leftmost ancestor.
1095
Will be None if the record is not a delta.
1096
:param keys: The keys to build a map for
1097
:param allow_missing: If some records are missing, rather than
1098
error, just return the data that could be generated.
1099
"""
1100
position_map = self._get_components_positions(keys,
1101
allow_missing=allow_missing)
1102
# key = component_id, r = record_details, i_m = index_memo, n = next
1103
records = [(key, i_m) for key, (r, i_m, n)
1104
in position_map.iteritems()]
1105
record_map = {}
1106
for key, record, digest in \
1107
self._read_records_iter(records):
1108
(record_details, index_memo, next) = position_map[key]
1109
record_map[key] = record, record_details, digest, next
1110
return record_map
1111
1112
def get_record_stream(self, keys, ordering, include_delta_closure):
1113
"""Get a stream of records for keys.
1114
1115
:param keys: The keys to include.
1116
:param ordering: Either 'unordered' or 'topological'. A topologically
1117
sorted stream has compression parents strictly before their
1118
children.
1119
:param include_delta_closure: If True then the closure across any
1120
compression parents will be included (in the opaque data).
1121
:return: An iterator of ContentFactory objects, each of which is only
1122
valid until the iterator is advanced.
1123
"""
1124
# keys might be a generator
1125
keys = set(keys)
1126
if not keys:
1127
return
1128
if not self._index.has_graph:
1129
# Cannot topological order when no graph has been stored.
1130
ordering = 'unordered'
1131
if include_delta_closure:
1132
positions = self._get_components_positions(keys, allow_missing=True)
1133
else:
1134
build_details = self._index.get_build_details(keys)
1135
# map from key to
1136
# (record_details, access_memo, compression_parent_key)
1137
positions = dict((key, self._build_details_to_components(details))
1138
for key, details in build_details.iteritems())
1139
absent_keys = keys.difference(set(positions))
1140
# There may be more absent keys : if we're missing the basis component
1141
# and are trying to include the delta closure.
1142
if include_delta_closure:
1143
needed_from_fallback = set()
1144
# Build up reconstructable_keys dict. key:True in this dict means
1145
# the key can be reconstructed.
1146
reconstructable_keys = {}
1147
for key in keys:
1148
# the delta chain
1149
try:
1150
chain = [key, positions[key][2]]
1151
except KeyError:
1152
needed_from_fallback.add(key)
1153
continue
1154
result = True
1155
while chain[-1] is not None:
1156
if chain[-1] in reconstructable_keys:
1157
result = reconstructable_keys[chain[-1]]
1158
break
1159
else:
1160
try:
1161
chain.append(positions[chain[-1]][2])
1162
except KeyError:
1163
# missing basis component
1164
needed_from_fallback.add(chain[-1])
1165
result = True
1166
break
1167
for chain_key in chain[:-1]:
1168
reconstructable_keys[chain_key] = result
1169
if not result:
1170
needed_from_fallback.add(key)
1171
# Double index lookups here : need a unified api ?
1172
global_map, parent_maps = self._get_parent_map_with_sources(keys)
1173
if ordering == 'topological':
1174
# Global topological sort
1175
present_keys = tsort.topo_sort(global_map)
1176
# Now group by source:
1177
source_keys = []
1178
current_source = None
1179
for key in present_keys:
1180
for parent_map in parent_maps:
1181
if key in parent_map:
1182
key_source = parent_map
1183
break
1184
if current_source is not key_source:
1185
source_keys.append((key_source, []))
1186
current_source = key_source
1187
source_keys[-1][1].append(key)
1188
else:
1189
# Just group by source; remote sources first.
1190
present_keys = []
1191
source_keys = []
1192
for parent_map in reversed(parent_maps):
1193
source_keys.append((parent_map, []))
1194
for key in parent_map:
1195
present_keys.append(key)
1196
source_keys[-1][1].append(key)
1197
absent_keys = keys - set(global_map)
1198
for key in absent_keys:
1199
yield AbsentContentFactory(key)
1200
# restrict our view to the keys we can answer.
1201
# XXX: Memory: TODO: batch data here to cap buffered data at (say) 1MB.
1202
# XXX: At that point we need to consider the impact of double reads by
1203
# utilising components multiple times.
1204
if include_delta_closure:
1205
# XXX: get_content_maps performs its own index queries; allow state
1206
# to be passed in.
1207
text_map, _ = self._get_content_maps(present_keys,
1208
needed_from_fallback - absent_keys)
1209
for key in present_keys:
1210
yield FulltextContentFactory(key, global_map[key], None,
1211
''.join(text_map[key]))
1212
else:
1213
for source, keys in source_keys:
1214
if source is parent_maps[0]:
1215
# this KnitVersionedFiles
1216
records = [(key, positions[key][1]) for key in keys]
1217
for key, raw_data, sha1 in self._read_records_iter_raw(records):
1218
(record_details, index_memo, _) = positions[key]
1219
yield KnitContentFactory(key, global_map[key],
1220
record_details, sha1, raw_data, self._factory.annotated, None)
1221
else:
1222
vf = self._fallback_vfs[parent_maps.index(source) - 1]
1223
for record in vf.get_record_stream(keys, ordering,
1224
include_delta_closure):
1225
yield record
1226
1227
def get_sha1s(self, keys):
1228
"""See VersionedFiles.get_sha1s()."""
1229
missing = set(keys)
1230
record_map = self._get_record_map(missing, allow_missing=True)
1231
result = {}
1232
for key, details in record_map.iteritems():
1233
if key not in missing:
1234
continue
1235
# record entry 2 is the 'digest'.
1236
result[key] = details[2]
1237
missing.difference_update(set(result))
1238
for source in self._fallback_vfs:
1239
if not missing:
1240
break
1241
new_result = source.get_sha1s(missing)
1242
result.update(new_result)
1243
missing.difference_update(set(new_result))
1244
return result
1245
1246
def insert_record_stream(self, stream):
1247
"""Insert a record stream into this container.
1248
1249
:param stream: A stream of records to insert.
1250
:return: None
1251
:seealso VersionedFiles.get_record_stream:
1252
"""
1253
def get_adapter(adapter_key):
1254
try:
1255
return adapters[adapter_key]
1256
except KeyError:
1257
adapter_factory = adapter_registry.get(adapter_key)
1258
adapter = adapter_factory(self)
1259
adapters[adapter_key] = adapter
1260
return adapter
1261
if self._factory.annotated:
1262
# self is annotated, we need annotated knits to use directly.
1263
annotated = "annotated-"
1264
convertibles = []
1265
else:
1266
# self is not annotated, but we can strip annotations cheaply.
1267
annotated = ""
1268
convertibles = set(["knit-annotated-ft-gz"])
1269
if self._max_delta_chain:
1270
convertibles.add("knit-annotated-delta-gz")
1271
# The set of types we can cheaply adapt without needing basis texts.
1272
native_types = set()
1273
if self._max_delta_chain:
1274
native_types.add("knit-%sdelta-gz" % annotated)
1275
native_types.add("knit-%sft-gz" % annotated)
1276
knit_types = native_types.union(convertibles)
1277
adapters = {}
1278
# Buffer all index entries that we can't add immediately because their
1279
# basis parent is missing. We don't buffer all because generating
1280
# annotations may require access to some of the new records. However we
1281
# can't generate annotations from new deltas until their basis parent
1282
# is present anyway, so we get away with not needing an index that
1283
# includes the new keys.
1284
# key = basis_parent, value = index entry to add
1285
buffered_index_entries = {}
1286
for record in stream:
1287
parents = record.parents
1288
# Raise an error when a record is missing.
1289
if record.storage_kind == 'absent':
1290
raise RevisionNotPresent([record.key], self)
1291
if record.storage_kind in knit_types:
1292
if record.storage_kind not in native_types:
1293
try:
1294
adapter_key = (record.storage_kind, "knit-delta-gz")
1295
adapter = get_adapter(adapter_key)
1296
except KeyError:
1297
adapter_key = (record.storage_kind, "knit-ft-gz")
1298
adapter = get_adapter(adapter_key)
1299
bytes = adapter.get_bytes(
1300
record, record.get_bytes_as(record.storage_kind))
1301
else:
1302
bytes = record.get_bytes_as(record.storage_kind)
1303
options = [record._build_details[0]]
1304
if record._build_details[1]:
1305
options.append('no-eol')
1306
# Just blat it across.
1307
# Note: This does end up adding data on duplicate keys. As
1308
# modern repositories use atomic insertions this should not
1309
# lead to excessive growth in the event of interrupted fetches.
1310
# 'knit' repositories may suffer excessive growth, but as a
1311
# deprecated format this is tolerable. It can be fixed if
1312
# needed by in the kndx index support raising on a duplicate
1313
# add with identical parents and options.
1314
access_memo = self._access.add_raw_records(
1315
[(record.key, len(bytes))], bytes)[0]
1316
index_entry = (record.key, options, access_memo, parents)
1317
buffered = False
1318
if 'fulltext' not in options:
1319
basis_parent = parents[0]
1320
# Note that pack backed knits don't need to buffer here
1321
# because they buffer all writes to the transaction level,
1322
# but we don't expose that difference at the index level. If
1323
# the query here has sufficient cost to show up in
1324
# profiling we should do that.
1325
if basis_parent not in self.get_parent_map([basis_parent]):
1326
pending = buffered_index_entries.setdefault(
1327
basis_parent, [])
1328
pending.append(index_entry)
1329
buffered = True
1330
if not buffered:
1331
self._index.add_records([index_entry])
1332
elif record.storage_kind == 'fulltext':
1333
self.add_lines(record.key, parents,
1334
split_lines(record.get_bytes_as('fulltext')))
1335
else:
1336
adapter_key = record.storage_kind, 'fulltext'
1337
adapter = get_adapter(adapter_key)
1338
lines = split_lines(adapter.get_bytes(
1339
record, record.get_bytes_as(record.storage_kind)))
1340
try:
1341
self.add_lines(record.key, parents, lines)
1342
except errors.RevisionAlreadyPresent:
1343
pass
1344
# Add any records whose basis parent is now available.
1345
added_keys = [record.key]
1346
while added_keys:
1347
key = added_keys.pop(0)
1348
if key in buffered_index_entries:
1349
index_entries = buffered_index_entries[key]
1350
self._index.add_records(index_entries)
1351
added_keys.extend(
1352
[index_entry[0] for index_entry in index_entries])
1353
del buffered_index_entries[key]
1354
# If there were any deltas which had a missing basis parent, error.
1355
if buffered_index_entries:
1356
raise errors.RevisionNotPresent(buffered_index_entries.keys()[0],
1357
self)
1358
1359
def iter_lines_added_or_present_in_keys(self, keys, pb=None):
1360
"""Iterate over the lines in the versioned files from keys.
1361
1362
This may return lines from other keys. Each item the returned
1363
iterator yields is a tuple of a line and a text version that that line
1364
is present in (not introduced in).
1365
1366
Ordering of results is in whatever order is most suitable for the
1367
underlying storage format.
1368
1369
If a progress bar is supplied, it may be used to indicate progress.
1370
The caller is responsible for cleaning up progress bars (because this
1371
is an iterator).
1372
1373
NOTES:
1374
* Lines are normalised by the underlying store: they will all have \n
1375
terminators.
1376
* Lines are returned in arbitrary order.
1377
1378
:return: An iterator over (line, key).
1379
"""
856
if sha_strings(text) != digest:
857
raise KnitCorrupt(self.filename,
858
'sha-1 does not match %s' % version_id)
859
860
text_map[version_id] = text
861
return text_map, final_content
862
863
def iter_lines_added_or_present_in_versions(self, version_ids=None,
864
pb=None):
865
"""See VersionedFile.iter_lines_added_or_present_in_versions()."""
866
if version_ids is None:
867
version_ids = self.versions()
1380
868
if pb is None:
1381
869
pb = progress.DummyProgress()
1382
keys = set(keys)
1383
total = len(keys)
1384
870
# we don't care about inclusions, the caller cares.
1385
871
# but we need to setup a list of records to visit.
1386
# we need key, position, length
1387
key_records = []
1388
build_details = self._index.get_build_details(keys)
1389
for key, details in build_details.iteritems():
1390
if key in keys:
1391
key_records.append((key, details[0]))
1392
keys.remove(key)
1393
records_iter = enumerate(self._read_records_iter(key_records))
1394
for (key_idx, (key, data, sha_value)) in records_iter:
1395
pb.update('Walking content.', key_idx, total)
1396
compression_parent = build_details[key][1]
1397
if compression_parent is None:
1398
# fulltext
1399
line_iterator = self._factory.get_fulltext_content(data)
872
# we need version_id, position, length
873
version_id_records = []
874
requested_versions = set(version_ids)
875
# filter for available versions
876
for version_id in requested_versions:
877
if not self.has_version(version_id):
878
raise RevisionNotPresent(version_id, self.filename)
879
# get a in-component-order queue:
880
for version_id in self.versions():
881
if version_id in requested_versions:
882
data_pos, length = self._index.get_position(version_id)
883
version_id_records.append((version_id, data_pos, length))
884
885
total = len(version_id_records)
886
for version_idx, (version_id, data, sha_value) in \
887
enumerate(self._data.read_records_iter(version_id_records)):
888
pb.update('Walking content.', version_idx, total)
889
method = self._index.get_method(version_id)
890
version_idx = self._index.lookup(version_id)
891
892
assert method in ('fulltext', 'line-delta')
893
if method == 'fulltext':
894
line_iterator = self.factory.get_fulltext_content(data)
1400
895
else:
1401
# Delta
1402
line_iterator = self._factory.get_linedelta_content(data)
1403
# XXX: It might be more efficient to yield (key,
1404
# line_iterator) in the future. However for now, this is a simpler
1405
# change to integrate into the rest of the codebase. RBC 20071110
896
line_iterator = self.factory.get_linedelta_content(data)
1406
897
for line in line_iterator:
1407
yield line, key
1408
for source in self._fallback_vfs:
1409
if not keys:
1410
break
1411
source_keys = set()
1412
for line, key in source.iter_lines_added_or_present_in_keys(keys):
1413
source_keys.add(key)
1414
yield line, key
1415
keys.difference_update(source_keys)
1416
if keys:
1417
raise RevisionNotPresent(keys, self.filename)
898
yield line
899
1418
900
pb.update('Walking content.', total, total)
1419
1420
def _make_line_delta(self, delta_seq, new_content):
1421
"""Generate a line delta from delta_seq and new_content."""
1422
diff_hunks = []
1423
for op in delta_seq.get_opcodes():
1424
if op[0] == 'equal':
1425
continue
1426
diff_hunks.append((op[1], op[2], op[4]-op[3], new_content._lines[op[3]:op[4]]))
1427
return diff_hunks
1428
1429
def _merge_annotations(self, content, parents, parent_texts={},
1430
delta=None, annotated=None,
1431
left_matching_blocks=None):
1432
"""Merge annotations for content and generate deltas.
1433
1434
This is done by comparing the annotations based on changes to the text
1435
and generating a delta on the resulting full texts. If annotations are
1436
not being created then a simple delta is created.
1437
"""
1438
if left_matching_blocks is not None:
1439
delta_seq = diff._PrematchedMatcher(left_matching_blocks)
1440
else:
1441
delta_seq = None
1442
if annotated:
1443
for parent_key in parents:
1444
merge_content = self._get_content(parent_key, parent_texts)
1445
if (parent_key == parents[0] and delta_seq is not None):
1446
seq = delta_seq
1447
else:
1448
seq = patiencediff.PatienceSequenceMatcher(
1449
None, merge_content.text(), content.text())
1450
for i, j, n in seq.get_matching_blocks():
1451
if n == 0:
1452
continue
1453
# this copies (origin, text) pairs across to the new
1454
# content for any line that matches the last-checked
1455
# parent.
1456
content._lines[j:j+n] = merge_content._lines[i:i+n]
1457
# XXX: Robert says the following block is a workaround for a
1458
# now-fixed bug and it can probably be deleted. -- mbp 20080618
1459
if content._lines and content._lines[-1][1][-1] != '\n':
1460
# The copied annotation was from a line without a trailing EOL,
1461
# reinstate one for the content object, to ensure correct
1462
# serialization.
1463
line = content._lines[-1][1] + '\n'
1464
content._lines[-1] = (content._lines[-1][0], line)
1465
if delta:
1466
if delta_seq is None:
1467
reference_content = self._get_content(parents[0], parent_texts)
1468
new_texts = content.text()
1469
old_texts = reference_content.text()
1470
delta_seq = patiencediff.PatienceSequenceMatcher(
1471
None, old_texts, new_texts)
1472
return self._make_line_delta(delta_seq, content)
1473
1474
def _parse_record(self, version_id, data):
1475
"""Parse an original format knit record.
1476
1477
These have the last element of the key only present in the stored data.
1478
"""
1479
rec, record_contents = self._parse_record_unchecked(data)
1480
self._check_header_version(rec, version_id)
1481
return record_contents, rec[3]
1482
1483
def _parse_record_header(self, key, raw_data):
1484
"""Parse a record header for consistency.
1485
1486
:return: the header and the decompressor stream.
1487
as (stream, header_record)
1488
"""
1489
df = tuned_gzip.GzipFile(mode='rb', fileobj=StringIO(raw_data))
1490
try:
1491
# Current serialise
1492
rec = self._check_header(key, df.readline())
1493
except Exception, e:
1494
raise KnitCorrupt(self,
1495
"While reading {%s} got %s(%s)"
1496
% (key, e.__class__.__name__, str(e)))
1497
return df, rec
1498
1499
def _parse_record_unchecked(self, data):
1500
# profiling notes:
1501
# 4168 calls in 2880 217 internal
1502
# 4168 calls to _parse_record_header in 2121
1503
# 4168 calls to readlines in 330
1504
df = tuned_gzip.GzipFile(mode='rb', fileobj=StringIO(data))
1505
try:
1506
record_contents = df.readlines()
1507
except Exception, e:
1508
raise KnitCorrupt(self, "Corrupt compressed record %r, got %s(%s)" %
1509
(data, e.__class__.__name__, str(e)))
1510
header = record_contents.pop(0)
1511
rec = self._split_header(header)
1512
last_line = record_contents.pop()
1513
if len(record_contents) != int(rec[2]):
1514
raise KnitCorrupt(self,
1515
'incorrect number of lines %s != %s'
1516
' for version {%s} %s'
1517
% (len(record_contents), int(rec[2]),
1518
rec[1], record_contents))
1519
if last_line != 'end %s\n' % rec[1]:
1520
raise KnitCorrupt(self,
1521
'unexpected version end line %r, wanted %r'
1522
% (last_line, rec[1]))
1523
df.close()
1524
return rec, record_contents
1525
1526
def _read_records_iter(self, records):
1527
"""Read text records from data file and yield result.
1528
1529
The result will be returned in whatever is the fastest to read.
1530
Not by the order requested. Also, multiple requests for the same
1531
record will only yield 1 response.
1532
:param records: A list of (key, access_memo) entries
1533
:return: Yields (key, contents, digest) in the order
1534
read, not the order requested
1535
"""
1536
if not records:
1537
return
1538
1539
# XXX: This smells wrong, IO may not be getting ordered right.
1540
needed_records = sorted(set(records), key=operator.itemgetter(1))
1541
if not needed_records:
1542
return
1543
1544
# The transport optimizes the fetching as well
1545
# (ie, reads continuous ranges.)
1546
raw_data = self._access.get_raw_records(
1547
[index_memo for key, index_memo in needed_records])
1548
1549
for (key, index_memo), data in \
1550
izip(iter(needed_records), raw_data):
1551
content, digest = self._parse_record(key[-1], data)
1552
yield key, content, digest
1553
1554
def _read_records_iter_raw(self, records):
1555
"""Read text records from data file and yield raw data.
1556
1557
This unpacks enough of the text record to validate the id is
1558
as expected but thats all.
1559
1560
Each item the iterator yields is (key, bytes, sha1_of_full_text).
1561
"""
1562
# setup an iterator of the external records:
1563
# uses readv so nice and fast we hope.
1564
if len(records):
1565
# grab the disk data needed.
1566
needed_offsets = [index_memo for key, index_memo
1567
in records]
1568
raw_records = self._access.get_raw_records(needed_offsets)
1569
1570
for key, index_memo in records:
1571
data = raw_records.next()
1572
# validate the header (note that we can only use the suffix in
1573
# current knit records).
1574
df, rec = self._parse_record_header(key, data)
1575
df.close()
1576
yield key, data, rec[3]
1577
1578
def _record_to_data(self, key, digest, lines, dense_lines=None):
1579
"""Convert key, digest, lines into a raw data block.
1580
1581
:param key: The key of the record. Currently keys are always serialised
1582
using just the trailing component.
1583
:param dense_lines: The bytes of lines but in a denser form. For
1584
instance, if lines is a list of 1000 bytestrings each ending in \n,
1585
dense_lines may be a list with one line in it, containing all the
1586
1000's lines and their \n's. Using dense_lines if it is already
1587
known is a win because the string join to create bytes in this
1588
function spends less time resizing the final string.
1589
:return: (len, a StringIO instance with the raw data ready to read.)
1590
"""
1591
# Note: using a string copy here increases memory pressure with e.g.
1592
# ISO's, but it is about 3 seconds faster on a 1.2Ghz intel machine
1593
# when doing the initial commit of a mozilla tree. RBC 20070921
1594
bytes = ''.join(chain(
1595
["version %s %d %s\n" % (key[-1],
1596
len(lines),
1597
digest)],
1598
dense_lines or lines,
1599
["end %s\n" % key[-1]]))
1600
if type(bytes) != str:
1601
raise AssertionError(
1602
'data must be plain bytes was %s' % type(bytes))
1603
if lines and lines[-1][-1] != '\n':
1604
raise ValueError('corrupt lines value %r' % lines)
1605
compressed_bytes = tuned_gzip.bytes_to_gzip(bytes)
1606
return len(compressed_bytes), compressed_bytes
1607
1608
def _split_header(self, line):
1609
rec = line.split()
1610
if len(rec) != 4:
1611
raise KnitCorrupt(self,
1612
'unexpected number of elements in record header')
1613
return rec
1614
1615
def keys(self):
1616
"""See VersionedFiles.keys."""
1617
if 'evil' in debug.debug_flags:
1618
trace.mutter_callsite(2, "keys scales with size of history")
1619
sources = [self._index] + self._fallback_vfs
1620
result = set()
1621
for source in sources:
1622
result.update(source.keys())
1623
return result
1624
1625
1626
1627
class _KndxIndex(object):
1628
"""Manages knit index files
1629
1630
The index is kept in memory and read on startup, to enable
901
902
def num_versions(self):
903
"""See VersionedFile.num_versions()."""
904
return self._index.num_versions()
905
906
__len__ = num_versions
907
908
def annotate_iter(self, version_id):
909
"""See VersionedFile.annotate_iter."""
910
content = self._get_content(version_id)
911
for origin, text in content.annotate_iter():
912
yield origin, text
913
914
def get_parents(self, version_id):
915
"""See VersionedFile.get_parents."""
916
# perf notes:
917
# optimism counts!
918
# 52554 calls in 1264 872 internal down from 3674
919
try:
920
return self._index.get_parents(version_id)
921
except KeyError:
922
raise RevisionNotPresent(version_id, self.filename)
923
924
def get_parents_with_ghosts(self, version_id):
925
"""See VersionedFile.get_parents."""
926
try:
927
return self._index.get_parents_with_ghosts(version_id)
928
except KeyError:
929
raise RevisionNotPresent(version_id, self.filename)
930
931
def get_ancestry(self, versions):
932
"""See VersionedFile.get_ancestry."""
933
if isinstance(versions, basestring):
934
versions = [versions]
935
if not versions:
936
return []
937
return self._index.get_ancestry(versions)
938
939
def get_ancestry_with_ghosts(self, versions):
940
"""See VersionedFile.get_ancestry_with_ghosts."""
941
if isinstance(versions, basestring):
942
versions = [versions]
943
if not versions:
944
return []
945
return self._index.get_ancestry_with_ghosts(versions)
946
947
#@deprecated_method(zero_eight)
948
def walk(self, version_ids):
949
"""See VersionedFile.walk."""
950
# We take the short path here, and extract all relevant texts
951
# and put them in a weave and let that do all the work. Far
952
# from optimal, but is much simpler.
953
# FIXME RB 20060228 this really is inefficient!
954
from bzrlib.weave import Weave
955
956
w = Weave(self.filename)
957
ancestry = self.get_ancestry(version_ids)
958
sorted_graph = topo_sort(self._index.get_graph())
959
version_list = [vid for vid in sorted_graph if vid in ancestry]
960
961
for version_id in version_list:
962
lines = self.get_lines(version_id)
963
w.add_lines(version_id, self.get_parents(version_id), lines)
964
965
for lineno, insert_id, dset, line in w.walk(version_ids):
966
yield lineno, insert_id, dset, line
967
968
def plan_merge(self, ver_a, ver_b):
969
"""See VersionedFile.plan_merge."""
970
ancestors_b = set(self.get_ancestry(ver_b))
971
def status_a(revision, text):
972
if revision in ancestors_b:
973
return 'killed-b', text
974
else:
975
return 'new-a', text
976
977
ancestors_a = set(self.get_ancestry(ver_a))
978
def status_b(revision, text):
979
if revision in ancestors_a:
980
return 'killed-a', text
981
else:
982
return 'new-b', text
983
984
annotated_a = self.annotate(ver_a)
985
annotated_b = self.annotate(ver_b)
986
plain_a = [t for (a, t) in annotated_a]
987
plain_b = [t for (a, t) in annotated_b]
988
blocks = KnitSequenceMatcher(None, plain_a, plain_b).get_matching_blocks()
989
a_cur = 0
990
b_cur = 0
991
for ai, bi, l in blocks:
992
# process all mismatched sections
993
# (last mismatched section is handled because blocks always
994
# includes a 0-length last block)
995
for revision, text in annotated_a[a_cur:ai]:
996
yield status_a(revision, text)
997
for revision, text in annotated_b[b_cur:bi]:
998
yield status_b(revision, text)
999
1000
# and now the matched section
1001
a_cur = ai + l
1002
b_cur = bi + l
1003
for text_a, text_b in zip(plain_a[ai:a_cur], plain_b[bi:b_cur]):
1004
assert text_a == text_b
1005
yield "unchanged", text_a
1006
1007
1008
class _KnitComponentFile(object):
1009
"""One of the files used to implement a knit database"""
1010
1011
def __init__(self, transport, filename, mode, file_mode=None,
1012
create_parent_dir=False, dir_mode=None):
1013
self._transport = transport
1014
self._filename = filename
1015
self._mode = mode
1016
self._file_mode = file_mode
1017
self._dir_mode = dir_mode
1018
self._create_parent_dir = create_parent_dir
1019
self._need_to_create = False
1020
1021
def _full_path(self):
1022
"""Return the full path to this file."""
1023
return self._transport.base + self._filename
1024
1025
def check_header(self, fp):
1026
line = fp.readline()
1027
if line == '':
1028
# An empty file can actually be treated as though the file doesn't
1029
# exist yet.
1030
raise errors.NoSuchFile(self._full_path())
1031
if line != self.HEADER:
1032
raise KnitHeaderError(badline=line,
1033
filename=self._transport.abspath(self._filename))
1034
1035
def commit(self):
1036
"""Commit is a nop."""
1037
1038
def __repr__(self):
1039
return '%s(%s)' % (self.__class__.__name__, self._filename)
1040
1041
1042
class _KnitIndex(_KnitComponentFile):
1043
"""Manages knit index file.
1044
1045
The index is already kept in memory and read on startup, to enable
1631
1046
fast lookups of revision information. The cursor of the index
1632
1047
file is always pointing to the end, making it easy to append
1633
1048
entries.
1675
1090
to ensure that records always start on new lines even if the last write was
1676
1091
interrupted. As a result its normal for the last line in the index to be
1677
1092
missing a trailing newline. One can be added with no harmful effects.
1678
1679
:ivar _kndx_cache: dict from prefix to the old state of KnitIndex objects,
1680
where prefix is e.g. the (fileid,) for .texts instances or () for
1681
constant-mapped things like .revisions, and the old state is
1682
tuple(history_vector, cache_dict). This is used to prevent having an
1683
ABI change with the C extension that reads .kndx files.
1684
1093
"""
1685
1094
1686
1095
HEADER = "# bzr knit index 8\n"
1687
1096
1688
def __init__(self, transport, mapper, get_scope, allow_writes, is_locked):
1689
"""Create a _KndxIndex on transport using mapper."""
1690
self._transport = transport
1691
self._mapper = mapper
1692
self._get_scope = get_scope
1693
self._allow_writes = allow_writes
1694
self._is_locked = is_locked
1695
self._reset_cache()
1696
self.has_graph = True
1697
1698
def add_records(self, records, random_id=False):
1699
"""Add multiple records to the index.
1700
1701
:param records: a list of tuples:
1702
(key, options, access_memo, parents).
1703
:param random_id: If True the ids being added were randomly generated
1704
and no check for existence will be performed.
1705
"""
1706
paths = {}
1707
for record in records:
1708
key = record[0]
1709
prefix = key[:-1]
1710
path = self._mapper.map(key) + '.kndx'
1711
path_keys = paths.setdefault(path, (prefix, []))
1712
path_keys[1].append(record)
1713
for path in sorted(paths):
1714
prefix, path_keys = paths[path]
1715
self._load_prefixes([prefix])
1716
lines = []
1717
orig_history = self._kndx_cache[prefix][1][:]
1718
orig_cache = self._kndx_cache[prefix][0].copy()
1719
1720
try:
1721
for key, options, (_, pos, size), parents in path_keys:
1722
if parents is None:
1723
# kndx indices cannot be parentless.
1724
parents = ()
1725
line = "\n%s %s %s %s %s :" % (
1726
key[-1], ','.join(options), pos, size,
1727
self._dictionary_compress(parents))
1728
if type(line) != str:
1729
raise AssertionError(
1730
'data must be utf8 was %s' % type(line))
1731
lines.append(line)
1732
self._cache_key(key, options, pos, size, parents)
1733
if len(orig_history):
1734
self._transport.append_bytes(path, ''.join(lines))
1735
else:
1736
self._init_index(path, lines)
1737
except:
1738
# If any problems happen, restore the original values and re-raise
1739
self._kndx_cache[prefix] = (orig_cache, orig_history)
1740
raise
1741
1742
def _cache_key(self, key, options, pos, size, parent_keys):
1097
# speed of knit parsing went from 280 ms to 280 ms with slots addition.
1098
# __slots__ = ['_cache', '_history', '_transport', '_filename']
1099
1100
def _cache_version(self, version_id, options, pos, size, parents):
1743
1101
"""Cache a version record in the history array and index cache.
1744
1102
1745
1103
This is inlined into _load_data for performance. KEEP IN SYNC.
1746
1104
(It saves 60ms, 25% of the __init__ overhead on local 4000 record
1747
1105
indexes).
1748
1106
"""
1749
prefix = key[:-1]
1750
version_id = key[-1]
1751
# last-element only for compatibilty with the C load_data.
1752
parents = tuple(parent[-1] for parent in parent_keys)
1753
for parent in parent_keys:
1754
if parent[:-1] != prefix:
1755
raise ValueError("mismatched prefixes for %r, %r" % (
1756
key, parent_keys))
1757
cache, history = self._kndx_cache[prefix]
1758
1107
# only want the _history index to reference the 1st index entry
1759
1108
# for version_id
1760
if version_id not in cache:
1761
index = len(history)
1762
history.append(version_id)
1109
if version_id not in self._cache:
1110
index = len(self._history)
1111
self._history.append(version_id)
1763
1112
else:
1764
index = cache[version_id][5]
1765
cache[version_id] = (version_id,
1113
index = self._cache[version_id][5]
1114
self._cache[version_id] = (version_id,
1766
1115
options,
1767
1116
pos,
1768
1117
size,
1769
1118
parents,
1770
1119
index)
1771
1120
1772
def check_header(self, fp):
1773
line = fp.readline()
1774
if line == '':
1775
# An empty file can actually be treated as though the file doesn't
1776
# exist yet.
1777
raise errors.NoSuchFile(self)
1778
if line != self.HEADER:
1779
raise KnitHeaderError(badline=line, filename=self)
1780
1781
def _check_read(self):
1782
if not self._is_locked():
1783
raise errors.ObjectNotLocked(self)
1784
if self._get_scope() != self._scope:
1785
self._reset_cache()
1786
1787
def _check_write_ok(self):
1788
"""Assert if not writes are permitted."""
1789
if not self._is_locked():
1790
raise errors.ObjectNotLocked(self)
1791
if self._get_scope() != self._scope:
1792
self._reset_cache()
1793
if self._mode != 'w':
1794
raise errors.ReadOnlyObjectDirtiedError(self)
1795
1796
def get_build_details(self, keys):
1797
"""Get the method, index_memo and compression parent for keys.
1798
1799
Ghosts are omitted from the result.
1800
1801
:param keys: An iterable of keys.
1802
:return: A dict of key:(index_memo, compression_parent, parents,
1803
record_details).
1804
index_memo
1805
opaque structure to pass to read_records to extract the raw
1806
data
1807
compression_parent
1808
Content that this record is built upon, may be None
1809
parents
1810
Logical parents of this node
1811
record_details
1812
extra information about the content which needs to be passed to
1813
Factory.parse_record
1814
"""
1815
prefixes = self._partition_keys(keys)
1816
parent_map = self.get_parent_map(keys)
1817
result = {}
1818
for key in keys:
1819
if key not in parent_map:
1820
continue # Ghost
1821
method = self.get_method(key)
1822
parents = parent_map[key]
1823
if method == 'fulltext':
1824
compression_parent = None
1825
else:
1826
compression_parent = parents[0]
1827
noeol = 'no-eol' in self.get_options(key)
1828
index_memo = self.get_position(key)
1829
result[key] = (index_memo, compression_parent,
1830
parents, (method, noeol))
1831
return result
1832
1833
def get_method(self, key):
1834
"""Return compression method of specified key."""
1835
options = self.get_options(key)
1836
if 'fulltext' in options:
1837
return 'fulltext'
1838
elif 'line-delta' in options:
1839
return 'line-delta'
1840
else:
1841
raise errors.KnitIndexUnknownMethod(self, options)
1842
1843
def get_options(self, key):
1844
"""Return a list representing options.
1845
1846
e.g. ['foo', 'bar']
1847
"""
1848
prefix, suffix = self._split_key(key)
1849
self._load_prefixes([prefix])
1121
def __init__(self, transport, filename, mode, create=False, file_mode=None,
1122
create_parent_dir=False, delay_create=False, dir_mode=None):
1123
_KnitComponentFile.__init__(self, transport, filename, mode,
1124
file_mode=file_mode,
1125
create_parent_dir=create_parent_dir,
1126
dir_mode=dir_mode)
1127
self._cache = {}
1128
# position in _history is the 'official' index for a revision
1129
# but the values may have come from a newer entry.
1130
# so - wc -l of a knit index is != the number of unique names
1131
# in the knit.
1132
self._history = []
1850
1133
try:
1851
return self._kndx_cache[prefix][0][suffix][1]
1852
except KeyError:
1853
raise RevisionNotPresent(key, self)
1854
1855
def get_parent_map(self, keys):
1856
"""Get a map of the parents of keys.
1857
1858
:param keys: The keys to look up parents for.
1859
:return: A mapping from keys to parents. Absent keys are absent from
1860
the mapping.
1861
"""
1862
# Parse what we need to up front, this potentially trades off I/O
1863
# locality (.kndx and .knit in the same block group for the same file
1864
# id) for less checking in inner loops.
1865
prefixes = set(key[:-1] for key in keys)
1866
self._load_prefixes(prefixes)
1867
result = {}
1868
for key in keys:
1869
prefix = key[:-1]
1870
try:
1871
suffix_parents = self._kndx_cache[prefix][0][key[-1]][4]
1872
except KeyError:
1873
pass
1874
else:
1875
result[key] = tuple(prefix + (suffix,) for
1876
suffix in suffix_parents)
1877
return result
1878
1879
def get_position(self, key):
1880
"""Return details needed to access the version.
1881
1882
:return: a tuple (key, data position, size) to hand to the access
1883
logic to get the record.
1884
"""
1885
prefix, suffix = self._split_key(key)
1886
self._load_prefixes([prefix])
1887
entry = self._kndx_cache[prefix][0][suffix]
1888
return key, entry[2], entry[3]
1889
1890
def _init_index(self, path, extra_lines=[]):
1891
"""Initialize an index."""
1892
sio = StringIO()
1893
sio.write(self.HEADER)
1894
sio.writelines(extra_lines)
1895
sio.seek(0)
1896
self._transport.put_file_non_atomic(path, sio,
1897
create_parent_dir=True)
1898
# self._create_parent_dir)
1899
# mode=self._file_mode,
1900
# dir_mode=self._dir_mode)
1901
1902
def keys(self):
1903
"""Get all the keys in the collection.
1904
1905
The keys are not ordered.
1906
"""
1907
result = set()
1908
# Identify all key prefixes.
1909
# XXX: A bit hacky, needs polish.
1910
if type(self._mapper) == ConstantMapper:
1911
prefixes = [()]
1912
else:
1913
relpaths = set()
1914
for quoted_relpath in self._transport.iter_files_recursive():
1915
path, ext = os.path.splitext(quoted_relpath)
1916
relpaths.add(path)
1917
prefixes = [self._mapper.unmap(path) for path in relpaths]
1918
self._load_prefixes(prefixes)
1919
for prefix in prefixes:
1920
for suffix in self._kndx_cache[prefix][1]:
1921
result.add(prefix + (suffix,))
1922
return result
1923
1924
def _load_prefixes(self, prefixes):
1925
"""Load the indices for prefixes."""
1926
self._check_read()
1927
for prefix in prefixes:
1928
if prefix not in self._kndx_cache:
1929
# the load_data interface writes to these variables.
1930
self._cache = {}
1931
self._history = []
1932
self._filename = prefix
1933
try:
1934
path = self._mapper.map(prefix) + '.kndx'
1935
fp = self._transport.get(path)
1936
try:
1937
# _load_data may raise NoSuchFile if the target knit is
1938
# completely empty.
1939
_load_data(self, fp)
1940
finally:
1941
fp.close()
1942
self._kndx_cache[prefix] = (self._cache, self._history)
1943
del self._cache
1944
del self._filename
1945
del self._history
1946
except NoSuchFile:
1947
self._kndx_cache[prefix] = ({}, [])
1948
if type(self._mapper) == ConstantMapper:
1949
# preserve behaviour for revisions.kndx etc.
1950
self._init_index(path)
1951
del self._cache
1952
del self._filename
1953
del self._history
1954
1955
def _partition_keys(self, keys):
1956
"""Turn keys into a dict of prefix:suffix_list."""
1957
result = {}
1958
for key in keys:
1959
prefix_keys = result.setdefault(key[:-1], [])
1960
prefix_keys.append(key[-1])
1961
return result
1962
1963
def _dictionary_compress(self, keys):
1964
"""Dictionary compress keys.
1965
1966
:param keys: The keys to generate references to.
1967
:return: A string representation of keys. keys which are present are
1968
dictionary compressed, and others are emitted as fulltext with a
1969
'.' prefix.
1970
"""
1971
if not keys:
1972
return ''
1134
fp = self._transport.get(self._filename)
1135
try:
1136
# _load_data may raise NoSuchFile if the target knit is
1137
# completely empty.
1138
self._load_data(fp)
1139
finally:
1140
fp.close()
1141
except NoSuchFile:
1142
if mode != 'w' or not create:
1143
raise
1144
elif delay_create:
1145
self._need_to_create = True
1146
else:
1147
self._transport.put_bytes_non_atomic(
1148
self._filename, self.HEADER, mode=self._file_mode)
1149
1150
def _load_data(self, fp):
1151
cache = self._cache
1152
history = self._history
1153
decode_utf8 = cache_utf8.decode
1154
1155
self.check_header(fp)
1156
# readlines reads the whole file at once:
1157
# bad for transports like http, good for local disk
1158
# we save 60 ms doing this one change (
1159
# from calling readline each time to calling
1160
# readlines once.
1161
# probably what we want for nice behaviour on
1162
# http is a incremental readlines that yields, or
1163
# a check for local vs non local indexes,
1164
history_top = len(history) - 1
1165
for line in fp.readlines():
1166
rec = line.split()
1167
if len(rec) < 5 or rec[-1] != ':':
1168
# corrupt line.
1169
# FIXME: in the future we should determine if its a
1170
# short write - and ignore it
1171
# or a different failure, and raise. RBC 20060407
1172
continue
1173
1174
parents = []
1175
for value in rec[4:-1]:
1176
if value[0] == '.':
1177
# uncompressed reference
1178
parents.append(decode_utf8(value[1:]))
1179
else:
1180
parents.append(history[int(value)])
1181
1182
version_id, options, pos, size = rec[:4]
1183
version_id = decode_utf8(version_id)
1184
1185
# See self._cache_version
1186
# only want the _history index to reference the 1st
1187
# index entry for version_id
1188
if version_id not in cache:
1189
history_top += 1
1190
index = history_top
1191
history.append(version_id)
1192
else:
1193
index = cache[version_id][5]
1194
cache[version_id] = (version_id,
1195
options.split(','),
1196
int(pos),
1197
int(size),
1198
parents,
1199
index)
1200
# end self._cache_version
1201
1202
def get_graph(self):
1203
return [(vid, idx[4]) for vid, idx in self._cache.iteritems()]
1204
1205
def get_ancestry(self, versions):
1206
"""See VersionedFile.get_ancestry."""
1207
# get a graph of all the mentioned versions:
1208
graph = {}
1209
pending = set(versions)
1210
cache = self._cache
1211
while pending:
1212
version = pending.pop()
1213
# trim ghosts
1214
try:
1215
parents = [p for p in cache[version][4] if p in cache]
1216
except KeyError:
1217
raise RevisionNotPresent(version, self._filename)
1218
# if not completed and not a ghost
1219
pending.update([p for p in parents if p not in graph])
1220
graph[version] = parents
1221
return topo_sort(graph.items())
1222
1223
def get_ancestry_with_ghosts(self, versions):
1224
"""See VersionedFile.get_ancestry_with_ghosts."""
1225
# get a graph of all the mentioned versions:
1226
self.check_versions_present(versions)
1227
cache = self._cache
1228
graph = {}
1229
pending = set(versions)
1230
while pending:
1231
version = pending.pop()
1232
try:
1233
parents = cache[version][4]
1234
except KeyError:
1235
# ghost, fake it
1236
graph[version] = []
1237
else:
1238
# if not completed
1239
pending.update([p for p in parents if p not in graph])
1240
graph[version] = parents
1241
return topo_sort(graph.items())
1242
1243
def num_versions(self):
1244
return len(self._history)
1245
1246
__len__ = num_versions
1247
1248
def get_versions(self):
1249
return self._history
1250
1251
def idx_to_name(self, idx):
1252
return self._history[idx]
1253
1254
def lookup(self, version_id):
1255
assert version_id in self._cache
1256
return self._cache[version_id][5]
1257
1258
def _version_list_to_index(self, versions):
1259
encode_utf8 = cache_utf8.encode
1973
1260
result_list = []
1974
prefix = keys[0][:-1]
1975
cache = self._kndx_cache[prefix][0]
1976
for key in keys:
1977
if key[:-1] != prefix:
1978
# kndx indices cannot refer across partitioned storage.
1979
raise ValueError("mismatched prefixes for %r" % keys)
1980
if key[-1] in cache:
1261
cache = self._cache
1262
for version in versions:
1263
if version in cache:
1981
1264
# -- inlined lookup() --
1982
result_list.append(str(cache[key[-1]][5]))
1265
result_list.append(str(cache[version][5]))
1983
1266
# -- end lookup () --
1984
1267
else:
1985
result_list.append('.' + key[-1])
1268
result_list.append('.' + encode_utf8(version))
1986
1269
return ' '.join(result_list)
1987
1270
1988
def _reset_cache(self):
1989
# Possibly this should be a LRU cache. A dictionary from key_prefix to
1990
# (cache_dict, history_vector) for parsed kndx files.
1991
self._kndx_cache = {}
1992
self._scope = self._get_scope()
1993
allow_writes = self._allow_writes()
1994
if allow_writes:
1995
self._mode = 'w'
1996
else:
1997
self._mode = 'r'
1998
1999
def _split_key(self, key):
2000
"""Split key into a prefix and suffix."""
2001
return key[:-1], key[-1]
2002
2003
2004
class _KnitGraphIndex(object):
2005
"""A KnitVersionedFiles index layered on GraphIndex."""
2006
2007
def __init__(self, graph_index, is_locked, deltas=False, parents=True,
2008
add_callback=None):
2009
"""Construct a KnitGraphIndex on a graph_index.
2010
2011
:param graph_index: An implementation of bzrlib.index.GraphIndex.
2012
:param is_locked: A callback to check whether the object should answer
2013
queries.
2014
:param deltas: Allow delta-compressed records.
2015
:param parents: If True, record knits parents, if not do not record
2016
parents.
2017
:param add_callback: If not None, allow additions to the index and call
2018
this callback with a list of added GraphIndex nodes:
2019
[(node, value, node_refs), ...]
2020
:param is_locked: A callback, returns True if the index is locked and
2021
thus usable.
2022
"""
2023
self._add_callback = add_callback
2024
self._graph_index = graph_index
2025
self._deltas = deltas
2026
self._parents = parents
2027
if deltas and not parents:
2028
# XXX: TODO: Delta tree and parent graph should be conceptually
2029
# separate.
2030
raise KnitCorrupt(self, "Cannot do delta compression without "
2031
"parent tracking.")
2032
self.has_graph = parents
2033
self._is_locked = is_locked
2034
2035
def __repr__(self):
2036
return "%s(%r)" % (self.__class__.__name__, self._graph_index)
2037
2038
def add_records(self, records, random_id=False):
2039
"""Add multiple records to the index.
2040
2041
This function does not insert data into the Immutable GraphIndex
2042
backing the KnitGraphIndex, instead it prepares data for insertion by
2043
the caller and checks that it is safe to insert then calls
2044
self._add_callback with the prepared GraphIndex nodes.
2045
2046
:param records: a list of tuples:
2047
(key, options, access_memo, parents).
2048
:param random_id: If True the ids being added were randomly generated
2049
and no check for existence will be performed.
2050
"""
2051
if not self._add_callback:
2052
raise errors.ReadOnlyError(self)
2053
# we hope there are no repositories with inconsistent parentage
2054
# anymore.
2055
2056
keys = {}
2057
for (key, options, access_memo, parents) in records:
2058
if self._parents:
2059
parents = tuple(parents)
2060
index, pos, size = access_memo
2061
if 'no-eol' in options:
2062
value = 'N'
2063
else:
2064
value = ' '
2065
value += "%d %d" % (pos, size)
2066
if not self._deltas:
2067
if 'line-delta' in options:
2068
raise KnitCorrupt(self, "attempt to add line-delta in non-delta knit")
2069
if self._parents:
2070
if self._deltas:
2071
if 'line-delta' in options:
2072
node_refs = (parents, (parents[0],))
2073
else:
2074
node_refs = (parents, ())
2075
else:
2076
node_refs = (parents, )
2077
else:
2078
if parents:
2079
raise KnitCorrupt(self, "attempt to add node with parents "
2080
"in parentless index.")
2081
node_refs = ()
2082
keys[key] = (value, node_refs)
2083
# check for dups
2084
if not random_id:
2085
present_nodes = self._get_entries(keys)
2086
for (index, key, value, node_refs) in present_nodes:
2087
if (value[0] != keys[key][0][0] or
2088
node_refs != keys[key][1]):
2089
raise KnitCorrupt(self, "inconsistent details in add_records"
2090
": %s %s" % ((value, node_refs), keys[key]))
2091
del keys[key]
2092
result = []
2093
if self._parents:
2094
for key, (value, node_refs) in keys.iteritems():
2095
result.append((key, value, node_refs))
2096
else:
2097
for key, (value, node_refs) in keys.iteritems():
2098
result.append((key, value))
2099
self._add_callback(result)
2100
2101
def _check_read(self):
2102
"""raise if reads are not permitted."""
2103
if not self._is_locked():
2104
raise errors.ObjectNotLocked(self)
2105
2106
def _check_write_ok(self):
2107
"""Assert if writes are not permitted."""
2108
if not self._is_locked():
2109
raise errors.ObjectNotLocked(self)
2110
2111
def _compression_parent(self, an_entry):
2112
# return the key that an_entry is compressed against, or None
2113
# Grab the second parent list (as deltas implies parents currently)
2114
compression_parents = an_entry[3][1]
2115
if not compression_parents:
2116
return None
2117
if len(compression_parents) != 1:
2118
raise AssertionError(
2119
"Too many compression parents: %r" % compression_parents)
2120
return compression_parents[0]
2121
2122
def get_build_details(self, keys):
2123
"""Get the method, index_memo and compression parent for version_ids.
2124
2125
Ghosts are omitted from the result.
2126
2127
:param keys: An iterable of keys.
2128
:return: A dict of key:
2129
(index_memo, compression_parent, parents, record_details).
2130
index_memo
2131
opaque structure to pass to read_records to extract the raw
2132
data
2133
compression_parent
2134
Content that this record is built upon, may be None
2135
parents
2136
Logical parents of this node
2137
record_details
2138
extra information about the content which needs to be passed to
2139
Factory.parse_record
2140
"""
2141
self._check_read()
2142
result = {}
2143
entries = self._get_entries(keys, False)
2144
for entry in entries:
2145
key = entry[1]
2146
if not self._parents:
2147
parents = ()
2148
else:
2149
parents = entry[3][0]
2150
if not self._deltas:
2151
compression_parent_key = None
2152
else:
2153
compression_parent_key = self._compression_parent(entry)
2154
noeol = (entry[2][0] == 'N')
2155
if compression_parent_key:
2156
method = 'line-delta'
2157
else:
2158
method = 'fulltext'
2159
result[key] = (self._node_to_position(entry),
2160
compression_parent_key, parents,
2161
(method, noeol))
2162
return result
2163
2164
def _get_entries(self, keys, check_present=False):
2165
"""Get the entries for keys.
2166
2167
:param keys: An iterable of index key tuples.
2168
"""
2169
keys = set(keys)
2170
found_keys = set()
2171
if self._parents:
2172
for node in self._graph_index.iter_entries(keys):
2173
yield node
2174
found_keys.add(node[1])
2175
else:
2176
# adapt parentless index to the rest of the code.
2177
for node in self._graph_index.iter_entries(keys):
2178
yield node[0], node[1], node[2], ()
2179
found_keys.add(node[1])
2180
if check_present:
2181
missing_keys = keys.difference(found_keys)
2182
if missing_keys:
2183
raise RevisionNotPresent(missing_keys.pop(), self)
2184
2185
def get_method(self, key):
2186
"""Return compression method of specified key."""
2187
return self._get_method(self._get_node(key))
2188
2189
def _get_method(self, node):
2190
if not self._deltas:
1271
def add_version(self, version_id, options, pos, size, parents):
1272
"""Add a version record to the index."""
1273
self.add_versions(((version_id, options, pos, size, parents),))
1274
1275
def add_versions(self, versions):
1276
"""Add multiple versions to the index.
1277
1278
:param versions: a list of tuples:
1279
(version_id, options, pos, size, parents).
1280
"""
1281
lines = []
1282
encode_utf8 = cache_utf8.encode
1283
orig_history = self._history[:]
1284
orig_cache = self._cache.copy()
1285
1286
try:
1287
for version_id, options, pos, size, parents in versions:
1288
line = "\n%s %s %s %s %s :" % (encode_utf8(version_id),
1289
','.join(options),
1290
pos,
1291
size,
1292
self._version_list_to_index(parents))
1293
assert isinstance(line, str), \
1294
'content must be utf-8 encoded: %r' % (line,)
1295
lines.append(line)
1296
self._cache_version(version_id, options, pos, size, parents)
1297
if not self._need_to_create:
1298
self._transport.append_bytes(self._filename, ''.join(lines))
1299
else:
1300
sio = StringIO()
1301
sio.write(self.HEADER)
1302
sio.writelines(lines)
1303
sio.seek(0)
1304
self._transport.put_file_non_atomic(self._filename, sio,
1305
create_parent_dir=self._create_parent_dir,
1306
mode=self._file_mode,
1307
dir_mode=self._dir_mode)
1308
self._need_to_create = False
1309
except:
1310
# If any problems happen, restore the original values and re-raise
1311
self._history = orig_history
1312
self._cache = orig_cache
1313
raise
1314
1315
def has_version(self, version_id):
1316
"""True if the version is in the index."""
1317
return version_id in self._cache
1318
1319
def get_position(self, version_id):
1320
"""Return data position and size of specified version."""
1321
entry = self._cache[version_id]
1322
return entry[2], entry[3]
1323
1324
def get_method(self, version_id):
1325
"""Return compression method of specified version."""
1326
options = self._cache[version_id][1]
1327
if 'fulltext' in options:
2191
1328
return 'fulltext'
2192
if self._compression_parent(node):
1329
else:
1330
if 'line-delta' not in options:
1331
raise errors.KnitIndexUnknownMethod(self._full_path(), options)
2193
1332
return 'line-delta'
2194
else:
2195
return 'fulltext'
2196
2197
def _get_node(self, key):
2198
try:
2199
return list(self._get_entries([key]))[0]
2200
except IndexError:
2201
raise RevisionNotPresent(key, self)
2202
2203
def get_options(self, key):
2204
"""Return a list representing options.
2205
2206
e.g. ['foo', 'bar']
2207
"""
2208
node = self._get_node(key)
2209
options = [self._get_method(node)]
2210
if node[2][0] == 'N':
2211
options.append('no-eol')
2212
return options
2213
2214
def get_parent_map(self, keys):
2215
"""Get a map of the parents of keys.
2216
2217
:param keys: The keys to look up parents for.
2218
:return: A mapping from keys to parents. Absent keys are absent from
2219
the mapping.
2220
"""
2221
self._check_read()
2222
nodes = self._get_entries(keys)
2223
result = {}
2224
if self._parents:
2225
for node in nodes:
2226
result[node[1]] = node[3][0]
2227
else:
2228
for node in nodes:
2229
result[node[1]] = None
2230
return result
2231
2232
def get_position(self, key):
2233
"""Return details needed to access the version.
2234
2235
:return: a tuple (index, data position, size) to hand to the access
2236
logic to get the record.
2237
"""
2238
node = self._get_node(key)
2239
return self._node_to_position(node)
2240
2241
def keys(self):
2242
"""Get all the keys in the collection.
2243
2244
The keys are not ordered.
2245
"""
2246
self._check_read()
2247
return [node[1] for node in self._graph_index.iter_all_entries()]
2248
2249
def _node_to_position(self, node):
2250
"""Convert an index value to position details."""
2251
bits = node[2][1:].split(' ')
2252
return node[0], int(bits[0]), int(bits[1])
2253
2254
2255
class _KnitKeyAccess(object):
2256
"""Access to records in .knit files."""
2257
2258
def __init__(self, transport, mapper):
2259
"""Create a _KnitKeyAccess with transport and mapper.
2260
2261
:param transport: The transport the access object is rooted at.
2262
:param mapper: The mapper used to map keys to .knit files.
2263
"""
2264
self._transport = transport
2265
self._mapper = mapper
2266
2267
def add_raw_records(self, key_sizes, raw_data):
2268
"""Add raw knit bytes to a storage area.
2269
2270
The data is spooled to the container writer in one bytes-record per
2271
raw data item.
2272
2273
:param sizes: An iterable of tuples containing the key and size of each
2274
raw data segment.
2275
:param raw_data: A bytestring containing the data.
2276
:return: A list of memos to retrieve the record later. Each memo is an
2277
opaque index memo. For _KnitKeyAccess the memo is (key, pos,
2278
length), where the key is the record key.
2279
"""
2280
if type(raw_data) != str:
2281
raise AssertionError(
2282
'data must be plain bytes was %s' % type(raw_data))
2283
result = []
2284
offset = 0
2285
# TODO: This can be tuned for writing to sftp and other servers where
2286
# append() is relatively expensive by grouping the writes to each key
2287
# prefix.
2288
for key, size in key_sizes:
2289
path = self._mapper.map(key)
2290
try:
2291
base = self._transport.append_bytes(path + '.knit',
2292
raw_data[offset:offset+size])
2293
except errors.NoSuchFile:
2294
self._transport.mkdir(osutils.dirname(path))
2295
base = self._transport.append_bytes(path + '.knit',
2296
raw_data[offset:offset+size])
2297
# if base == 0:
2298
# chmod.
2299
offset += size
2300
result.append((key, base, size))
2301
return result
2302
2303
def get_raw_records(self, memos_for_retrieval):
2304
"""Get the raw bytes for a records.
2305
2306
:param memos_for_retrieval: An iterable containing the access memo for
2307
retrieving the bytes.
2308
:return: An iterator over the bytes of the records.
2309
"""
2310
# first pass, group into same-index request to minimise readv's issued.
2311
request_lists = []
2312
current_prefix = None
2313
for (key, offset, length) in memos_for_retrieval:
2314
if current_prefix == key[:-1]:
2315
current_list.append((offset, length))
2316
else:
2317
if current_prefix is not None:
2318
request_lists.append((current_prefix, current_list))
2319
current_prefix = key[:-1]
2320
current_list = [(offset, length)]
2321
# handle the last entry
2322
if current_prefix is not None:
2323
request_lists.append((current_prefix, current_list))
2324
for prefix, read_vector in request_lists:
2325
path = self._mapper.map(prefix) + '.knit'
2326
for pos, data in self._transport.readv(path, read_vector):
2327
yield data
2328
2329
2330
class _DirectPackAccess(object):
2331
"""Access to data in one or more packs with less translation."""
2332
2333
def __init__(self, index_to_packs):
2334
"""Create a _DirectPackAccess object.
2335
2336
:param index_to_packs: A dict mapping index objects to the transport
2337
and file names for obtaining data.
2338
"""
2339
self._container_writer = None
2340
self._write_index = None
2341
self._indices = index_to_packs
2342
2343
def add_raw_records(self, key_sizes, raw_data):
2344
"""Add raw knit bytes to a storage area.
2345
2346
The data is spooled to the container writer in one bytes-record per
2347
raw data item.
2348
2349
:param sizes: An iterable of tuples containing the key and size of each
2350
raw data segment.
2351
:param raw_data: A bytestring containing the data.
2352
:return: A list of memos to retrieve the record later. Each memo is an
2353
opaque index memo. For _DirectPackAccess the memo is (index, pos,
2354
length), where the index field is the write_index object supplied
2355
to the PackAccess object.
2356
"""
2357
if type(raw_data) != str:
2358
raise AssertionError(
2359
'data must be plain bytes was %s' % type(raw_data))
2360
result = []
2361
offset = 0
2362
for key, size in key_sizes:
2363
p_offset, p_length = self._container_writer.add_bytes_record(
2364
raw_data[offset:offset+size], [])
2365
offset += size
2366
result.append((self._write_index, p_offset, p_length))
2367
return result
2368
2369
def get_raw_records(self, memos_for_retrieval):
2370
"""Get the raw bytes for a records.
2371
2372
:param memos_for_retrieval: An iterable containing the (index, pos,
2373
length) memo for retrieving the bytes. The Pack access method
2374
looks up the pack to use for a given record in its index_to_pack
2375
map.
2376
:return: An iterator over the bytes of the records.
2377
"""
2378
# first pass, group into same-index requests
2379
request_lists = []
2380
current_index = None
2381
for (index, offset, length) in memos_for_retrieval:
2382
if current_index == index:
2383
current_list.append((offset, length))
2384
else:
2385
if current_index is not None:
2386
request_lists.append((current_index, current_list))
2387
current_index = index
2388
current_list = [(offset, length)]
2389
# handle the last entry
2390
if current_index is not None:
2391
request_lists.append((current_index, current_list))
2392
for index, offsets in request_lists:
2393
transport, path = self._indices[index]
2394
reader = pack.make_readv_reader(transport, path, offsets)
2395
for names, read_func in reader.iter_records():
2396
yield read_func(None)
2397
2398
def set_writer(self, writer, index, transport_packname):
2399
"""Set a writer to use for adding data."""
2400
if index is not None:
2401
self._indices[index] = transport_packname
2402
self._container_writer = writer
2403
self._write_index = index
2404
2405
2406
# Deprecated, use PatienceSequenceMatcher instead
2407
KnitSequenceMatcher = patiencediff.PatienceSequenceMatcher
2408
2409
2410
def annotate_knit(knit, revision_id):
2411
"""Annotate a knit with no cached annotations.
2412
2413
This implementation is for knits with no cached annotations.
2414
It will work for knits with cached annotations, but this is not
2415
recommended.
1333
1334
def get_options(self, version_id):
1335
return self._cache[version_id][1]
1336
1337
def get_parents(self, version_id):
1338
"""Return parents of specified version ignoring ghosts."""
1339
return [parent for parent in self._cache[version_id][4]
1340
if parent in self._cache]
1341
1342
def get_parents_with_ghosts(self, version_id):
1343
"""Return parents of specified version with ghosts."""
1344
return self._cache[version_id][4]
1345
1346
def check_versions_present(self, version_ids):
1347
"""Check that all specified versions are present."""
1348
cache = self._cache
1349
for version_id in version_ids:
1350
if version_id not in cache:
1351
raise RevisionNotPresent(version_id, self._filename)
1352
1353
1354
class _KnitData(_KnitComponentFile):
1355
"""Contents of the knit data file"""
1356
1357
def __init__(self, transport, filename, mode, create=False, file_mode=None,
1358
create_parent_dir=False, delay_create=False,
1359
dir_mode=None):
1360
_KnitComponentFile.__init__(self, transport, filename, mode,
1361
file_mode=file_mode,
1362
create_parent_dir=create_parent_dir,
1363
dir_mode=dir_mode)
1364
self._checked = False
1365
# TODO: jam 20060713 conceptually, this could spill to disk
1366
# if the cached size gets larger than a certain amount
1367
# but it complicates the model a bit, so for now just use
1368
# a simple dictionary
1369
self._cache = {}
1370
self._do_cache = False
1371
if create:
1372
if delay_create:
1373
self._need_to_create = create
1374
else:
1375
self._transport.put_bytes_non_atomic(self._filename, '',
1376
mode=self._file_mode)
1377
1378
def enable_cache(self):
1379
"""Enable caching of reads."""
1380
self._do_cache = True
1381
1382
def clear_cache(self):
1383
"""Clear the record cache."""
1384
self._do_cache = False
1385
self._cache = {}
1386
1387
def _open_file(self):
1388
try:
1389
return self._transport.get(self._filename)
1390
except NoSuchFile:
1391
pass
1392
return None
1393
1394
def _record_to_data(self, version_id, digest, lines):
1395
"""Convert version_id, digest, lines into a raw data block.
1396
1397
:return: (len, a StringIO instance with the raw data ready to read.)
1398
"""
1399
sio = StringIO()
1400
data_file = GzipFile(None, mode='wb', fileobj=sio)
1401
1402
version_id_utf8 = cache_utf8.encode(version_id)
1403
data_file.writelines(chain(
1404
["version %s %d %s\n" % (version_id_utf8,
1405
len(lines),
1406
digest)],
1407
lines,
1408
["end %s\n" % version_id_utf8]))
1409
data_file.close()
1410
length= sio.tell()
1411
1412
sio.seek(0)
1413
return length, sio
1414
1415
def add_raw_record(self, raw_data):
1416
"""Append a prepared record to the data file.
1417
1418
:return: the offset in the data file raw_data was written.
1419
"""
1420
assert isinstance(raw_data, str), 'data must be plain bytes'
1421
if not self._need_to_create:
1422
return self._transport.append_bytes(self._filename, raw_data)
1423
else:
1424
self._transport.put_bytes_non_atomic(self._filename, raw_data,
1425
create_parent_dir=self._create_parent_dir,
1426
mode=self._file_mode,
1427
dir_mode=self._dir_mode)
1428
self._need_to_create = False
1429
return 0
1430
1431
def add_record(self, version_id, digest, lines):
1432
"""Write new text record to disk. Returns the position in the
1433
file where it was written."""
1434
size, sio = self._record_to_data(version_id, digest, lines)
1435
# write to disk
1436
if not self._need_to_create:
1437
start_pos = self._transport.append_file(self._filename, sio)
1438
else:
1439
self._transport.put_file_non_atomic(self._filename, sio,
1440
create_parent_dir=self._create_parent_dir,
1441
mode=self._file_mode,
1442
dir_mode=self._dir_mode)
1443
self._need_to_create = False
1444
start_pos = 0
1445
if self._do_cache:
1446
self._cache[version_id] = sio.getvalue()
1447
return start_pos, size
1448
1449
def _parse_record_header(self, version_id, raw_data):
1450
"""Parse a record header for consistency.
1451
1452
:return: the header and the decompressor stream.
1453
as (stream, header_record)
1454
"""
1455
df = GzipFile(mode='rb', fileobj=StringIO(raw_data))
1456
rec = self._check_header(version_id, df.readline())
1457
return df, rec
1458
1459
def _check_header(self, version_id, line):
1460
rec = line.split()
1461
if len(rec) != 4:
1462
raise KnitCorrupt(self._filename,
1463
'unexpected number of elements in record header')
1464
if cache_utf8.decode(rec[1]) != version_id:
1465
raise KnitCorrupt(self._filename,
1466
'unexpected version, wanted %r, got %r'
1467
% (version_id, rec[1]))
1468
return rec
1469
1470
def _parse_record(self, version_id, data):
1471
# profiling notes:
1472
# 4168 calls in 2880 217 internal
1473
# 4168 calls to _parse_record_header in 2121
1474
# 4168 calls to readlines in 330
1475
df = GzipFile(mode='rb', fileobj=StringIO(data))
1476
1477
record_contents = df.readlines()
1478
header = record_contents.pop(0)
1479
rec = self._check_header(version_id, header)
1480
1481
last_line = record_contents.pop()
1482
assert len(record_contents) == int(rec[2])
1483
if last_line != 'end %s\n' % rec[1]:
1484
raise KnitCorrupt(self._filename,
1485
'unexpected version end line %r, wanted %r'
1486
% (last_line, version_id))
1487
df.close()
1488
return record_contents, rec[3]
1489
1490
def read_records_iter_raw(self, records):
1491
"""Read text records from data file and yield raw data.
1492
1493
This unpacks enough of the text record to validate the id is
1494
as expected but thats all.
1495
"""
1496
# setup an iterator of the external records:
1497
# uses readv so nice and fast we hope.
1498
if len(records):
1499
# grab the disk data needed.
1500
if self._cache:
1501
# Don't check _cache if it is empty
1502
needed_offsets = [(pos, size) for version_id, pos, size
1503
in records
1504
if version_id not in self._cache]
1505
else:
1506
needed_offsets = [(pos, size) for version_id, pos, size
1507
in records]
1508
1509
raw_records = self._transport.readv(self._filename, needed_offsets)
1510
1511
for version_id, pos, size in records:
1512
if version_id in self._cache:
1513
# This data has already been validated
1514
data = self._cache[version_id]
1515
else:
1516
pos, data = raw_records.next()
1517
if self._do_cache:
1518
self._cache[version_id] = data
1519
1520
# validate the header
1521
df, rec = self._parse_record_header(version_id, data)
1522
df.close()
1523
yield version_id, data
1524
1525
def read_records_iter(self, records):
1526
"""Read text records from data file and yield result.
1527
1528
The result will be returned in whatever is the fastest to read.
1529
Not by the order requested. Also, multiple requests for the same
1530
record will only yield 1 response.
1531
:param records: A list of (version_id, pos, len) entries
1532
:return: Yields (version_id, contents, digest) in the order
1533
read, not the order requested
1534
"""
1535
if not records:
1536
return
1537
1538
if self._cache:
1539
# Skip records we have alread seen
1540
yielded_records = set()
1541
needed_records = set()
1542
for record in records:
1543
if record[0] in self._cache:
1544
if record[0] in yielded_records:
1545
continue
1546
yielded_records.add(record[0])
1547
data = self._cache[record[0]]
1548
content, digest = self._parse_record(record[0], data)
1549
yield (record[0], content, digest)
1550
else:
1551
needed_records.add(record)
1552
needed_records = sorted(needed_records, key=operator.itemgetter(1))
1553
else:
1554
needed_records = sorted(set(records), key=operator.itemgetter(1))
1555
1556
if not needed_records:
1557
return
1558
1559
# The transport optimizes the fetching as well
1560
# (ie, reads continuous ranges.)
1561
readv_response = self._transport.readv(self._filename,
1562
[(pos, size) for version_id, pos, size in needed_records])
1563
1564
for (version_id, pos, size), (pos, data) in \
1565
izip(iter(needed_records), readv_response):
1566
content, digest = self._parse_record(version_id, data)
1567
if self._do_cache:
1568
self._cache[version_id] = data
1569
yield version_id, content, digest
1570
1571
def read_records(self, records):
1572
"""Read records into a dictionary."""
1573
components = {}
1574
for record_id, content, digest in \
1575
self.read_records_iter(records):
1576
components[record_id] = (content, digest)
1577
return components
1578
1579
1580
class InterKnit(InterVersionedFile):
1581
"""Optimised code paths for knit to knit operations."""
1582
1583
_matching_file_from_factory = KnitVersionedFile
1584
_matching_file_to_factory = KnitVersionedFile
1585
1586
@staticmethod
1587
def is_compatible(source, target):
1588
"""Be compatible with knits. """
1589
try:
1590
return (isinstance(source, KnitVersionedFile) and
1591
isinstance(target, KnitVersionedFile))
1592
except AttributeError:
1593
return False
1594
1595
def join(self, pb=None, msg=None, version_ids=None, ignore_missing=False):
1596
"""See InterVersionedFile.join."""
1597
assert isinstance(self.source, KnitVersionedFile)
1598
assert isinstance(self.target, KnitVersionedFile)
1599
1600
version_ids = self._get_source_version_ids(version_ids, ignore_missing)
1601
1602
if not version_ids:
1603
return 0
1604
1605
pb = ui.ui_factory.nested_progress_bar()
1606
try:
1607
version_ids = list(version_ids)
1608
if None in version_ids:
1609
version_ids.remove(None)
1610
1611
self.source_ancestry = set(self.source.get_ancestry(version_ids))
1612
this_versions = set(self.target._index.get_versions())
1613
needed_versions = self.source_ancestry - this_versions
1614
cross_check_versions = self.source_ancestry.intersection(this_versions)
1615
mismatched_versions = set()
1616
for version in cross_check_versions:
1617
# scan to include needed parents.
1618
n1 = set(self.target.get_parents_with_ghosts(version))
1619
n2 = set(self.source.get_parents_with_ghosts(version))
1620
if n1 != n2:
1621
# FIXME TEST this check for cycles being introduced works
1622
# the logic is we have a cycle if in our graph we are an
1623
# ancestor of any of the n2 revisions.
1624
for parent in n2:
1625
if parent in n1:
1626
# safe
1627
continue
1628
else:
1629
parent_ancestors = self.source.get_ancestry(parent)
1630
if version in parent_ancestors:
1631
raise errors.GraphCycleError([parent, version])
1632
# ensure this parent will be available later.
1633
new_parents = n2.difference(n1)
1634
needed_versions.update(new_parents.difference(this_versions))
1635
mismatched_versions.add(version)
1636
1637
if not needed_versions and not mismatched_versions:
1638
return 0
1639
full_list = topo_sort(self.source.get_graph())
1640
1641
version_list = [i for i in full_list if (not self.target.has_version(i)
1642
and i in needed_versions)]
1643
1644
# plan the join:
1645
copy_queue = []
1646
copy_queue_records = []
1647
copy_set = set()
1648
for version_id in version_list:
1649
options = self.source._index.get_options(version_id)
1650
parents = self.source._index.get_parents_with_ghosts(version_id)
1651
# check that its will be a consistent copy:
1652
for parent in parents:
1653
# if source has the parent, we must :
1654
# * already have it or
1655
# * have it scheduled already
1656
# otherwise we don't care
1657
assert (self.target.has_version(parent) or
1658
parent in copy_set or
1659
not self.source.has_version(parent))
1660
data_pos, data_size = self.source._index.get_position(version_id)
1661
copy_queue_records.append((version_id, data_pos, data_size))
1662
copy_queue.append((version_id, options, parents))
1663
copy_set.add(version_id)
1664
1665
# data suck the join:
1666
count = 0
1667
total = len(version_list)
1668
raw_datum = []
1669
raw_records = []
1670
for (version_id, raw_data), \
1671
(version_id2, options, parents) in \
1672
izip(self.source._data.read_records_iter_raw(copy_queue_records),
1673
copy_queue):
1674
assert version_id == version_id2, 'logic error, inconsistent results'
1675
count = count + 1
1676
pb.update("Joining knit", count, total)
1677
raw_records.append((version_id, options, parents, len(raw_data)))
1678
raw_datum.append(raw_data)
1679
self.target._add_raw_records(raw_records, ''.join(raw_datum))
1680
1681
for version in mismatched_versions:
1682
# FIXME RBC 20060309 is this needed?
1683
n1 = set(self.target.get_parents_with_ghosts(version))
1684
n2 = set(self.source.get_parents_with_ghosts(version))
1685
# write a combined record to our history preserving the current
1686
# parents as first in the list
1687
new_parents = self.target.get_parents_with_ghosts(version) + list(n2.difference(n1))
1688
self.target.fix_parents(version, new_parents)
1689
return count
1690
finally:
1691
pb.finished()
1692
1693
1694
InterVersionedFile.register_optimiser(InterKnit)
1695
1696
1697
class WeaveToKnit(InterVersionedFile):
1698
"""Optimised code paths for weave to knit operations."""
1699
1700
_matching_file_from_factory = bzrlib.weave.WeaveFile
1701
_matching_file_to_factory = KnitVersionedFile
1702
1703
@staticmethod
1704
def is_compatible(source, target):
1705
"""Be compatible with weaves to knits."""
1706
try:
1707
return (isinstance(source, bzrlib.weave.Weave) and
1708
isinstance(target, KnitVersionedFile))
1709
except AttributeError:
1710
return False
1711
1712
def join(self, pb=None, msg=None, version_ids=None, ignore_missing=False):
1713
"""See InterVersionedFile.join."""
1714
assert isinstance(self.source, bzrlib.weave.Weave)
1715
assert isinstance(self.target, KnitVersionedFile)
1716
1717
version_ids = self._get_source_version_ids(version_ids, ignore_missing)
1718
1719
if not version_ids:
1720
return 0
1721
1722
pb = ui.ui_factory.nested_progress_bar()
1723
try:
1724
version_ids = list(version_ids)
1725
1726
self.source_ancestry = set(self.source.get_ancestry(version_ids))
1727
this_versions = set(self.target._index.get_versions())
1728
needed_versions = self.source_ancestry - this_versions
1729
cross_check_versions = self.source_ancestry.intersection(this_versions)
1730
mismatched_versions = set()
1731
for version in cross_check_versions:
1732
# scan to include needed parents.
1733
n1 = set(self.target.get_parents_with_ghosts(version))
1734
n2 = set(self.source.get_parents(version))
1735
# if all of n2's parents are in n1, then its fine.
1736
if n2.difference(n1):
1737
# FIXME TEST this check for cycles being introduced works
1738
# the logic is we have a cycle if in our graph we are an
1739
# ancestor of any of the n2 revisions.
1740
for parent in n2:
1741
if parent in n1:
1742
# safe
1743
continue
1744
else:
1745
parent_ancestors = self.source.get_ancestry(parent)
1746
if version in parent_ancestors:
1747
raise errors.GraphCycleError([parent, version])
1748
# ensure this parent will be available later.
1749
new_parents = n2.difference(n1)
1750
needed_versions.update(new_parents.difference(this_versions))
1751
mismatched_versions.add(version)
1752
1753
if not needed_versions and not mismatched_versions:
1754
return 0
1755
full_list = topo_sort(self.source.get_graph())
1756
1757
version_list = [i for i in full_list if (not self.target.has_version(i)
1758
and i in needed_versions)]
1759
1760
# do the join:
1761
count = 0
1762
total = len(version_list)
1763
for version_id in version_list:
1764
pb.update("Converting to knit", count, total)
1765
parents = self.source.get_parents(version_id)
1766
# check that its will be a consistent copy:
1767
for parent in parents:
1768
# if source has the parent, we must already have it
1769
assert (self.target.has_version(parent))
1770
self.target.add_lines(
1771
version_id, parents, self.source.get_lines(version_id))
1772
count = count + 1
1773
1774
for version in mismatched_versions:
1775
# FIXME RBC 20060309 is this needed?
1776
n1 = set(self.target.get_parents_with_ghosts(version))
1777
n2 = set(self.source.get_parents(version))
1778
# write a combined record to our history preserving the current
1779
# parents as first in the list
1780
new_parents = self.target.get_parents_with_ghosts(version) + list(n2.difference(n1))
1781
self.target.fix_parents(version, new_parents)
1782
return count
1783
finally:
1784
pb.finished()
1785
1786
1787
InterVersionedFile.register_optimiser(WeaveToKnit)
1788
1789
1790
class KnitSequenceMatcher(difflib.SequenceMatcher):
1791
"""Knit tuned sequence matcher.
1792
1793
This is based on profiling of difflib which indicated some improvements
1794
for our usage pattern.
2416
1795
"""
2417
annotator = _KnitAnnotator(knit)
2418
return iter(annotator.annotate(revision_id))
2419
2420
2421
class _KnitAnnotator(object):
2422
"""Build up the annotations for a text."""
2423
2424
def __init__(self, knit):
2425
self._knit = knit
2426
2427
# Content objects, differs from fulltexts because of how final newlines
2428
# are treated by knits. the content objects here will always have a
2429
# final newline
2430
self._fulltext_contents = {}
2431
2432
# Annotated lines of specific revisions
2433
self._annotated_lines = {}
2434
2435
# Track the raw data for nodes that we could not process yet.
2436
# This maps the revision_id of the base to a list of children that will
2437
# annotated from it.
2438
self._pending_children = {}
2439
2440
# Nodes which cannot be extracted
2441
self._ghosts = set()
2442
2443
# Track how many children this node has, so we know if we need to keep
2444
# it
2445
self._annotate_children = {}
2446
self._compression_children = {}
2447
2448
self._all_build_details = {}
2449
# The children => parent revision_id graph
2450
self._revision_id_graph = {}
2451
2452
self._heads_provider = None
2453
2454
self._nodes_to_keep_annotations = set()
2455
self._generations_until_keep = 100
2456
2457
def set_generations_until_keep(self, value):
2458
"""Set the number of generations before caching a node.
2459
2460
Setting this to -1 will cache every merge node, setting this higher
2461
will cache fewer nodes.
2462
"""
2463
self._generations_until_keep = value
2464
2465
def _add_fulltext_content(self, revision_id, content_obj):
2466
self._fulltext_contents[revision_id] = content_obj
2467
# TODO: jam 20080305 It might be good to check the sha1digest here
2468
return content_obj.text()
2469
2470
def _check_parents(self, child, nodes_to_annotate):
2471
"""Check if all parents have been processed.
2472
2473
:param child: A tuple of (rev_id, parents, raw_content)
2474
:param nodes_to_annotate: If child is ready, add it to
2475
nodes_to_annotate, otherwise put it back in self._pending_children
2476
"""
2477
for parent_id in child[1]:
2478
if (parent_id not in self._annotated_lines):
2479
# This parent is present, but another parent is missing
2480
self._pending_children.setdefault(parent_id,
2481
[]).append(child)
2482
break
2483
else:
2484
# This one is ready to be processed
2485
nodes_to_annotate.append(child)
2486
2487
def _add_annotation(self, revision_id, fulltext, parent_ids,
2488
left_matching_blocks=None):
2489
"""Add an annotation entry.
2490
2491
All parents should already have been annotated.
2492
:return: A list of children that now have their parents satisfied.
2493
"""
2494
a = self._annotated_lines
2495
annotated_parent_lines = [a[p] for p in parent_ids]
2496
annotated_lines = list(annotate.reannotate(annotated_parent_lines,
2497
fulltext, revision_id, left_matching_blocks,
2498
heads_provider=self._get_heads_provider()))
2499
self._annotated_lines[revision_id] = annotated_lines
2500
for p in parent_ids:
2501
ann_children = self._annotate_children[p]
2502
ann_children.remove(revision_id)
2503
if (not ann_children
2504
and p not in self._nodes_to_keep_annotations):
2505
del self._annotated_lines[p]
2506
del self._all_build_details[p]
2507
if p in self._fulltext_contents:
2508
del self._fulltext_contents[p]
2509
# Now that we've added this one, see if there are any pending
2510
# deltas to be done, certainly this parent is finished
2511
nodes_to_annotate = []
2512
for child in self._pending_children.pop(revision_id, []):
2513
self._check_parents(child, nodes_to_annotate)
2514
return nodes_to_annotate
2515
2516
def _get_build_graph(self, key):
2517
"""Get the graphs for building texts and annotations.
2518
2519
The data you need for creating a full text may be different than the
2520
data you need to annotate that text. (At a minimum, you need both
2521
parents to create an annotation, but only need 1 parent to generate the
2522
fulltext.)
2523
2524
:return: A list of (key, index_memo) records, suitable for
2525
passing to read_records_iter to start reading in the raw data fro/
2526
the pack file.
2527
"""
2528
if key in self._annotated_lines:
2529
# Nothing to do
2530
return []
2531
pending = set([key])
2532
records = []
2533
generation = 0
2534
kept_generation = 0
2535
while pending:
2536
# get all pending nodes
2537
generation += 1
2538
this_iteration = pending
2539
build_details = self._knit._index.get_build_details(this_iteration)
2540
self._all_build_details.update(build_details)
2541
# new_nodes = self._knit._index._get_entries(this_iteration)
2542
pending = set()
2543
for key, details in build_details.iteritems():
2544
(index_memo, compression_parent, parents,
2545
record_details) = details
2546
self._revision_id_graph[key] = parents
2547
records.append((key, index_memo))
2548
# Do we actually need to check _annotated_lines?
2549
pending.update(p for p in parents
2550
if p not in self._all_build_details)
2551
if compression_parent:
2552
self._compression_children.setdefault(compression_parent,
2553
[]).append(key)
2554
if parents:
2555
for parent in parents:
2556
self._annotate_children.setdefault(parent,
2557
[]).append(key)
2558
num_gens = generation - kept_generation
2559
if ((num_gens >= self._generations_until_keep)
2560
and len(parents) > 1):
2561
kept_generation = generation
2562
self._nodes_to_keep_annotations.add(key)
2563
2564
missing_versions = this_iteration.difference(build_details.keys())
2565
self._ghosts.update(missing_versions)
2566
for missing_version in missing_versions:
2567
# add a key, no parents
2568
self._revision_id_graph[missing_version] = ()
2569
pending.discard(missing_version) # don't look for it
2570
if self._ghosts.intersection(self._compression_children):
2571
raise KnitCorrupt(
2572
"We cannot have nodes which have a ghost compression parent:\n"
2573
"ghosts: %r\n"
2574
"compression children: %r"
2575
% (self._ghosts, self._compression_children))
2576
# Cleanout anything that depends on a ghost so that we don't wait for
2577
# the ghost to show up
2578
for node in self._ghosts:
2579
if node in self._annotate_children:
2580
# We won't be building this node
2581
del self._annotate_children[node]
2582
# Generally we will want to read the records in reverse order, because
2583
# we find the parent nodes after the children
2584
records.reverse()
2585
return records
2586
2587
def _annotate_records(self, records):
2588
"""Build the annotations for the listed records."""
2589
# We iterate in the order read, rather than a strict order requested
2590
# However, process what we can, and put off to the side things that
2591
# still need parents, cleaning them up when those parents are
2592
# processed.
2593
for (rev_id, record,
2594
digest) in self._knit._read_records_iter(records):
2595
if rev_id in self._annotated_lines:
2596
continue
2597
parent_ids = self._revision_id_graph[rev_id]
2598
parent_ids = [p for p in parent_ids if p not in self._ghosts]
2599
details = self._all_build_details[rev_id]
2600
(index_memo, compression_parent, parents,
2601
record_details) = details
2602
nodes_to_annotate = []
2603
# TODO: Remove the punning between compression parents, and
2604
# parent_ids, we should be able to do this without assuming
2605
# the build order
2606
if len(parent_ids) == 0:
2607
# There are no parents for this node, so just add it
2608
# TODO: This probably needs to be decoupled
2609
fulltext_content, delta = self._knit._factory.parse_record(
2610
rev_id, record, record_details, None)
2611
fulltext = self._add_fulltext_content(rev_id, fulltext_content)
2612
nodes_to_annotate.extend(self._add_annotation(rev_id, fulltext,
2613
parent_ids, left_matching_blocks=None))
2614
else:
2615
child = (rev_id, parent_ids, record)
2616
# Check if all the parents are present
2617
self._check_parents(child, nodes_to_annotate)
2618
while nodes_to_annotate:
2619
# Should we use a queue here instead of a stack?
2620
(rev_id, parent_ids, record) = nodes_to_annotate.pop()
2621
(index_memo, compression_parent, parents,
2622
record_details) = self._all_build_details[rev_id]
2623
if compression_parent is not None:
2624
comp_children = self._compression_children[compression_parent]
2625
if rev_id not in comp_children:
2626
raise AssertionError("%r not in compression children %r"
2627
% (rev_id, comp_children))
2628
# If there is only 1 child, it is safe to reuse this
2629
# content
2630
reuse_content = (len(comp_children) == 1
2631
and compression_parent not in
2632
self._nodes_to_keep_annotations)
2633
if reuse_content:
2634
# Remove it from the cache since it will be changing
2635
parent_fulltext_content = self._fulltext_contents.pop(compression_parent)
2636
# Make sure to copy the fulltext since it might be
2637
# modified
2638
parent_fulltext = list(parent_fulltext_content.text())
2639
else:
2640
parent_fulltext_content = self._fulltext_contents[compression_parent]
2641
parent_fulltext = parent_fulltext_content.text()
2642
comp_children.remove(rev_id)
2643
fulltext_content, delta = self._knit._factory.parse_record(
2644
rev_id, record, record_details,
2645
parent_fulltext_content,
2646
copy_base_content=(not reuse_content))
2647
fulltext = self._add_fulltext_content(rev_id,
2648
fulltext_content)
2649
blocks = KnitContent.get_line_delta_blocks(delta,
2650
parent_fulltext, fulltext)
2651
else:
2652
fulltext_content = self._knit._factory.parse_fulltext(
2653
record, rev_id)
2654
fulltext = self._add_fulltext_content(rev_id,
2655
fulltext_content)
2656
blocks = None
2657
nodes_to_annotate.extend(
2658
self._add_annotation(rev_id, fulltext, parent_ids,
2659
left_matching_blocks=blocks))
2660
2661
def _get_heads_provider(self):
2662
"""Create a heads provider for resolving ancestry issues."""
2663
if self._heads_provider is not None:
2664
return self._heads_provider
2665
parent_provider = _mod_graph.DictParentsProvider(
2666
self._revision_id_graph)
2667
graph_obj = _mod_graph.Graph(parent_provider)
2668
head_cache = _mod_graph.FrozenHeadsCache(graph_obj)
2669
self._heads_provider = head_cache
2670
return head_cache
2671
2672
def annotate(self, key):
2673
"""Return the annotated fulltext at the given key.
2674
2675
:param key: The key to annotate.
2676
"""
2677
if True or len(self._knit._fallback_vfs) > 0:
2678
# stacked knits can't use the fast path at present.
2679
return self._simple_annotate(key)
2680
records = self._get_build_graph(key)
2681
if key in self._ghosts:
2682
raise errors.RevisionNotPresent(key, self._knit)
2683
self._annotate_records(records)
2684
return self._annotated_lines[key]
2685
2686
def _simple_annotate(self, key):
2687
"""Return annotated fulltext, rediffing from the full texts.
2688
2689
This is slow but makes no assumptions about the repository
2690
being able to produce line deltas.
2691
"""
2692
# TODO: this code generates a parent maps of present ancestors; it
2693
# could be split out into a separate method, and probably should use
2694
# iter_ancestry instead. -- mbp and robertc 20080704
2695
graph = _mod_graph.Graph(self._knit)
2696
head_cache = _mod_graph.FrozenHeadsCache(graph)
2697
search = graph._make_breadth_first_searcher([key])
2698
keys = set()
2699
while True:
1796
1797
def find_longest_match(self, alo, ahi, blo, bhi):
1798
"""Find longest matching block in a[alo:ahi] and b[blo:bhi].
1799
1800
If isjunk is not defined:
1801
1802
Return (i,j,k) such that a[i:i+k] is equal to b[j:j+k], where
1803
alo <= i <= i+k <= ahi
1804
blo <= j <= j+k <= bhi
1805
and for all (i',j',k') meeting those conditions,
1806
k >= k'
1807
i <= i'
1808
and if i == i', j <= j'
1809
1810
In other words, of all maximal matching blocks, return one that
1811
starts earliest in a, and of all those maximal matching blocks that
1812
start earliest in a, return the one that starts earliest in b.
1813
1814
>>> s = SequenceMatcher(None, " abcd", "abcd abcd")
1815
>>> s.find_longest_match(0, 5, 0, 9)
1816
(0, 4, 5)
1817
1818
If isjunk is defined, first the longest matching block is
1819
determined as above, but with the additional restriction that no
1820
junk element appears in the block. Then that block is extended as
1821
far as possible by matching (only) junk elements on both sides. So
1822
the resulting block never matches on junk except as identical junk
1823
happens to be adjacent to an "interesting" match.
1824
1825
Here's the same example as before, but considering blanks to be
1826
junk. That prevents " abcd" from matching the " abcd" at the tail
1827
end of the second sequence directly. Instead only the "abcd" can
1828
match, and matches the leftmost "abcd" in the second sequence:
1829
1830
>>> s = SequenceMatcher(lambda x: x==" ", " abcd", "abcd abcd")
1831
>>> s.find_longest_match(0, 5, 0, 9)
1832
(1, 0, 4)
1833
1834
If no blocks match, return (alo, blo, 0).
1835
1836
>>> s = SequenceMatcher(None, "ab", "c")
1837
>>> s.find_longest_match(0, 2, 0, 1)
1838
(0, 0, 0)
1839
"""
1840
1841
# CAUTION: stripping common prefix or suffix would be incorrect.
1842
# E.g.,
1843
# ab
1844
# acab
1845
# Longest matching block is "ab", but if common prefix is
1846
# stripped, it's "a" (tied with "b"). UNIX(tm) diff does so
1847
# strip, so ends up claiming that ab is changed to acab by
1848
# inserting "ca" in the middle. That's minimal but unintuitive:
1849
# "it's obvious" that someone inserted "ac" at the front.
1850
# Windiff ends up at the same place as diff, but by pairing up
1851
# the unique 'b's and then matching the first two 'a's.
1852
1853
a, b, b2j, isbjunk = self.a, self.b, self.b2j, self.isbjunk
1854
besti, bestj, bestsize = alo, blo, 0
1855
# find longest junk-free match
1856
# during an iteration of the loop, j2len[j] = length of longest
1857
# junk-free match ending with a[i-1] and b[j]
1858
j2len = {}
1859
# nothing = []
1860
b2jget = b2j.get
1861
for i in xrange(alo, ahi):
1862
# look at all instances of a[i] in b; note that because
1863
# b2j has no junk keys, the loop is skipped if a[i] is junk
1864
j2lenget = j2len.get
1865
newj2len = {}
1866
1867
# changing b2j.get(a[i], nothing) to a try:KeyError pair produced the
1868
# following improvement
1869
# 704 0 4650.5320 2620.7410 bzrlib.knit:1336(find_longest_match)
1870
# +326674 0 1655.1210 1655.1210 +<method 'get' of 'dict' objects>
1871
# +76519 0 374.6700 374.6700 +<method 'has_key' of 'dict' objects>
1872
# to
1873
# 704 0 3733.2820 2209.6520 bzrlib.knit:1336(find_longest_match)
1874
# +211400 0 1147.3520 1147.3520 +<method 'get' of 'dict' objects>
1875
# +76519 0 376.2780 376.2780 +<method 'has_key' of 'dict' objects>
1876
2700
1877
try:
2701
present, ghosts = search.next_with_ghosts()
2702
except StopIteration:
2703
break
2704
keys.update(present)
2705
parent_map = self._knit.get_parent_map(keys)
2706
parent_cache = {}
2707
reannotate = annotate.reannotate
2708
for record in self._knit.get_record_stream(keys, 'topological', True):
2709
key = record.key
2710
fulltext = split_lines(record.get_bytes_as('fulltext'))
2711
parents = parent_map[key]
2712
if parents is not None:
2713
parent_lines = [parent_cache[parent] for parent in parent_map[key]]
1878
js = b2j[a[i]]
1879
except KeyError:
1880
pass
2714
1881
else:
2715
parent_lines = []
2716
parent_cache[key] = list(
2717
reannotate(parent_lines, fulltext, key, None, head_cache))
2718
try:
2719
return parent_cache[key]
2720
except KeyError, e:
2721
raise errors.RevisionNotPresent(key, self._knit)
2722
2723
2724
try:
2725
from bzrlib._knit_load_data_c import _load_data_c as _load_data
2726
except ImportError:
2727
from bzrlib._knit_load_data_py import _load_data_py as _load_data
1882
for j in js:
1883
# a[i] matches b[j]
1884
if j >= blo:
1885
if j >= bhi:
1886
break
1887
k = newj2len[j] = 1 + j2lenget(-1 + j, 0)
1888
if k > bestsize:
1889
besti, bestj, bestsize = 1 + i-k, 1 + j-k, k
1890
j2len = newj2len
1891
1892
# Extend the best by non-junk elements on each end. In particular,
1893
# "popular" non-junk elements aren't in b2j, which greatly speeds
1894
# the inner loop above, but also means "the best" match so far
1895
# doesn't contain any junk *or* popular non-junk elements.
1896
while besti > alo and bestj > blo and \
1897
not isbjunk(b[bestj-1]) and \
1898
a[besti-1] == b[bestj-1]:
1899
besti, bestj, bestsize = besti-1, bestj-1, bestsize+1
1900
while besti+bestsize < ahi and bestj+bestsize < bhi and \
1901
not isbjunk(b[bestj+bestsize]) and \
1902
a[besti+bestsize] == b[bestj+bestsize]:
1903
bestsize += 1
1904
1905
# Now that we have a wholly interesting match (albeit possibly
1906
# empty!), we may as well suck up the matching junk on each
1907
# side of it too. Can't think of a good reason not to, and it
1908
# saves post-processing the (possibly considerable) expense of
1909
# figuring out what to do with it. In the case of an empty
1910
# interesting match, this is clearly the right thing to do,
1911
# because no other kind of match is possible in the regions.
1912
while besti > alo and bestj > blo and \
1913
isbjunk(b[bestj-1]) and \
1914
a[besti-1] == b[bestj-1]:
1915
besti, bestj, bestsize = besti-1, bestj-1, bestsize+1
1916
while besti+bestsize < ahi and bestj+bestsize < bhi and \
1917
isbjunk(b[bestj+bestsize]) and \
1918
a[besti+bestsize] == b[bestj+bestsize]:
1919
bestsize = bestsize + 1
1920
1921
return besti, bestj, bestsize
Loggerhead is a web-based interface for Breezy
Version: 2.0.1