← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/knit.py

Better readv coalescing, now with test, and progress during knit index reading.

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/knit.py
69
69
from gzip import GzipFile
70
70
import os
71
71
 
 
72
import bzrlib
72
73
import bzrlib.errors as errors
73
74
from bzrlib.errors import FileExists, NoSuchFile, KnitError, \
74
75
        InvalidRevisionId, KnitCorrupt, KnitHeaderError, \
532
533
                raise RevisionNotPresent(version_id, self.filename)
533
534
            data_pos, length = self._index.get_position(version_id)
534
535
            version_id_records.append((version_id, data_pos, length))
535
 
        for version_id, data, sha_value in \
536
 
            self._data.read_records_iter(version_id_records):
537
 
            method = self._index.get_method(version_id)
538
 
            version_idx = self._index.lookup(version_id)
539
 
            assert method in ('fulltext', 'line-delta')
540
 
            if method == 'fulltext':
541
 
                content = self.factory.parse_fulltext(data, version_idx)
542
 
                for line in content.text():
543
 
                    yield line
544
 
            else:
545
 
                delta = self.factory.parse_line_delta(data, version_idx)
546
 
                for start, end, count, lines in delta:
547
 
                    for origin, line in lines:
 
536
        pb = bzrlib.ui.ui_factory.nested_progress_bar()
 
537
        count = 0
 
538
        total = len(version_id_records)
 
539
        try:
 
540
            for version_id, data, sha_value in \
 
541
                self._data.read_records_iter(version_id_records):
 
542
                pb.update('Walking content.', count, total)
 
543
                method = self._index.get_method(version_id)
 
544
                version_idx = self._index.lookup(version_id)
 
545
                assert method in ('fulltext', 'line-delta')
 
546
                if method == 'fulltext':
 
547
                    content = self.factory.parse_fulltext(data, version_idx)
 
548
                    for line in content.text():
548
549
                        yield line
 
550
                else:
 
551
                    delta = self.factory.parse_line_delta(data, version_idx)
 
552
                    for start, end, count, lines in delta:
 
553
                        for origin, line in lines:
 
554
                            yield line
 
555
                count +=1
 
556
        except:
 
557
            pb.update('Walking content.', total, total)
 
558
            pb.finished()
 
559
            raise
549
560
        
550
561
    def num_versions(self):
551
562
        """See VersionedFile.num_versions()."""
700
711
            self._history.append(version_id)
701
712
 
702
713
    def _iter_index(self, fp):
703
 
        lines = fp.read()
704
 
        for l in lines.splitlines(False):
 
714
        for l in fp.readlines():
705
715
            yield l.split()
 
716
        #lines = fp.read()
 
717
        #for l in lines.splitlines(False):
 
718
        #    yield l.split()
706
719
 
707
720
    def __init__(self, transport, filename, mode, create=False):
708
721
        _KnitComponentFile.__init__(self, transport, filename, mode)
712
725
        # so - wc -l of a knit index is != the number of uniqe names
713
726
        # in the weave.
714
727
        self._history = []
 
728
        pb = bzrlib.ui.ui_factory.nested_progress_bar()
715
729
        try:
716
 
            fp = self._transport.get(self._filename)
717
 
            self.check_header(fp)
718
 
            for rec in self._iter_index(fp):
719
 
                parents = self._parse_parents(rec[4:])
720
 
                self._cache_version(rec[0], rec[1].split(','), int(rec[2]), int(rec[3]),
721
 
                    parents)
722
 
        except NoSuchFile, e:
723
 
            if mode != 'w' or not create:
724
 
                raise
725
 
            self.write_header()
 
730
            count = 0
 
731
            total = 1
 
732
            try:
 
733
                pb.update('read knit index', count, total)
 
734
                fp = self._transport.get(self._filename)
 
735
                self.check_header(fp)
 
736
                for rec in self._iter_index(fp):
 
737
                    count += 1
 
738
                    total += 1
 
739
                    pb.update('read knit index', count, total)
 
740
                    parents = self._parse_parents(rec[4:])
 
741
                    self._cache_version(rec[0], rec[1].split(','), int(rec[2]), int(rec[3]),
 
742
                        parents)
 
743
            except NoSuchFile, e:
 
744
                if mode != 'w' or not create:
 
745
                    raise
 
746
                self.write_header()
 
747
        finally:
 
748
            pb.update('read knit index', total, total)
 
749
            pb.finished()
726
750
 
727
751
    def _parse_parents(self, compressed_parents):
728
752
        """convert a list of string parent values into version ids.