← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/groupcompress.py

  • Committer: John Arbash Meinel
  • Date: 2009-03-16 21:15:19 UTC
  • mto: (3735.2.156 brisbane-core)
  • mto: This revision was merged to the branch mainline in revision 4280.
  • Revision ID: john@arbash-meinel.com-20090316211519-3r3zx1fgg167gl3z
Add groupcompress-block[-ref] as valid stream types.

However, we now encounter that the versionedfile tests assume that Factory
objects know the content's sha1 sum if we request them with delta_closure=False.
I assume this is because we had been reading the knit content headers to
get the sha1sum, but I think Andrew was planning on changing that anyway.
The VF tests expose the lack of sha1, but otherwise things seem to
work if we pass back the lazy block, rather than extracting it
ahead of time.

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/groupcompress.py
255
255
            # If we have nothing left to decomp, we ran out of decomp bytes
256
256
            assert remaining_decomp
257
257
            needed_bytes = num_bytes - len(self._content)
258
 
            # We always set max_size to 32kB over the minimum needed, so that zlib
259
 
            # will give us as much as we really want.
260
 
            # TODO: If this isn't good enough, we could make a loop here, that
261
 
            #       keeps expanding the request until we get enough
 
258
            # We always set max_size to 32kB over the minimum needed, so that
 
259
            # zlib will give us as much as we really want.
 
260
            # TODO: If this isn't good enough, we could make a loop here,
 
261
            #       that keeps expanding the request until we get enough
262
262
            self._content += self._z_content_decompressor.decompress(
263
263
                remaining_decomp, needed_bytes + _ZLIB_DECOMP_WINDOW)
264
264
            assert len(self._content) >= num_bytes
456
456
        self._start = start
457
457
        self._end = end
458
458
 
 
459
    def __repr__(self):
 
460
        return '%s(%s, first=%s)' % (self.__class__.__name__,
 
461
            self.key, self._first)
 
462
 
459
463
    def get_bytes_as(self, storage_kind):
460
464
        if storage_kind == self.storage_kind:
461
465
            if self._first:
965
969
            source = key_to_source_map.get(key, self)
966
970
            if source is not current_source:
967
971
                source_keys.append((source, []))
 
972
                current_source = source
968
973
            source_keys[-1][1].append(key)
969
974
        return source_keys
970
975
 
981
986
                continue
982
987
            if source is not current_source:
983
988
                source_keys.append((source, []))
 
989
                current_source = source
984
990
            source_keys[-1][1].append(key)
985
991
        return source_keys
986
992
 
1039
1045
                unadded_keys, source_result)
1040
1046
        for key in missing:
1041
1047
            yield AbsentContentFactory(key)
 
1048
        last_block = None
 
1049
        lazy_buffered = None
1042
1050
        for source, keys in source_keys:
1043
1051
            if source is self:
1044
1052
                for key in keys:
1045
1053
                    if key in self._unadded_refs:
 
1054
                        if lazy_buffered:
 
1055
                            for factory in lazy_buffered:
 
1056
                                yield factory
 
1057
                            lazy_buffered = None
 
1058
                            last_block = None
1046
1059
                        bytes, sha1 = self._compressor.extract(key)
1047
1060
                        parents = self._unadded_refs[key]
 
1061
                        yield FulltextContentFactory(key, parents, sha1, bytes)
1048
1062
                    else:
1049
1063
                        index_memo, _, parents, (method, _) = locations[key]
1050
1064
                        block = self._get_block(index_memo)
1051
1065
                        start, end = index_memo[3:5]
1052
 
                        entry, bytes = block.extract(key, start, end)
1053
 
                        sha1 = entry.sha1
1054
 
                        # TODO: If we don't have labels, then the sha1 here is
1055
 
                        #       computed from the data, so we don't want to
1056
 
                        #       re-sha the string.
1057
 
                        if not _FAST and sha_string(bytes) != sha1:
1058
 
                            raise AssertionError('sha1 sum did not match')
1059
 
                    yield FulltextContentFactory(key, parents, sha1, bytes)
 
1066
                        if block is last_block:
 
1067
                            first = False
 
1068
                        else:
 
1069
                            first = True
 
1070
                        factory = LazyGroupCompressFactory(key,
 
1071
                            parents, block, start, end, first)
 
1072
                        if first:
 
1073
                            # The first entry in a new group
 
1074
                            if lazy_buffered:
 
1075
                                for old in lazy_buffered:
 
1076
                                    yield old
 
1077
                            lazy_buffered = [factory]
 
1078
                        else:
 
1079
                            lazy_buffered.append(factory)
 
1080
                        last_block = block
1060
1081
            else:
 
1082
                # We need to flush everything we have buffered so far
 
1083
                if lazy_buffered:
 
1084
                    for factory in lazy_buffered:
 
1085
                        yield factory
 
1086
                    lazy_buffered = None
 
1087
                    last_block = None
1061
1088
                for record in source.get_record_stream(keys, ordering,
1062
1089
                                                       include_delta_closure):
1063
1090
                    yield record
 
1091
        if lazy_buffered:
 
1092
            for factory in lazy_buffered:
 
1093
                yield factory
 
1094
            lazy_buffered = None
 
1095
            last_block = None
1064
1096
 
1065
1097
    def get_sha1s(self, keys):
1066
1098
        """See VersionedFiles.get_sha1s()."""
1070
1102
                result[record.key] = record.sha1
1071
1103
            else:
1072
1104
                if record.storage_kind != 'absent':
1073
 
                    result[record.key] == sha_string(record.get_bytes_as(
 
1105
                    result[record.key] = sha_string(record.get_bytes_as(
1074
1106
                        'fulltext'))
1075
1107
        return result
1076
1108