← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/knit.py

Merge merge-into2 into direct-patching

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/knit.py
1124
1124
            record_map[key] = record, record_details, digest, next
1125
1125
        return record_map
1126
1126
 
 
1127
    def _split_by_prefix(self, keys):
 
1128
        """For the given keys, split them up based on their prefix.
 
1129
 
 
1130
        To keep memory pressure somewhat under control, split the
 
1131
        requests back into per-file-id requests, otherwise "bzr co"
 
1132
        extracts the full tree into memory before writing it to disk.
 
1133
        This should be revisited if _get_content_maps() can ever cross
 
1134
        file-id boundaries.
 
1135
 
 
1136
        :param keys: An iterable of key tuples
 
1137
        :return: A dict of {prefix: [key_list]}
 
1138
        """
 
1139
        split_by_prefix = {}
 
1140
        for key in keys:
 
1141
            if len(key) == 1:
 
1142
                split_by_prefix.setdefault('', []).append(key)
 
1143
            else:
 
1144
                split_by_prefix.setdefault(key[0], []).append(key)
 
1145
        return split_by_prefix
 
1146
 
1127
1147
    def get_record_stream(self, keys, ordering, include_delta_closure):
1128
1148
        """Get a stream of records for keys.
1129
1149
 
1223
1243
        if include_delta_closure:
1224
1244
            # XXX: get_content_maps performs its own index queries; allow state
1225
1245
            # to be passed in.
1226
 
            text_map, _ = self._get_content_maps(present_keys,
1227
 
                needed_from_fallback - absent_keys)
1228
 
            for key in present_keys:
1229
 
                yield FulltextContentFactory(key, global_map[key], None,
1230
 
                    ''.join(text_map[key]))
 
1246
            non_local_keys = needed_from_fallback - absent_keys
 
1247
            prefix_split_keys = self._split_by_prefix(present_keys)
 
1248
            prefix_split_non_local_keys = self._split_by_prefix(non_local_keys)
 
1249
            for prefix, keys in prefix_split_keys.iteritems():
 
1250
                non_local = prefix_split_non_local_keys.get(prefix, [])
 
1251
                non_local = set(non_local)
 
1252
                text_map, _ = self._get_content_maps(keys, non_local)
 
1253
                for key in keys:
 
1254
                    lines = text_map.pop(key)
 
1255
                    text = ''.join(lines)
 
1256
                    yield FulltextContentFactory(key, global_map[key], None,
 
1257
                                                 text)
1231
1258
        else:
1232
1259
            for source, keys in source_keys:
1233
1260
                if source is parent_maps[0]: