← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/repofmt/groupcompress_repo.py

  • Committer: John Arbash Meinel
  • Date: 2010-08-04 04:33:24 UTC
  • mto: This revision was merged to the branch mainline in revision 5390.
  • Revision ID: john@arbash-meinel.com-20100804043324-1ldc2v2w1kza7ox4
get into the nitty gritty for the _key_to_sha1 function.

It doesn't seem to help to op-out of the actual unhexlify call, so at least
that doesn't seem to be the performance overhead.
We get down to around 19.9us for _key_to_sha1 across 120 keys, which is
0.166us per key.

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/repofmt/groupcompress_repo.py
262
262
        remaining_keys = set(keys)
263
263
        counter = [0]
264
264
        if self._gather_text_refs:
265
 
            bytes_to_info = inventory.CHKInventory._bytes_to_utf8name_key
266
265
            self._text_refs = set()
267
266
        def _get_referenced_stream(root_keys, parse_leaf_nodes=False):
268
267
            cur_keys = root_keys
289
288
                    # Store is None, because we know we have a LeafNode, and we
290
289
                    # just want its entries
291
290
                    for file_id, bytes in node.iteritems(None):
292
 
                        name_utf8, file_id, revision_id = bytes_to_info(bytes)
293
 
                        self._text_refs.add((file_id, revision_id))
 
291
                        self._text_refs.add(chk_map._bytes_to_text_key(bytes))
294
292
                def next_stream():
295
293
                    stream = source_vf.get_record_stream(cur_keys,
296
294
                                                         'as-requested', True)
647
645
        chk_diff = chk_map.iter_interesting_nodes(
648
646
            chk_bytes_no_fallbacks, root_key_info.interesting_root_keys,
649
647
            root_key_info.uninteresting_root_keys)
650
 
        bytes_to_info = inventory.CHKInventory._bytes_to_utf8name_key
651
648
        text_keys = set()
652
649
        try:
653
 
            for record in _filter_text_keys(chk_diff, text_keys, bytes_to_info):
 
650
            for record in _filter_text_keys(chk_diff, text_keys,
 
651
                                            chk_map._bytes_to_text_key):
654
652
                pass
655
653
        except errors.NoSuchRevision, e:
656
654
            # XXX: It would be nice if we could give a more precise error here.
864
862
        if basis_inv is None:
865
863
            if basis_revision_id == _mod_revision.NULL_REVISION:
866
864
                new_inv = self._create_inv_from_null(delta, new_revision_id)
 
865
                if new_inv.root_id is None:
 
866
                    raise errors.RootMissing()
867
867
                inv_lines = new_inv.to_lines()
868
868
                return self._inventory_add_lines(new_revision_id, parents,
869
869
                    inv_lines, check_content=False), new_inv
1087
1087
                uninteresting_root_keys.add(inv.id_to_entry.key())
1088
1088
                uninteresting_pid_root_keys.add(
1089
1089
                    inv.parent_id_basename_to_file_id.key())
1090
 
        bytes_to_info = inventory.CHKInventory._bytes_to_utf8name_key
1091
1090
        chk_bytes = self.from_repository.chk_bytes
1092
1091
        def _filter_id_to_entry():
1093
1092
            interesting_nodes = chk_map.iter_interesting_nodes(chk_bytes,
1094
1093
                        self._chk_id_roots, uninteresting_root_keys)
1095
1094
            for record in _filter_text_keys(interesting_nodes, self._text_keys,
1096
 
                    bytes_to_info):
 
1095
                    chk_map._bytes_to_text_key):
1097
1096
                if record is not None:
1098
1097
                    yield record
1099
1098
            # Consumed
1109
1108
        yield 'chk_bytes', _get_parent_id_basename_to_file_id_pages()
1110
1109
 
1111
1110
    def get_stream(self, search):
 
1111
        def wrap_and_count(pb, rc, stream):
 
1112
            """Yield records from stream while showing progress."""
 
1113
            count = 0
 
1114
            for record in stream:
 
1115
                if count == rc.STEP:
 
1116
                    rc.increment(count)
 
1117
                    pb.update('Estimate', rc.current, rc.max)
 
1118
                    count = 0
 
1119
                count += 1
 
1120
                yield record
 
1121
 
1112
1122
        revision_ids = search.get_keys()
 
1123
        pb = ui.ui_factory.nested_progress_bar()
 
1124
        rc = self._record_counter
 
1125
        self._record_counter.setup(len(revision_ids))
1113
1126
        for stream_info in self._fetch_revision_texts(revision_ids):
1114
 
            yield stream_info
 
1127
            yield (stream_info[0],
 
1128
                wrap_and_count(pb, rc, stream_info[1]))
1115
1129
        self._revision_keys = [(rev_id,) for rev_id in revision_ids]
1116
1130
        self.from_repository.revisions.clear_cache()
1117
1131
        self.from_repository.signatures.clear_cache()
1118
 
        yield self._get_inventory_stream(self._revision_keys)
 
1132
        s = self._get_inventory_stream(self._revision_keys)
 
1133
        yield (s[0], wrap_and_count(pb, rc, s[1]))
1119
1134
        self.from_repository.inventories.clear_cache()
1120
1135
        # TODO: The keys to exclude might be part of the search recipe
1121
1136
        # For now, exclude all parents that are at the edge of ancestry, for
1124
1139
        parent_keys = from_repo._find_parent_keys_of_revisions(
1125
1140
                        self._revision_keys)
1126
1141
        for stream_info in self._get_filtered_chk_streams(parent_keys):
1127
 
            yield stream_info
 
1142
            yield (stream_info[0], wrap_and_count(pb, rc, stream_info[1]))
1128
1143
        self.from_repository.chk_bytes.clear_cache()
1129
 
        yield self._get_text_stream()
 
1144
        s = self._get_text_stream()
 
1145
        yield (s[0], wrap_and_count(pb, rc, s[1]))
1130
1146
        self.from_repository.texts.clear_cache()
 
1147
        pb.update('Done', rc.max, rc.max)
 
1148
        pb.finished()
1131
1149
 
1132
1150
    def get_stream_for_missing_keys(self, missing_keys):
1133
1151
        # missing keys can only occur when we are byte copying and not
1187
1205
    return result
1188
1206
 
1189
1207
 
1190
 
def _filter_text_keys(interesting_nodes_iterable, text_keys, bytes_to_info):
 
1208
def _filter_text_keys(interesting_nodes_iterable, text_keys, bytes_to_text_key):
1191
1209
    """Iterate the result of iter_interesting_nodes, yielding the records
1192
1210
    and adding to text_keys.
1193
1211
    """
 
1212
    text_keys_update = text_keys.update
1194
1213
    for record, items in interesting_nodes_iterable:
1195
 
        for name, bytes in items:
1196
 
            # Note: we don't care about name_utf8, because groupcompress repos
1197
 
            # are always rich-root, so there are no synthesised root records to
1198
 
            # ignore.
1199
 
            _, file_id, revision_id = bytes_to_info(bytes)
1200
 
            file_id = intern(file_id)
1201
 
            revision_id = intern(revision_id)
1202
 
            text_keys.add(StaticTuple(file_id, revision_id).intern())
 
1214
        text_keys_update([bytes_to_text_key(b) for n,b in items])
1203
1215
        yield record
1204
1216
 
1205
1217