~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/repofmt/groupcompress_repo.py

• browse files at revision 5027
• compare with another revision
• download diff
• download tarball
• view history from revision 5027

• reverse the comparison (5027 to 5401)
• stop comparing with revision 5401

Show diffs side-by-side

added

removed

bzrlib/repofmt/groupcompress_repo.py

29

29

    knit,

30

30

    osutils,

31

31

    pack,

 

32

    remote,

32

33

    revision as _mod_revision,

33

34

    trace,

34

35

    ui,

35

 

    versionedfile,

36

36

    )

37

37

from bzrlib.btree_index import (

38

38

    BTreeGraphIndex,

39

39

    BTreeBuilder,

40

40

    )

41

 

from bzrlib.decorators import needs_write_lock

42

41

from bzrlib.groupcompress import (

43

42

    _GCGraphIndex,

44

43

    GroupCompressVersionedFiles,

264

263

        remaining_keys = set(keys)

265

264

        counter = [0]

266

265

        if self._gather_text_refs:

 

266

            bytes_to_info = inventory.CHKInventory._bytes_to_utf8name_key

267

267

            self._text_refs = set()

268

268

        def _get_referenced_stream(root_keys, parse_leaf_nodes=False):

269

269

            cur_keys = root_keys

290

290

                    # Store is None, because we know we have a LeafNode, and we

291

291

                    # just want its entries

292

292

                    for file_id, bytes in node.iteritems(None):

293

 

                        self._text_refs.add(chk_map._bytes_to_text_key(bytes))

 

293

                        name_utf8, file_id, revision_id = bytes_to_info(bytes)

 

294

                        self._text_refs.add((file_id, revision_id))

294

295

                def next_stream():

295

296

                    stream = source_vf.get_record_stream(cur_keys,

296

297

                                                         'as-requested', True)

427

428

        self._copy_stream(source_vf, target_vf, inventory_keys,

428

429

                          'inventories', self._get_filtered_inv_stream, 2)

429

430

 

430

 

    def _get_chk_vfs_for_copy(self):

431

 

        return self._build_vfs('chk', False, False)

432

 

 

433

431

    def _copy_chk_texts(self):

434

 

        source_vf, target_vf = self._get_chk_vfs_for_copy()

 

432

        source_vf, target_vf = self._build_vfs('chk', False, False)

435

433

        # TODO: This is technically spurious... if it is a performance issue,

436

434

        #       remove it

437

435

        total_keys = source_vf.keys()

583

581

        return new_pack.data_inserted() and self._data_changed

584

582

 

585

583

 

586

 

class GCCHKCanonicalizingPacker(GCCHKPacker):

587

 

    """A packer that ensures inventories have canonical-form CHK maps.

588

 

    

589

 

    Ideally this would be part of reconcile, but it's very slow and rarely

590

 

    needed.  (It repairs repositories affected by

591

 

    https://bugs.launchpad.net/bzr/+bug/522637).

592

 

    """

593

 

 

594

 

    def __init__(self, *args, **kwargs):

595

 

        super(GCCHKCanonicalizingPacker, self).__init__(*args, **kwargs)

596

 

        self._data_changed = False

597

 

    

598

 

    def _exhaust_stream(self, source_vf, keys, message, vf_to_stream, pb_offset):

599

 

        """Create and exhaust a stream, but don't insert it.

600

 

        

601

 

        This is useful to get the side-effects of generating a stream.

602

 

        """

603

 

        self.pb.update('scanning %s' % (message,), pb_offset)

604

 

        child_pb = ui.ui_factory.nested_progress_bar()

605

 

        try:

606

 

            list(vf_to_stream(source_vf, keys, message, child_pb))

607

 

        finally:

608

 

            child_pb.finished()

609

 

 

610

 

    def _copy_inventory_texts(self):

611

 

        source_vf, target_vf = self._build_vfs('inventory', True, True)

612

 

        source_chk_vf, target_chk_vf = self._get_chk_vfs_for_copy()

613

 

        inventory_keys = source_vf.keys()

614

 

        # First, copy the existing CHKs on the assumption that most of them

615

 

        # will be correct.  This will save us from having to reinsert (and

616

 

        # recompress) these records later at the cost of perhaps preserving a

617

 

        # few unused CHKs. 

618

 

        # (Iterate but don't insert _get_filtered_inv_stream to populate the

619

 

        # variables needed by GCCHKPacker._copy_chk_texts.)

620

 

        self._exhaust_stream(source_vf, inventory_keys, 'inventories',

621

 

                self._get_filtered_inv_stream, 2)

622

 

        GCCHKPacker._copy_chk_texts(self)

623

 

        # Now copy and fix the inventories, and any regenerated CHKs.

624

 

        def chk_canonicalizing_inv_stream(source_vf, keys, message, pb=None):

625

 

            return self._get_filtered_canonicalizing_inv_stream(

626

 

                source_vf, keys, message, pb, source_chk_vf, target_chk_vf)

627

 

        self._copy_stream(source_vf, target_vf, inventory_keys,

628

 

                          'inventories', chk_canonicalizing_inv_stream, 4)

629

 

 

630

 

    def _copy_chk_texts(self):

631

 

        # No-op; in this class this happens during _copy_inventory_texts.

632

 

        pass

633

 

 

634

 

    def _get_filtered_canonicalizing_inv_stream(self, source_vf, keys, message,

635

 

            pb=None, source_chk_vf=None, target_chk_vf=None):

636

 

        """Filter the texts of inventories, regenerating CHKs to make sure they

637

 

        are canonical.

638

 

        """

639

 

        total_keys = len(keys)

640

 

        target_chk_vf = versionedfile.NoDupeAddLinesDecorator(target_chk_vf)

641

 

        def _filtered_inv_stream():

642

 

            stream = source_vf.get_record_stream(keys, 'groupcompress', True)

643

 

            search_key_name = None

644

 

            for idx, record in enumerate(stream):

645

 

                # Inventories should always be with revisions; assume success.

646

 

                bytes = record.get_bytes_as('fulltext')

647

 

                chk_inv = inventory.CHKInventory.deserialise(

648

 

                    source_chk_vf, bytes, record.key)

649

 

                if pb is not None:

650

 

                    pb.update('inv', idx, total_keys)

651

 

                chk_inv.id_to_entry._ensure_root()

652

 

                if search_key_name is None:

653

 

                    # Find the name corresponding to the search_key_func

654

 

                    search_key_reg = chk_map.search_key_registry

655

 

                    for search_key_name, func in search_key_reg.iteritems():

656

 

                        if func == chk_inv.id_to_entry._search_key_func:

657

 

                            break

658

 

                canonical_inv = inventory.CHKInventory.from_inventory(

659

 

                    target_chk_vf, chk_inv,

660

 

                    maximum_size=chk_inv.id_to_entry._root_node._maximum_size,

661

 

                    search_key_name=search_key_name)

662

 

                if chk_inv.id_to_entry.key() != canonical_inv.id_to_entry.key():

663

 

                    trace.mutter(

664

 

                        'Non-canonical CHK map for id_to_entry of inv: %s '

665

 

                        '(root is %s, should be %s)' % (chk_inv.revision_id,

666

 

                        chk_inv.id_to_entry.key()[0],

667

 

                        canonical_inv.id_to_entry.key()[0]))

668

 

                    self._data_changed = True

669

 

                p_id_map = chk_inv.parent_id_basename_to_file_id

670

 

                p_id_map._ensure_root()

671

 

                canon_p_id_map = canonical_inv.parent_id_basename_to_file_id

672

 

                if p_id_map.key() != canon_p_id_map.key():

673

 

                    trace.mutter(

674

 

                        'Non-canonical CHK map for parent_id_to_basename of '

675

 

                        'inv: %s (root is %s, should be %s)'

676

 

                        % (chk_inv.revision_id, p_id_map.key()[0],

677

 

                           canon_p_id_map.key()[0]))

678

 

                    self._data_changed = True

679

 

                yield versionedfile.ChunkedContentFactory(record.key,

680

 

                        record.parents, record.sha1,

681

 

                        canonical_inv.to_lines())

682

 

            # We have finished processing all of the inventory records, we

683

 

            # don't need these sets anymore

684

 

        return _filtered_inv_stream()

685

 

 

686

 

    def _use_pack(self, new_pack):

687

 

        """Override _use_pack to check for reconcile having changed content."""

688

 

        return new_pack.data_inserted() and self._data_changed

689

 

 

690

 

 

691

584

class GCRepositoryPackCollection(RepositoryPackCollection):

692

585

 

693

586

    pack_factory = GCPack

755

648

        chk_diff = chk_map.iter_interesting_nodes(

756

649

            chk_bytes_no_fallbacks, root_key_info.interesting_root_keys,

757

650

            root_key_info.uninteresting_root_keys)

 

651

        bytes_to_info = inventory.CHKInventory._bytes_to_utf8name_key

758

652

        text_keys = set()

759

653

        try:

760

 

            for record in _filter_text_keys(chk_diff, text_keys,

761

 

                                            chk_map._bytes_to_text_key):

 

654

            for record in _filter_text_keys(chk_diff, text_keys, bytes_to_info):

762

655

                pass

763

656

        except errors.NoSuchRevision, e:

764

657

            # XXX: It would be nice if we could give a more precise error here.

972

865

        if basis_inv is None:

973

866

            if basis_revision_id == _mod_revision.NULL_REVISION:

974

867

                new_inv = self._create_inv_from_null(delta, new_revision_id)

975

 

                if new_inv.root_id is None:

976

 

                    raise errors.RootMissing()

977

868

                inv_lines = new_inv.to_lines()

978

869

                return self._inventory_add_lines(new_revision_id, parents,

979

870

                    inv_lines, check_content=False), new_inv

1109

1000

        finally:

1110

1001

            pb.finished()

1111

1002

 

1112

 

    @needs_write_lock

1113

 

    def reconcile_canonicalize_chks(self):

1114

 

        """Reconcile this repository to make sure all CHKs are in canonical

1115

 

        form.

1116

 

        """

1117

 

        from bzrlib.reconcile import PackReconciler

1118

 

        reconciler = PackReconciler(self, thorough=True, canonicalize_chks=True)

1119

 

        reconciler.reconcile()

1120

 

        return reconciler

1121

 

 

1122

1003

    def _reconcile_pack(self, collection, packs, extension, revs, pb):

1123

1004

        packer = GCCHKReconcilePacker(collection, packs, extension)

1124

1005

        return packer.pack(pb)

1125

1006

 

1126

 

    def _canonicalize_chks_pack(self, collection, packs, extension, revs, pb):

1127

 

        packer = GCCHKCanonicalizingPacker(collection, packs, extension, revs)

1128

 

        return packer.pack(pb)

1129

 

 

1130

1007

    def _get_source(self, to_format):

1131

1008

        """Return a source for streaming from this repository."""

1132

1009

        if self._format._serializer == to_format._serializer:

1211

1088

                uninteresting_root_keys.add(inv.id_to_entry.key())

1212

1089

                uninteresting_pid_root_keys.add(

1213

1090

                    inv.parent_id_basename_to_file_id.key())

 

1091

        bytes_to_info = inventory.CHKInventory._bytes_to_utf8name_key

1214

1092

        chk_bytes = self.from_repository.chk_bytes

1215

1093

        def _filter_id_to_entry():

1216

1094

            interesting_nodes = chk_map.iter_interesting_nodes(chk_bytes,

1217

1095

                        self._chk_id_roots, uninteresting_root_keys)

1218

1096

            for record in _filter_text_keys(interesting_nodes, self._text_keys,

1219

 

                    chk_map._bytes_to_text_key):

 

1097

                    bytes_to_info):

1220

1098

                if record is not None:

1221

1099

                    yield record

1222

1100

            # Consumed

1232

1110

        yield 'chk_bytes', _get_parent_id_basename_to_file_id_pages()

1233

1111

 

1234

1112

    def get_stream(self, search):

1235

 

        def wrap_and_count(pb, rc, stream):

1236

 

            """Yield records from stream while showing progress."""

1237

 

            count = 0

1238

 

            for record in stream:

1239

 

                if count == rc.STEP:

1240

 

                    rc.increment(count)

1241

 

                    pb.update('Estimate', rc.current, rc.max)

1242

 

                    count = 0

1243

 

                count += 1

1244

 

                yield record

1245

 

 

1246

1113

        revision_ids = search.get_keys()

1247

 

        pb = ui.ui_factory.nested_progress_bar()

1248

 

        rc = self._record_counter

1249

 

        self._record_counter.setup(len(revision_ids))

1250

1114

        for stream_info in self._fetch_revision_texts(revision_ids):

1251

 

            yield (stream_info[0],

1252

 

                wrap_and_count(pb, rc, stream_info[1]))

 

1115

            yield stream_info

1253

1116

        self._revision_keys = [(rev_id,) for rev_id in revision_ids]

1254

1117

        self.from_repository.revisions.clear_cache()

1255

1118

        self.from_repository.signatures.clear_cache()

1256

 

        s = self._get_inventory_stream(self._revision_keys)

1257

 

        yield (s[0], wrap_and_count(pb, rc, s[1]))

 

1119

        yield self._get_inventory_stream(self._revision_keys)

1258

1120

        self.from_repository.inventories.clear_cache()

1259

1121

        # TODO: The keys to exclude might be part of the search recipe

1260

1122

        # For now, exclude all parents that are at the edge of ancestry, for

1263

1125

        parent_keys = from_repo._find_parent_keys_of_revisions(

1264

1126

                        self._revision_keys)

1265

1127

        for stream_info in self._get_filtered_chk_streams(parent_keys):

1266

 

            yield (stream_info[0], wrap_and_count(pb, rc, stream_info[1]))

 

1128

            yield stream_info

1267

1129

        self.from_repository.chk_bytes.clear_cache()

1268

 

        s = self._get_text_stream()

1269

 

        yield (s[0], wrap_and_count(pb, rc, s[1]))

 

1130

        yield self._get_text_stream()

1270

1131

        self.from_repository.texts.clear_cache()

1271

 

        pb.update('Done', rc.max, rc.max)

1272

 

        pb.finished()

1273

1132

 

1274

1133

    def get_stream_for_missing_keys(self, missing_keys):

1275

1134

        # missing keys can only occur when we are byte copying and not

1329

1188

    return result

1330

1189

 

1331

1190

 

1332

 

def _filter_text_keys(interesting_nodes_iterable, text_keys, bytes_to_text_key):

 

1191

def _filter_text_keys(interesting_nodes_iterable, text_keys, bytes_to_info):

1333

1192

    """Iterate the result of iter_interesting_nodes, yielding the records

1334

1193

    and adding to text_keys.

1335

1194

    """

1336

 

    text_keys_update = text_keys.update

1337

1195

    for record, items in interesting_nodes_iterable:

1338

 

        text_keys_update([bytes_to_text_key(b) for n,b in items])

 

1196

        for name, bytes in items:

 

1197

            # Note: we don't care about name_utf8, because groupcompress repos

 

1198

            # are always rich-root, so there are no synthesised root records to

 

1199

            # ignore.

 

1200

            _, file_id, revision_id = bytes_to_info(bytes)

 

1201

            file_id = intern(file_id)

 

1202

            revision_id = intern(revision_id)

 

1203

            text_keys.add(StaticTuple(file_id, revision_id).intern())

1339

1204

        yield record

1340

1205

 

1341

1206

 

• Older »

Loggerhead is a web-based interface for Breezy
Version: 2.0.1