~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/dirstate.py

  • Committer: Samuel Bronson
  • Date: 2009-07-14 02:15:03 UTC
  • mto: (4547.1.1 integration)
  • mto: This revision was merged to the branch mainline in revision 4548.
  • Revision ID: naesten@gmail.com-20090714021503-t8g55xpm2tp53fhs
Clarify comment about bzrlib.foreign.VcsMapping's variable revid_prefix.

Remove the oxymoron "native foreign revisions".

Show diffs side-by-side

added added

removed removed

Lines of Context:
1
 
# Copyright (C) 2006-2010 Canonical Ltd
 
1
# Copyright (C) 2006, 2007, 2008 Canonical Ltd
2
2
#
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
204
204
import bisect
205
205
import binascii
206
206
import errno
207
 
import operator
208
207
import os
209
208
from stat import S_IEXEC
210
209
import stat
1278
1277
    def update_by_delta(self, delta):
1279
1278
        """Apply an inventory delta to the dirstate for tree 0
1280
1279
 
1281
 
        This is the workhorse for apply_inventory_delta in dirstate based
1282
 
        trees.
1283
 
 
1284
1280
        :param delta: An inventory delta.  See Inventory.apply_delta for
1285
1281
            details.
1286
1282
        """
1287
1283
        self._read_dirblocks_if_needed()
1288
 
        encode = cache_utf8.encode
1289
1284
        insertions = {}
1290
1285
        removals = {}
1291
 
        # Accumulate parent references (path_utf8, id), to check for parentless
1292
 
        # items or items placed under files/links/tree-references. We get
1293
 
        # references from every item in the delta that is not a deletion and
1294
 
        # is not itself the root.
1295
 
        parents = set()
1296
 
        # Added ids must not be in the dirstate already. This set holds those
1297
 
        # ids.
1298
 
        new_ids = set()
1299
 
        # This loop transforms the delta to single atomic operations that can
1300
 
        # be executed and validated.
1301
 
        for old_path, new_path, file_id, inv_entry in sorted(
1302
 
            inventory._check_delta_unique_old_paths(
1303
 
            inventory._check_delta_unique_new_paths(
1304
 
            inventory._check_delta_ids_match_entry(
1305
 
            inventory._check_delta_ids_are_valid(
1306
 
            inventory._check_delta_new_path_entry_both_or_None(delta))))),
1307
 
            reverse=True):
 
1286
        for old_path, new_path, file_id, inv_entry in sorted(delta, reverse=True):
1308
1287
            if (file_id in insertions) or (file_id in removals):
1309
1288
                raise errors.InconsistentDelta(old_path or new_path, file_id,
1310
1289
                    "repeated file_id")
1311
1290
            if old_path is not None:
1312
1291
                old_path = old_path.encode('utf-8')
1313
1292
                removals[file_id] = old_path
1314
 
            else:
1315
 
                new_ids.add(file_id)
1316
1293
            if new_path is not None:
1317
 
                if inv_entry is None:
1318
 
                    raise errors.InconsistentDelta(new_path, file_id,
1319
 
                        "new_path with no entry")
1320
1294
                new_path = new_path.encode('utf-8')
1321
 
                dirname_utf8, basename = osutils.split(new_path)
1322
 
                if basename:
1323
 
                    parents.add((dirname_utf8, inv_entry.parent_id))
1324
 
                key = (dirname_utf8, basename, file_id)
 
1295
                dirname, basename = osutils.split(new_path)
 
1296
                key = (dirname, basename, file_id)
1325
1297
                minikind = DirState._kind_to_minikind[inv_entry.kind]
1326
1298
                if minikind == 't':
1327
 
                    fingerprint = inv_entry.reference_revision or ''
 
1299
                    fingerprint = inv_entry.reference_revision
1328
1300
                else:
1329
1301
                    fingerprint = ''
1330
1302
                insertions[file_id] = (key, minikind, inv_entry.executable,
1339
1311
                    minikind = child[1][0][0]
1340
1312
                    fingerprint = child[1][0][4]
1341
1313
                    executable = child[1][0][3]
1342
 
                    old_child_path = osutils.pathjoin(child_dirname,
1343
 
                                                      child_basename)
 
1314
                    old_child_path = osutils.pathjoin(child[0][0],
 
1315
                                                      child[0][1])
1344
1316
                    removals[child[0][2]] = old_child_path
1345
1317
                    child_suffix = child_dirname[len(old_path):]
1346
1318
                    new_child_dirname = (new_path + child_suffix)
1347
1319
                    key = (new_child_dirname, child_basename, child[0][2])
1348
 
                    new_child_path = osutils.pathjoin(new_child_dirname,
1349
 
                                                      child_basename)
 
1320
                    new_child_path = os.path.join(new_child_dirname,
 
1321
                                                  child_basename)
1350
1322
                    insertions[child[0][2]] = (key, minikind, executable,
1351
1323
                                               fingerprint, new_child_path)
1352
 
        self._check_delta_ids_absent(new_ids, delta, 0)
1353
 
        try:
1354
 
            self._apply_removals(removals.iteritems())
1355
 
            self._apply_insertions(insertions.values())
1356
 
            # Validate parents
1357
 
            self._after_delta_check_parents(parents, 0)
1358
 
        except errors.BzrError, e:
1359
 
            self._changes_aborted = True
1360
 
            if 'integrity error' not in str(e):
1361
 
                raise
1362
 
            # _get_entry raises BzrError when a request is inconsistent; we
1363
 
            # want such errors to be shown as InconsistentDelta - and that 
1364
 
            # fits the behaviour we trigger.
1365
 
            raise errors.InconsistentDeltaDelta(delta, "error from _get_entry.")
 
1324
        self._apply_removals(removals.values())
 
1325
        self._apply_insertions(insertions.values())
1366
1326
 
1367
1327
    def _apply_removals(self, removals):
1368
 
        for file_id, path in sorted(removals, reverse=True,
1369
 
            key=operator.itemgetter(1)):
 
1328
        for path in sorted(removals, reverse=True):
1370
1329
            dirname, basename = osutils.split(path)
1371
1330
            block_i, entry_i, d_present, f_present = \
1372
1331
                self._get_block_entry_index(dirname, basename, 0)
1373
 
            try:
1374
 
                entry = self._dirblocks[block_i][1][entry_i]
1375
 
            except IndexError:
1376
 
                self._changes_aborted = True
1377
 
                raise errors.InconsistentDelta(path, file_id,
1378
 
                    "Wrong path for old path.")
1379
 
            if not f_present or entry[1][0][0] in 'ar':
1380
 
                self._changes_aborted = True
1381
 
                raise errors.InconsistentDelta(path, file_id,
1382
 
                    "Wrong path for old path.")
1383
 
            if file_id != entry[0][2]:
1384
 
                self._changes_aborted = True
1385
 
                raise errors.InconsistentDelta(path, file_id,
1386
 
                    "Attempt to remove path has wrong id - found %r."
1387
 
                    % entry[0][2])
 
1332
            entry = self._dirblocks[block_i][1][entry_i]
1388
1333
            self._make_absent(entry)
1389
1334
            # See if we have a malformed delta: deleting a directory must not
1390
1335
            # leave crud behind. This increases the number of bisects needed
1398
1343
                # be due to it being in a parent tree, or a corrupt delta.
1399
1344
                for child_entry in self._dirblocks[block_i][1]:
1400
1345
                    if child_entry[1][0][0] not in ('r', 'a'):
1401
 
                        self._changes_aborted = True
1402
1346
                        raise errors.InconsistentDelta(path, entry[0][2],
1403
1347
                            "The file id was deleted but its children were "
1404
1348
                            "not deleted.")
1405
1349
 
1406
1350
    def _apply_insertions(self, adds):
1407
 
        try:
1408
 
            for key, minikind, executable, fingerprint, path_utf8 in sorted(adds):
1409
 
                self.update_minimal(key, minikind, executable, fingerprint,
1410
 
                                    path_utf8=path_utf8)
1411
 
        except errors.NotVersionedError:
1412
 
            self._changes_aborted = True
1413
 
            raise errors.InconsistentDelta(path_utf8.decode('utf8'), key[2],
1414
 
                "Missing parent")
 
1351
        for key, minikind, executable, fingerprint, path_utf8 in sorted(adds):
 
1352
            self.update_minimal(key, minikind, executable, fingerprint,
 
1353
                                path_utf8=path_utf8)
1415
1354
 
1416
1355
    def update_basis_by_delta(self, delta, new_revid):
1417
1356
        """Update the parents of this tree after a commit.
1461
1400
        # At the same time, to reduce interface friction we convert the input
1462
1401
        # inventory entries to dirstate.
1463
1402
        root_only = ('', '')
1464
 
        # Accumulate parent references (path_utf8, id), to check for parentless
 
1403
        # Accumulate parent references (path and id), to check for parentless
1465
1404
        # items or items placed under files/links/tree-references. We get
1466
1405
        # references from every item in the delta that is not a deletion and
1467
1406
        # is not itself the root.
1468
1407
        parents = set()
1469
 
        # Added ids must not be in the dirstate already. This set holds those
1470
 
        # ids.
1471
 
        new_ids = set()
1472
1408
        for old_path, new_path, file_id, inv_entry in delta:
1473
1409
            if inv_entry is not None and file_id != inv_entry.file_id:
1474
1410
                raise errors.InconsistentDelta(new_path, file_id,
1475
1411
                    "mismatched entry file_id %r" % inv_entry)
1476
1412
            if new_path is not None:
1477
 
                if inv_entry is None:
1478
 
                    raise errors.InconsistentDelta(new_path, file_id,
1479
 
                        "new_path with no entry")
1480
1413
                new_path_utf8 = encode(new_path)
1481
1414
                # note the parent for validation
1482
1415
                dirname_utf8, basename_utf8 = osutils.split(new_path_utf8)
1485
1418
            if old_path is None:
1486
1419
                adds.append((None, encode(new_path), file_id,
1487
1420
                    inv_to_entry(inv_entry), True))
1488
 
                new_ids.add(file_id)
1489
1421
            elif new_path is None:
1490
1422
                deletes.append((encode(old_path), None, file_id, None, True))
1491
1423
            elif (old_path, new_path) != root_only:
1534
1466
                # of everything.
1535
1467
                changes.append((encode(old_path), encode(new_path), file_id,
1536
1468
                    inv_to_entry(inv_entry)))
1537
 
        self._check_delta_ids_absent(new_ids, delta, 1)
 
1469
 
1538
1470
        try:
1539
1471
            # Finish expunging deletes/first half of renames.
1540
1472
            self._update_basis_apply_deletes(deletes)
1543
1475
            # Apply in-situ changes.
1544
1476
            self._update_basis_apply_changes(changes)
1545
1477
            # Validate parents
1546
 
            self._after_delta_check_parents(parents, 1)
 
1478
            self._update_basis_check_parents(parents)
1547
1479
        except errors.BzrError, e:
1548
 
            self._changes_aborted = True
1549
1480
            if 'integrity error' not in str(e):
1550
1481
                raise
1551
1482
            # _get_entry raises BzrError when a request is inconsistent; we
1553
1484
            # fits the behaviour we trigger. Partof this is driven by dirstate
1554
1485
            # only supporting deltas that turn the basis into a closer fit to
1555
1486
            # the active tree.
 
1487
            self._changes_aborted = True
1556
1488
            raise errors.InconsistentDeltaDelta(delta, "error from _get_entry.")
1557
1489
 
1558
1490
        self._dirblock_state = DirState.IN_MEMORY_MODIFIED
1560
1492
        self._id_index = None
1561
1493
        return
1562
1494
 
1563
 
    def _check_delta_ids_absent(self, new_ids, delta, tree_index):
1564
 
        """Check that none of the file_ids in new_ids are present in a tree."""
1565
 
        if not new_ids:
1566
 
            return
1567
 
        id_index = self._get_id_index()
1568
 
        for file_id in new_ids:
1569
 
            for key in id_index.get(file_id, []):
1570
 
                block_i, entry_i, d_present, f_present = \
1571
 
                    self._get_block_entry_index(key[0], key[1], tree_index)
1572
 
                if not f_present:
1573
 
                    # In a different tree
1574
 
                    continue
1575
 
                entry = self._dirblocks[block_i][1][entry_i]
1576
 
                if entry[0][2] != file_id:
1577
 
                    # Different file_id, so not what we want.
1578
 
                    continue
1579
 
                # NB: No changes made before this helper is called, so no need
1580
 
                # to set the _changes_aborted flag.
1581
 
                raise errors.InconsistentDelta(
1582
 
                    ("%s/%s" % key[0:2]).decode('utf8'), file_id,
1583
 
                    "This file_id is new in the delta but already present in "
1584
 
                    "the target")
1585
 
 
1586
1495
    def _update_basis_apply_adds(self, adds):
1587
1496
        """Apply a sequence of adds to tree 1 during update_basis_by_delta.
1588
1497
 
1653
1562
        null = DirState.NULL_PARENT_DETAILS
1654
1563
        for old_path, new_path, file_id, _, real_delete in deletes:
1655
1564
            if real_delete != (new_path is None):
1656
 
                self._changes_aborted = True
1657
1565
                raise AssertionError("bad delete delta")
1658
1566
            # the entry for this file_id must be in tree 1.
1659
1567
            dirname, basename = osutils.split(old_path)
1692
1600
                    # it is being resurrected here, so blank it out temporarily.
1693
1601
                    self._dirblocks[block_index][1][entry_index][1][1] = null
1694
1602
 
1695
 
    def _after_delta_check_parents(self, parents, index):
1696
 
        """Check that parents required by the delta are all intact.
1697
 
        
1698
 
        :param parents: An iterable of (path_utf8, file_id) tuples which are
1699
 
            required to be present in tree 'index' at path_utf8 with id file_id
1700
 
            and be a directory.
1701
 
        :param index: The column in the dirstate to check for parents in.
1702
 
        """
 
1603
    def _update_basis_check_parents(self, parents):
 
1604
        """Check that parents required by the delta are all intact."""
1703
1605
        for dirname_utf8, file_id in parents:
1704
1606
            # Get the entry - the ensures that file_id, dirname_utf8 exists and
1705
1607
            # has the right file id.
1706
 
            entry = self._get_entry(index, file_id, dirname_utf8)
 
1608
            entry = self._get_entry(1, file_id, dirname_utf8)
1707
1609
            if entry[1] is None:
1708
1610
                self._changes_aborted = True
1709
1611
                raise errors.InconsistentDelta(dirname_utf8.decode('utf8'),
1710
1612
                    file_id, "This parent is not present.")
1711
1613
            # Parents of things must be directories
1712
 
            if entry[1][index][0] != 'd':
 
1614
            if entry[1][1][0] != 'd':
1713
1615
                self._changes_aborted = True
1714
1616
                raise errors.InconsistentDelta(dirname_utf8.decode('utf8'),
1715
1617
                    file_id, "This parent is not a directory.")
1997
1899
                entry_index, present = self._find_entry_index(key, block)
1998
1900
                if present:
1999
1901
                    entry = self._dirblocks[block_index][1][entry_index]
2000
 
                    # TODO: We might want to assert that entry[0][2] ==
2001
 
                    #       fileid_utf8.
2002
1902
                    if entry[1][tree_index][0] in 'fdlt':
2003
1903
                        # this is the result we are looking for: the
2004
1904
                        # real home of this file_id in this tree.
2356
2256
        self.update_minimal(('', '', new_id), 'd',
2357
2257
            path_utf8='', packed_stat=entry[1][0][4])
2358
2258
        self._dirblock_state = DirState.IN_MEMORY_MODIFIED
 
2259
        if self._id_index is not None:
 
2260
            self._id_index.setdefault(new_id, set()).add(entry[0])
2359
2261
 
2360
2262
    def set_parent_trees(self, trees, ghosts):
2361
2263
        """Set the parent trees for the dirstate.
2520
2422
        if 'evil' in debug.debug_flags:
2521
2423
            trace.mutter_callsite(1,
2522
2424
                "set_state_from_inventory called; please mutate the tree instead")
2523
 
        tracing = 'dirstate' in debug.debug_flags
2524
 
        if tracing:
2525
 
            trace.mutter("set_state_from_inventory trace:")
2526
2425
        self._read_dirblocks_if_needed()
2527
2426
        # sketch:
2528
2427
        # Two iterators: current data and new data, both in dirblock order.
2537
2436
        new_iterator = new_inv.iter_entries_by_dir()
2538
2437
        # we will be modifying the dirstate, so we need a stable iterator. In
2539
2438
        # future we might write one, for now we just clone the state into a
2540
 
        # list using a copy so that we see every original item and don't have
2541
 
        # to adjust the position when items are inserted or deleted in the
2542
 
        # underlying dirstate.
 
2439
        # list - which is a shallow copy.
2543
2440
        old_iterator = iter(list(self._iter_entries()))
2544
2441
        # both must have roots so this is safe:
2545
2442
        current_new = new_iterator.next()
2579
2476
            # we make both end conditions explicit
2580
2477
            if not current_old:
2581
2478
                # old is finished: insert current_new into the state.
2582
 
                if tracing:
2583
 
                    trace.mutter("Appending from new '%s'.",
2584
 
                        new_path_utf8.decode('utf8'))
2585
2479
                self.update_minimal(new_entry_key, current_new_minikind,
2586
2480
                    executable=current_new[1].executable,
2587
 
                    path_utf8=new_path_utf8, fingerprint=fingerprint,
2588
 
                    fullscan=True)
 
2481
                    path_utf8=new_path_utf8, fingerprint=fingerprint)
2589
2482
                current_new = advance(new_iterator)
2590
2483
            elif not current_new:
2591
2484
                # new is finished
2592
 
                if tracing:
2593
 
                    trace.mutter("Truncating from old '%s/%s'.",
2594
 
                        current_old[0][0].decode('utf8'),
2595
 
                        current_old[0][1].decode('utf8'))
2596
2485
                self._make_absent(current_old)
2597
2486
                current_old = advance(old_iterator)
2598
2487
            elif new_entry_key == current_old[0]:
2605
2494
                # kind has changed.
2606
2495
                if (current_old[1][0][3] != current_new[1].executable or
2607
2496
                    current_old[1][0][0] != current_new_minikind):
2608
 
                    if tracing:
2609
 
                        trace.mutter("Updating in-place change '%s'.",
2610
 
                            new_path_utf8.decode('utf8'))
2611
2497
                    self.update_minimal(current_old[0], current_new_minikind,
2612
2498
                        executable=current_new[1].executable,
2613
 
                        path_utf8=new_path_utf8, fingerprint=fingerprint,
2614
 
                        fullscan=True)
 
2499
                        path_utf8=new_path_utf8, fingerprint=fingerprint)
2615
2500
                # both sides are dealt with, move on
2616
2501
                current_old = advance(old_iterator)
2617
2502
                current_new = advance(new_iterator)
2620
2505
                      and new_entry_key[1:] < current_old[0][1:])):
2621
2506
                # new comes before:
2622
2507
                # add a entry for this and advance new
2623
 
                if tracing:
2624
 
                    trace.mutter("Inserting from new '%s'.",
2625
 
                        new_path_utf8.decode('utf8'))
2626
2508
                self.update_minimal(new_entry_key, current_new_minikind,
2627
2509
                    executable=current_new[1].executable,
2628
 
                    path_utf8=new_path_utf8, fingerprint=fingerprint,
2629
 
                    fullscan=True)
 
2510
                    path_utf8=new_path_utf8, fingerprint=fingerprint)
2630
2511
                current_new = advance(new_iterator)
2631
2512
            else:
2632
2513
                # we've advanced past the place where the old key would be,
2633
2514
                # without seeing it in the new list.  so it must be gone.
2634
 
                if tracing:
2635
 
                    trace.mutter("Deleting from old '%s/%s'.",
2636
 
                        current_old[0][0].decode('utf8'),
2637
 
                        current_old[0][1].decode('utf8'))
2638
2515
                self._make_absent(current_old)
2639
2516
                current_old = advance(old_iterator)
2640
2517
        self._dirblock_state = DirState.IN_MEMORY_MODIFIED
2641
2518
        self._id_index = None
2642
2519
        self._packed_stat_index = None
2643
 
        if tracing:
2644
 
            trace.mutter("set_state_from_inventory complete.")
2645
2520
 
2646
2521
    def _make_absent(self, current_old):
2647
2522
        """Mark current_old - an entry - as absent for tree 0.
2696
2571
        return last_reference
2697
2572
 
2698
2573
    def update_minimal(self, key, minikind, executable=False, fingerprint='',
2699
 
        packed_stat=None, size=0, path_utf8=None, fullscan=False):
 
2574
                       packed_stat=None, size=0, path_utf8=None):
2700
2575
        """Update an entry to the state in tree 0.
2701
2576
 
2702
2577
        This will either create a new entry at 'key' or update an existing one.
2713
2588
        :param size: Size information for new entry
2714
2589
        :param path_utf8: key[0] + '/' + key[1], just passed in to avoid doing
2715
2590
                extra computation.
2716
 
        :param fullscan: If True then a complete scan of the dirstate is being
2717
 
            done and checking for duplicate rows should not be done. This
2718
 
            should only be set by set_state_from_inventory and similar methods.
2719
2591
 
2720
2592
        If packed_stat and fingerprint are not given, they're invalidated in
2721
2593
        the entry.
2730
2602
        new_details = (minikind, fingerprint, size, executable, packed_stat)
2731
2603
        id_index = self._get_id_index()
2732
2604
        if not present:
2733
 
            # New record. Check there isn't a entry at this path already.
2734
 
            if not fullscan:
2735
 
                low_index, _ = self._find_entry_index(key[0:2] + ('',), block)
2736
 
                while low_index < len(block):
2737
 
                    entry = block[low_index]
2738
 
                    if entry[0][0:2] == key[0:2]:
2739
 
                        if entry[1][0][0] not in 'ar':
2740
 
                            # This entry has the same path (but a different id) as
2741
 
                            # the new entry we're adding, and is present in ths
2742
 
                            # tree.
2743
 
                            raise errors.InconsistentDelta(
2744
 
                                ("%s/%s" % key[0:2]).decode('utf8'), key[2],
2745
 
                                "Attempt to add item at path already occupied by "
2746
 
                                "id %r" % entry[0][2])
2747
 
                        low_index += 1
2748
 
                    else:
2749
 
                        break
2750
2605
            # new entry, synthesis cross reference here,
2751
2606
            existing_keys = id_index.setdefault(key[2], set())
2752
2607
            if not existing_keys:
2757
2612
                # grab one of them and use it to generate parent
2758
2613
                # relocation/absent entries.
2759
2614
                new_entry = key, [new_details]
2760
 
                # existing_keys can be changed as we iterate.
2761
 
                for other_key in tuple(existing_keys):
 
2615
                for other_key in existing_keys:
2762
2616
                    # change the record at other to be a pointer to this new
2763
2617
                    # record. The loop looks similar to the change to
2764
2618
                    # relocations when updating an existing record but its not:
2765
2619
                    # the test for existing kinds is different: this can be
2766
2620
                    # factored out to a helper though.
2767
 
                    other_block_index, present = self._find_block_index_from_key(
2768
 
                        other_key)
2769
 
                    if not present:
2770
 
                        raise AssertionError('could not find block for %s' % (
2771
 
                            other_key,))
2772
 
                    other_block = self._dirblocks[other_block_index][1]
2773
 
                    other_entry_index, present = self._find_entry_index(
2774
 
                        other_key, other_block)
2775
 
                    if not present:
2776
 
                        raise AssertionError(
2777
 
                            'update_minimal: could not find other entry for %s'
2778
 
                            % (other_key,))
 
2621
                    other_block_index, present = self._find_block_index_from_key(other_key)
 
2622
                    if not present:
 
2623
                        raise AssertionError('could not find block for %s' % (other_key,))
 
2624
                    other_entry_index, present = self._find_entry_index(other_key,
 
2625
                                            self._dirblocks[other_block_index][1])
 
2626
                    if not present:
 
2627
                        raise AssertionError('could not find entry for %s' % (other_key,))
2779
2628
                    if path_utf8 is None:
2780
2629
                        raise AssertionError('no path')
2781
 
                    # Turn this other location into a reference to the new
2782
 
                    # location. This also updates the aliased iterator
2783
 
                    # (current_old in set_state_from_inventory) so that the old
2784
 
                    # entry, if not already examined, is skipped over by that
2785
 
                    # loop.
2786
 
                    other_entry = other_block[other_entry_index]
2787
 
                    other_entry[1][0] = ('r', path_utf8, 0, False, '')
2788
 
                    self._maybe_remove_row(other_block, other_entry_index,
2789
 
                        id_index)
 
2630
                    self._dirblocks[other_block_index][1][other_entry_index][1][0] = \
 
2631
                        ('r', path_utf8, 0, False, '')
2790
2632
 
2791
 
                # This loop:
2792
 
                # adds a tuple to the new details for each column
2793
 
                #  - either by copying an existing relocation pointer inside that column
2794
 
                #  - or by creating a new pointer to the right row inside that column
2795
2633
                num_present_parents = self._num_present_parents()
2796
 
                if num_present_parents:
2797
 
                    other_key = list(existing_keys)[0]
2798
2634
                for lookup_index in xrange(1, num_present_parents + 1):
2799
2635
                    # grab any one entry, use it to find the right path.
2800
2636
                    # TODO: optimise this to reduce memory use in highly
2807
2643
                    update_entry_index, present = \
2808
2644
                        self._find_entry_index(other_key, self._dirblocks[update_block_index][1])
2809
2645
                    if not present:
2810
 
                        raise AssertionError('update_minimal: could not find entry for %s' % (other_key,))
 
2646
                        raise AssertionError('could not find entry for %s' % (other_key,))
2811
2647
                    update_details = self._dirblocks[update_block_index][1][update_entry_index][1][lookup_index]
2812
2648
                    if update_details[0] in 'ar': # relocated, absent
2813
2649
                        # its a pointer or absent in lookup_index's tree, use
2859
2695
 
2860
2696
        self._dirblock_state = DirState.IN_MEMORY_MODIFIED
2861
2697
 
2862
 
    def _maybe_remove_row(self, block, index, id_index):
2863
 
        """Remove index if it is absent or relocated across the row.
2864
 
        
2865
 
        id_index is updated accordingly.
2866
 
        """
2867
 
        present_in_row = False
2868
 
        entry = block[index]
2869
 
        for column in entry[1]:
2870
 
            if column[0] not in 'ar':
2871
 
                present_in_row = True
2872
 
                break
2873
 
        if not present_in_row:
2874
 
            block.pop(index)
2875
 
            id_index[entry[0][2]].remove(entry[0])
2876
 
 
2877
2698
    def _validate(self):
2878
2699
        """Check that invariants on the dirblock are correct.
2879
2700
 
3013
2834
            if absent_positions == tree_count:
3014
2835
                raise AssertionError(
3015
2836
                    "entry %r has no data for any tree." % (entry,))
3016
 
        if self._id_index is not None:
3017
 
            for file_id, entry_keys in self._id_index.iteritems():
3018
 
                for entry_key in entry_keys:
3019
 
                    if entry_key[2] != file_id:
3020
 
                        raise AssertionError(
3021
 
                            'file_id %r did not match entry key %s'
3022
 
                            % (file_id, entry_key))
3023
2837
 
3024
2838
    def _wipe_state(self):
3025
2839
        """Forget all state information about the dirstate."""
3171
2985
 
3172
2986
class ProcessEntryPython(object):
3173
2987
 
3174
 
    __slots__ = ["old_dirname_to_file_id", "new_dirname_to_file_id",
 
2988
    __slots__ = ["old_dirname_to_file_id", "new_dirname_to_file_id", "uninteresting",
3175
2989
        "last_source_parent", "last_target_parent", "include_unchanged",
3176
 
        "partial", "use_filesystem_for_exec", "utf8_decode",
3177
 
        "searched_specific_files", "search_specific_files",
3178
 
        "searched_exact_paths", "search_specific_file_parents", "seen_ids",
3179
 
        "state", "source_index", "target_index", "want_unversioned", "tree"]
 
2990
        "use_filesystem_for_exec", "utf8_decode", "searched_specific_files",
 
2991
        "search_specific_files", "state", "source_index", "target_index",
 
2992
        "want_unversioned", "tree"]
3180
2993
 
3181
2994
    def __init__(self, include_unchanged, use_filesystem_for_exec,
3182
2995
        search_specific_files, state, source_index, target_index,
3183
2996
        want_unversioned, tree):
3184
2997
        self.old_dirname_to_file_id = {}
3185
2998
        self.new_dirname_to_file_id = {}
3186
 
        # Are we doing a partial iter_changes?
3187
 
        self.partial = search_specific_files != set([''])
 
2999
        # Just a sentry, so that _process_entry can say that this
 
3000
        # record is handled, but isn't interesting to process (unchanged)
 
3001
        self.uninteresting = object()
3188
3002
        # Using a list so that we can access the values and change them in
3189
3003
        # nested scope. Each one is [path, file_id, entry]
3190
3004
        self.last_source_parent = [None, None]
3193
3007
        self.use_filesystem_for_exec = use_filesystem_for_exec
3194
3008
        self.utf8_decode = cache_utf8._utf8_decode
3195
3009
        # for all search_indexs in each path at or under each element of
3196
 
        # search_specific_files, if the detail is relocated: add the id, and
3197
 
        # add the relocated path as one to search if its not searched already.
3198
 
        # If the detail is not relocated, add the id.
 
3010
        # search_specific_files, if the detail is relocated: add the id, and add the
 
3011
        # relocated path as one to search if its not searched already. If the
 
3012
        # detail is not relocated, add the id.
3199
3013
        self.searched_specific_files = set()
3200
 
        # When we search exact paths without expanding downwards, we record
3201
 
        # that here.
3202
 
        self.searched_exact_paths = set()
3203
3014
        self.search_specific_files = search_specific_files
3204
 
        # The parents up to the root of the paths we are searching.
3205
 
        # After all normal paths are returned, these specific items are returned.
3206
 
        self.search_specific_file_parents = set()
3207
 
        # The ids we've sent out in the delta.
3208
 
        self.seen_ids = set()
3209
3015
        self.state = state
3210
3016
        self.source_index = source_index
3211
3017
        self.target_index = target_index
3212
 
        if target_index != 0:
3213
 
            # A lot of code in here depends on target_index == 0
3214
 
            raise errors.BzrError('unsupported target index')
3215
3018
        self.want_unversioned = want_unversioned
3216
3019
        self.tree = tree
3217
3020
 
3219
3022
        """Compare an entry and real disk to generate delta information.
3220
3023
 
3221
3024
        :param path_info: top_relpath, basename, kind, lstat, abspath for
3222
 
            the path of entry. If None, then the path is considered absent in 
3223
 
            the target (Perhaps we should pass in a concrete entry for this ?)
 
3025
            the path of entry. If None, then the path is considered absent.
 
3026
            (Perhaps we should pass in a concrete entry for this ?)
3224
3027
            Basename is returned as a utf8 string because we expect this
3225
3028
            tuple will be ignored, and don't want to take the time to
3226
3029
            decode.
3227
 
        :return: (iter_changes_result, changed). If the entry has not been
3228
 
            handled then changed is None. Otherwise it is False if no content
3229
 
            or metadata changes have occurred, and True if any content or
3230
 
            metadata change has occurred. If self.include_unchanged is True then
3231
 
            if changed is not None, iter_changes_result will always be a result
3232
 
            tuple. Otherwise, iter_changes_result is None unless changed is
3233
 
            True.
 
3030
        :return: None if these don't match
 
3031
                 A tuple of information about the change, or
 
3032
                 the object 'uninteresting' if these match, but are
 
3033
                 basically identical.
3234
3034
        """
3235
3035
        if self.source_index is None:
3236
3036
            source_details = DirState.NULL_PARENT_DETAILS
3335
3135
                        content_change = False
3336
3136
                    target_exec = False
3337
3137
                else:
3338
 
                    if path is None:
3339
 
                        path = pathjoin(old_dirname, old_basename)
3340
 
                    raise errors.BadFileKindError(path, path_info[2])
 
3138
                    raise Exception, "unknown kind %s" % path_info[2]
3341
3139
            if source_minikind == 'd':
3342
3140
                if path is None:
3343
3141
                    old_path = path = pathjoin(old_dirname, old_basename)
3344
3142
                self.old_dirname_to_file_id[old_path] = file_id
3345
3143
            # parent id is the entry for the path in the target tree
3346
 
            if old_basename and old_dirname == self.last_source_parent[0]:
 
3144
            if old_dirname == self.last_source_parent[0]:
3347
3145
                source_parent_id = self.last_source_parent[1]
3348
3146
            else:
3349
3147
                try:
3359
3157
                    self.last_source_parent[0] = old_dirname
3360
3158
                    self.last_source_parent[1] = source_parent_id
3361
3159
            new_dirname = entry[0][0]
3362
 
            if entry[0][1] and new_dirname == self.last_target_parent[0]:
 
3160
            if new_dirname == self.last_target_parent[0]:
3363
3161
                target_parent_id = self.last_target_parent[1]
3364
3162
            else:
3365
3163
                try:
3382
3180
                    self.last_target_parent[1] = target_parent_id
3383
3181
 
3384
3182
            source_exec = source_details[3]
3385
 
            changed = (content_change
 
3183
            if (self.include_unchanged
 
3184
                or content_change
3386
3185
                or source_parent_id != target_parent_id
3387
3186
                or old_basename != entry[0][1]
3388
3187
                or source_exec != target_exec
3389
 
                )
3390
 
            if not changed and not self.include_unchanged:
3391
 
                return None, False
3392
 
            else:
 
3188
                ):
3393
3189
                if old_path is None:
3394
3190
                    old_path = path = pathjoin(old_dirname, old_basename)
3395
3191
                    old_path_u = self.utf8_decode(old_path)[0]
3408
3204
                       (source_parent_id, target_parent_id),
3409
3205
                       (self.utf8_decode(old_basename)[0], self.utf8_decode(entry[0][1])[0]),
3410
3206
                       (source_kind, target_kind),
3411
 
                       (source_exec, target_exec)), changed
 
3207
                       (source_exec, target_exec))
 
3208
            else:
 
3209
                return self.uninteresting
3412
3210
        elif source_minikind in 'a' and target_minikind in 'fdlt':
3413
3211
            # looks like a new file
3414
3212
            path = pathjoin(entry[0][0], entry[0][1])
3435
3233
                       (None, parent_id),
3436
3234
                       (None, self.utf8_decode(entry[0][1])[0]),
3437
3235
                       (None, path_info[2]),
3438
 
                       (None, target_exec)), True
 
3236
                       (None, target_exec))
3439
3237
            else:
3440
3238
                # Its a missing file, report it as such.
3441
3239
                return (entry[0][2],
3445
3243
                       (None, parent_id),
3446
3244
                       (None, self.utf8_decode(entry[0][1])[0]),
3447
3245
                       (None, None),
3448
 
                       (None, False)), True
 
3246
                       (None, False))
3449
3247
        elif source_minikind in 'fdlt' and target_minikind in 'a':
3450
3248
            # unversioned, possibly, or possibly not deleted: we dont care.
3451
3249
            # if its still on disk, *and* theres no other entry at this
3463
3261
                   (parent_id, None),
3464
3262
                   (self.utf8_decode(entry[0][1])[0], None),
3465
3263
                   (DirState._minikind_to_kind[source_minikind], None),
3466
 
                   (source_details[3], None)), True
 
3264
                   (source_details[3], None))
3467
3265
        elif source_minikind in 'fdlt' and target_minikind in 'r':
3468
3266
            # a rename; could be a true rename, or a rename inherited from
3469
3267
            # a renamed parent. TODO: handle this efficiently. Its not
3481
3279
                "source_minikind=%r, target_minikind=%r"
3482
3280
                % (source_minikind, target_minikind))
3483
3281
            ## import pdb;pdb.set_trace()
3484
 
        return None, None
 
3282
        return None
3485
3283
 
3486
3284
    def __iter__(self):
3487
3285
        return self
3488
3286
 
3489
 
    def _gather_result_for_consistency(self, result):
3490
 
        """Check a result we will yield to make sure we are consistent later.
3491
 
        
3492
 
        This gathers result's parents into a set to output later.
3493
 
 
3494
 
        :param result: A result tuple.
3495
 
        """
3496
 
        if not self.partial or not result[0]:
3497
 
            return
3498
 
        self.seen_ids.add(result[0])
3499
 
        new_path = result[1][1]
3500
 
        if new_path:
3501
 
            # Not the root and not a delete: queue up the parents of the path.
3502
 
            self.search_specific_file_parents.update(
3503
 
                osutils.parent_directories(new_path.encode('utf8')))
3504
 
            # Add the root directory which parent_directories does not
3505
 
            # provide.
3506
 
            self.search_specific_file_parents.add('')
3507
 
 
3508
3287
    def iter_changes(self):
3509
3288
        """Iterate over the changes."""
3510
3289
        utf8_decode = cache_utf8._utf8_decode
3511
3290
        _cmp_by_dirs = cmp_by_dirs
3512
3291
        _process_entry = self._process_entry
 
3292
        uninteresting = self.uninteresting
3513
3293
        search_specific_files = self.search_specific_files
3514
3294
        searched_specific_files = self.searched_specific_files
3515
3295
        splitpath = osutils.splitpath
3585
3365
                continue
3586
3366
            path_handled = False
3587
3367
            for entry in root_entries:
3588
 
                result, changed = _process_entry(entry, root_dir_info)
3589
 
                if changed is not None:
 
3368
                result = _process_entry(entry, root_dir_info)
 
3369
                if result is not None:
3590
3370
                    path_handled = True
3591
 
                    if changed:
3592
 
                        self._gather_result_for_consistency(result)
3593
 
                    if changed or self.include_unchanged:
 
3371
                    if result is not uninteresting:
3594
3372
                        yield result
3595
3373
            if self.want_unversioned and not path_handled and root_dir_info:
3596
3374
                new_executable = bool(
3706
3484
                        for current_entry in current_block[1]:
3707
3485
                            # entry referring to file not present on disk.
3708
3486
                            # advance the entry only, after processing.
3709
 
                            result, changed = _process_entry(current_entry, None)
3710
 
                            if changed is not None:
3711
 
                                if changed:
3712
 
                                    self._gather_result_for_consistency(result)
3713
 
                                if changed or self.include_unchanged:
 
3487
                            result = _process_entry(current_entry, None)
 
3488
                            if result is not None:
 
3489
                                if result is not uninteresting:
3714
3490
                                    yield result
3715
3491
                        block_index +=1
3716
3492
                        if (block_index < len(self.state._dirblocks) and
3746
3522
                        pass
3747
3523
                    elif current_path_info is None:
3748
3524
                        # no path is fine: the per entry code will handle it.
3749
 
                        result, changed = _process_entry(current_entry, current_path_info)
3750
 
                        if changed is not None:
3751
 
                            if changed:
3752
 
                                self._gather_result_for_consistency(result)
3753
 
                            if changed or self.include_unchanged:
 
3525
                        result = _process_entry(current_entry, current_path_info)
 
3526
                        if result is not None:
 
3527
                            if result is not uninteresting:
3754
3528
                                yield result
3755
3529
                    elif (current_entry[0][1] != current_path_info[1]
3756
3530
                          or current_entry[1][self.target_index][0] in 'ar'):
3769
3543
                        else:
3770
3544
                            # entry referring to file not present on disk.
3771
3545
                            # advance the entry only, after processing.
3772
 
                            result, changed = _process_entry(current_entry, None)
3773
 
                            if changed is not None:
3774
 
                                if changed:
3775
 
                                    self._gather_result_for_consistency(result)
3776
 
                                if changed or self.include_unchanged:
 
3546
                            result = _process_entry(current_entry, None)
 
3547
                            if result is not None:
 
3548
                                if result is not uninteresting:
3777
3549
                                    yield result
3778
3550
                            advance_path = False
3779
3551
                    else:
3780
 
                        result, changed = _process_entry(current_entry, current_path_info)
3781
 
                        if changed is not None:
 
3552
                        result = _process_entry(current_entry, current_path_info)
 
3553
                        if result is not None:
3782
3554
                            path_handled = True
3783
 
                            if changed:
3784
 
                                self._gather_result_for_consistency(result)
3785
 
                            if changed or self.include_unchanged:
 
3555
                            if result is not uninteresting:
3786
3556
                                yield result
3787
3557
                    if advance_entry and current_entry is not None:
3788
3558
                        entry_index += 1
3847
3617
                        current_dir_info = dir_iterator.next()
3848
3618
                    except StopIteration:
3849
3619
                        current_dir_info = None
3850
 
        for result in self._iter_specific_file_parents():
3851
 
            yield result
3852
 
 
3853
 
    def _iter_specific_file_parents(self):
3854
 
        """Iter over the specific file parents."""
3855
 
        while self.search_specific_file_parents:
3856
 
            # Process the parent directories for the paths we were iterating.
3857
 
            # Even in extremely large trees this should be modest, so currently
3858
 
            # no attempt is made to optimise.
3859
 
            path_utf8 = self.search_specific_file_parents.pop()
3860
 
            if osutils.is_inside_any(self.searched_specific_files, path_utf8):
3861
 
                # We've examined this path.
3862
 
                continue
3863
 
            if path_utf8 in self.searched_exact_paths:
3864
 
                # We've examined this path.
3865
 
                continue
3866
 
            path_entries = self.state._entries_for_path(path_utf8)
3867
 
            # We need either one or two entries. If the path in
3868
 
            # self.target_index has moved (so the entry in source_index is in
3869
 
            # 'ar') then we need to also look for the entry for this path in
3870
 
            # self.source_index, to output the appropriate delete-or-rename.
3871
 
            selected_entries = []
3872
 
            found_item = False
3873
 
            for candidate_entry in path_entries:
3874
 
                # Find entries present in target at this path:
3875
 
                if candidate_entry[1][self.target_index][0] not in 'ar':
3876
 
                    found_item = True
3877
 
                    selected_entries.append(candidate_entry)
3878
 
                # Find entries present in source at this path:
3879
 
                elif (self.source_index is not None and
3880
 
                    candidate_entry[1][self.source_index][0] not in 'ar'):
3881
 
                    found_item = True
3882
 
                    if candidate_entry[1][self.target_index][0] == 'a':
3883
 
                        # Deleted, emit it here.
3884
 
                        selected_entries.append(candidate_entry)
3885
 
                    else:
3886
 
                        # renamed, emit it when we process the directory it
3887
 
                        # ended up at.
3888
 
                        self.search_specific_file_parents.add(
3889
 
                            candidate_entry[1][self.target_index][1])
3890
 
            if not found_item:
3891
 
                raise AssertionError(
3892
 
                    "Missing entry for specific path parent %r, %r" % (
3893
 
                    path_utf8, path_entries))
3894
 
            path_info = self._path_info(path_utf8, path_utf8.decode('utf8'))
3895
 
            for entry in selected_entries:
3896
 
                if entry[0][2] in self.seen_ids:
3897
 
                    continue
3898
 
                result, changed = self._process_entry(entry, path_info)
3899
 
                if changed is None:
3900
 
                    raise AssertionError(
3901
 
                        "Got entry<->path mismatch for specific path "
3902
 
                        "%r entry %r path_info %r " % (
3903
 
                        path_utf8, entry, path_info))
3904
 
                # Only include changes - we're outside the users requested
3905
 
                # expansion.
3906
 
                if changed:
3907
 
                    self._gather_result_for_consistency(result)
3908
 
                    if (result[6][0] == 'directory' and
3909
 
                        result[6][1] != 'directory'):
3910
 
                        # This stopped being a directory, the old children have
3911
 
                        # to be included.
3912
 
                        if entry[1][self.source_index][0] == 'r':
3913
 
                            # renamed, take the source path
3914
 
                            entry_path_utf8 = entry[1][self.source_index][1]
3915
 
                        else:
3916
 
                            entry_path_utf8 = path_utf8
3917
 
                        initial_key = (entry_path_utf8, '', '')
3918
 
                        block_index, _ = self.state._find_block_index_from_key(
3919
 
                            initial_key)
3920
 
                        if block_index == 0:
3921
 
                            # The children of the root are in block index 1.
3922
 
                            block_index +=1
3923
 
                        current_block = None
3924
 
                        if block_index < len(self.state._dirblocks):
3925
 
                            current_block = self.state._dirblocks[block_index]
3926
 
                            if not osutils.is_inside(
3927
 
                                entry_path_utf8, current_block[0]):
3928
 
                                # No entries for this directory at all.
3929
 
                                current_block = None
3930
 
                        if current_block is not None:
3931
 
                            for entry in current_block[1]:
3932
 
                                if entry[1][self.source_index][0] in 'ar':
3933
 
                                    # Not in the source tree, so doesn't have to be
3934
 
                                    # included.
3935
 
                                    continue
3936
 
                                # Path of the entry itself.
3937
 
 
3938
 
                                self.search_specific_file_parents.add(
3939
 
                                    osutils.pathjoin(*entry[0][:2]))
3940
 
                if changed or self.include_unchanged:
3941
 
                    yield result
3942
 
            self.searched_exact_paths.add(path_utf8)
3943
 
 
3944
 
    def _path_info(self, utf8_path, unicode_path):
3945
 
        """Generate path_info for unicode_path.
3946
 
 
3947
 
        :return: None if unicode_path does not exist, or a path_info tuple.
3948
 
        """
3949
 
        abspath = self.tree.abspath(unicode_path)
3950
 
        try:
3951
 
            stat = os.lstat(abspath)
3952
 
        except OSError, e:
3953
 
            if e.errno == errno.ENOENT:
3954
 
                # the path does not exist.
3955
 
                return None
3956
 
            else:
3957
 
                raise
3958
 
        utf8_basename = utf8_path.rsplit('/', 1)[-1]
3959
 
        dir_info = (utf8_path, utf8_basename,
3960
 
            osutils.file_kind_from_stat_mode(stat.st_mode), stat,
3961
 
            abspath)
3962
 
        if dir_info[2] == 'directory':
3963
 
            if self.tree._directory_is_tree_reference(
3964
 
                unicode_path):
3965
 
                self.root_dir_info = self.root_dir_info[:2] + \
3966
 
                    ('tree-reference',) + self.root_dir_info[3:]
3967
 
        return dir_info
3968
3620
 
3969
3621
 
3970
3622
# Try to load the compiled form if possible
3978
3630
        ProcessEntryC as _process_entry,
3979
3631
        update_entry as update_entry,
3980
3632
        )
3981
 
except ImportError, e:
3982
 
    osutils.failed_to_load_extension(e)
 
3633
except ImportError:
3983
3634
    from bzrlib._dirstate_helpers_py import (
3984
3635
        _read_dirblocks,
3985
3636
        bisect_dirblock,