← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/dirstate.py

(jameinel) Bug #380202,
 don't bother rewriting the dirstate if <10 items have changed. (John A
 Meinel)

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/dirstate.py
366
366
    NOT_IN_MEMORY = 0
367
367
    IN_MEMORY_UNMODIFIED = 1
368
368
    IN_MEMORY_MODIFIED = 2
 
369
    IN_MEMORY_HASH_MODIFIED = 3 # Only hash-cache updates
369
370
 
370
371
    # A pack_stat (the x's) that is just noise and will never match the output
371
372
    # of base64 encode.
375
376
    HEADER_FORMAT_2 = '#bazaar dirstate flat format 2\n'
376
377
    HEADER_FORMAT_3 = '#bazaar dirstate flat format 3\n'
377
378
 
378
 
    def __init__(self, path, sha1_provider):
 
379
    def __init__(self, path, sha1_provider, worth_saving_limit=0):
379
380
        """Create a  DirState object.
380
381
 
381
382
        :param path: The path at which the dirstate file on disk should live.
382
383
        :param sha1_provider: an object meeting the SHA1Provider interface.
 
384
        :param worth_saving_limit: when the exact number of hash changed
 
385
            entries is known, only bother saving the dirstate if more than
 
386
            this count of entries have changed.
 
387
            -1 means never save hash changes, 0 means always save hash changes.
383
388
        """
384
389
        # _header_state and _dirblock_state represent the current state
385
390
        # of the dirstate metadata and the per-row data respectiely.
422
427
        # during commit.
423
428
        self._last_block_index = None
424
429
        self._last_entry_index = None
 
430
        # The set of known hash changes
 
431
        self._known_hash_changes = set()
 
432
        # How many hash changed entries can we have without saving
 
433
        self._worth_saving_limit = worth_saving_limit
425
434
 
426
435
    def __repr__(self):
427
436
        return "%s(%r)" % \
428
437
            (self.__class__.__name__, self._filename)
429
438
 
 
439
    def _mark_modified(self, hash_changed_entries=None, header_modified=False):
 
440
        """Mark this dirstate as modified.
 
441
 
 
442
        :param hash_changed_entries: if non-None, mark just these entries as
 
443
            having their hash modified.
 
444
        :param header_modified: mark the header modified as well, not just the
 
445
            dirblocks.
 
446
        """
 
447
        #trace.mutter_callsite(3, "modified hash entries: %s", hash_changed_entries)
 
448
        if hash_changed_entries:
 
449
            self._known_hash_changes.update([e[0] for e in hash_changed_entries])
 
450
            if self._dirblock_state in (DirState.NOT_IN_MEMORY,
 
451
                                        DirState.IN_MEMORY_UNMODIFIED):
 
452
                # If the dirstate is already marked a IN_MEMORY_MODIFIED, then
 
453
                # that takes precedence.
 
454
                self._dirblock_state = DirState.IN_MEMORY_HASH_MODIFIED
 
455
        else:
 
456
            # TODO: Since we now have a IN_MEMORY_HASH_MODIFIED state, we
 
457
            #       should fail noisily if someone tries to set
 
458
            #       IN_MEMORY_MODIFIED but we don't have a write-lock!
 
459
            # We don't know exactly what changed so disable smart saving
 
460
            self._dirblock_state = DirState.IN_MEMORY_MODIFIED
 
461
        if header_modified:
 
462
            self._header_state = DirState.IN_MEMORY_MODIFIED
 
463
 
 
464
    def _mark_unmodified(self):
 
465
        """Mark this dirstate as unmodified."""
 
466
        self._header_state = DirState.IN_MEMORY_UNMODIFIED
 
467
        self._dirblock_state = DirState.IN_MEMORY_UNMODIFIED
 
468
        self._known_hash_changes = set()
 
469
 
430
470
    def add(self, path, file_id, kind, stat, fingerprint):
431
471
        """Add a path to be tracked.
432
472
 
558
598
        if kind == 'directory':
559
599
           # insert a new dirblock
560
600
           self._ensure_block(block_index, entry_index, utf8path)
561
 
        self._dirblock_state = DirState.IN_MEMORY_MODIFIED
 
601
        self._mark_modified()
562
602
        if self._id_index:
563
603
            self._add_to_id_index(self._id_index, entry_key)
564
604
 
1030
1070
 
1031
1071
        self._ghosts = []
1032
1072
        self._parents = [parents[0]]
1033
 
        self._dirblock_state = DirState.IN_MEMORY_MODIFIED
1034
 
        self._header_state = DirState.IN_MEMORY_MODIFIED
 
1073
        self._mark_modified(header_modified=True)
1035
1074
 
1036
1075
    def _empty_parent_info(self):
1037
1076
        return [DirState.NULL_PARENT_DETAILS] * (len(self._parents) -
1567
1606
            # the active tree.
1568
1607
            raise errors.InconsistentDeltaDelta(delta, "error from _get_entry.")
1569
1608
 
1570
 
        self._dirblock_state = DirState.IN_MEMORY_MODIFIED
1571
 
        self._header_state = DirState.IN_MEMORY_MODIFIED
 
1609
        self._mark_modified(header_modified=True)
1572
1610
        self._id_index = None
1573
1611
        return
1574
1612
 
1747
1785
                and stat_value.st_ctime < self._cutoff_time):
1748
1786
                entry[1][0] = ('f', sha1, stat_value.st_size, entry[1][0][3],
1749
1787
                               packed_stat)
1750
 
                self._dirblock_state = DirState.IN_MEMORY_MODIFIED
 
1788
                self._mark_modified([entry])
1751
1789
 
1752
1790
    def _sha_cutoff_time(self):
1753
1791
        """Return cutoff time.
1811
1849
        """Serialise the entire dirstate to a sequence of lines."""
1812
1850
        if (self._header_state == DirState.IN_MEMORY_UNMODIFIED and
1813
1851
            self._dirblock_state == DirState.IN_MEMORY_UNMODIFIED):
1814
 
            # read whats on disk.
 
1852
            # read what's on disk.
1815
1853
            self._state_file.seek(0)
1816
1854
            return self._state_file.readlines()
1817
1855
        lines = []
1818
1856
        lines.append(self._get_parents_line(self.get_parent_ids()))
1819
1857
        lines.append(self._get_ghosts_line(self._ghosts))
1820
 
        # append the root line which is special cased
1821
 
        lines.extend(map(self._entry_to_line, self._iter_entries()))
 
1858
        lines.extend(self._get_entry_lines())
1822
1859
        return self._get_output_lines(lines)
1823
1860
 
1824
1861
    def _get_ghosts_line(self, ghost_ids):
1829
1866
        """Create a line for the state file for parents information."""
1830
1867
        return '\0'.join([str(len(parent_ids))] + parent_ids)
1831
1868
 
 
1869
    def _get_entry_lines(self):
 
1870
        """Create lines for entries."""
 
1871
        return map(self._entry_to_line, self._iter_entries())
 
1872
 
1832
1873
    def _get_fields_to_entry(self):
1833
1874
        """Get a function which converts entry fields into a entry record.
1834
1875
 
2222
2263
        """The number of parent entries in each record row."""
2223
2264
        return len(self._parents) - len(self._ghosts)
2224
2265
 
2225
 
    @staticmethod
2226
 
    def on_file(path, sha1_provider=None):
 
2266
    @classmethod
 
2267
    def on_file(cls, path, sha1_provider=None, worth_saving_limit=0):
2227
2268
        """Construct a DirState on the file at path "path".
2228
2269
 
2229
2270
        :param path: The path at which the dirstate file on disk should live.
2230
2271
        :param sha1_provider: an object meeting the SHA1Provider interface.
2231
2272
            If None, a DefaultSHA1Provider is used.
 
2273
        :param worth_saving_limit: when the exact number of hash changed
 
2274
            entries is known, only bother saving the dirstate if more than
 
2275
            this count of entries have changed. -1 means never save.
2232
2276
        :return: An unlocked DirState object, associated with the given path.
2233
2277
        """
2234
2278
        if sha1_provider is None:
2235
2279
            sha1_provider = DefaultSHA1Provider()
2236
 
        result = DirState(path, sha1_provider)
 
2280
        result = cls(path, sha1_provider,
 
2281
                     worth_saving_limit=worth_saving_limit)
2237
2282
        return result
2238
2283
 
2239
2284
    def _read_dirblocks_if_needed(self):
2331
2376
            trace.mutter('Not saving DirState because '
2332
2377
                    '_changes_aborted is set.')
2333
2378
            return
2334
 
        if (self._header_state == DirState.IN_MEMORY_MODIFIED or
2335
 
            self._dirblock_state == DirState.IN_MEMORY_MODIFIED):
2336
 
 
 
2379
        # TODO: Since we now distinguish IN_MEMORY_MODIFIED from
 
2380
        #       IN_MEMORY_HASH_MODIFIED, we should only fail quietly if we fail
 
2381
        #       to save an IN_MEMORY_HASH_MODIFIED, and fail *noisily* if we
 
2382
        #       fail to save IN_MEMORY_MODIFIED
 
2383
        if self._worth_saving():
2337
2384
            grabbed_write_lock = False
2338
2385
            if self._lock_state != 'w':
2339
2386
                grabbed_write_lock, new_lock = self._lock_token.temporary_write_lock()
2347
2394
                    # We couldn't grab a write lock, so we switch back to a read one
2348
2395
                    return
2349
2396
            try:
 
2397
                lines = self.get_lines()
2350
2398
                self._state_file.seek(0)
2351
 
                self._state_file.writelines(self.get_lines())
 
2399
                self._state_file.writelines(lines)
2352
2400
                self._state_file.truncate()
2353
2401
                self._state_file.flush()
2354
 
                self._header_state = DirState.IN_MEMORY_UNMODIFIED
2355
 
                self._dirblock_state = DirState.IN_MEMORY_UNMODIFIED
 
2402
                self._mark_unmodified()
2356
2403
            finally:
2357
2404
                if grabbed_write_lock:
2358
2405
                    self._lock_token = self._lock_token.restore_read_lock()
2361
2408
                    #       not changed contents. Since restore_read_lock may
2362
2409
                    #       not be an atomic operation.
2363
2410
 
 
2411
    def _worth_saving(self):
 
2412
        """Is it worth saving the dirstate or not?"""
 
2413
        if (self._header_state == DirState.IN_MEMORY_MODIFIED
 
2414
            or self._dirblock_state == DirState.IN_MEMORY_MODIFIED):
 
2415
            return True
 
2416
        if self._dirblock_state == DirState.IN_MEMORY_HASH_MODIFIED:
 
2417
            if self._worth_saving_limit == -1:
 
2418
                # We never save hash changes when the limit is -1
 
2419
                return False
 
2420
            # If we're using smart saving and only a small number of
 
2421
            # entries have changed their hash, don't bother saving. John has
 
2422
            # suggested using a heuristic here based on the size of the
 
2423
            # changed files and/or tree. For now, we go with a configurable
 
2424
            # number of changes, keeping the calculation time
 
2425
            # as low overhead as possible. (This also keeps all existing
 
2426
            # tests passing as the default is 0, i.e. always save.)
 
2427
            if len(self._known_hash_changes) >= self._worth_saving_limit:
 
2428
                return True
 
2429
        return False
 
2430
 
2364
2431
    def _set_data(self, parent_ids, dirblocks):
2365
2432
        """Set the full dirstate data in memory.
2366
2433
 
2374
2441
        """
2375
2442
        # our memory copy is now authoritative.
2376
2443
        self._dirblocks = dirblocks
2377
 
        self._header_state = DirState.IN_MEMORY_MODIFIED
2378
 
        self._dirblock_state = DirState.IN_MEMORY_MODIFIED
 
2444
        self._mark_modified(header_modified=True)
2379
2445
        self._parents = list(parent_ids)
2380
2446
        self._id_index = None
2381
2447
        self._packed_stat_index = None
2401
2467
        self._make_absent(entry)
2402
2468
        self.update_minimal(('', '', new_id), 'd',
2403
2469
            path_utf8='', packed_stat=entry[1][0][4])
2404
 
        self._dirblock_state = DirState.IN_MEMORY_MODIFIED
 
2470
        self._mark_modified()
 
2471
        # XXX: This was added by Ian, we need to make sure there
 
2472
        #      are tests for it, because it isn't in bzr.dev TRUNK
 
2473
        #      It looks like the only place it is called is in setting the root
 
2474
        #      id of the tree. So probably we never had an _id_index when we
 
2475
        #      don't even have a root yet.
 
2476
        if self._id_index is not None:
 
2477
            self._add_to_id_index(self._id_index, entry[0])
2405
2478
 
2406
2479
    def set_parent_trees(self, trees, ghosts):
2407
2480
        """Set the parent trees for the dirstate.
2542
2615
        self._entries_to_current_state(new_entries)
2543
2616
        self._parents = [rev_id for rev_id, tree in trees]
2544
2617
        self._ghosts = list(ghosts)
2545
 
        self._header_state = DirState.IN_MEMORY_MODIFIED
2546
 
        self._dirblock_state = DirState.IN_MEMORY_MODIFIED
 
2618
        self._mark_modified(header_modified=True)
2547
2619
        self._id_index = id_index
2548
2620
 
2549
2621
    def _sort_entries(self, entry_list):
2686
2758
                        current_old[0][1].decode('utf8'))
2687
2759
                self._make_absent(current_old)
2688
2760
                current_old = advance(old_iterator)
2689
 
        self._dirblock_state = DirState.IN_MEMORY_MODIFIED
 
2761
        self._mark_modified()
2690
2762
        self._id_index = None
2691
2763
        self._packed_stat_index = None
2692
2764
        if tracing:
2758
2830
            if update_tree_details[0][0] == 'a': # absent
2759
2831
                raise AssertionError('bad row %r' % (update_tree_details,))
2760
2832
            update_tree_details[0] = DirState.NULL_PARENT_DETAILS
2761
 
        self._dirblock_state = DirState.IN_MEMORY_MODIFIED
 
2833
        self._mark_modified()
2762
2834
        return last_reference
2763
2835
 
2764
2836
    def update_minimal(self, key, minikind, executable=False, fingerprint='',
2933
3005
            if not present:
2934
3006
                self._dirblocks.insert(block_index, (subdir_key[0], []))
2935
3007
 
2936
 
        self._dirblock_state = DirState.IN_MEMORY_MODIFIED
 
3008
        self._mark_modified()
2937
3009
 
2938
3010
    def _maybe_remove_row(self, block, index, id_index):
2939
3011
        """Remove index if it is absent or relocated across the row.
3242
3314
        else:
3243
3315
            worth_saving = False
3244
3316
    elif minikind == 'l':
 
3317
        if saved_minikind == 'l':
 
3318
            worth_saving = False
3245
3319
        link_or_sha1 = state._read_link(abspath, saved_link_or_sha1)
3246
3320
        if state._cutoff_time is None:
3247
3321
            state._sha_cutoff_time()
3253
3327
            entry[1][0] = ('l', '', stat_value.st_size,
3254
3328
                           False, DirState.NULLSTAT)
3255
3329
    if worth_saving:
3256
 
        state._dirblock_state = DirState.IN_MEMORY_MODIFIED
 
3330
        state._mark_modified([entry])
3257
3331
    return link_or_sha1
3258
3332
 
3259
3333