2052.3.1
by John Arbash Meinel
Add tests to cleanup the copyright of all source files |
1 |
# Copyright (C) 2005, 2006 Canonical Ltd
|
1570.1.2
by Robert Collins
Import bzrtools' 'fix' command as 'bzr reconcile.' |
2 |
#
|
3 |
# This program is free software; you can redistribute it and/or modify
|
|
4 |
# it under the terms of the GNU General Public License as published by
|
|
5 |
# the Free Software Foundation; either version 2 of the License, or
|
|
6 |
# (at your option) any later version.
|
|
7 |
#
|
|
8 |
# This program is distributed in the hope that it will be useful,
|
|
9 |
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
10 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
11 |
# GNU General Public License for more details.
|
|
12 |
#
|
|
13 |
# You should have received a copy of the GNU General Public License
|
|
14 |
# along with this program; if not, write to the Free Software
|
|
15 |
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
16 |
||
1570.1.7
by Robert Collins
Replace the slow topo_sort routine with a much faster one for non trivial datasets. |
17 |
"""Reconcilers are able to fix some potential data errors in a branch."""
|
1570.1.2
by Robert Collins
Import bzrtools' 'fix' command as 'bzr reconcile.' |
18 |
|
19 |
||
2592.3.80
by Robert Collins
Make reconcile work, and pass tests. |
20 |
__all__ = [ |
21 |
'KnitReconciler', |
|
22 |
'PackReconciler', |
|
23 |
'reconcile', |
|
24 |
'Reconciler', |
|
25 |
'RepoReconciler', |
|
26 |
]
|
|
1570.1.14
by Robert Collins
Enforce repository consistency during 'fetch' operations. |
27 |
|
28 |
||
2745.6.11
by Aaron Bentley
Fix knit file parents to follow parentage from revision/inventory XML |
29 |
from bzrlib import ( |
2745.6.16
by Aaron Bentley
Update from review |
30 |
errors, |
2745.6.11
by Aaron Bentley
Fix knit file parents to follow parentage from revision/inventory XML |
31 |
ui, |
32 |
repository, |
|
33 |
)
|
|
2819.2.5
by Andrew Bennetts
Make reconcile abort gracefully if the revision index has bad parents. |
34 |
from bzrlib.trace import mutter, note |
1570.1.7
by Robert Collins
Replace the slow topo_sort routine with a much faster one for non trivial datasets. |
35 |
from bzrlib.tsort import TopoSorter |
1570.1.2
by Robert Collins
Import bzrtools' 'fix' command as 'bzr reconcile.' |
36 |
|
37 |
||
1692.1.1
by Robert Collins
* Repository.reconcile now takes a thorough keyword parameter to allow |
38 |
def reconcile(dir, other=None): |
1570.1.2
by Robert Collins
Import bzrtools' 'fix' command as 'bzr reconcile.' |
39 |
"""Reconcile the data in dir.
|
40 |
||
41 |
Currently this is limited to a inventory 'reweave'.
|
|
42 |
||
1570.1.8
by Robert Collins
Only reconcile if doing so will perform gc or correct ancestry. |
43 |
This is a convenience method, for using a Reconciler object.
|
44 |
||
45 |
Directly using Reconciler is recommended for library users that
|
|
46 |
desire fine grained control or analysis of the found issues.
|
|
1692.1.1
by Robert Collins
* Repository.reconcile now takes a thorough keyword parameter to allow |
47 |
|
48 |
:param other: another bzrdir to reconcile against.
|
|
1570.1.2
by Robert Collins
Import bzrtools' 'fix' command as 'bzr reconcile.' |
49 |
"""
|
1692.1.1
by Robert Collins
* Repository.reconcile now takes a thorough keyword parameter to allow |
50 |
reconciler = Reconciler(dir, other=other) |
1570.1.2
by Robert Collins
Import bzrtools' 'fix' command as 'bzr reconcile.' |
51 |
reconciler.reconcile() |
52 |
||
53 |
||
1570.1.6
by Robert Collins
Update fast topological_sort to be a function and to have the topo_sort tests run against it. |
54 |
class Reconciler(object): |
1570.1.14
by Robert Collins
Enforce repository consistency during 'fetch' operations. |
55 |
"""Reconcilers are used to reconcile existing data."""
|
1570.1.6
by Robert Collins
Update fast topological_sort to be a function and to have the topo_sort tests run against it. |
56 |
|
1692.1.1
by Robert Collins
* Repository.reconcile now takes a thorough keyword parameter to allow |
57 |
def __init__(self, dir, other=None): |
58 |
"""Create a Reconciler."""
|
|
1570.1.6
by Robert Collins
Update fast topological_sort to be a function and to have the topo_sort tests run against it. |
59 |
self.bzrdir = dir |
60 |
||
61 |
def reconcile(self): |
|
1570.1.8
by Robert Collins
Only reconcile if doing so will perform gc or correct ancestry. |
62 |
"""Perform reconciliation.
|
63 |
|
|
64 |
After reconciliation the following attributes document found issues:
|
|
65 |
inconsistent_parents: The number of revisions in the repository whose
|
|
66 |
ancestry was being reported incorrectly.
|
|
67 |
garbage_inventories: The number of inventory objects without revisions
|
|
68 |
that were garbage collected.
|
|
69 |
"""
|
|
1594.1.3
by Robert Collins
Fixup pb usage to use nested_progress_bar. |
70 |
self.pb = ui.ui_factory.nested_progress_bar() |
71 |
try: |
|
72 |
self._reconcile() |
|
73 |
finally: |
|
74 |
self.pb.finished() |
|
75 |
||
76 |
def _reconcile(self): |
|
77 |
"""Helper function for performing reconciliation."""
|
|
1570.1.11
by Robert Collins
Make reconcile work with shared repositories. |
78 |
self.repo = self.bzrdir.find_repository() |
1570.1.14
by Robert Collins
Enforce repository consistency during 'fetch' operations. |
79 |
self.pb.note('Reconciling repository %s', |
80 |
self.repo.bzrdir.root_transport.base) |
|
1692.1.1
by Robert Collins
* Repository.reconcile now takes a thorough keyword parameter to allow |
81 |
repo_reconciler = self.repo.reconcile(thorough=True) |
1570.1.14
by Robert Collins
Enforce repository consistency during 'fetch' operations. |
82 |
self.inconsistent_parents = repo_reconciler.inconsistent_parents |
83 |
self.garbage_inventories = repo_reconciler.garbage_inventories |
|
2819.2.5
by Andrew Bennetts
Make reconcile abort gracefully if the revision index has bad parents. |
84 |
if repo_reconciler.aborted: |
85 |
self.pb.note( |
|
86 |
'Reconcile aborted: revision index has inconsistent parents.') |
|
87 |
self.pb.note( |
|
88 |
'Run "bzr check" for more details.') |
|
89 |
else: |
|
90 |
self.pb.note('Reconciliation complete.') |
|
1570.1.14
by Robert Collins
Enforce repository consistency during 'fetch' operations. |
91 |
|
92 |
||
93 |
class RepoReconciler(object): |
|
94 |
"""Reconciler that reconciles a repository.
|
|
95 |
||
2857.1.2
by Robert Collins
Review feedback. |
96 |
The goal of repository reconciliation is to make any derived data
|
2592.3.80
by Robert Collins
Make reconcile work, and pass tests. |
97 |
consistent with the core data committed by a user. This can involve
|
98 |
reindexing, or removing unreferenced data if that can interfere with
|
|
99 |
queries in a given repository.
|
|
100 |
||
1570.1.14
by Robert Collins
Enforce repository consistency during 'fetch' operations. |
101 |
Currently this consists of an inventory reweave with revision cross-checks.
|
102 |
"""
|
|
103 |
||
1692.1.1
by Robert Collins
* Repository.reconcile now takes a thorough keyword parameter to allow |
104 |
def __init__(self, repo, other=None, thorough=False): |
105 |
"""Construct a RepoReconciler.
|
|
106 |
||
107 |
:param thorough: perform a thorough check which may take longer but
|
|
108 |
will correct non-data loss issues such as incorrect
|
|
109 |
cached data.
|
|
110 |
"""
|
|
111 |
self.garbage_inventories = 0 |
|
112 |
self.inconsistent_parents = 0 |
|
2819.2.5
by Andrew Bennetts
Make reconcile abort gracefully if the revision index has bad parents. |
113 |
self.aborted = False |
1570.1.14
by Robert Collins
Enforce repository consistency during 'fetch' operations. |
114 |
self.repo = repo |
1692.1.1
by Robert Collins
* Repository.reconcile now takes a thorough keyword parameter to allow |
115 |
self.thorough = thorough |
1570.1.14
by Robert Collins
Enforce repository consistency during 'fetch' operations. |
116 |
|
117 |
def reconcile(self): |
|
118 |
"""Perform reconciliation.
|
|
119 |
|
|
120 |
After reconciliation the following attributes document found issues:
|
|
121 |
inconsistent_parents: The number of revisions in the repository whose
|
|
122 |
ancestry was being reported incorrectly.
|
|
123 |
garbage_inventories: The number of inventory objects without revisions
|
|
124 |
that were garbage collected.
|
|
125 |
"""
|
|
1570.1.6
by Robert Collins
Update fast topological_sort to be a function and to have the topo_sort tests run against it. |
126 |
self.repo.lock_write() |
127 |
try: |
|
1594.1.3
by Robert Collins
Fixup pb usage to use nested_progress_bar. |
128 |
self.pb = ui.ui_factory.nested_progress_bar() |
129 |
try: |
|
1594.2.7
by Robert Collins
Add versionedfile.fix_parents api for correcting data post hoc. |
130 |
self._reconcile_steps() |
1594.1.3
by Robert Collins
Fixup pb usage to use nested_progress_bar. |
131 |
finally: |
132 |
self.pb.finished() |
|
1570.1.6
by Robert Collins
Update fast topological_sort to be a function and to have the topo_sort tests run against it. |
133 |
finally: |
134 |
self.repo.unlock() |
|
135 |
||
1594.2.7
by Robert Collins
Add versionedfile.fix_parents api for correcting data post hoc. |
136 |
def _reconcile_steps(self): |
137 |
"""Perform the steps to reconcile this repository."""
|
|
1692.1.3
by Robert Collins
Finish the reconcile tweak: filled in ghosts are a data loss issue and need to be checked during fast reconciles. |
138 |
self._reweave_inventory() |
1594.2.7
by Robert Collins
Add versionedfile.fix_parents api for correcting data post hoc. |
139 |
|
1570.1.6
by Robert Collins
Update fast topological_sort to be a function and to have the topo_sort tests run against it. |
140 |
def _reweave_inventory(self): |
1692.1.3
by Robert Collins
Finish the reconcile tweak: filled in ghosts are a data loss issue and need to be checked during fast reconciles. |
141 |
"""Regenerate the inventory weave for the repository from scratch.
|
142 |
|
|
143 |
This is a smart function: it will only do the reweave if doing it
|
|
144 |
will correct data issues. The self.thorough flag controls whether
|
|
145 |
only data-loss causing issues (!self.thorough) or all issues
|
|
146 |
(self.thorough) are treated as requiring the reweave.
|
|
147 |
"""
|
|
148 |
# local because needing to know about WeaveFile is a wart we want to hide
|
|
1563.2.42
by Robert Collins
Stop reconcile on weaves being quadratic. |
149 |
from bzrlib.weave import WeaveFile, Weave |
1563.2.29
by Robert Collins
Remove all but fetch references to repository.revision_store. |
150 |
transaction = self.repo.get_transaction() |
1570.1.6
by Robert Collins
Update fast topological_sort to be a function and to have the topo_sort tests run against it. |
151 |
self.pb.update('Reading inventory data.') |
152 |
self.inventory = self.repo.get_inventory_weave() |
|
153 |
# the total set of revisions to process
|
|
1563.2.29
by Robert Collins
Remove all but fetch references to repository.revision_store. |
154 |
self.pending = set([rev_id for rev_id in self.repo._revision_store.all_revision_ids(transaction)]) |
1570.1.6
by Robert Collins
Update fast topological_sort to be a function and to have the topo_sort tests run against it. |
155 |
|
156 |
# mapping from revision_id to parents
|
|
157 |
self._rev_graph = {} |
|
1570.1.8
by Robert Collins
Only reconcile if doing so will perform gc or correct ancestry. |
158 |
# errors that we detect
|
159 |
self.inconsistent_parents = 0 |
|
1570.1.6
by Robert Collins
Update fast topological_sort to be a function and to have the topo_sort tests run against it. |
160 |
# we need the revision id of each revision and its available parents list
|
1570.1.10
by Robert Collins
UI tweaks to reconcile - show progress for inventory backup. |
161 |
self._setup_steps(len(self.pending)) |
1570.1.6
by Robert Collins
Update fast topological_sort to be a function and to have the topo_sort tests run against it. |
162 |
for rev_id in self.pending: |
163 |
# put a revision into the graph.
|
|
164 |
self._graph_revision(rev_id) |
|
1594.2.2
by Robert Collins
Trivial change to reconcile to mutter the cause of reconciliation to bzr.log |
165 |
self._check_garbage_inventories() |
1692.1.3
by Robert Collins
Finish the reconcile tweak: filled in ghosts are a data loss issue and need to be checked during fast reconciles. |
166 |
# if there are no inconsistent_parents and
|
167 |
# (no garbage inventories or we are not doing a thorough check)
|
|
168 |
if (not self.inconsistent_parents and |
|
169 |
(not self.garbage_inventories or not self.thorough)): |
|
1570.1.8
by Robert Collins
Only reconcile if doing so will perform gc or correct ancestry. |
170 |
self.pb.note('Inventory ok.') |
171 |
return
|
|
1570.1.10
by Robert Collins
UI tweaks to reconcile - show progress for inventory backup. |
172 |
self.pb.update('Backing up inventory...', 0, 0) |
1563.2.25
by Robert Collins
Merge in upstream. |
173 |
self.repo.control_weaves.copy(self.inventory, 'inventory.backup', self.repo.get_transaction()) |
1570.1.8
by Robert Collins
Only reconcile if doing so will perform gc or correct ancestry. |
174 |
self.pb.note('Backup Inventory created.') |
175 |
# asking for '' should never return a non-empty weave
|
|
1616.1.1
by Martin Pool
[merge] robertc |
176 |
new_inventory_vf = self.repo.control_weaves.get_empty('inventory.new', |
1570.1.8
by Robert Collins
Only reconcile if doing so will perform gc or correct ancestry. |
177 |
self.repo.get_transaction()) |
1570.1.6
by Robert Collins
Update fast topological_sort to be a function and to have the topo_sort tests run against it. |
178 |
|
1570.1.4
by Robert Collins
Somewhat optimised version of reconciler. |
179 |
# we have topological order of revisions and non ghost parents ready.
|
1570.1.10
by Robert Collins
UI tweaks to reconcile - show progress for inventory backup. |
180 |
self._setup_steps(len(self._rev_graph)) |
1570.1.7
by Robert Collins
Replace the slow topo_sort routine with a much faster one for non trivial datasets. |
181 |
for rev_id in TopoSorter(self._rev_graph.items()).iter_topo_order(): |
182 |
parents = self._rev_graph[rev_id] |
|
1570.1.4
by Robert Collins
Somewhat optimised version of reconciler. |
183 |
# double check this really is in topological order.
|
1616.1.1
by Martin Pool
[merge] robertc |
184 |
unavailable = [p for p in parents if p not in new_inventory_vf] |
1570.1.4
by Robert Collins
Somewhat optimised version of reconciler. |
185 |
assert len(unavailable) == 0 |
186 |
# this entry has all the non ghost parents in the inventory
|
|
187 |
# file already.
|
|
188 |
self._reweave_step('adding inventories') |
|
1616.1.1
by Martin Pool
[merge] robertc |
189 |
if isinstance(new_inventory_vf, WeaveFile): |
190 |
# It's really a WeaveFile, but we call straight into the
|
|
191 |
# Weave's add method to disable the auto-write-out behaviour.
|
|
1607.1.11
by Robert Collins
Merge from bzr.dev |
192 |
# This is done to avoid a revision_count * time-to-write additional overhead on
|
193 |
# reconcile.
|
|
1616.1.1
by Martin Pool
[merge] robertc |
194 |
new_inventory_vf._check_write_ok() |
2794.1.1
by Robert Collins
Allow knits to be instructed not to add a text based on a sha, for commit. |
195 |
Weave._add_lines(new_inventory_vf, rev_id, parents, |
2805.6.7
by Robert Collins
Review feedback. |
196 |
self.inventory.get_lines(rev_id), None, None, None, False, True) |
1563.2.42
by Robert Collins
Stop reconcile on weaves being quadratic. |
197 |
else: |
1616.1.1
by Martin Pool
[merge] robertc |
198 |
new_inventory_vf.add_lines(rev_id, parents, self.inventory.get_lines(rev_id)) |
1570.1.4
by Robert Collins
Somewhat optimised version of reconciler. |
199 |
|
1616.1.1
by Martin Pool
[merge] robertc |
200 |
if isinstance(new_inventory_vf, WeaveFile): |
201 |
new_inventory_vf._save() |
|
202 |
# if this worked, the set of new_inventory_vf.names should equal
|
|
1570.1.4
by Robert Collins
Somewhat optimised version of reconciler. |
203 |
# self.pending
|
1616.1.1
by Martin Pool
[merge] robertc |
204 |
assert set(new_inventory_vf.versions()) == self.pending |
1570.1.2
by Robert Collins
Import bzrtools' 'fix' command as 'bzr reconcile.' |
205 |
self.pb.update('Writing weave') |
1616.1.1
by Martin Pool
[merge] robertc |
206 |
self.repo.control_weaves.copy(new_inventory_vf, 'inventory', self.repo.get_transaction()) |
1563.2.25
by Robert Collins
Merge in upstream. |
207 |
self.repo.control_weaves.delete('inventory.new', self.repo.get_transaction()) |
1570.1.3
by Robert Collins
Optimise reconcilation to only hit each revision once. |
208 |
self.inventory = None |
1570.1.2
by Robert Collins
Import bzrtools' 'fix' command as 'bzr reconcile.' |
209 |
self.pb.note('Inventory regenerated.') |
1570.1.3
by Robert Collins
Optimise reconcilation to only hit each revision once. |
210 |
|
1570.1.10
by Robert Collins
UI tweaks to reconcile - show progress for inventory backup. |
211 |
def _setup_steps(self, new_total): |
212 |
"""Setup the markers we need to control the progress bar."""
|
|
213 |
self.total = new_total |
|
214 |
self.count = 0 |
|
215 |
||
1570.1.4
by Robert Collins
Somewhat optimised version of reconciler. |
216 |
def _graph_revision(self, rev_id): |
217 |
"""Load a revision into the revision graph."""
|
|
218 |
# pick a random revision
|
|
219 |
# analyse revision id rev_id and put it in the stack.
|
|
220 |
self._reweave_step('loading revisions') |
|
1570.1.13
by Robert Collins
Check for incorrect revision parentage in the weave during revision access. |
221 |
rev = self.repo.get_revision_reconcile(rev_id) |
1570.1.3
by Robert Collins
Optimise reconcilation to only hit each revision once. |
222 |
assert rev.revision_id == rev_id |
223 |
parents = [] |
|
224 |
for parent in rev.parent_ids: |
|
1570.1.14
by Robert Collins
Enforce repository consistency during 'fetch' operations. |
225 |
if self._parent_is_available(parent): |
1570.1.3
by Robert Collins
Optimise reconcilation to only hit each revision once. |
226 |
parents.append(parent) |
227 |
else: |
|
228 |
mutter('found ghost %s', parent) |
|
1570.1.4
by Robert Collins
Somewhat optimised version of reconciler. |
229 |
self._rev_graph[rev_id] = parents |
1692.1.3
by Robert Collins
Finish the reconcile tweak: filled in ghosts are a data loss issue and need to be checked during fast reconciles. |
230 |
if self._parents_are_inconsistent(rev_id, parents): |
1570.1.8
by Robert Collins
Only reconcile if doing so will perform gc or correct ancestry. |
231 |
self.inconsistent_parents += 1 |
1594.2.2
by Robert Collins
Trivial change to reconcile to mutter the cause of reconciliation to bzr.log |
232 |
mutter('Inconsistent inventory parents: id {%s} ' |
233 |
'inventory claims %r, ' |
|
234 |
'available parents are %r, ' |
|
235 |
'unavailable parents are %r', |
|
236 |
rev_id, |
|
1563.2.39
by Robert Collins
Merge from integration. |
237 |
set(self.inventory.get_parents(rev_id)), |
1594.2.2
by Robert Collins
Trivial change to reconcile to mutter the cause of reconciliation to bzr.log |
238 |
set(parents), |
239 |
set(rev.parent_ids).difference(set(parents))) |
|
240 |
||
1692.1.3
by Robert Collins
Finish the reconcile tweak: filled in ghosts are a data loss issue and need to be checked during fast reconciles. |
241 |
def _parents_are_inconsistent(self, rev_id, parents): |
242 |
"""Return True if the parents list of rev_id does not match the weave.
|
|
243 |
||
1759.2.2
by Jelmer Vernooij
Revert some of my spelling fixes and fix some typos after review by Aaron. |
244 |
This detects inconsistencies based on the self.thorough value:
|
1692.1.3
by Robert Collins
Finish the reconcile tweak: filled in ghosts are a data loss issue and need to be checked during fast reconciles. |
245 |
if thorough is on, the first parent value is checked as well as ghost
|
246 |
differences.
|
|
247 |
Otherwise only the ghost differences are evaluated.
|
|
248 |
"""
|
|
249 |
weave_parents = self.inventory.get_parents(rev_id) |
|
250 |
weave_missing_old_ghosts = set(weave_parents) != set(parents) |
|
251 |
first_parent_is_wrong = ( |
|
252 |
len(weave_parents) and len(parents) and |
|
253 |
parents[0] != weave_parents[0]) |
|
254 |
if self.thorough: |
|
255 |
return weave_missing_old_ghosts or first_parent_is_wrong |
|
256 |
else: |
|
257 |
return weave_missing_old_ghosts |
|
258 |
||
1594.2.2
by Robert Collins
Trivial change to reconcile to mutter the cause of reconciliation to bzr.log |
259 |
def _check_garbage_inventories(self): |
260 |
"""Check for garbage inventories which we cannot trust
|
|
261 |
||
262 |
We cant trust them because their pre-requisite file data may not
|
|
263 |
be present - all we know is that their revision was not installed.
|
|
264 |
"""
|
|
1692.1.3
by Robert Collins
Finish the reconcile tweak: filled in ghosts are a data loss issue and need to be checked during fast reconciles. |
265 |
if not self.thorough: |
266 |
return
|
|
1563.2.39
by Robert Collins
Merge from integration. |
267 |
inventories = set(self.inventory.versions()) |
1594.2.2
by Robert Collins
Trivial change to reconcile to mutter the cause of reconciliation to bzr.log |
268 |
revisions = set(self._rev_graph.keys()) |
269 |
garbage = inventories.difference(revisions) |
|
270 |
self.garbage_inventories = len(garbage) |
|
271 |
for revision_id in garbage: |
|
272 |
mutter('Garbage inventory {%s} found.', revision_id) |
|
1570.1.4
by Robert Collins
Somewhat optimised version of reconciler. |
273 |
|
1570.1.14
by Robert Collins
Enforce repository consistency during 'fetch' operations. |
274 |
def _parent_is_available(self, parent): |
275 |
"""True if parent is a fully available revision
|
|
276 |
||
277 |
A fully available revision has a inventory and a revision object in the
|
|
278 |
repository.
|
|
279 |
"""
|
|
280 |
return (parent in self._rev_graph or |
|
281 |
(parent in self.inventory and self.repo.has_revision(parent))) |
|
282 |
||
1570.1.4
by Robert Collins
Somewhat optimised version of reconciler. |
283 |
def _reweave_step(self, message): |
284 |
"""Mark a single step of regeneration complete."""
|
|
285 |
self.pb.update(message, self.count, self.total) |
|
286 |
self.count += 1 |
|
1594.2.7
by Robert Collins
Add versionedfile.fix_parents api for correcting data post hoc. |
287 |
|
288 |
||
289 |
class KnitReconciler(RepoReconciler): |
|
290 |
"""Reconciler that reconciles a knit format repository.
|
|
291 |
||
2592.3.80
by Robert Collins
Make reconcile work, and pass tests. |
292 |
This will detect garbage inventories and remove them in thorough mode.
|
1594.2.7
by Robert Collins
Add versionedfile.fix_parents api for correcting data post hoc. |
293 |
"""
|
294 |
||
295 |
def _reconcile_steps(self): |
|
296 |
"""Perform the steps to reconcile this repository."""
|
|
1692.1.1
by Robert Collins
* Repository.reconcile now takes a thorough keyword parameter to allow |
297 |
if self.thorough: |
2819.2.5
by Andrew Bennetts
Make reconcile abort gracefully if the revision index has bad parents. |
298 |
try: |
299 |
self._load_indexes() |
|
300 |
except errors.BzrCheckError: |
|
301 |
self.aborted = True |
|
302 |
return
|
|
1692.1.1
by Robert Collins
* Repository.reconcile now takes a thorough keyword parameter to allow |
303 |
# knits never suffer this
|
304 |
self._gc_inventory() |
|
2745.6.13
by Aaron Bentley
Misc cleanup |
305 |
self._fix_text_parents() |
1594.2.7
by Robert Collins
Add versionedfile.fix_parents api for correcting data post hoc. |
306 |
|
307 |
def _load_indexes(self): |
|
308 |
"""Load indexes for the reconciliation."""
|
|
309 |
self.transaction = self.repo.get_transaction() |
|
310 |
self.pb.update('Reading indexes.', 0, 2) |
|
311 |
self.inventory = self.repo.get_inventory_weave() |
|
312 |
self.pb.update('Reading indexes.', 1, 2) |
|
2819.2.5
by Andrew Bennetts
Make reconcile abort gracefully if the revision index has bad parents. |
313 |
self.repo._check_for_inconsistent_revision_parents() |
1594.2.7
by Robert Collins
Add versionedfile.fix_parents api for correcting data post hoc. |
314 |
self.revisions = self.repo._revision_store.get_revision_file(self.transaction) |
315 |
self.pb.update('Reading indexes.', 2, 2) |
|
316 |
||
317 |
def _gc_inventory(self): |
|
318 |
"""Remove inventories that are not referenced from the revision store."""
|
|
319 |
self.pb.update('Checking unused inventories.', 0, 1) |
|
320 |
self._check_garbage_inventories() |
|
321 |
self.pb.update('Checking unused inventories.', 1, 3) |
|
322 |
if not self.garbage_inventories: |
|
323 |
self.pb.note('Inventory ok.') |
|
324 |
return
|
|
325 |
self.pb.update('Backing up inventory...', 0, 0) |
|
326 |
self.repo.control_weaves.copy(self.inventory, 'inventory.backup', self.transaction) |
|
327 |
self.pb.note('Backup Inventory created.') |
|
328 |
# asking for '' should never return a non-empty weave
|
|
1616.1.1
by Martin Pool
[merge] robertc |
329 |
new_inventory_vf = self.repo.control_weaves.get_empty('inventory.new', |
1594.2.7
by Robert Collins
Add versionedfile.fix_parents api for correcting data post hoc. |
330 |
self.transaction) |
331 |
||
332 |
# we have topological order of revisions and non ghost parents ready.
|
|
1594.2.9
by Robert Collins
Teach Knit repositories how to handle ghosts without corrupting at all. |
333 |
self._setup_steps(len(self.revisions)) |
334 |
for rev_id in TopoSorter(self.revisions.get_graph().items()).iter_topo_order(): |
|
335 |
parents = self.revisions.get_parents(rev_id) |
|
1594.2.7
by Robert Collins
Add versionedfile.fix_parents api for correcting data post hoc. |
336 |
# double check this really is in topological order.
|
1616.1.1
by Martin Pool
[merge] robertc |
337 |
unavailable = [p for p in parents if p not in new_inventory_vf] |
1594.2.7
by Robert Collins
Add versionedfile.fix_parents api for correcting data post hoc. |
338 |
assert len(unavailable) == 0 |
339 |
# this entry has all the non ghost parents in the inventory
|
|
340 |
# file already.
|
|
341 |
self._reweave_step('adding inventories') |
|
342 |
# ugly but needed, weaves are just way tooooo slow else.
|
|
1616.1.1
by Martin Pool
[merge] robertc |
343 |
new_inventory_vf.add_lines(rev_id, parents, self.inventory.get_lines(rev_id)) |
1594.2.7
by Robert Collins
Add versionedfile.fix_parents api for correcting data post hoc. |
344 |
|
1616.1.1
by Martin Pool
[merge] robertc |
345 |
# if this worked, the set of new_inventory_vf.names should equal
|
1594.2.7
by Robert Collins
Add versionedfile.fix_parents api for correcting data post hoc. |
346 |
# self.pending
|
1616.1.1
by Martin Pool
[merge] robertc |
347 |
assert set(new_inventory_vf.versions()) == set(self.revisions.versions()) |
1594.2.7
by Robert Collins
Add versionedfile.fix_parents api for correcting data post hoc. |
348 |
self.pb.update('Writing weave') |
1616.1.1
by Martin Pool
[merge] robertc |
349 |
self.repo.control_weaves.copy(new_inventory_vf, 'inventory', self.transaction) |
1594.2.7
by Robert Collins
Add versionedfile.fix_parents api for correcting data post hoc. |
350 |
self.repo.control_weaves.delete('inventory.new', self.transaction) |
351 |
self.inventory = None |
|
352 |
self.pb.note('Inventory regenerated.') |
|
353 |
||
354 |
def _check_garbage_inventories(self): |
|
355 |
"""Check for garbage inventories which we cannot trust
|
|
356 |
||
357 |
We cant trust them because their pre-requisite file data may not
|
|
358 |
be present - all we know is that their revision was not installed.
|
|
359 |
"""
|
|
360 |
inventories = set(self.inventory.versions()) |
|
361 |
revisions = set(self.revisions.versions()) |
|
362 |
garbage = inventories.difference(revisions) |
|
363 |
self.garbage_inventories = len(garbage) |
|
364 |
for revision_id in garbage: |
|
365 |
mutter('Garbage inventory {%s} found.', revision_id) |
|
2745.6.11
by Aaron Bentley
Fix knit file parents to follow parentage from revision/inventory XML |
366 |
|
367 |
def _fix_text_parents(self): |
|
2745.6.13
by Aaron Bentley
Misc cleanup |
368 |
"""Fix bad versionedfile parent entries.
|
369 |
||
2745.6.16
by Aaron Bentley
Update from review |
370 |
It is possible for the parents entry in a versionedfile entry to be
|
2745.6.13
by Aaron Bentley
Misc cleanup |
371 |
inconsistent with the values in the revision and inventory.
|
372 |
||
373 |
This method finds entries with such inconsistencies, corrects their
|
|
374 |
parent lists, and replaces the versionedfile with a corrected version.
|
|
375 |
"""
|
|
2745.6.11
by Aaron Bentley
Fix knit file parents to follow parentage from revision/inventory XML |
376 |
transaction = self.repo.get_transaction() |
2745.6.16
by Aaron Bentley
Update from review |
377 |
revision_versions = repository._RevisionTextVersionCache(self.repo) |
2906.1.1
by Andrew Bennetts
Speed up reconcile by not repeatedly fetching the full inventories, by cache heads and parents queries, and by fetching revision trees in batches. |
378 |
versions = self.revisions.versions() |
379 |
revision_versions.prepopulate_revs(versions) |
|
2745.6.12
by Aaron Bentley
Do topological sorting when adding new records to VersionedFile |
380 |
for num, file_id in enumerate(self.repo.weave_store): |
381 |
self.pb.update('Fixing text parents', num, |
|
382 |
len(self.repo.weave_store)) |
|
2745.6.11
by Aaron Bentley
Fix knit file parents to follow parentage from revision/inventory XML |
383 |
vf = self.repo.weave_store.get_weave(file_id, transaction) |
2745.6.47
by Andrew Bennetts
Move check_parents out of VersionedFile. |
384 |
vf_checker = self.repo.get_versioned_file_checker( |
2906.1.1
by Andrew Bennetts
Speed up reconcile by not repeatedly fetching the full inventories, by cache heads and parents queries, and by fetching revision trees in batches. |
385 |
versions, revision_versions) |
2745.6.47
by Andrew Bennetts
Move check_parents out of VersionedFile. |
386 |
versions_with_bad_parents = vf_checker.check_file_version_parents( |
2745.6.49
by Andrew Bennetts
Get rid of bzrlib.repository._RevisionParentsProvider. |
387 |
vf, file_id) |
2745.6.34
by Andrew Bennetts
Remove _find_correct_parents from reconcile; use the result of vf.check_parents() instead. |
388 |
if len(versions_with_bad_parents) == 0: |
2745.6.11
by Aaron Bentley
Fix knit file parents to follow parentage from revision/inventory XML |
389 |
continue
|
2745.6.53
by Andrew Bennetts
Some more changes suggested by review. |
390 |
self._fix_text_parent(file_id, vf, versions_with_bad_parents) |
391 |
||
392 |
def _fix_text_parent(self, file_id, vf, versions_with_bad_parents): |
|
393 |
"""Fix bad versionedfile entries in a single versioned file."""
|
|
394 |
new_vf = self.repo.weave_store.get_empty('temp:%s' % file_id, |
|
395 |
self.transaction) |
|
396 |
new_parents = {} |
|
397 |
for version in vf.versions(): |
|
398 |
if version in versions_with_bad_parents: |
|
399 |
parents = versions_with_bad_parents[version][1] |
|
400 |
else: |
|
401 |
parents = vf.get_parents(version) |
|
402 |
new_parents[version] = parents |
|
2592.3.214
by Robert Collins
Merge bzr.dev. |
403 |
for version in TopoSorter(new_parents.items()).iter_topo_order(): |
2745.6.53
by Andrew Bennetts
Some more changes suggested by review. |
404 |
new_vf.add_lines(version, new_parents[version], |
405 |
vf.get_lines(version)) |
|
406 |
self.repo.weave_store.copy(new_vf, file_id, self.transaction) |
|
407 |
self.repo.weave_store.delete('temp:%s' % file_id, self.transaction) |
|
2745.6.11
by Aaron Bentley
Fix knit file parents to follow parentage from revision/inventory XML |
408 |
|
2592.3.80
by Robert Collins
Make reconcile work, and pass tests. |
409 |
|
410 |
class PackReconciler(RepoReconciler): |
|
411 |
"""Reconciler that reconciles a pack based repository.
|
|
412 |
||
413 |
Garbage inventories do not affect ancestry queries, and removal is
|
|
414 |
considerably more expensive as there is no separate versioned file for
|
|
415 |
them, so they are not cleaned. In short it is currently a no-op.
|
|
416 |
||
417 |
In future this may be a good place to hook in annotation cache checking,
|
|
418 |
index recreation etc.
|
|
419 |
"""
|
|
420 |
||
2592.3.239
by Martin Pool
doc |
421 |
# XXX: The index corruption that _fix_text_parents performs is needed for
|
422 |
# packs, but not yet implemented. The basic approach is to:
|
|
423 |
# - lock the names list
|
|
424 |
# - perform a customised pack() that regenerates data as needed
|
|
425 |
# - unlock the names list
|
|
426 |
# https://bugs.edge.launchpad.net/bzr/+bug/154173
|
|
427 |
||
2592.3.80
by Robert Collins
Make reconcile work, and pass tests. |
428 |
def _reconcile_steps(self): |
429 |
"""Perform the steps to reconcile this repository."""
|