3376.2.12
by Martin Pool
pyflakes corrections (thanks spiv) |
1 |
# Copyright (C) 2005, 2006, 2007, 2008 Canonical Ltd
|
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
2 |
#
|
3 |
# This program is free software; you can redistribute it and/or modify
|
|
4 |
# it under the terms of the GNU General Public License as published by
|
|
5 |
# the Free Software Foundation; either version 2 of the License, or
|
|
6 |
# (at your option) any later version.
|
|
7 |
#
|
|
8 |
# This program is distributed in the hope that it will be useful,
|
|
9 |
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
10 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
11 |
# GNU General Public License for more details.
|
|
12 |
#
|
|
13 |
# You should have received a copy of the GNU General Public License
|
|
14 |
# along with this program; if not, write to the Free Software
|
|
15 |
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
16 |
||
3789.2.20
by John Arbash Meinel
The autopack code can now trigger itself to retry when _copy_revision_texts fails. |
17 |
import sys |
18 |
||
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
19 |
from bzrlib.lazy_import import lazy_import |
20 |
lazy_import(globals(), """ |
|
21 |
from itertools import izip
|
|
2592.3.91
by Robert Collins
Incrementally closing in on a correct fetch for packs. |
22 |
import time
|
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
23 |
|
24 |
from bzrlib import (
|
|
3603.2.1
by Andrew Bennetts
Remove duplicated class definitions, remove unused imports. |
25 |
debug,
|
26 |
graph,
|
|
3734.2.4
by Vincent Ladeuil
Fix python2.6 deprecation warnings related to hashlib. |
27 |
osutils,
|
3603.2.1
by Andrew Bennetts
Remove duplicated class definitions, remove unused imports. |
28 |
pack,
|
29 |
transactions,
|
|
30 |
ui,
|
|
3224.5.16
by Andrew Bennetts
Merge from bzr.dev. |
31 |
xml5,
|
32 |
xml6,
|
|
33 |
xml7,
|
|
3603.2.1
by Andrew Bennetts
Remove duplicated class definitions, remove unused imports. |
34 |
)
|
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
35 |
from bzrlib.index import (
|
3734.2.4
by Vincent Ladeuil
Fix python2.6 deprecation warnings related to hashlib. |
36 |
CombinedGraphIndex,
|
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
37 |
GraphIndex,
|
38 |
GraphIndexBuilder,
|
|
3734.2.4
by Vincent Ladeuil
Fix python2.6 deprecation warnings related to hashlib. |
39 |
GraphIndexPrefixAdapter,
|
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
40 |
InMemoryGraphIndex,
|
41 |
)
|
|
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
42 |
from bzrlib.knit import (
|
43 |
KnitPlainFactory,
|
|
44 |
KnitVersionedFiles,
|
|
45 |
_KnitGraphIndex,
|
|
46 |
_DirectPackAccess,
|
|
47 |
)
|
|
3063.2.1
by Robert Collins
Solve reconciling erroring when multiple portions of a single delta chain are being reinserted. |
48 |
from bzrlib import tsort
|
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
49 |
""") |
50 |
from bzrlib import ( |
|
51 |
bzrdir, |
|
52 |
errors, |
|
53 |
lockable_files, |
|
54 |
lockdir, |
|
3099.3.3
by John Arbash Meinel
Deprecate get_parents() in favor of get_parent_map() |
55 |
symbol_versioning, |
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
56 |
)
|
57 |
||
3603.2.1
by Andrew Bennetts
Remove duplicated class definitions, remove unused imports. |
58 |
from bzrlib.decorators import needs_write_lock |
3735.1.1
by Robert Collins
Add development2 formats using BTree indices. |
59 |
from bzrlib.btree_index import ( |
60 |
BTreeGraphIndex, |
|
61 |
BTreeBuilder, |
|
62 |
)
|
|
63 |
from bzrlib.index import ( |
|
64 |
GraphIndex, |
|
65 |
InMemoryGraphIndex, |
|
66 |
)
|
|
2592.3.166
by Robert Collins
Merge KnitRepository3 removal branch. |
67 |
from bzrlib.repofmt.knitrepo import KnitRepository |
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
68 |
from bzrlib.repository import ( |
2592.3.135
by Robert Collins
Do not create many transient knit objects, saving 4% on commit. |
69 |
CommitBuilder, |
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
70 |
MetaDirRepositoryFormat, |
3376.2.12
by Martin Pool
pyflakes corrections (thanks spiv) |
71 |
RepositoryFormat, |
2592.3.135
by Robert Collins
Do not create many transient knit objects, saving 4% on commit. |
72 |
RootCommitBuilder, |
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
73 |
)
|
74 |
import bzrlib.revision as _mod_revision |
|
3376.2.12
by Martin Pool
pyflakes corrections (thanks spiv) |
75 |
from bzrlib.trace import ( |
76 |
mutter, |
|
77 |
warning, |
|
78 |
)
|
|
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
79 |
|
80 |
||
2592.3.135
by Robert Collins
Do not create many transient knit objects, saving 4% on commit. |
81 |
class PackCommitBuilder(CommitBuilder): |
82 |
"""A subclass of CommitBuilder to add texts with pack semantics.
|
|
83 |
|
|
84 |
Specifically this uses one knit object rather than one knit object per
|
|
85 |
added text, reducing memory and object pressure.
|
|
86 |
"""
|
|
87 |
||
2979.2.2
by Robert Collins
Per-file graph heads detection during commit for pack repositories. |
88 |
def __init__(self, repository, parents, config, timestamp=None, |
89 |
timezone=None, committer=None, revprops=None, |
|
90 |
revision_id=None): |
|
91 |
CommitBuilder.__init__(self, repository, parents, config, |
|
92 |
timestamp=timestamp, timezone=timezone, committer=committer, |
|
93 |
revprops=revprops, revision_id=revision_id) |
|
3099.3.1
by John Arbash Meinel
Implement get_parent_map for ParentProviders |
94 |
self._file_graph = graph.Graph( |
2979.2.2
by Robert Collins
Per-file graph heads detection during commit for pack repositories. |
95 |
repository._pack_collection.text_index.combined_index) |
96 |
||
2979.2.5
by Robert Collins
Make CommitBuilder.heads be _heads as its internal to CommitBuilder only. |
97 |
def _heads(self, file_id, revision_ids): |
2979.2.2
by Robert Collins
Per-file graph heads detection during commit for pack repositories. |
98 |
keys = [(file_id, revision_id) for revision_id in revision_ids] |
99 |
return set([key[1] for key in self._file_graph.heads(keys)]) |
|
100 |
||
2592.3.135
by Robert Collins
Do not create many transient knit objects, saving 4% on commit. |
101 |
|
102 |
class PackRootCommitBuilder(RootCommitBuilder): |
|
103 |
"""A subclass of RootCommitBuilder to add texts with pack semantics.
|
|
104 |
|
|
105 |
Specifically this uses one knit object rather than one knit object per
|
|
106 |
added text, reducing memory and object pressure.
|
|
107 |
"""
|
|
108 |
||
2979.2.2
by Robert Collins
Per-file graph heads detection during commit for pack repositories. |
109 |
def __init__(self, repository, parents, config, timestamp=None, |
110 |
timezone=None, committer=None, revprops=None, |
|
111 |
revision_id=None): |
|
112 |
CommitBuilder.__init__(self, repository, parents, config, |
|
113 |
timestamp=timestamp, timezone=timezone, committer=committer, |
|
114 |
revprops=revprops, revision_id=revision_id) |
|
3099.3.1
by John Arbash Meinel
Implement get_parent_map for ParentProviders |
115 |
self._file_graph = graph.Graph( |
2979.2.2
by Robert Collins
Per-file graph heads detection during commit for pack repositories. |
116 |
repository._pack_collection.text_index.combined_index) |
117 |
||
2979.2.5
by Robert Collins
Make CommitBuilder.heads be _heads as its internal to CommitBuilder only. |
118 |
def _heads(self, file_id, revision_ids): |
2979.2.2
by Robert Collins
Per-file graph heads detection during commit for pack repositories. |
119 |
keys = [(file_id, revision_id) for revision_id in revision_ids] |
120 |
return set([key[1] for key in self._file_graph.heads(keys)]) |
|
121 |
||
2592.3.135
by Robert Collins
Do not create many transient knit objects, saving 4% on commit. |
122 |
|
2592.3.90
by Robert Collins
Slightly broken, but branch and fetch performance is now roughly on par (for bzr.dev) with knits - should be much faster for large repos. |
123 |
class Pack(object): |
2592.3.191
by Robert Collins
Give Pack responsibility for index naming, and two concrete classes - NewPack for new packs and ExistingPack for packs we read from disk. |
124 |
"""An in memory proxy for a pack and its indices.
|
125 |
||
126 |
This is a base class that is not directly used, instead the classes
|
|
127 |
ExistingPack and NewPack are used.
|
|
128 |
"""
|
|
129 |
||
2592.3.197
by Robert Collins
Hand over signature index creation to NewPack. |
130 |
def __init__(self, revision_index, inventory_index, text_index, |
131 |
signature_index): |
|
2592.3.192
by Robert Collins
Move new revision index management to NewPack. |
132 |
"""Create a pack instance.
|
133 |
||
134 |
:param revision_index: A GraphIndex for determining what revisions are
|
|
135 |
present in the Pack and accessing the locations of their texts.
|
|
2592.3.195
by Robert Collins
Move some inventory index logic to NewPack. |
136 |
:param inventory_index: A GraphIndex for determining what inventories are
|
2592.3.196
by Robert Collins
Move some text index logic to NewPack. |
137 |
present in the Pack and accessing the locations of their
|
138 |
texts/deltas.
|
|
139 |
:param text_index: A GraphIndex for determining what file texts
|
|
2592.3.197
by Robert Collins
Hand over signature index creation to NewPack. |
140 |
are present in the pack and accessing the locations of their
|
141 |
texts/deltas (via (fileid, revisionid) tuples).
|
|
3495.3.1
by Martin Pool
doc correction from SuperMMX |
142 |
:param signature_index: A GraphIndex for determining what signatures are
|
2592.3.197
by Robert Collins
Hand over signature index creation to NewPack. |
143 |
present in the Pack and accessing the locations of their texts.
|
2592.3.192
by Robert Collins
Move new revision index management to NewPack. |
144 |
"""
|
145 |
self.revision_index = revision_index |
|
2592.3.195
by Robert Collins
Move some inventory index logic to NewPack. |
146 |
self.inventory_index = inventory_index |
2592.3.196
by Robert Collins
Move some text index logic to NewPack. |
147 |
self.text_index = text_index |
2592.3.197
by Robert Collins
Hand over signature index creation to NewPack. |
148 |
self.signature_index = signature_index |
2592.3.191
by Robert Collins
Give Pack responsibility for index naming, and two concrete classes - NewPack for new packs and ExistingPack for packs we read from disk. |
149 |
|
2592.3.208
by Robert Collins
Start refactoring the knit-pack thunking to be clearer. |
150 |
def access_tuple(self): |
151 |
"""Return a tuple (transport, name) for the pack content."""
|
|
152 |
return self.pack_transport, self.file_name() |
|
153 |
||
2592.3.200
by Robert Collins
Make NewPack reopen the index files, separating out the task of refreshing the index maps in the repository and managing the completion of writing a single pack to disk. |
154 |
def file_name(self): |
155 |
"""Get the file name for the pack on disk."""
|
|
156 |
return self.name + '.pack' |
|
157 |
||
2592.3.194
by Robert Collins
Output the revision index from NewPack.finish |
158 |
def get_revision_count(self): |
159 |
return self.revision_index.key_count() |
|
160 |
||
161 |
def inventory_index_name(self, name): |
|
162 |
"""The inv index is the name + .iix."""
|
|
2592.3.200
by Robert Collins
Make NewPack reopen the index files, separating out the task of refreshing the index maps in the repository and managing the completion of writing a single pack to disk. |
163 |
return self.index_name('inventory', name) |
2592.3.194
by Robert Collins
Output the revision index from NewPack.finish |
164 |
|
2592.3.191
by Robert Collins
Give Pack responsibility for index naming, and two concrete classes - NewPack for new packs and ExistingPack for packs we read from disk. |
165 |
def revision_index_name(self, name): |
166 |
"""The revision index is the name + .rix."""
|
|
2592.3.200
by Robert Collins
Make NewPack reopen the index files, separating out the task of refreshing the index maps in the repository and managing the completion of writing a single pack to disk. |
167 |
return self.index_name('revision', name) |
2592.3.191
by Robert Collins
Give Pack responsibility for index naming, and two concrete classes - NewPack for new packs and ExistingPack for packs we read from disk. |
168 |
|
169 |
def signature_index_name(self, name): |
|
170 |
"""The signature index is the name + .six."""
|
|
2592.3.200
by Robert Collins
Make NewPack reopen the index files, separating out the task of refreshing the index maps in the repository and managing the completion of writing a single pack to disk. |
171 |
return self.index_name('signature', name) |
2592.3.191
by Robert Collins
Give Pack responsibility for index naming, and two concrete classes - NewPack for new packs and ExistingPack for packs we read from disk. |
172 |
|
173 |
def text_index_name(self, name): |
|
174 |
"""The text index is the name + .tix."""
|
|
2592.3.200
by Robert Collins
Make NewPack reopen the index files, separating out the task of refreshing the index maps in the repository and managing the completion of writing a single pack to disk. |
175 |
return self.index_name('text', name) |
2592.3.191
by Robert Collins
Give Pack responsibility for index naming, and two concrete classes - NewPack for new packs and ExistingPack for packs we read from disk. |
176 |
|
3035.2.5
by John Arbash Meinel
Rename function to remove _new_ (per Robert's suggestion) |
177 |
def _external_compression_parents_of_texts(self): |
3035.2.4
by John Arbash Meinel
Fix bug #165290 by having the fetch code check that all external references are satisfied before it allows the data to be committed. |
178 |
keys = set() |
179 |
refs = set() |
|
180 |
for node in self.text_index.iter_all_entries(): |
|
181 |
keys.add(node[1]) |
|
182 |
refs.update(node[3][1]) |
|
183 |
return refs - keys |
|
184 |
||
2592.3.191
by Robert Collins
Give Pack responsibility for index naming, and two concrete classes - NewPack for new packs and ExistingPack for packs we read from disk. |
185 |
|
186 |
class ExistingPack(Pack): |
|
2592.3.222
by Robert Collins
More review feedback. |
187 |
"""An in memory proxy for an existing .pack and its disk indices."""
|
2592.3.90
by Robert Collins
Slightly broken, but branch and fetch performance is now roughly on par (for bzr.dev) with knits - should be much faster for large repos. |
188 |
|
2592.3.200
by Robert Collins
Make NewPack reopen the index files, separating out the task of refreshing the index maps in the repository and managing the completion of writing a single pack to disk. |
189 |
def __init__(self, pack_transport, name, revision_index, inventory_index, |
2592.3.177
by Robert Collins
Make all parameters to Pack objects mandatory. |
190 |
text_index, signature_index): |
2592.3.200
by Robert Collins
Make NewPack reopen the index files, separating out the task of refreshing the index maps in the repository and managing the completion of writing a single pack to disk. |
191 |
"""Create an ExistingPack object.
|
192 |
||
193 |
:param pack_transport: The transport where the pack file resides.
|
|
194 |
:param name: The name of the pack on disk in the pack_transport.
|
|
195 |
"""
|
|
2592.3.197
by Robert Collins
Hand over signature index creation to NewPack. |
196 |
Pack.__init__(self, revision_index, inventory_index, text_index, |
197 |
signature_index) |
|
2592.3.173
by Robert Collins
Basic implementation of all_packs. |
198 |
self.name = name |
2592.3.200
by Robert Collins
Make NewPack reopen the index files, separating out the task of refreshing the index maps in the repository and managing the completion of writing a single pack to disk. |
199 |
self.pack_transport = pack_transport |
3376.2.4
by Martin Pool
Remove every assert statement from bzrlib! |
200 |
if None in (revision_index, inventory_index, text_index, |
201 |
signature_index, name, pack_transport): |
|
202 |
raise AssertionError() |
|
2592.3.173
by Robert Collins
Basic implementation of all_packs. |
203 |
|
204 |
def __eq__(self, other): |
|
205 |
return self.__dict__ == other.__dict__ |
|
206 |
||
207 |
def __ne__(self, other): |
|
208 |
return not self.__eq__(other) |
|
209 |
||
210 |
def __repr__(self): |
|
211 |
return "<bzrlib.repofmt.pack_repo.Pack object at 0x%x, %s, %s" % ( |
|
3221.12.4
by Robert Collins
Implement basic repository supporting external references. |
212 |
id(self), self.pack_transport, self.name) |
2592.3.90
by Robert Collins
Slightly broken, but branch and fetch performance is now roughly on par (for bzr.dev) with knits - should be much faster for large repos. |
213 |
|
214 |
||
2592.3.191
by Robert Collins
Give Pack responsibility for index naming, and two concrete classes - NewPack for new packs and ExistingPack for packs we read from disk. |
215 |
class NewPack(Pack): |
216 |
"""An in memory proxy for a pack which is being created."""
|
|
217 |
||
2592.3.200
by Robert Collins
Make NewPack reopen the index files, separating out the task of refreshing the index maps in the repository and managing the completion of writing a single pack to disk. |
218 |
# A map of index 'type' to the file extension and position in the
|
219 |
# index_sizes array.
|
|
2592.3.227
by Martin Pool
Rename NewPack.indices to NewPack.index_definitions |
220 |
index_definitions = { |
2592.3.226
by Martin Pool
formatting and docstrings |
221 |
'revision': ('.rix', 0), |
222 |
'inventory': ('.iix', 1), |
|
223 |
'text': ('.tix', 2), |
|
224 |
'signature': ('.six', 3), |
|
2592.3.200
by Robert Collins
Make NewPack reopen the index files, separating out the task of refreshing the index maps in the repository and managing the completion of writing a single pack to disk. |
225 |
}
|
226 |
||
2592.3.194
by Robert Collins
Output the revision index from NewPack.finish |
227 |
def __init__(self, upload_transport, index_transport, pack_transport, |
3735.1.1
by Robert Collins
Add development2 formats using BTree indices. |
228 |
upload_suffix='', file_mode=None, index_builder_class=None, |
229 |
index_class=None): |
|
2592.3.194
by Robert Collins
Output the revision index from NewPack.finish |
230 |
"""Create a NewPack instance.
|
231 |
||
232 |
:param upload_transport: A writable transport for the pack to be
|
|
233 |
incrementally uploaded to.
|
|
234 |
:param index_transport: A writable transport for the pack's indices to
|
|
235 |
be written to when the pack is finished.
|
|
236 |
:param pack_transport: A writable transport for the pack to be renamed
|
|
2592.3.206
by Robert Collins
Move pack rename-into-place into NewPack.finish and document hash-collision cases somewhat better. |
237 |
to when the upload is complete. This *must* be the same as
|
238 |
upload_transport.clone('../packs').
|
|
2592.3.194
by Robert Collins
Output the revision index from NewPack.finish |
239 |
:param upload_suffix: An optional suffix to be given to any temporary
|
240 |
files created during the pack creation. e.g '.autopack'
|
|
3010.1.11
by Robert Collins
Provide file modes to files created by pack repositories |
241 |
:param file_mode: An optional file mode to create the new files with.
|
3735.1.1
by Robert Collins
Add development2 formats using BTree indices. |
242 |
:param index_builder_class: Required keyword parameter - the class of
|
243 |
index builder to use.
|
|
244 |
:param index_class: Required keyword parameter - the class of index
|
|
245 |
object to use.
|
|
2592.3.194
by Robert Collins
Output the revision index from NewPack.finish |
246 |
"""
|
2592.3.228
by Martin Pool
docstrings and error messages from review |
247 |
# The relative locations of the packs are constrained, but all are
|
248 |
# passed in because the caller has them, so as to avoid object churn.
|
|
2592.3.195
by Robert Collins
Move some inventory index logic to NewPack. |
249 |
Pack.__init__(self, |
250 |
# Revisions: parents list, no text compression.
|
|
3735.1.1
by Robert Collins
Add development2 formats using BTree indices. |
251 |
index_builder_class(reference_lists=1), |
2592.3.195
by Robert Collins
Move some inventory index logic to NewPack. |
252 |
# Inventory: We want to map compression only, but currently the
|
253 |
# knit code hasn't been updated enough to understand that, so we
|
|
254 |
# have a regular 2-list index giving parents and compression
|
|
255 |
# source.
|
|
3735.1.1
by Robert Collins
Add development2 formats using BTree indices. |
256 |
index_builder_class(reference_lists=2), |
2592.3.196
by Robert Collins
Move some text index logic to NewPack. |
257 |
# Texts: compression and per file graph, for all fileids - so two
|
258 |
# reference lists and two elements in the key tuple.
|
|
3735.1.1
by Robert Collins
Add development2 formats using BTree indices. |
259 |
index_builder_class(reference_lists=2, key_elements=2), |
2592.3.197
by Robert Collins
Hand over signature index creation to NewPack. |
260 |
# Signatures: Just blobs to store, no compression, no parents
|
261 |
# listing.
|
|
3735.1.1
by Robert Collins
Add development2 formats using BTree indices. |
262 |
index_builder_class(reference_lists=0), |
2592.3.196
by Robert Collins
Move some text index logic to NewPack. |
263 |
)
|
3735.1.1
by Robert Collins
Add development2 formats using BTree indices. |
264 |
# When we make readonly indices, we need this.
|
265 |
self.index_class = index_class |
|
2592.3.194
by Robert Collins
Output the revision index from NewPack.finish |
266 |
# where should the new pack be opened
|
267 |
self.upload_transport = upload_transport |
|
268 |
# where are indices written out to
|
|
269 |
self.index_transport = index_transport |
|
270 |
# where is the pack renamed to when it is finished?
|
|
271 |
self.pack_transport = pack_transport |
|
3010.1.11
by Robert Collins
Provide file modes to files created by pack repositories |
272 |
# What file mode to upload the pack and indices with.
|
273 |
self._file_mode = file_mode |
|
2592.3.193
by Robert Collins
Move hash tracking of new packs into NewPack. |
274 |
# tracks the content written to the .pack file.
|
3734.2.4
by Vincent Ladeuil
Fix python2.6 deprecation warnings related to hashlib. |
275 |
self._hash = osutils.md5() |
2592.3.194
by Robert Collins
Output the revision index from NewPack.finish |
276 |
# a four-tuple with the length in bytes of the indices, once the pack
|
2592.3.195
by Robert Collins
Move some inventory index logic to NewPack. |
277 |
# is finalised. (rev, inv, text, sigs)
|
2592.3.194
by Robert Collins
Output the revision index from NewPack.finish |
278 |
self.index_sizes = None |
2592.3.203
by Robert Collins
Teach NewPack how to buffer for pack operations. |
279 |
# How much data to cache when writing packs. Note that this is not
|
2592.3.222
by Robert Collins
More review feedback. |
280 |
# synchronised with reads, because it's not in the transport layer, so
|
2592.3.203
by Robert Collins
Teach NewPack how to buffer for pack operations. |
281 |
# is not safe unless the client knows it won't be reading from the pack
|
282 |
# under creation.
|
|
283 |
self._cache_limit = 0 |
|
2592.3.194
by Robert Collins
Output the revision index from NewPack.finish |
284 |
# the temporary pack file name.
|
3734.2.4
by Vincent Ladeuil
Fix python2.6 deprecation warnings related to hashlib. |
285 |
self.random_name = osutils.rand_chars(20) + upload_suffix |
2592.3.194
by Robert Collins
Output the revision index from NewPack.finish |
286 |
# when was this pack started ?
|
287 |
self.start_time = time.time() |
|
2592.3.202
by Robert Collins
Move write stream management into NewPack. |
288 |
# open an output stream for the data added to the pack.
|
289 |
self.write_stream = self.upload_transport.open_write_stream( |
|
3010.1.11
by Robert Collins
Provide file modes to files created by pack repositories |
290 |
self.random_name, mode=self._file_mode) |
2592.3.234
by Martin Pool
Use -Dpack not -Dfetch for pack traces |
291 |
if 'pack' in debug.debug_flags: |
2592.3.202
by Robert Collins
Move write stream management into NewPack. |
292 |
mutter('%s: create_pack: pack stream open: %s%s t+%6.3fs', |
293 |
time.ctime(), self.upload_transport.base, self.random_name, |
|
294 |
time.time() - self.start_time) |
|
2592.3.233
by Martin Pool
Review cleanups |
295 |
# A list of byte sequences to be written to the new pack, and the
|
296 |
# aggregate size of them. Stored as a list rather than separate
|
|
297 |
# variables so that the _write_data closure below can update them.
|
|
2592.3.203
by Robert Collins
Teach NewPack how to buffer for pack operations. |
298 |
self._buffer = [[], 0] |
2592.3.233
by Martin Pool
Review cleanups |
299 |
# create a callable for adding data
|
300 |
#
|
|
301 |
# robertc says- this is a closure rather than a method on the object
|
|
302 |
# so that the variables are locals, and faster than accessing object
|
|
303 |
# members.
|
|
2592.3.203
by Robert Collins
Teach NewPack how to buffer for pack operations. |
304 |
def _write_data(bytes, flush=False, _buffer=self._buffer, |
305 |
_write=self.write_stream.write, _update=self._hash.update): |
|
306 |
_buffer[0].append(bytes) |
|
307 |
_buffer[1] += len(bytes) |
|
2592.3.222
by Robert Collins
More review feedback. |
308 |
# buffer cap
|
2592.3.203
by Robert Collins
Teach NewPack how to buffer for pack operations. |
309 |
if _buffer[1] > self._cache_limit or flush: |
310 |
bytes = ''.join(_buffer[0]) |
|
311 |
_write(bytes) |
|
312 |
_update(bytes) |
|
313 |
_buffer[:] = [[], 0] |
|
2592.3.202
by Robert Collins
Move write stream management into NewPack. |
314 |
# expose this on self, for the occasion when clients want to add data.
|
315 |
self._write_data = _write_data |
|
2592.3.205
by Robert Collins
Move the pack ContainerWriter instance into NewPack. |
316 |
# a pack writer object to serialise pack records.
|
317 |
self._writer = pack.ContainerWriter(self._write_data) |
|
318 |
self._writer.begin() |
|
2592.3.208
by Robert Collins
Start refactoring the knit-pack thunking to be clearer. |
319 |
# what state is the pack in? (open, finished, aborted)
|
320 |
self._state = 'open' |
|
2592.3.202
by Robert Collins
Move write stream management into NewPack. |
321 |
|
322 |
def abort(self): |
|
323 |
"""Cancel creating this pack."""
|
|
2592.3.208
by Robert Collins
Start refactoring the knit-pack thunking to be clearer. |
324 |
self._state = 'aborted' |
2938.1.1
by Robert Collins
trivial fix for packs@win32: explicitly close file before deleting |
325 |
self.write_stream.close() |
2592.3.202
by Robert Collins
Move write stream management into NewPack. |
326 |
# Remove the temporary pack file.
|
327 |
self.upload_transport.delete(self.random_name) |
|
328 |
# The indices have no state on disk.
|
|
2592.3.194
by Robert Collins
Output the revision index from NewPack.finish |
329 |
|
2592.3.208
by Robert Collins
Start refactoring the knit-pack thunking to be clearer. |
330 |
def access_tuple(self): |
331 |
"""Return a tuple (transport, name) for the pack content."""
|
|
332 |
if self._state == 'finished': |
|
333 |
return Pack.access_tuple(self) |
|
3376.2.4
by Martin Pool
Remove every assert statement from bzrlib! |
334 |
elif self._state == 'open': |
2592.3.208
by Robert Collins
Start refactoring the knit-pack thunking to be clearer. |
335 |
return self.upload_transport, self.random_name |
3376.2.4
by Martin Pool
Remove every assert statement from bzrlib! |
336 |
else: |
337 |
raise AssertionError(self._state) |
|
2592.3.208
by Robert Collins
Start refactoring the knit-pack thunking to be clearer. |
338 |
|
2592.3.198
by Robert Collins
Factor out data_inserted to reduce code duplication in detecting empty packs. |
339 |
def data_inserted(self): |
340 |
"""True if data has been added to this pack."""
|
|
2592.3.233
by Martin Pool
Review cleanups |
341 |
return bool(self.get_revision_count() or |
342 |
self.inventory_index.key_count() or |
|
343 |
self.text_index.key_count() or |
|
344 |
self.signature_index.key_count()) |
|
2592.3.198
by Robert Collins
Factor out data_inserted to reduce code duplication in detecting empty packs. |
345 |
|
2592.3.194
by Robert Collins
Output the revision index from NewPack.finish |
346 |
def finish(self): |
347 |
"""Finish the new pack.
|
|
348 |
||
349 |
This:
|
|
350 |
- finalises the content
|
|
351 |
- assigns a name (the md5 of the content, currently)
|
|
352 |
- writes out the associated indices
|
|
353 |
- renames the pack into place.
|
|
354 |
- stores the index size tuple for the pack in the index_sizes
|
|
355 |
attribute.
|
|
356 |
"""
|
|
2592.3.205
by Robert Collins
Move the pack ContainerWriter instance into NewPack. |
357 |
self._writer.end() |
2592.3.203
by Robert Collins
Teach NewPack how to buffer for pack operations. |
358 |
if self._buffer[1]: |
359 |
self._write_data('', flush=True) |
|
2592.3.199
by Robert Collins
Store the name of a NewPack in the object upon finish(). |
360 |
self.name = self._hash.hexdigest() |
2592.3.203
by Robert Collins
Teach NewPack how to buffer for pack operations. |
361 |
# write indices
|
2592.3.233
by Martin Pool
Review cleanups |
362 |
# XXX: It'd be better to write them all to temporary names, then
|
363 |
# rename them all into place, so that the window when only some are
|
|
364 |
# visible is smaller. On the other hand none will be seen until
|
|
365 |
# they're in the names list.
|
|
2592.3.194
by Robert Collins
Output the revision index from NewPack.finish |
366 |
self.index_sizes = [None, None, None, None] |
2592.3.200
by Robert Collins
Make NewPack reopen the index files, separating out the task of refreshing the index maps in the repository and managing the completion of writing a single pack to disk. |
367 |
self._write_index('revision', self.revision_index, 'revision') |
368 |
self._write_index('inventory', self.inventory_index, 'inventory') |
|
369 |
self._write_index('text', self.text_index, 'file texts') |
|
370 |
self._write_index('signature', self.signature_index, |
|
371 |
'revision signatures') |
|
2592.3.202
by Robert Collins
Move write stream management into NewPack. |
372 |
self.write_stream.close() |
2592.3.206
by Robert Collins
Move pack rename-into-place into NewPack.finish and document hash-collision cases somewhat better. |
373 |
# Note that this will clobber an existing pack with the same name,
|
374 |
# without checking for hash collisions. While this is undesirable this
|
|
375 |
# is something that can be rectified in a subsequent release. One way
|
|
376 |
# to rectify it may be to leave the pack at the original name, writing
|
|
377 |
# its pack-names entry as something like 'HASH: index-sizes
|
|
378 |
# temporary-name'. Allocate that and check for collisions, if it is
|
|
379 |
# collision free then rename it into place. If clients know this scheme
|
|
380 |
# they can handle missing-file errors by:
|
|
381 |
# - try for HASH.pack
|
|
382 |
# - try for temporary-name
|
|
383 |
# - refresh the pack-list to see if the pack is now absent
|
|
384 |
self.upload_transport.rename(self.random_name, |
|
385 |
'../packs/' + self.name + '.pack') |
|
2592.3.211
by Robert Collins
Pack inventory index management cleaned up. |
386 |
self._state = 'finished' |
2592.3.234
by Martin Pool
Use -Dpack not -Dfetch for pack traces |
387 |
if 'pack' in debug.debug_flags: |
2592.3.219
by Robert Collins
Review feedback. |
388 |
# XXX: size might be interesting?
|
389 |
mutter('%s: create_pack: pack renamed into place: %s%s->%s%s t+%6.3fs', |
|
390 |
time.ctime(), self.upload_transport.base, self.random_name, |
|
391 |
self.pack_transport, self.name, |
|
392 |
time.time() - self.start_time) |
|
2592.3.200
by Robert Collins
Make NewPack reopen the index files, separating out the task of refreshing the index maps in the repository and managing the completion of writing a single pack to disk. |
393 |
|
2951.2.9
by Robert Collins
* ``pack-0.92`` repositories can now be reconciled. |
394 |
def flush(self): |
395 |
"""Flush any current data."""
|
|
396 |
if self._buffer[1]: |
|
397 |
bytes = ''.join(self._buffer[0]) |
|
398 |
self.write_stream.write(bytes) |
|
399 |
self._hash.update(bytes) |
|
400 |
self._buffer[:] = [[], 0] |
|
401 |
||
2592.3.200
by Robert Collins
Make NewPack reopen the index files, separating out the task of refreshing the index maps in the repository and managing the completion of writing a single pack to disk. |
402 |
def index_name(self, index_type, name): |
403 |
"""Get the disk name of an index type for pack name 'name'."""
|
|
2592.3.227
by Martin Pool
Rename NewPack.indices to NewPack.index_definitions |
404 |
return name + NewPack.index_definitions[index_type][0] |
2592.3.200
by Robert Collins
Make NewPack reopen the index files, separating out the task of refreshing the index maps in the repository and managing the completion of writing a single pack to disk. |
405 |
|
406 |
def index_offset(self, index_type): |
|
407 |
"""Get the position in a index_size array for a given index type."""
|
|
2592.3.227
by Martin Pool
Rename NewPack.indices to NewPack.index_definitions |
408 |
return NewPack.index_definitions[index_type][1] |
2592.3.200
by Robert Collins
Make NewPack reopen the index files, separating out the task of refreshing the index maps in the repository and managing the completion of writing a single pack to disk. |
409 |
|
2592.3.233
by Martin Pool
Review cleanups |
410 |
def _replace_index_with_readonly(self, index_type): |
411 |
setattr(self, index_type + '_index', |
|
3735.1.1
by Robert Collins
Add development2 formats using BTree indices. |
412 |
self.index_class(self.index_transport, |
2592.3.233
by Martin Pool
Review cleanups |
413 |
self.index_name(index_type, self.name), |
414 |
self.index_sizes[self.index_offset(index_type)])) |
|
415 |
||
2592.3.203
by Robert Collins
Teach NewPack how to buffer for pack operations. |
416 |
def set_write_cache_size(self, size): |
417 |
self._cache_limit = size |
|
418 |
||
2592.3.200
by Robert Collins
Make NewPack reopen the index files, separating out the task of refreshing the index maps in the repository and managing the completion of writing a single pack to disk. |
419 |
def _write_index(self, index_type, index, label): |
2592.3.196
by Robert Collins
Move some text index logic to NewPack. |
420 |
"""Write out an index.
|
421 |
||
2592.3.222
by Robert Collins
More review feedback. |
422 |
:param index_type: The type of index to write - e.g. 'revision'.
|
2592.3.196
by Robert Collins
Move some text index logic to NewPack. |
423 |
:param index: The index object to serialise.
|
424 |
:param label: What label to give the index e.g. 'revision'.
|
|
425 |
"""
|
|
2592.3.200
by Robert Collins
Make NewPack reopen the index files, separating out the task of refreshing the index maps in the repository and managing the completion of writing a single pack to disk. |
426 |
index_name = self.index_name(index_type, self.name) |
427 |
self.index_sizes[self.index_offset(index_type)] = \ |
|
3010.1.11
by Robert Collins
Provide file modes to files created by pack repositories |
428 |
self.index_transport.put_file(index_name, index.finish(), |
429 |
mode=self._file_mode) |
|
2592.3.234
by Martin Pool
Use -Dpack not -Dfetch for pack traces |
430 |
if 'pack' in debug.debug_flags: |
2592.3.196
by Robert Collins
Move some text index logic to NewPack. |
431 |
# XXX: size might be interesting?
|
432 |
mutter('%s: create_pack: wrote %s index: %s%s t+%6.3fs', |
|
433 |
time.ctime(), label, self.upload_transport.base, |
|
434 |
self.random_name, time.time() - self.start_time) |
|
2592.3.233
by Martin Pool
Review cleanups |
435 |
# Replace the writable index on this object with a readonly,
|
436 |
# presently unloaded index. We should alter
|
|
437 |
# the index layer to make its finish() error if add_node is
|
|
2592.3.200
by Robert Collins
Make NewPack reopen the index files, separating out the task of refreshing the index maps in the repository and managing the completion of writing a single pack to disk. |
438 |
# subsequently used. RBC
|
2592.3.233
by Martin Pool
Review cleanups |
439 |
self._replace_index_with_readonly(index_type) |
2592.3.195
by Robert Collins
Move some inventory index logic to NewPack. |
440 |
|
2592.3.191
by Robert Collins
Give Pack responsibility for index naming, and two concrete classes - NewPack for new packs and ExistingPack for packs we read from disk. |
441 |
|
2592.3.208
by Robert Collins
Start refactoring the knit-pack thunking to be clearer. |
442 |
class AggregateIndex(object): |
443 |
"""An aggregated index for the RepositoryPackCollection.
|
|
444 |
||
445 |
AggregateIndex is reponsible for managing the PackAccess object,
|
|
446 |
Index-To-Pack mapping, and all indices list for a specific type of index
|
|
447 |
such as 'revision index'.
|
|
2592.3.228
by Martin Pool
docstrings and error messages from review |
448 |
|
449 |
A CombinedIndex provides an index on a single key space built up
|
|
450 |
from several on-disk indices. The AggregateIndex builds on this
|
|
451 |
to provide a knit access layer, and allows having up to one writable
|
|
452 |
index within the collection.
|
|
2592.3.208
by Robert Collins
Start refactoring the knit-pack thunking to be clearer. |
453 |
"""
|
2592.3.235
by Martin Pool
Review cleanups |
454 |
# XXX: Probably 'can be written to' could/should be separated from 'acts
|
455 |
# like a knit index' -- mbp 20071024
|
|
2592.3.208
by Robert Collins
Start refactoring the knit-pack thunking to be clearer. |
456 |
|
3789.1.3
by John Arbash Meinel
CombinedGraphIndex can now reload when calling key_count(). |
457 |
def __init__(self, reload_func=None): |
458 |
"""Create an AggregateIndex.
|
|
459 |
||
460 |
:param reload_func: A function to call if we find we are missing an
|
|
461 |
index. Should have the form reload_func() => True/False to indicate
|
|
462 |
if reloading actually changed anything.
|
|
463 |
"""
|
|
464 |
self._reload_func = reload_func |
|
2592.3.208
by Robert Collins
Start refactoring the knit-pack thunking to be clearer. |
465 |
self.index_to_pack = {} |
3789.1.3
by John Arbash Meinel
CombinedGraphIndex can now reload when calling key_count(). |
466 |
self.combined_index = CombinedGraphIndex([], reload_func=reload_func) |
3789.2.14
by John Arbash Meinel
Update AggregateIndex to pass the reload_func into _DirectPackAccess |
467 |
self.data_access = _DirectPackAccess(self.index_to_pack, |
468 |
reload_func=reload_func) |
|
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
469 |
self.add_callback = None |
2592.3.208
by Robert Collins
Start refactoring the knit-pack thunking to be clearer. |
470 |
|
471 |
def replace_indices(self, index_to_pack, indices): |
|
472 |
"""Replace the current mappings with fresh ones.
|
|
473 |
||
474 |
This should probably not be used eventually, rather incremental add and
|
|
475 |
removal of indices. It has been added during refactoring of existing
|
|
476 |
code.
|
|
477 |
||
478 |
:param index_to_pack: A mapping from index objects to
|
|
479 |
(transport, name) tuples for the pack file data.
|
|
480 |
:param indices: A list of indices.
|
|
481 |
"""
|
|
482 |
# refresh the revision pack map dict without replacing the instance.
|
|
483 |
self.index_to_pack.clear() |
|
484 |
self.index_to_pack.update(index_to_pack) |
|
485 |
# XXX: API break - clearly a 'replace' method would be good?
|
|
486 |
self.combined_index._indices[:] = indices |
|
2592.3.209
by Robert Collins
Revision index management looking sane for packs. |
487 |
# the current add nodes callback for the current writable index if
|
488 |
# there is one.
|
|
489 |
self.add_callback = None |
|
2592.3.208
by Robert Collins
Start refactoring the knit-pack thunking to be clearer. |
490 |
|
491 |
def add_index(self, index, pack): |
|
492 |
"""Add index to the aggregate, which is an index for Pack pack.
|
|
2592.3.226
by Martin Pool
formatting and docstrings |
493 |
|
494 |
Future searches on the aggregate index will seach this new index
|
|
495 |
before all previously inserted indices.
|
|
2592.3.208
by Robert Collins
Start refactoring the knit-pack thunking to be clearer. |
496 |
|
2592.3.226
by Martin Pool
formatting and docstrings |
497 |
:param index: An Index for the pack.
|
2592.3.208
by Robert Collins
Start refactoring the knit-pack thunking to be clearer. |
498 |
:param pack: A Pack instance.
|
499 |
"""
|
|
500 |
# expose it to the index map
|
|
501 |
self.index_to_pack[index] = pack.access_tuple() |
|
502 |
# put it at the front of the linear index list
|
|
503 |
self.combined_index.insert_index(0, index) |
|
504 |
||
2592.3.209
by Robert Collins
Revision index management looking sane for packs. |
505 |
def add_writable_index(self, index, pack): |
506 |
"""Add an index which is able to have data added to it.
|
|
2592.3.235
by Martin Pool
Review cleanups |
507 |
|
508 |
There can be at most one writable index at any time. Any
|
|
509 |
modifications made to the knit are put into this index.
|
|
2592.3.209
by Robert Collins
Revision index management looking sane for packs. |
510 |
|
511 |
:param index: An index from the pack parameter.
|
|
512 |
:param pack: A Pack instance.
|
|
513 |
"""
|
|
3376.2.4
by Martin Pool
Remove every assert statement from bzrlib! |
514 |
if self.add_callback is not None: |
515 |
raise AssertionError( |
|
516 |
"%s already has a writable index through %s" % \ |
|
517 |
(self, self.add_callback)) |
|
2592.3.209
by Robert Collins
Revision index management looking sane for packs. |
518 |
# allow writing: queue writes to a new index
|
519 |
self.add_index(index, pack) |
|
520 |
# Updates the index to packs mapping as a side effect,
|
|
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
521 |
self.data_access.set_writer(pack._writer, index, pack.access_tuple()) |
2592.3.209
by Robert Collins
Revision index management looking sane for packs. |
522 |
self.add_callback = index.add_nodes |
523 |
||
524 |
def clear(self): |
|
525 |
"""Reset all the aggregate data to nothing."""
|
|
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
526 |
self.data_access.set_writer(None, None, (None, None)) |
2592.3.209
by Robert Collins
Revision index management looking sane for packs. |
527 |
self.index_to_pack.clear() |
528 |
del self.combined_index._indices[:] |
|
529 |
self.add_callback = None |
|
530 |
||
531 |
def remove_index(self, index, pack): |
|
532 |
"""Remove index from the indices used to answer queries.
|
|
533 |
|
|
534 |
:param index: An index from the pack parameter.
|
|
535 |
:param pack: A Pack instance.
|
|
536 |
"""
|
|
537 |
del self.index_to_pack[index] |
|
538 |
self.combined_index._indices.remove(index) |
|
539 |
if (self.add_callback is not None and |
|
540 |
getattr(index, 'add_nodes', None) == self.add_callback): |
|
541 |
self.add_callback = None |
|
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
542 |
self.data_access.set_writer(None, None, (None, None)) |
2592.3.209
by Robert Collins
Revision index management looking sane for packs. |
543 |
|
2592.3.208
by Robert Collins
Start refactoring the knit-pack thunking to be clearer. |
544 |
|
2951.1.2
by Robert Collins
Partial refactoring of pack_repo to create a Packer object for packing. |
545 |
class Packer(object): |
546 |
"""Create a pack from packs."""
|
|
547 |
||
3789.2.20
by John Arbash Meinel
The autopack code can now trigger itself to retry when _copy_revision_texts fails. |
548 |
def __init__(self, pack_collection, packs, suffix, revision_ids=None, |
549 |
reload_func=None): |
|
2951.1.3
by Robert Collins
Partial support for native reconcile with packs. |
550 |
"""Create a Packer.
|
551 |
||
552 |
:param pack_collection: A RepositoryPackCollection object where the
|
|
553 |
new pack is being written to.
|
|
554 |
:param packs: The packs to combine.
|
|
555 |
:param suffix: The suffix to use on the temporary files for the pack.
|
|
556 |
:param revision_ids: Revision ids to limit the pack to.
|
|
3789.2.20
by John Arbash Meinel
The autopack code can now trigger itself to retry when _copy_revision_texts fails. |
557 |
:param reload_func: A function to call if a pack file/index goes
|
558 |
missing. The side effect of calling this function should be to
|
|
559 |
update self.packs. See also AggregateIndex
|
|
2951.1.3
by Robert Collins
Partial support for native reconcile with packs. |
560 |
"""
|
2951.1.2
by Robert Collins
Partial refactoring of pack_repo to create a Packer object for packing. |
561 |
self.packs = packs |
562 |
self.suffix = suffix |
|
563 |
self.revision_ids = revision_ids |
|
2951.2.1
by Robert Collins
Factor out revision text copying in Packer to a single helper method. |
564 |
# The pack object we are creating.
|
565 |
self.new_pack = None |
|
2951.1.2
by Robert Collins
Partial refactoring of pack_repo to create a Packer object for packing. |
566 |
self._pack_collection = pack_collection |
3789.2.20
by John Arbash Meinel
The autopack code can now trigger itself to retry when _copy_revision_texts fails. |
567 |
self._reload_func = reload_func |
2951.2.1
by Robert Collins
Factor out revision text copying in Packer to a single helper method. |
568 |
# The index layer keys for the revisions being copied. None for 'all
|
569 |
# objects'.
|
|
570 |
self._revision_keys = None |
|
2951.2.2
by Robert Collins
Factor out inventory text copying in Packer to a single helper method. |
571 |
# What text keys to copy. None for 'all texts'. This is set by
|
572 |
# _copy_inventory_texts
|
|
573 |
self._text_filter = None |
|
2951.2.9
by Robert Collins
* ``pack-0.92`` repositories can now be reconciled. |
574 |
self._extra_init() |
575 |
||
576 |
def _extra_init(self): |
|
577 |
"""A template hook to allow extending the constructor trivially."""
|
|
2951.1.2
by Robert Collins
Partial refactoring of pack_repo to create a Packer object for packing. |
578 |
|
2951.1.3
by Robert Collins
Partial support for native reconcile with packs. |
579 |
def pack(self, pb=None): |
2592.3.90
by Robert Collins
Slightly broken, but branch and fetch performance is now roughly on par (for bzr.dev) with knits - should be much faster for large repos. |
580 |
"""Create a new pack by reading data from other packs.
|
581 |
||
582 |
This does little more than a bulk copy of data. One key difference
|
|
583 |
is that data with the same item key across multiple packs is elided
|
|
584 |
from the output. The new pack is written into the current pack store
|
|
585 |
along with its indices, and the name added to the pack names. The
|
|
2592.3.182
by Robert Collins
Eliminate the need to use a transport,name tuple to represent a pack during fetch. |
586 |
source packs are not altered and are not required to be in the current
|
587 |
pack collection.
|
|
2592.3.90
by Robert Collins
Slightly broken, but branch and fetch performance is now roughly on par (for bzr.dev) with knits - should be much faster for large repos. |
588 |
|
2951.1.3
by Robert Collins
Partial support for native reconcile with packs. |
589 |
:param pb: An optional progress bar to use. A nested bar is created if
|
590 |
this is None.
|
|
2592.3.91
by Robert Collins
Incrementally closing in on a correct fetch for packs. |
591 |
:return: A Pack object, or None if nothing was copied.
|
2592.3.90
by Robert Collins
Slightly broken, but branch and fetch performance is now roughly on par (for bzr.dev) with knits - should be much faster for large repos. |
592 |
"""
|
593 |
# open a pack - using the same name as the last temporary file
|
|
594 |
# - which has already been flushed, so its safe.
|
|
595 |
# XXX: - duplicate code warning with start_write_group; fix before
|
|
596 |
# considering 'done'.
|
|
2951.1.2
by Robert Collins
Partial refactoring of pack_repo to create a Packer object for packing. |
597 |
if self._pack_collection._new_pack is not None: |
3789.2.22
by John Arbash Meinel
We need the Packer class to cleanup if it is getting a Retry it isn't handling. |
598 |
raise errors.BzrError('call to %s.pack() while another pack is' |
599 |
' being written.'
|
|
600 |
% (self.__class__.__name__,)) |
|
2951.1.2
by Robert Collins
Partial refactoring of pack_repo to create a Packer object for packing. |
601 |
if self.revision_ids is not None: |
602 |
if len(self.revision_ids) == 0: |
|
2947.1.3
by Robert Collins
Unbreak autopack. Doh. |
603 |
# silly fetch request.
|
604 |
return None |
|
605 |
else: |
|
2951.1.2
by Robert Collins
Partial refactoring of pack_repo to create a Packer object for packing. |
606 |
self.revision_ids = frozenset(self.revision_ids) |
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
607 |
self.revision_keys = frozenset((revid,) for revid in |
608 |
self.revision_ids) |
|
2951.1.3
by Robert Collins
Partial support for native reconcile with packs. |
609 |
if pb is None: |
610 |
self.pb = ui.ui_factory.nested_progress_bar() |
|
611 |
else: |
|
612 |
self.pb = pb |
|
2592.6.11
by Robert Collins
* A progress bar has been added for knitpack -> knitpack fetching. |
613 |
try: |
2951.1.2
by Robert Collins
Partial refactoring of pack_repo to create a Packer object for packing. |
614 |
return self._create_pack_from_packs() |
2592.6.11
by Robert Collins
* A progress bar has been added for knitpack -> knitpack fetching. |
615 |
finally: |
2951.1.3
by Robert Collins
Partial support for native reconcile with packs. |
616 |
if pb is None: |
617 |
self.pb.finished() |
|
2951.1.2
by Robert Collins
Partial refactoring of pack_repo to create a Packer object for packing. |
618 |
|
619 |
def open_pack(self): |
|
620 |
"""Open a pack for the pack we are creating."""
|
|
621 |
return NewPack(self._pack_collection._upload_transport, |
|
622 |
self._pack_collection._index_transport, |
|
3010.1.11
by Robert Collins
Provide file modes to files created by pack repositories |
623 |
self._pack_collection._pack_transport, upload_suffix=self.suffix, |
3735.1.1
by Robert Collins
Add development2 formats using BTree indices. |
624 |
file_mode=self._pack_collection.repo.bzrdir._get_file_mode(), |
625 |
index_builder_class=self._pack_collection._index_builder_class, |
|
626 |
index_class=self._pack_collection._index_class) |
|
2951.1.2
by Robert Collins
Partial refactoring of pack_repo to create a Packer object for packing. |
627 |
|
2951.2.1
by Robert Collins
Factor out revision text copying in Packer to a single helper method. |
628 |
def _copy_revision_texts(self): |
629 |
"""Copy revision data to the new pack."""
|
|
630 |
# select revisions
|
|
631 |
if self.revision_ids: |
|
632 |
revision_keys = [(revision_id,) for revision_id in self.revision_ids] |
|
633 |
else: |
|
634 |
revision_keys = None |
|
3789.2.17
by John Arbash Meinel
Start getting _copy_revision_texts to retry. |
635 |
completed_keys = [] |
636 |
while True: |
|
637 |
try: |
|
638 |
# select revision keys
|
|
639 |
revision_index_map = self._pack_collection._packs_list_to_pack_map_and_index_list( |
|
640 |
self.packs, 'revision_index')[0] |
|
3789.2.20
by John Arbash Meinel
The autopack code can now trigger itself to retry when _copy_revision_texts fails. |
641 |
revision_nodes = self._pack_collection._index_contents( |
642 |
revision_index_map, revision_keys) |
|
3789.2.17
by John Arbash Meinel
Start getting _copy_revision_texts to retry. |
643 |
# copy revision keys and adjust values
|
644 |
self.pb.update("Copying revision texts", 1) |
|
645 |
total_items, readv_group_iter = self._revision_node_readv(revision_nodes) |
|
646 |
list(self._copy_nodes_graph(revision_index_map, self.new_pack._writer, |
|
647 |
self.new_pack.revision_index, readv_group_iter, |
|
648 |
total_items, completed_keys=completed_keys)) |
|
649 |
break
|
|
650 |
except errors.NoSuchFile: |
|
3789.2.20
by John Arbash Meinel
The autopack code can now trigger itself to retry when _copy_revision_texts fails. |
651 |
if self._reload_func is None: |
652 |
raise
|
|
3789.2.17
by John Arbash Meinel
Start getting _copy_revision_texts to retry. |
653 |
# A pack file went missing, try reloading in case it was just
|
654 |
# someone else repacking the repo.
|
|
3789.2.20
by John Arbash Meinel
The autopack code can now trigger itself to retry when _copy_revision_texts fails. |
655 |
if not self._reload_func(): |
3789.2.17
by John Arbash Meinel
Start getting _copy_revision_texts to retry. |
656 |
raise
|
657 |
||
2951.2.1
by Robert Collins
Factor out revision text copying in Packer to a single helper method. |
658 |
if 'pack' in debug.debug_flags: |
659 |
mutter('%s: create_pack: revisions copied: %s%s %d items t+%6.3fs', |
|
660 |
time.ctime(), self._pack_collection._upload_transport.base, |
|
661 |
self.new_pack.random_name, |
|
662 |
self.new_pack.revision_index.key_count(), |
|
663 |
time.time() - self.new_pack.start_time) |
|
664 |
self._revision_keys = revision_keys |
|
665 |
||
2951.2.2
by Robert Collins
Factor out inventory text copying in Packer to a single helper method. |
666 |
def _copy_inventory_texts(self): |
667 |
"""Copy the inventory texts to the new pack.
|
|
668 |
||
669 |
self._revision_keys is used to determine what inventories to copy.
|
|
670 |
||
671 |
Sets self._text_filter appropriately.
|
|
672 |
"""
|
|
2592.3.90
by Robert Collins
Slightly broken, but branch and fetch performance is now roughly on par (for bzr.dev) with knits - should be much faster for large repos. |
673 |
# select inventory keys
|
2951.2.1
by Robert Collins
Factor out revision text copying in Packer to a single helper method. |
674 |
inv_keys = self._revision_keys # currently the same keyspace, and note that |
2592.3.145
by Robert Collins
Fix test_fetch_missing_text_other_location_fails for pack repositories. |
675 |
# querying for keys here could introduce a bug where an inventory item
|
676 |
# is missed, so do not change it to query separately without cross
|
|
677 |
# checking like the text key check below.
|
|
2951.1.2
by Robert Collins
Partial refactoring of pack_repo to create a Packer object for packing. |
678 |
inventory_index_map = self._pack_collection._packs_list_to_pack_map_and_index_list( |
679 |
self.packs, 'inventory_index')[0] |
|
680 |
inv_nodes = self._pack_collection._index_contents(inventory_index_map, inv_keys) |
|
2592.3.90
by Robert Collins
Slightly broken, but branch and fetch performance is now roughly on par (for bzr.dev) with knits - should be much faster for large repos. |
681 |
# copy inventory keys and adjust values
|
2592.3.104
by Robert Collins
hackish fix, but all tests passing again. |
682 |
# XXX: Should be a helper function to allow different inv representation
|
683 |
# at this point.
|
|
2951.1.2
by Robert Collins
Partial refactoring of pack_repo to create a Packer object for packing. |
684 |
self.pb.update("Copying inventory texts", 2) |
3070.1.1
by Robert Collins
* ``bzr pack`` now orders revision texts in topological order, with newest |
685 |
total_items, readv_group_iter = self._least_readv_node_readv(inv_nodes) |
3253.1.1
by John Arbash Meinel
Reduce memory consumption during autopack. |
686 |
# Only grab the output lines if we will be processing them
|
687 |
output_lines = bool(self.revision_ids) |
|
3070.1.1
by Robert Collins
* ``bzr pack`` now orders revision texts in topological order, with newest |
688 |
inv_lines = self._copy_nodes_graph(inventory_index_map, |
689 |
self.new_pack._writer, self.new_pack.inventory_index, |
|
3253.1.1
by John Arbash Meinel
Reduce memory consumption during autopack. |
690 |
readv_group_iter, total_items, output_lines=output_lines) |
2951.1.2
by Robert Collins
Partial refactoring of pack_repo to create a Packer object for packing. |
691 |
if self.revision_ids: |
2951.2.9
by Robert Collins
* ``pack-0.92`` repositories can now be reconciled. |
692 |
self._process_inventory_lines(inv_lines) |
2592.3.110
by Robert Collins
Filter out texts and signatures not referenced by the revisions being copied during pack to pack fetching. |
693 |
else: |
2592.3.145
by Robert Collins
Fix test_fetch_missing_text_other_location_fails for pack repositories. |
694 |
# eat the iterator to cause it to execute.
|
2592.3.110
by Robert Collins
Filter out texts and signatures not referenced by the revisions being copied during pack to pack fetching. |
695 |
list(inv_lines) |
2951.2.9
by Robert Collins
* ``pack-0.92`` repositories can now be reconciled. |
696 |
self._text_filter = None |
2592.3.234
by Martin Pool
Use -Dpack not -Dfetch for pack traces |
697 |
if 'pack' in debug.debug_flags: |
2592.3.91
by Robert Collins
Incrementally closing in on a correct fetch for packs. |
698 |
mutter('%s: create_pack: inventories copied: %s%s %d items t+%6.3fs', |
2951.2.2
by Robert Collins
Factor out inventory text copying in Packer to a single helper method. |
699 |
time.ctime(), self._pack_collection._upload_transport.base, |
700 |
self.new_pack.random_name, |
|
701 |
self.new_pack.inventory_index.key_count(), |
|
3231.3.1
by James Westby
Make -Dpack not cause a error trying to use an unkown variable. |
702 |
time.time() - self.new_pack.start_time) |
2951.2.2
by Robert Collins
Factor out inventory text copying in Packer to a single helper method. |
703 |
|
2951.2.9
by Robert Collins
* ``pack-0.92`` repositories can now be reconciled. |
704 |
def _copy_text_texts(self): |
2592.3.90
by Robert Collins
Slightly broken, but branch and fetch performance is now roughly on par (for bzr.dev) with knits - should be much faster for large repos. |
705 |
# select text keys
|
2951.2.9
by Robert Collins
* ``pack-0.92`` repositories can now be reconciled. |
706 |
text_index_map, text_nodes = self._get_text_nodes() |
2951.2.2
by Robert Collins
Factor out inventory text copying in Packer to a single helper method. |
707 |
if self._text_filter is not None: |
2592.3.149
by Robert Collins
Unbreak pack to pack fetching properly, with missing-text detection really working. |
708 |
# We could return the keys copied as part of the return value from
|
709 |
# _copy_nodes_graph but this doesn't work all that well with the
|
|
710 |
# need to get line output too, so we check separately, and as we're
|
|
711 |
# going to buffer everything anyway, we check beforehand, which
|
|
712 |
# saves reading knit data over the wire when we know there are
|
|
713 |
# mising records.
|
|
714 |
text_nodes = set(text_nodes) |
|
715 |
present_text_keys = set(_node[1] for _node in text_nodes) |
|
2951.2.2
by Robert Collins
Factor out inventory text copying in Packer to a single helper method. |
716 |
missing_text_keys = set(self._text_filter) - present_text_keys |
2592.3.149
by Robert Collins
Unbreak pack to pack fetching properly, with missing-text detection really working. |
717 |
if missing_text_keys: |
718 |
# TODO: raise a specific error that can handle many missing
|
|
719 |
# keys.
|
|
720 |
a_missing_key = missing_text_keys.pop() |
|
721 |
raise errors.RevisionNotPresent(a_missing_key[1], |
|
722 |
a_missing_key[0]) |
|
2592.3.90
by Robert Collins
Slightly broken, but branch and fetch performance is now roughly on par (for bzr.dev) with knits - should be much faster for large repos. |
723 |
# copy text keys and adjust values
|
2951.1.2
by Robert Collins
Partial refactoring of pack_repo to create a Packer object for packing. |
724 |
self.pb.update("Copying content texts", 3) |
3070.1.1
by Robert Collins
* ``bzr pack`` now orders revision texts in topological order, with newest |
725 |
total_items, readv_group_iter = self._least_readv_node_readv(text_nodes) |
726 |
list(self._copy_nodes_graph(text_index_map, self.new_pack._writer, |
|
727 |
self.new_pack.text_index, readv_group_iter, total_items)) |
|
2951.2.9
by Robert Collins
* ``pack-0.92`` repositories can now be reconciled. |
728 |
self._log_copied_texts() |
729 |
||
3035.2.6
by John Arbash Meinel
Suggested by Robert: Move the missing externals check into part of Packer.pack() |
730 |
def _check_references(self): |
731 |
"""Make sure our external refereneces are present."""
|
|
732 |
external_refs = self.new_pack._external_compression_parents_of_texts() |
|
733 |
if external_refs: |
|
734 |
index = self._pack_collection.text_index.combined_index |
|
735 |
found_items = list(index.iter_entries(external_refs)) |
|
736 |
if len(found_items) != len(external_refs): |
|
737 |
found_keys = set(k for idx, k, refs, value in found_items) |
|
738 |
missing_items = external_refs - found_keys |
|
739 |
missing_file_id, missing_revision_id = missing_items.pop() |
|
740 |
raise errors.RevisionNotPresent(missing_revision_id, |
|
741 |
missing_file_id) |
|
742 |
||
2951.2.9
by Robert Collins
* ``pack-0.92`` repositories can now be reconciled. |
743 |
def _create_pack_from_packs(self): |
744 |
self.pb.update("Opening pack", 0, 5) |
|
745 |
self.new_pack = self.open_pack() |
|
746 |
new_pack = self.new_pack |
|
747 |
# buffer data - we won't be reading-back during the pack creation and
|
|
748 |
# this makes a significant difference on sftp pushes.
|
|
749 |
new_pack.set_write_cache_size(1024*1024) |
|
2592.3.234
by Martin Pool
Use -Dpack not -Dfetch for pack traces |
750 |
if 'pack' in debug.debug_flags: |
2951.2.9
by Robert Collins
* ``pack-0.92`` repositories can now be reconciled. |
751 |
plain_pack_list = ['%s%s' % (a_pack.pack_transport.base, a_pack.name) |
752 |
for a_pack in self.packs] |
|
753 |
if self.revision_ids is not None: |
|
754 |
rev_count = len(self.revision_ids) |
|
755 |
else: |
|
756 |
rev_count = 'all' |
|
757 |
mutter('%s: create_pack: creating pack from source packs: ' |
|
758 |
'%s%s %s revisions wanted %s t=0', |
|
2951.1.2
by Robert Collins
Partial refactoring of pack_repo to create a Packer object for packing. |
759 |
time.ctime(), self._pack_collection._upload_transport.base, new_pack.random_name, |
2951.2.9
by Robert Collins
* ``pack-0.92`` repositories can now be reconciled. |
760 |
plain_pack_list, rev_count) |
761 |
self._copy_revision_texts() |
|
762 |
self._copy_inventory_texts() |
|
763 |
self._copy_text_texts() |
|
2592.3.90
by Robert Collins
Slightly broken, but branch and fetch performance is now roughly on par (for bzr.dev) with knits - should be much faster for large repos. |
764 |
# select signature keys
|
2951.2.1
by Robert Collins
Factor out revision text copying in Packer to a single helper method. |
765 |
signature_filter = self._revision_keys # same keyspace |
2951.1.2
by Robert Collins
Partial refactoring of pack_repo to create a Packer object for packing. |
766 |
signature_index_map = self._pack_collection._packs_list_to_pack_map_and_index_list( |
767 |
self.packs, 'signature_index')[0] |
|
768 |
signature_nodes = self._pack_collection._index_contents(signature_index_map, |
|
2592.3.110
by Robert Collins
Filter out texts and signatures not referenced by the revisions being copied during pack to pack fetching. |
769 |
signature_filter) |
2592.3.90
by Robert Collins
Slightly broken, but branch and fetch performance is now roughly on par (for bzr.dev) with knits - should be much faster for large repos. |
770 |
# copy signature keys and adjust values
|
2951.1.2
by Robert Collins
Partial refactoring of pack_repo to create a Packer object for packing. |
771 |
self.pb.update("Copying signature texts", 4) |
2592.3.205
by Robert Collins
Move the pack ContainerWriter instance into NewPack. |
772 |
self._copy_nodes(signature_nodes, signature_index_map, new_pack._writer, |
773 |
new_pack.signature_index) |
|
2592.3.234
by Martin Pool
Use -Dpack not -Dfetch for pack traces |
774 |
if 'pack' in debug.debug_flags: |
2592.3.91
by Robert Collins
Incrementally closing in on a correct fetch for packs. |
775 |
mutter('%s: create_pack: revision signatures copied: %s%s %d items t+%6.3fs', |
2951.1.2
by Robert Collins
Partial refactoring of pack_repo to create a Packer object for packing. |
776 |
time.ctime(), self._pack_collection._upload_transport.base, new_pack.random_name, |
2592.3.197
by Robert Collins
Hand over signature index creation to NewPack. |
777 |
new_pack.signature_index.key_count(), |
2592.3.194
by Robert Collins
Output the revision index from NewPack.finish |
778 |
time.time() - new_pack.start_time) |
3035.2.6
by John Arbash Meinel
Suggested by Robert: Move the missing externals check into part of Packer.pack() |
779 |
self._check_references() |
2951.2.8
by Robert Collins
Test that reconciling a repository can be done twice in a row. |
780 |
if not self._use_pack(new_pack): |
2592.3.203
by Robert Collins
Teach NewPack how to buffer for pack operations. |
781 |
new_pack.abort() |
782 |
return None |
|
2951.1.2
by Robert Collins
Partial refactoring of pack_repo to create a Packer object for packing. |
783 |
self.pb.update("Finishing pack", 5) |
2592.3.194
by Robert Collins
Output the revision index from NewPack.finish |
784 |
new_pack.finish() |
2951.1.2
by Robert Collins
Partial refactoring of pack_repo to create a Packer object for packing. |
785 |
self._pack_collection.allocate(new_pack) |
2592.3.206
by Robert Collins
Move pack rename-into-place into NewPack.finish and document hash-collision cases somewhat better. |
786 |
return new_pack |
2592.3.90
by Robert Collins
Slightly broken, but branch and fetch performance is now roughly on par (for bzr.dev) with knits - should be much faster for large repos. |
787 |
|
2951.1.2
by Robert Collins
Partial refactoring of pack_repo to create a Packer object for packing. |
788 |
def _copy_nodes(self, nodes, index_map, writer, write_index): |
789 |
"""Copy knit nodes between packs with no graph references."""
|
|
790 |
pb = ui.ui_factory.nested_progress_bar() |
|
791 |
try: |
|
792 |
return self._do_copy_nodes(nodes, index_map, writer, |
|
793 |
write_index, pb) |
|
794 |
finally: |
|
795 |
pb.finished() |
|
796 |
||
797 |
def _do_copy_nodes(self, nodes, index_map, writer, write_index, pb): |
|
798 |
# for record verification
|
|
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
799 |
knit = KnitVersionedFiles(None, None) |
2951.1.2
by Robert Collins
Partial refactoring of pack_repo to create a Packer object for packing. |
800 |
# plan a readv on each source pack:
|
801 |
# group by pack
|
|
802 |
nodes = sorted(nodes) |
|
803 |
# how to map this into knit.py - or knit.py into this?
|
|
804 |
# we don't want the typical knit logic, we want grouping by pack
|
|
805 |
# at this point - perhaps a helper library for the following code
|
|
806 |
# duplication points?
|
|
807 |
request_groups = {} |
|
808 |
for index, key, value in nodes: |
|
809 |
if index not in request_groups: |
|
810 |
request_groups[index] = [] |
|
811 |
request_groups[index].append((key, value)) |
|
812 |
record_index = 0 |
|
813 |
pb.update("Copied record", record_index, len(nodes)) |
|
814 |
for index, items in request_groups.iteritems(): |
|
815 |
pack_readv_requests = [] |
|
816 |
for key, value in items: |
|
817 |
# ---- KnitGraphIndex.get_position
|
|
818 |
bits = value[1:].split(' ') |
|
819 |
offset, length = int(bits[0]), int(bits[1]) |
|
820 |
pack_readv_requests.append((offset, length, (key, value[0]))) |
|
821 |
# linear scan up the pack
|
|
822 |
pack_readv_requests.sort() |
|
823 |
# copy the data
|
|
824 |
transport, path = index_map[index] |
|
825 |
reader = pack.make_readv_reader(transport, path, |
|
826 |
[offset[0:2] for offset in pack_readv_requests]) |
|
827 |
for (names, read_func), (_1, _2, (key, eol_flag)) in \ |
|
828 |
izip(reader.iter_records(), pack_readv_requests): |
|
829 |
raw_data = read_func(None) |
|
830 |
# check the header only
|
|
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
831 |
df, _ = knit._parse_record_header(key, raw_data) |
2951.1.2
by Robert Collins
Partial refactoring of pack_repo to create a Packer object for packing. |
832 |
df.close() |
833 |
pos, size = writer.add_bytes_record(raw_data, names) |
|
834 |
write_index.add_node(key, eol_flag + "%d %d" % (pos, size)) |
|
835 |
pb.update("Copied record", record_index) |
|
836 |
record_index += 1 |
|
837 |
||
3070.1.1
by Robert Collins
* ``bzr pack`` now orders revision texts in topological order, with newest |
838 |
def _copy_nodes_graph(self, index_map, writer, write_index, |
3789.2.16
by John Arbash Meinel
Set up failing tests for _copy_revision_texts, _copy_inventory_texts, and _copy_text_texts. |
839 |
readv_group_iter, total_items, output_lines=False, completed_keys=None): |
2951.1.2
by Robert Collins
Partial refactoring of pack_repo to create a Packer object for packing. |
840 |
"""Copy knit nodes between packs.
|
841 |
||
842 |
:param output_lines: Return lines present in the copied data as
|
|
2975.3.1
by Robert Collins
Change (without backwards compatibility) the |
843 |
an iterator of line,version_id.
|
3789.2.16
by John Arbash Meinel
Set up failing tests for _copy_revision_texts, _copy_inventory_texts, and _copy_text_texts. |
844 |
:param completed_keys: If set to a list, we will fill it with the keys
|
845 |
that have been successfully written to the target repository. This
|
|
846 |
is used in case there is a fault and we need to restart.
|
|
2951.1.2
by Robert Collins
Partial refactoring of pack_repo to create a Packer object for packing. |
847 |
"""
|
848 |
pb = ui.ui_factory.nested_progress_bar() |
|
849 |
try: |
|
3070.1.1
by Robert Collins
* ``bzr pack`` now orders revision texts in topological order, with newest |
850 |
for result in self._do_copy_nodes_graph(index_map, writer, |
3789.2.16
by John Arbash Meinel
Set up failing tests for _copy_revision_texts, _copy_inventory_texts, and _copy_text_texts. |
851 |
write_index, output_lines, pb, readv_group_iter, total_items, |
852 |
completed_keys=completed_keys): |
|
3039.1.1
by Robert Collins
(robertc) Fix the text progress for pack to pack fetches. (Robert Collins). |
853 |
yield result |
3039.1.2
by Robert Collins
python2.4 'compatibility'. |
854 |
except Exception: |
3039.1.3
by Robert Collins
Document the try:except:else: rather than a finally: in pack_repo.._copy_nodes_graph. |
855 |
# Python 2.4 does not permit try:finally: in a generator.
|
3039.1.2
by Robert Collins
python2.4 'compatibility'. |
856 |
pb.finished() |
857 |
raise
|
|
858 |
else: |
|
2951.1.2
by Robert Collins
Partial refactoring of pack_repo to create a Packer object for packing. |
859 |
pb.finished() |
860 |
||
3070.1.1
by Robert Collins
* ``bzr pack`` now orders revision texts in topological order, with newest |
861 |
def _do_copy_nodes_graph(self, index_map, writer, write_index, |
3789.2.16
by John Arbash Meinel
Set up failing tests for _copy_revision_texts, _copy_inventory_texts, and _copy_text_texts. |
862 |
output_lines, pb, readv_group_iter, total_items, completed_keys=None): |
2951.1.2
by Robert Collins
Partial refactoring of pack_repo to create a Packer object for packing. |
863 |
# for record verification
|
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
864 |
knit = KnitVersionedFiles(None, None) |
2951.1.2
by Robert Collins
Partial refactoring of pack_repo to create a Packer object for packing. |
865 |
# for line extraction when requested (inventories only)
|
866 |
if output_lines: |
|
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
867 |
factory = KnitPlainFactory() |
2951.1.2
by Robert Collins
Partial refactoring of pack_repo to create a Packer object for packing. |
868 |
record_index = 0 |
3070.1.1
by Robert Collins
* ``bzr pack`` now orders revision texts in topological order, with newest |
869 |
pb.update("Copied record", record_index, total_items) |
3789.2.16
by John Arbash Meinel
Set up failing tests for _copy_revision_texts, _copy_inventory_texts, and _copy_text_texts. |
870 |
if completed_keys is None: |
871 |
completed_keys_append = None |
|
872 |
else: |
|
873 |
completed_keys_append = completed_keys.append |
|
3070.1.1
by Robert Collins
* ``bzr pack`` now orders revision texts in topological order, with newest |
874 |
for index, readv_vector, node_vector in readv_group_iter: |
2951.1.2
by Robert Collins
Partial refactoring of pack_repo to create a Packer object for packing. |
875 |
# copy the data
|
876 |
transport, path = index_map[index] |
|
3070.1.1
by Robert Collins
* ``bzr pack`` now orders revision texts in topological order, with newest |
877 |
reader = pack.make_readv_reader(transport, path, readv_vector) |
878 |
for (names, read_func), (key, eol_flag, references) in \ |
|
879 |
izip(reader.iter_records(), node_vector): |
|
2951.1.2
by Robert Collins
Partial refactoring of pack_repo to create a Packer object for packing. |
880 |
raw_data = read_func(None) |
881 |
if output_lines: |
|
882 |
# read the entire thing
|
|
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
883 |
content, _ = knit._parse_record(key[-1], raw_data) |
2951.1.2
by Robert Collins
Partial refactoring of pack_repo to create a Packer object for packing. |
884 |
if len(references[-1]) == 0: |
885 |
line_iterator = factory.get_fulltext_content(content) |
|
886 |
else: |
|
887 |
line_iterator = factory.get_linedelta_content(content) |
|
888 |
for line in line_iterator: |
|
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
889 |
yield line, key |
2951.1.2
by Robert Collins
Partial refactoring of pack_repo to create a Packer object for packing. |
890 |
else: |
891 |
# check the header only
|
|
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
892 |
df, _ = knit._parse_record_header(key, raw_data) |
2951.1.2
by Robert Collins
Partial refactoring of pack_repo to create a Packer object for packing. |
893 |
df.close() |
894 |
pos, size = writer.add_bytes_record(raw_data, names) |
|
895 |
write_index.add_node(key, eol_flag + "%d %d" % (pos, size), references) |
|
3789.2.16
by John Arbash Meinel
Set up failing tests for _copy_revision_texts, _copy_inventory_texts, and _copy_text_texts. |
896 |
if completed_keys_append is not None: |
897 |
completed_keys_append(key) |
|
2951.1.2
by Robert Collins
Partial refactoring of pack_repo to create a Packer object for packing. |
898 |
pb.update("Copied record", record_index) |
899 |
record_index += 1 |
|
900 |
||
2951.2.9
by Robert Collins
* ``pack-0.92`` repositories can now be reconciled. |
901 |
def _get_text_nodes(self): |
902 |
text_index_map = self._pack_collection._packs_list_to_pack_map_and_index_list( |
|
903 |
self.packs, 'text_index')[0] |
|
904 |
return text_index_map, self._pack_collection._index_contents(text_index_map, |
|
905 |
self._text_filter) |
|
906 |
||
3070.1.1
by Robert Collins
* ``bzr pack`` now orders revision texts in topological order, with newest |
907 |
def _least_readv_node_readv(self, nodes): |
908 |
"""Generate request groups for nodes using the least readv's.
|
|
909 |
|
|
910 |
:param nodes: An iterable of graph index nodes.
|
|
911 |
:return: Total node count and an iterator of the data needed to perform
|
|
912 |
readvs to obtain the data for nodes. Each item yielded by the
|
|
913 |
iterator is a tuple with:
|
|
914 |
index, readv_vector, node_vector. readv_vector is a list ready to
|
|
915 |
hand to the transport readv method, and node_vector is a list of
|
|
916 |
(key, eol_flag, references) for the the node retrieved by the
|
|
917 |
matching readv_vector.
|
|
918 |
"""
|
|
919 |
# group by pack so we do one readv per pack
|
|
920 |
nodes = sorted(nodes) |
|
921 |
total = len(nodes) |
|
922 |
request_groups = {} |
|
923 |
for index, key, value, references in nodes: |
|
924 |
if index not in request_groups: |
|
925 |
request_groups[index] = [] |
|
926 |
request_groups[index].append((key, value, references)) |
|
927 |
result = [] |
|
928 |
for index, items in request_groups.iteritems(): |
|
929 |
pack_readv_requests = [] |
|
930 |
for key, value, references in items: |
|
931 |
# ---- KnitGraphIndex.get_position
|
|
932 |
bits = value[1:].split(' ') |
|
933 |
offset, length = int(bits[0]), int(bits[1]) |
|
934 |
pack_readv_requests.append( |
|
935 |
((offset, length), (key, value[0], references))) |
|
936 |
# linear scan up the pack to maximum range combining.
|
|
937 |
pack_readv_requests.sort() |
|
938 |
# split out the readv and the node data.
|
|
939 |
pack_readv = [readv for readv, node in pack_readv_requests] |
|
940 |
node_vector = [node for readv, node in pack_readv_requests] |
|
941 |
result.append((index, pack_readv, node_vector)) |
|
942 |
return total, result |
|
943 |
||
2951.2.9
by Robert Collins
* ``pack-0.92`` repositories can now be reconciled. |
944 |
def _log_copied_texts(self): |
945 |
if 'pack' in debug.debug_flags: |
|
946 |
mutter('%s: create_pack: file texts copied: %s%s %d items t+%6.3fs', |
|
947 |
time.ctime(), self._pack_collection._upload_transport.base, |
|
948 |
self.new_pack.random_name, |
|
949 |
self.new_pack.text_index.key_count(), |
|
950 |
time.time() - self.new_pack.start_time) |
|
951 |
||
952 |
def _process_inventory_lines(self, inv_lines): |
|
953 |
"""Use up the inv_lines generator and setup a text key filter."""
|
|
954 |
repo = self._pack_collection.repo |
|
955 |
fileid_revisions = repo._find_file_ids_from_xml_inventory_lines( |
|
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
956 |
inv_lines, self.revision_keys) |
2951.2.9
by Robert Collins
* ``pack-0.92`` repositories can now be reconciled. |
957 |
text_filter = [] |
958 |
for fileid, file_revids in fileid_revisions.iteritems(): |
|
959 |
text_filter.extend([(fileid, file_revid) for file_revid in file_revids]) |
|
960 |
self._text_filter = text_filter |
|
961 |
||
3070.1.2
by John Arbash Meinel
Cleanup OptimizingPacker code according to my review feedback |
962 |
def _revision_node_readv(self, revision_nodes): |
3070.1.1
by Robert Collins
* ``bzr pack`` now orders revision texts in topological order, with newest |
963 |
"""Return the total revisions and the readv's to issue.
|
964 |
||
965 |
:param revision_nodes: The revision index contents for the packs being
|
|
966 |
incorporated into the new pack.
|
|
967 |
:return: As per _least_readv_node_readv.
|
|
968 |
"""
|
|
969 |
return self._least_readv_node_readv(revision_nodes) |
|
970 |
||
2951.2.8
by Robert Collins
Test that reconciling a repository can be done twice in a row. |
971 |
def _use_pack(self, new_pack): |
972 |
"""Return True if new_pack should be used.
|
|
973 |
||
974 |
:param new_pack: The pack that has just been created.
|
|
975 |
:return: True if the pack should be used.
|
|
976 |
"""
|
|
977 |
return new_pack.data_inserted() |
|
978 |
||
2951.1.2
by Robert Collins
Partial refactoring of pack_repo to create a Packer object for packing. |
979 |
|
3070.1.1
by Robert Collins
* ``bzr pack`` now orders revision texts in topological order, with newest |
980 |
class OptimisingPacker(Packer): |
981 |
"""A packer which spends more time to create better disk layouts."""
|
|
982 |
||
3070.1.2
by John Arbash Meinel
Cleanup OptimizingPacker code according to my review feedback |
983 |
def _revision_node_readv(self, revision_nodes): |
3070.1.1
by Robert Collins
* ``bzr pack`` now orders revision texts in topological order, with newest |
984 |
"""Return the total revisions and the readv's to issue.
|
985 |
||
986 |
This sort places revisions in topological order with the ancestors
|
|
987 |
after the children.
|
|
988 |
||
989 |
:param revision_nodes: The revision index contents for the packs being
|
|
990 |
incorporated into the new pack.
|
|
991 |
:return: As per _least_readv_node_readv.
|
|
992 |
"""
|
|
993 |
# build an ancestors dict
|
|
994 |
ancestors = {} |
|
995 |
by_key = {} |
|
996 |
for index, key, value, references in revision_nodes: |
|
997 |
ancestors[key] = references[0] |
|
998 |
by_key[key] = (index, value, references) |
|
999 |
order = tsort.topo_sort(ancestors) |
|
1000 |
total = len(order) |
|
1001 |
# Single IO is pathological, but it will work as a starting point.
|
|
1002 |
requests = [] |
|
1003 |
for key in reversed(order): |
|
1004 |
index, value, references = by_key[key] |
|
1005 |
# ---- KnitGraphIndex.get_position
|
|
1006 |
bits = value[1:].split(' ') |
|
1007 |
offset, length = int(bits[0]), int(bits[1]) |
|
1008 |
requests.append( |
|
1009 |
(index, [(offset, length)], [(key, value[0], references)])) |
|
1010 |
# TODO: combine requests in the same index that are in ascending order.
|
|
1011 |
return total, requests |
|
1012 |
||
3777.5.4
by John Arbash Meinel
OptimisingPacker now sets the optimize flags for the indexes being built. |
1013 |
def open_pack(self): |
1014 |
"""Open a pack for the pack we are creating."""
|
|
3777.5.5
by John Arbash Meinel
Up-call to the parent as suggested by Andrew. |
1015 |
new_pack = super(OptimisingPacker, self).open_pack() |
1016 |
# Turn on the optimization flags for all the index builders.
|
|
3777.5.4
by John Arbash Meinel
OptimisingPacker now sets the optimize flags for the indexes being built. |
1017 |
new_pack.revision_index.set_optimize(for_size=True) |
1018 |
new_pack.inventory_index.set_optimize(for_size=True) |
|
1019 |
new_pack.text_index.set_optimize(for_size=True) |
|
1020 |
new_pack.signature_index.set_optimize(for_size=True) |
|
1021 |
return new_pack |
|
1022 |
||
3070.1.1
by Robert Collins
* ``bzr pack`` now orders revision texts in topological order, with newest |
1023 |
|
2951.1.3
by Robert Collins
Partial support for native reconcile with packs. |
1024 |
class ReconcilePacker(Packer): |
1025 |
"""A packer which regenerates indices etc as it copies.
|
|
1026 |
|
|
1027 |
This is used by ``bzr reconcile`` to cause parent text pointers to be
|
|
1028 |
regenerated.
|
|
1029 |
"""
|
|
1030 |
||
2951.2.9
by Robert Collins
* ``pack-0.92`` repositories can now be reconciled. |
1031 |
def _extra_init(self): |
1032 |
self._data_changed = False |
|
1033 |
||
1034 |
def _process_inventory_lines(self, inv_lines): |
|
1035 |
"""Generate a text key reference map rather for reconciling with."""
|
|
1036 |
repo = self._pack_collection.repo |
|
1037 |
refs = repo._find_text_key_references_from_xml_inventory_lines( |
|
1038 |
inv_lines) |
|
1039 |
self._text_refs = refs |
|
1040 |
# during reconcile we:
|
|
1041 |
# - convert unreferenced texts to full texts
|
|
1042 |
# - correct texts which reference a text not copied to be full texts
|
|
1043 |
# - copy all others as-is but with corrected parents.
|
|
1044 |
# - so at this point we don't know enough to decide what becomes a full
|
|
1045 |
# text.
|
|
1046 |
self._text_filter = None |
|
1047 |
||
1048 |
def _copy_text_texts(self): |
|
1049 |
"""generate what texts we should have and then copy."""
|
|
1050 |
self.pb.update("Copying content texts", 3) |
|
1051 |
# we have three major tasks here:
|
|
1052 |
# 1) generate the ideal index
|
|
1053 |
repo = self._pack_collection.repo |
|
3063.2.1
by Robert Collins
Solve reconciling erroring when multiple portions of a single delta chain are being reinserted. |
1054 |
ancestors = dict([(key[0], tuple(ref[0] for ref in refs[0])) for |
3063.2.2
by Robert Collins
Review feedback. |
1055 |
_1, key, _2, refs in |
3063.2.1
by Robert Collins
Solve reconciling erroring when multiple portions of a single delta chain are being reinserted. |
1056 |
self.new_pack.revision_index.iter_all_entries()]) |
1057 |
ideal_index = repo._generate_text_key_index(self._text_refs, ancestors) |
|
2951.2.9
by Robert Collins
* ``pack-0.92`` repositories can now be reconciled. |
1058 |
# 2) generate a text_nodes list that contains all the deltas that can
|
1059 |
# be used as-is, with corrected parents.
|
|
1060 |
ok_nodes = [] |
|
1061 |
bad_texts = [] |
|
1062 |
discarded_nodes = [] |
|
1063 |
NULL_REVISION = _mod_revision.NULL_REVISION |
|
1064 |
text_index_map, text_nodes = self._get_text_nodes() |
|
1065 |
for node in text_nodes: |
|
1066 |
# 0 - index
|
|
1067 |
# 1 - key
|
|
1068 |
# 2 - value
|
|
1069 |
# 3 - refs
|
|
1070 |
try: |
|
1071 |
ideal_parents = tuple(ideal_index[node[1]]) |
|
1072 |
except KeyError: |
|
1073 |
discarded_nodes.append(node) |
|
1074 |
self._data_changed = True |
|
1075 |
else: |
|
1076 |
if ideal_parents == (NULL_REVISION,): |
|
1077 |
ideal_parents = () |
|
1078 |
if ideal_parents == node[3][0]: |
|
1079 |
# no change needed.
|
|
1080 |
ok_nodes.append(node) |
|
1081 |
elif ideal_parents[0:1] == node[3][0][0:1]: |
|
1082 |
# the left most parent is the same, or there are no parents
|
|
1083 |
# today. Either way, we can preserve the representation as
|
|
1084 |
# long as we change the refs to be inserted.
|
|
1085 |
self._data_changed = True |
|
1086 |
ok_nodes.append((node[0], node[1], node[2], |
|
1087 |
(ideal_parents, node[3][1]))) |
|
1088 |
self._data_changed = True |
|
1089 |
else: |
|
1090 |
# Reinsert this text completely
|
|
1091 |
bad_texts.append((node[1], ideal_parents)) |
|
1092 |
self._data_changed = True |
|
1093 |
# we're finished with some data.
|
|
1094 |
del ideal_index |
|
1095 |
del text_nodes |
|
3063.2.2
by Robert Collins
Review feedback. |
1096 |
# 3) bulk copy the ok data
|
3070.1.1
by Robert Collins
* ``bzr pack`` now orders revision texts in topological order, with newest |
1097 |
total_items, readv_group_iter = self._least_readv_node_readv(ok_nodes) |
1098 |
list(self._copy_nodes_graph(text_index_map, self.new_pack._writer, |
|
1099 |
self.new_pack.text_index, readv_group_iter, total_items)) |
|
3063.2.1
by Robert Collins
Solve reconciling erroring when multiple portions of a single delta chain are being reinserted. |
1100 |
# 4) adhoc copy all the other texts.
|
1101 |
# We have to topologically insert all texts otherwise we can fail to
|
|
1102 |
# reconcile when parts of a single delta chain are preserved intact,
|
|
1103 |
# and other parts are not. E.g. Discarded->d1->d2->d3. d1 will be
|
|
1104 |
# reinserted, and if d3 has incorrect parents it will also be
|
|
1105 |
# reinserted. If we insert d3 first, d2 is present (as it was bulk
|
|
1106 |
# copied), so we will try to delta, but d2 is not currently able to be
|
|
1107 |
# extracted because it's basis d1 is not present. Topologically sorting
|
|
1108 |
# addresses this. The following generates a sort for all the texts that
|
|
1109 |
# are being inserted without having to reference the entire text key
|
|
1110 |
# space (we only topo sort the revisions, which is smaller).
|
|
1111 |
topo_order = tsort.topo_sort(ancestors) |
|
1112 |
rev_order = dict(zip(topo_order, range(len(topo_order)))) |
|
1113 |
bad_texts.sort(key=lambda key:rev_order[key[0][1]]) |
|
2951.2.9
by Robert Collins
* ``pack-0.92`` repositories can now be reconciled. |
1114 |
transaction = repo.get_transaction() |
1115 |
file_id_index = GraphIndexPrefixAdapter( |
|
1116 |
self.new_pack.text_index, |
|
1117 |
('blank', ), 1, |
|
1118 |
add_nodes_callback=self.new_pack.text_index.add_nodes) |
|
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
1119 |
data_access = _DirectPackAccess( |
1120 |
{self.new_pack.text_index:self.new_pack.access_tuple()}) |
|
1121 |
data_access.set_writer(self.new_pack._writer, self.new_pack.text_index, |
|
1122 |
self.new_pack.access_tuple()) |
|
1123 |
output_texts = KnitVersionedFiles( |
|
1124 |
_KnitGraphIndex(self.new_pack.text_index, |
|
1125 |
add_callback=self.new_pack.text_index.add_nodes, |
|
1126 |
deltas=True, parents=True, is_locked=repo.is_locked), |
|
1127 |
data_access=data_access, max_delta_chain=200) |
|
2951.2.9
by Robert Collins
* ``pack-0.92`` repositories can now be reconciled. |
1128 |
for key, parent_keys in bad_texts: |
1129 |
# We refer to the new pack to delta data being output.
|
|
1130 |
# A possible improvement would be to catch errors on short reads
|
|
1131 |
# and only flush then.
|
|
1132 |
self.new_pack.flush() |
|
1133 |
parents = [] |
|
1134 |
for parent_key in parent_keys: |
|
1135 |
if parent_key[0] != key[0]: |
|
1136 |
# Graph parents must match the fileid
|
|
1137 |
raise errors.BzrError('Mismatched key parent %r:%r' % |
|
1138 |
(key, parent_keys)) |
|
1139 |
parents.append(parent_key[1]) |
|
3734.2.4
by Vincent Ladeuil
Fix python2.6 deprecation warnings related to hashlib. |
1140 |
text_lines = osutils.split_lines(repo.texts.get_record_stream( |
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
1141 |
[key], 'unordered', True).next().get_bytes_as('fulltext')) |
1142 |
output_texts.add_lines(key, parent_keys, text_lines, |
|
1143 |
random_id=True, check_content=False) |
|
3063.2.2
by Robert Collins
Review feedback. |
1144 |
# 5) check that nothing inserted has a reference outside the keyspace.
|
3035.2.5
by John Arbash Meinel
Rename function to remove _new_ (per Robert's suggestion) |
1145 |
missing_text_keys = self.new_pack._external_compression_parents_of_texts() |
2951.2.9
by Robert Collins
* ``pack-0.92`` repositories can now be reconciled. |
1146 |
if missing_text_keys: |
1147 |
raise errors.BzrError('Reference to missing compression parents %r' |
|
3376.2.12
by Martin Pool
pyflakes corrections (thanks spiv) |
1148 |
% (missing_text_keys,)) |
2951.2.9
by Robert Collins
* ``pack-0.92`` repositories can now be reconciled. |
1149 |
self._log_copied_texts() |
1150 |
||
2951.2.8
by Robert Collins
Test that reconciling a repository can be done twice in a row. |
1151 |
def _use_pack(self, new_pack): |
1152 |
"""Override _use_pack to check for reconcile having changed content."""
|
|
1153 |
# XXX: we might be better checking this at the copy time.
|
|
1154 |
original_inventory_keys = set() |
|
1155 |
inv_index = self._pack_collection.inventory_index.combined_index |
|
1156 |
for entry in inv_index.iter_all_entries(): |
|
1157 |
original_inventory_keys.add(entry[1]) |
|
1158 |
new_inventory_keys = set() |
|
1159 |
for entry in new_pack.inventory_index.iter_all_entries(): |
|
1160 |
new_inventory_keys.add(entry[1]) |
|
1161 |
if new_inventory_keys != original_inventory_keys: |
|
1162 |
self._data_changed = True |
|
1163 |
return new_pack.data_inserted() and self._data_changed |
|
1164 |
||
2951.1.2
by Robert Collins
Partial refactoring of pack_repo to create a Packer object for packing. |
1165 |
|
1166 |
class RepositoryPackCollection(object): |
|
3517.4.4
by Martin Pool
Document RepositoryPackCollection._names |
1167 |
"""Management of packs within a repository.
|
1168 |
|
|
1169 |
:ivar _names: map of {pack_name: (index_size,)}
|
|
1170 |
"""
|
|
2951.1.2
by Robert Collins
Partial refactoring of pack_repo to create a Packer object for packing. |
1171 |
|
1172 |
def __init__(self, repo, transport, index_transport, upload_transport, |
|
3735.1.1
by Robert Collins
Add development2 formats using BTree indices. |
1173 |
pack_transport, index_builder_class, index_class): |
2951.1.2
by Robert Collins
Partial refactoring of pack_repo to create a Packer object for packing. |
1174 |
"""Create a new RepositoryPackCollection.
|
1175 |
||
1176 |
:param transport: Addresses the repository base directory
|
|
1177 |
(typically .bzr/repository/).
|
|
1178 |
:param index_transport: Addresses the directory containing indices.
|
|
1179 |
:param upload_transport: Addresses the directory into which packs are written
|
|
1180 |
while they're being created.
|
|
1181 |
:param pack_transport: Addresses the directory of existing complete packs.
|
|
3735.1.1
by Robert Collins
Add development2 formats using BTree indices. |
1182 |
:param index_builder_class: The index builder class to use.
|
1183 |
:param index_class: The index class to use.
|
|
2951.1.2
by Robert Collins
Partial refactoring of pack_repo to create a Packer object for packing. |
1184 |
"""
|
1185 |
self.repo = repo |
|
1186 |
self.transport = transport |
|
1187 |
self._index_transport = index_transport |
|
1188 |
self._upload_transport = upload_transport |
|
1189 |
self._pack_transport = pack_transport |
|
3735.1.1
by Robert Collins
Add development2 formats using BTree indices. |
1190 |
self._index_builder_class = index_builder_class |
1191 |
self._index_class = index_class |
|
2951.1.2
by Robert Collins
Partial refactoring of pack_repo to create a Packer object for packing. |
1192 |
self._suffix_offsets = {'.rix': 0, '.iix': 1, '.tix': 2, '.six': 3} |
1193 |
self.packs = [] |
|
1194 |
# name:Pack mapping
|
|
1195 |
self._packs_by_name = {} |
|
1196 |
# the previous pack-names content
|
|
1197 |
self._packs_at_load = None |
|
1198 |
# when a pack is being created by this object, the state of that pack.
|
|
1199 |
self._new_pack = None |
|
1200 |
# aggregated revision index data
|
|
3789.1.8
by John Arbash Meinel
Change the api of reload_pack_names(). |
1201 |
self.revision_index = AggregateIndex(self.reload_pack_names) |
1202 |
self.inventory_index = AggregateIndex(self.reload_pack_names) |
|
1203 |
self.text_index = AggregateIndex(self.reload_pack_names) |
|
1204 |
self.signature_index = AggregateIndex(self.reload_pack_names) |
|
2951.1.2
by Robert Collins
Partial refactoring of pack_repo to create a Packer object for packing. |
1205 |
|
1206 |
def add_pack_to_memory(self, pack): |
|
1207 |
"""Make a Pack object available to the repository to satisfy queries.
|
|
1208 |
|
|
1209 |
:param pack: A Pack object.
|
|
1210 |
"""
|
|
3376.2.4
by Martin Pool
Remove every assert statement from bzrlib! |
1211 |
if pack.name in self._packs_by_name: |
1212 |
raise AssertionError() |
|
2951.1.2
by Robert Collins
Partial refactoring of pack_repo to create a Packer object for packing. |
1213 |
self.packs.append(pack) |
1214 |
self._packs_by_name[pack.name] = pack |
|
1215 |
self.revision_index.add_index(pack.revision_index, pack) |
|
1216 |
self.inventory_index.add_index(pack.inventory_index, pack) |
|
1217 |
self.text_index.add_index(pack.text_index, pack) |
|
1218 |
self.signature_index.add_index(pack.signature_index, pack) |
|
1219 |
||
1220 |
def all_packs(self): |
|
1221 |
"""Return a list of all the Pack objects this repository has.
|
|
1222 |
||
1223 |
Note that an in-progress pack being created is not returned.
|
|
1224 |
||
1225 |
:return: A list of Pack objects for all the packs in the repository.
|
|
1226 |
"""
|
|
1227 |
result = [] |
|
1228 |
for name in self.names(): |
|
1229 |
result.append(self.get_pack_by_name(name)) |
|
1230 |
return result |
|
1231 |
||
1232 |
def autopack(self): |
|
1233 |
"""Pack the pack collection incrementally.
|
|
1234 |
|
|
1235 |
This will not attempt global reorganisation or recompression,
|
|
1236 |
rather it will just ensure that the total number of packs does
|
|
1237 |
not grow without bound. It uses the _max_pack_count method to
|
|
1238 |
determine if autopacking is needed, and the pack_distribution
|
|
1239 |
method to determine the number of revisions in each pack.
|
|
1240 |
||
1241 |
If autopacking takes place then the packs name collection will have
|
|
1242 |
been flushed to disk - packing requires updating the name collection
|
|
1243 |
in synchronisation with certain steps. Otherwise the names collection
|
|
1244 |
is not flushed.
|
|
1245 |
||
1246 |
:return: True if packing took place.
|
|
1247 |
"""
|
|
3789.2.20
by John Arbash Meinel
The autopack code can now trigger itself to retry when _copy_revision_texts fails. |
1248 |
while True: |
1249 |
try: |
|
1250 |
return self._do_autopack() |
|
1251 |
except errors.RetryAutopack, e: |
|
3789.2.22
by John Arbash Meinel
We need the Packer class to cleanup if it is getting a Retry it isn't handling. |
1252 |
# If we get a RetryAutopack exception, we should abort the
|
1253 |
# current action, and retry.
|
|
3789.2.20
by John Arbash Meinel
The autopack code can now trigger itself to retry when _copy_revision_texts fails. |
1254 |
pass
|
1255 |
||
1256 |
def _do_autopack(self): |
|
2951.1.2
by Robert Collins
Partial refactoring of pack_repo to create a Packer object for packing. |
1257 |
# XXX: Should not be needed when the management of indices is sane.
|
3221.12.13
by Robert Collins
Implement generic stacking rather than pack-internals based stacking. |
1258 |
total_revisions = self.revision_index.combined_index.key_count() |
1259 |
total_packs = len(self._names) |
|
2951.1.2
by Robert Collins
Partial refactoring of pack_repo to create a Packer object for packing. |
1260 |
if self._max_pack_count(total_revisions) >= total_packs: |
1261 |
return False |
|
1262 |
# XXX: the following may want to be a class, to pack with a given
|
|
1263 |
# policy.
|
|
1264 |
mutter('Auto-packing repository %s, which has %d pack files, ' |
|
1265 |
'containing %d revisions into %d packs.', self, total_packs, |
|
1266 |
total_revisions, self._max_pack_count(total_revisions)) |
|
1267 |
# determine which packs need changing
|
|
1268 |
pack_distribution = self.pack_distribution(total_revisions) |
|
1269 |
existing_packs = [] |
|
1270 |
for pack in self.all_packs(): |
|
1271 |
revision_count = pack.get_revision_count() |
|
1272 |
if revision_count == 0: |
|
1273 |
# revision less packs are not generated by normal operation,
|
|
1274 |
# only by operations like sign-my-commits, and thus will not
|
|
1275 |
# tend to grow rapdily or without bound like commit containing
|
|
1276 |
# packs do - leave them alone as packing them really should
|
|
1277 |
# group their data with the relevant commit, and that may
|
|
1278 |
# involve rewriting ancient history - which autopack tries to
|
|
1279 |
# avoid. Alternatively we could not group the data but treat
|
|
1280 |
# each of these as having a single revision, and thus add
|
|
1281 |
# one revision for each to the total revision count, to get
|
|
1282 |
# a matching distribution.
|
|
1283 |
continue
|
|
1284 |
existing_packs.append((revision_count, pack)) |
|
1285 |
pack_operations = self.plan_autopack_combinations( |
|
1286 |
existing_packs, pack_distribution) |
|
3789.2.20
by John Arbash Meinel
The autopack code can now trigger itself to retry when _copy_revision_texts fails. |
1287 |
self._execute_pack_operations(pack_operations, |
1288 |
reload_func=self._restart_autopack) |
|
2951.1.2
by Robert Collins
Partial refactoring of pack_repo to create a Packer object for packing. |
1289 |
return True |
1290 |
||
3789.2.20
by John Arbash Meinel
The autopack code can now trigger itself to retry when _copy_revision_texts fails. |
1291 |
def _execute_pack_operations(self, pack_operations, _packer_class=Packer, |
1292 |
reload_func=None): |
|
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
1293 |
"""Execute a series of pack operations.
|
1294 |
||
1295 |
:param pack_operations: A list of [revision_count, packs_to_combine].
|
|
3070.1.2
by John Arbash Meinel
Cleanup OptimizingPacker code according to my review feedback |
1296 |
:param _packer_class: The class of packer to use (default: Packer).
|
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
1297 |
:return: None.
|
1298 |
"""
|
|
2592.3.187
by Robert Collins
Finish cleaning up the packing logic to take Pack objects - all tests pass. |
1299 |
for revision_count, packs in pack_operations: |
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
1300 |
# we may have no-ops from the setup logic
|
2592.3.187
by Robert Collins
Finish cleaning up the packing logic to take Pack objects - all tests pass. |
1301 |
if len(packs) == 0: |
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
1302 |
continue
|
3789.2.22
by John Arbash Meinel
We need the Packer class to cleanup if it is getting a Retry it isn't handling. |
1303 |
packer = _packer_class(self, packs, '.autopack', |
1304 |
reload_func=reload_func) |
|
1305 |
try: |
|
1306 |
packer.pack() |
|
1307 |
except errors.RetryWithNewPacks: |
|
1308 |
# An exception is propagating out of this context, make sure
|
|
3789.2.23
by John Arbash Meinel
Clarify the comment. |
1309 |
# this packer has cleaned up. Packer() doesn't set its new_pack
|
1310 |
# state into the RepositoryPackCollection object, so we only
|
|
1311 |
# have access to it directly here.
|
|
3789.2.22
by John Arbash Meinel
We need the Packer class to cleanup if it is getting a Retry it isn't handling. |
1312 |
if packer.new_pack is not None: |
1313 |
packer.new_pack.abort() |
|
1314 |
raise
|
|
2592.3.187
by Robert Collins
Finish cleaning up the packing logic to take Pack objects - all tests pass. |
1315 |
for pack in packs: |
2592.3.236
by Martin Pool
Make RepositoryPackCollection.remove_pack_from_memory private |
1316 |
self._remove_pack_from_memory(pack) |
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
1317 |
# record the newly available packs and stop advertising the old
|
1318 |
# packs
|
|
2948.1.1
by Robert Collins
* Obsolete packs are now cleaned up by pack and autopack operations. |
1319 |
self._save_pack_names(clear_obsolete_packs=True) |
2592.3.187
by Robert Collins
Finish cleaning up the packing logic to take Pack objects - all tests pass. |
1320 |
# Move the old packs out of the way now they are no longer referenced.
|
1321 |
for revision_count, packs in pack_operations: |
|
1322 |
self._obsolete_packs(packs) |
|
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
1323 |
|
2592.3.188
by Robert Collins
Allow pack repositories to have multiple writers active at one time, for greater concurrency. |
1324 |
def lock_names(self): |
1325 |
"""Acquire the mutex around the pack-names index.
|
|
1326 |
|
|
1327 |
This cannot be used in the middle of a read-only transaction on the
|
|
1328 |
repository.
|
|
1329 |
"""
|
|
1330 |
self.repo.control_files.lock_write() |
|
1331 |
||
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
1332 |
def pack(self): |
1333 |
"""Pack the pack collection totally."""
|
|
1334 |
self.ensure_loaded() |
|
3221.12.13
by Robert Collins
Implement generic stacking rather than pack-internals based stacking. |
1335 |
total_packs = len(self._names) |
2592.3.213
by Robert Collins
Retain packs and indices in memory within a lock, even when write groups are entered and exited. |
1336 |
if total_packs < 2: |
3070.1.1
by Robert Collins
* ``bzr pack`` now orders revision texts in topological order, with newest |
1337 |
# This is arguably wrong because we might not be optimal, but for
|
1338 |
# now lets leave it in. (e.g. reconcile -> one pack. But not
|
|
1339 |
# optimal.
|
|
2592.3.213
by Robert Collins
Retain packs and indices in memory within a lock, even when write groups are entered and exited. |
1340 |
return
|
3221.12.13
by Robert Collins
Implement generic stacking rather than pack-internals based stacking. |
1341 |
total_revisions = self.revision_index.combined_index.key_count() |
2592.3.213
by Robert Collins
Retain packs and indices in memory within a lock, even when write groups are entered and exited. |
1342 |
# XXX: the following may want to be a class, to pack with a given
|
1343 |
# policy.
|
|
1344 |
mutter('Packing repository %s, which has %d pack files, ' |
|
1345 |
'containing %d revisions into 1 packs.', self, total_packs, |
|
1346 |
total_revisions) |
|
1347 |
# determine which packs need changing
|
|
1348 |
pack_distribution = [1] |
|
1349 |
pack_operations = [[0, []]] |
|
1350 |
for pack in self.all_packs(): |
|
3070.1.1
by Robert Collins
* ``bzr pack`` now orders revision texts in topological order, with newest |
1351 |
pack_operations[-1][0] += pack.get_revision_count() |
2592.3.213
by Robert Collins
Retain packs and indices in memory within a lock, even when write groups are entered and exited. |
1352 |
pack_operations[-1][1].append(pack) |
3070.1.1
by Robert Collins
* ``bzr pack`` now orders revision texts in topological order, with newest |
1353 |
self._execute_pack_operations(pack_operations, OptimisingPacker) |
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
1354 |
|
1355 |
def plan_autopack_combinations(self, existing_packs, pack_distribution): |
|
2592.3.176
by Robert Collins
Various pack refactorings. |
1356 |
"""Plan a pack operation.
|
1357 |
||
2592.3.187
by Robert Collins
Finish cleaning up the packing logic to take Pack objects - all tests pass. |
1358 |
:param existing_packs: The packs to pack. (A list of (revcount, Pack)
|
1359 |
tuples).
|
|
2592.3.235
by Martin Pool
Review cleanups |
1360 |
:param pack_distribution: A list with the number of revisions desired
|
2592.3.176
by Robert Collins
Various pack refactorings. |
1361 |
in each pack.
|
1362 |
"""
|
|
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
1363 |
if len(existing_packs) <= len(pack_distribution): |
1364 |
return [] |
|
1365 |
existing_packs.sort(reverse=True) |
|
1366 |
pack_operations = [[0, []]] |
|
1367 |
# plan out what packs to keep, and what to reorganise
|
|
1368 |
while len(existing_packs): |
|
1369 |
# take the largest pack, and if its less than the head of the
|
|
3711.4.1
by John Arbash Meinel
Fix bug #242510, when determining the autopack sequence, |
1370 |
# distribution chart we will include its contents in the new pack
|
1371 |
# for that position. If its larger, we remove its size from the
|
|
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
1372 |
# distribution chart
|
2592.3.187
by Robert Collins
Finish cleaning up the packing logic to take Pack objects - all tests pass. |
1373 |
next_pack_rev_count, next_pack = existing_packs.pop(0) |
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
1374 |
if next_pack_rev_count >= pack_distribution[0]: |
1375 |
# this is already packed 'better' than this, so we can
|
|
1376 |
# not waste time packing it.
|
|
1377 |
while next_pack_rev_count > 0: |
|
1378 |
next_pack_rev_count -= pack_distribution[0] |
|
1379 |
if next_pack_rev_count >= 0: |
|
1380 |
# more to go
|
|
1381 |
del pack_distribution[0] |
|
1382 |
else: |
|
1383 |
# didn't use that entire bucket up
|
|
1384 |
pack_distribution[0] = -next_pack_rev_count |
|
1385 |
else: |
|
1386 |
# add the revisions we're going to add to the next output pack
|
|
1387 |
pack_operations[-1][0] += next_pack_rev_count |
|
1388 |
# allocate this pack to the next pack sub operation
|
|
2592.3.187
by Robert Collins
Finish cleaning up the packing logic to take Pack objects - all tests pass. |
1389 |
pack_operations[-1][1].append(next_pack) |
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
1390 |
if pack_operations[-1][0] >= pack_distribution[0]: |
1391 |
# this pack is used up, shift left.
|
|
1392 |
del pack_distribution[0] |
|
1393 |
pack_operations.append([0, []]) |
|
3711.4.3
by John Arbash Meinel
Small cleanups from Robert |
1394 |
# Now that we know which pack files we want to move, shove them all
|
1395 |
# into a single pack file.
|
|
3711.4.2
by John Arbash Meinel
Change the logic to solve it in a different way. |
1396 |
final_rev_count = 0 |
1397 |
final_pack_list = [] |
|
1398 |
for num_revs, pack_files in pack_operations: |
|
1399 |
final_rev_count += num_revs |
|
1400 |
final_pack_list.extend(pack_files) |
|
1401 |
if len(final_pack_list) == 1: |
|
1402 |
raise AssertionError('We somehow generated an autopack with a' |
|
3711.4.3
by John Arbash Meinel
Small cleanups from Robert |
1403 |
' single pack file being moved.') |
3711.4.2
by John Arbash Meinel
Change the logic to solve it in a different way. |
1404 |
return [] |
1405 |
return [[final_rev_count, final_pack_list]] |
|
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
1406 |
|
1407 |
def ensure_loaded(self): |
|
2592.3.214
by Robert Collins
Merge bzr.dev. |
1408 |
# NB: if you see an assertion error here, its probably access against
|
1409 |
# an unlocked repo. Naughty.
|
|
3052.1.6
by John Arbash Meinel
Change the lock check to raise ObjectNotLocked. |
1410 |
if not self.repo.is_locked(): |
1411 |
raise errors.ObjectNotLocked(self.repo) |
|
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
1412 |
if self._names is None: |
2592.3.118
by Robert Collins
Record the size of the index files in the pack-names index. |
1413 |
self._names = {} |
2592.3.188
by Robert Collins
Allow pack repositories to have multiple writers active at one time, for greater concurrency. |
1414 |
self._packs_at_load = set() |
1415 |
for index, key, value in self._iter_disk_pack_index(): |
|
2592.3.118
by Robert Collins
Record the size of the index files in the pack-names index. |
1416 |
name = key[0] |
3221.12.13
by Robert Collins
Implement generic stacking rather than pack-internals based stacking. |
1417 |
self._names[name] = self._parse_index_sizes(value) |
2592.3.188
by Robert Collins
Allow pack repositories to have multiple writers active at one time, for greater concurrency. |
1418 |
self._packs_at_load.add((key, value)) |
2592.3.209
by Robert Collins
Revision index management looking sane for packs. |
1419 |
# populate all the metadata.
|
1420 |
self.all_packs() |
|
1421 |
||
1422 |
def _parse_index_sizes(self, value): |
|
1423 |
"""Parse a string of index sizes."""
|
|
1424 |
return tuple([int(digits) for digits in value.split(' ')]) |
|
2592.3.118
by Robert Collins
Record the size of the index files in the pack-names index. |
1425 |
|
2592.3.176
by Robert Collins
Various pack refactorings. |
1426 |
def get_pack_by_name(self, name): |
1427 |
"""Get a Pack object by name.
|
|
1428 |
||
1429 |
:param name: The name of the pack - e.g. '123456'
|
|
1430 |
:return: A Pack object.
|
|
1431 |
"""
|
|
1432 |
try: |
|
2592.3.232
by Martin Pool
Disambiguate two member variables called _packs into _packs_by_name and _pack_collection |
1433 |
return self._packs_by_name[name] |
2592.3.176
by Robert Collins
Various pack refactorings. |
1434 |
except KeyError: |
1435 |
rev_index = self._make_index(name, '.rix') |
|
1436 |
inv_index = self._make_index(name, '.iix') |
|
1437 |
txt_index = self._make_index(name, '.tix') |
|
1438 |
sig_index = self._make_index(name, '.six') |
|
3221.12.13
by Robert Collins
Implement generic stacking rather than pack-internals based stacking. |
1439 |
result = ExistingPack(self._pack_transport, name, rev_index, |
2592.3.191
by Robert Collins
Give Pack responsibility for index naming, and two concrete classes - NewPack for new packs and ExistingPack for packs we read from disk. |
1440 |
inv_index, txt_index, sig_index) |
2592.3.178
by Robert Collins
Add pack objects to the api for PackCollection.create_pack_from_packs. |
1441 |
self.add_pack_to_memory(result) |
2592.3.176
by Robert Collins
Various pack refactorings. |
1442 |
return result |
1443 |
||
2592.3.201
by Robert Collins
Cleanup RepositoryPackCollection.allocate. |
1444 |
def allocate(self, a_new_pack): |
2592.3.118
by Robert Collins
Record the size of the index files in the pack-names index. |
1445 |
"""Allocate name in the list of packs.
|
1446 |
||
2592.3.201
by Robert Collins
Cleanup RepositoryPackCollection.allocate. |
1447 |
:param a_new_pack: A NewPack instance to be added to the collection of
|
1448 |
packs for this repository.
|
|
2592.3.118
by Robert Collins
Record the size of the index files in the pack-names index. |
1449 |
"""
|
2592.3.91
by Robert Collins
Incrementally closing in on a correct fetch for packs. |
1450 |
self.ensure_loaded() |
2592.3.201
by Robert Collins
Cleanup RepositoryPackCollection.allocate. |
1451 |
if a_new_pack.name in self._names: |
2951.2.7
by Robert Collins
Raise an error on duplicate pack name allocation. |
1452 |
raise errors.BzrError( |
1453 |
'Pack %r already exists in %s' % (a_new_pack.name, self)) |
|
3221.12.13
by Robert Collins
Implement generic stacking rather than pack-internals based stacking. |
1454 |
self._names[a_new_pack.name] = tuple(a_new_pack.index_sizes) |
2592.3.201
by Robert Collins
Cleanup RepositoryPackCollection.allocate. |
1455 |
self.add_pack_to_memory(a_new_pack) |
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
1456 |
|
2592.3.188
by Robert Collins
Allow pack repositories to have multiple writers active at one time, for greater concurrency. |
1457 |
def _iter_disk_pack_index(self): |
1458 |
"""Iterate over the contents of the pack-names index.
|
|
1459 |
|
|
1460 |
This is used when loading the list from disk, and before writing to
|
|
1461 |
detect updates from others during our write operation.
|
|
1462 |
:return: An iterator of the index contents.
|
|
1463 |
"""
|
|
3735.1.1
by Robert Collins
Add development2 formats using BTree indices. |
1464 |
return self._index_class(self.transport, 'pack-names', None |
2592.3.188
by Robert Collins
Allow pack repositories to have multiple writers active at one time, for greater concurrency. |
1465 |
).iter_all_entries() |
1466 |
||
2592.3.176
by Robert Collins
Various pack refactorings. |
1467 |
def _make_index(self, name, suffix): |
1468 |
size_offset = self._suffix_offsets[suffix] |
|
1469 |
index_name = name + suffix |
|
3221.12.13
by Robert Collins
Implement generic stacking rather than pack-internals based stacking. |
1470 |
index_size = self._names[name][size_offset] |
3735.1.1
by Robert Collins
Add development2 formats using BTree indices. |
1471 |
return self._index_class( |
3221.12.13
by Robert Collins
Implement generic stacking rather than pack-internals based stacking. |
1472 |
self._index_transport, index_name, index_size) |
2592.5.5
by Martin Pool
Make RepositoryPackCollection remember the index transport, and responsible for getting a map of indexes |
1473 |
|
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
1474 |
def _max_pack_count(self, total_revisions): |
1475 |
"""Return the maximum number of packs to use for total revisions.
|
|
1476 |
|
|
1477 |
:param total_revisions: The total number of revisions in the
|
|
1478 |
repository.
|
|
1479 |
"""
|
|
1480 |
if not total_revisions: |
|
1481 |
return 1 |
|
1482 |
digits = str(total_revisions) |
|
1483 |
result = 0 |
|
1484 |
for digit in digits: |
|
1485 |
result += int(digit) |
|
1486 |
return result |
|
1487 |
||
1488 |
def names(self): |
|
1489 |
"""Provide an order to the underlying names."""
|
|
2592.3.118
by Robert Collins
Record the size of the index files in the pack-names index. |
1490 |
return sorted(self._names.keys()) |
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
1491 |
|
2592.3.187
by Robert Collins
Finish cleaning up the packing logic to take Pack objects - all tests pass. |
1492 |
def _obsolete_packs(self, packs): |
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
1493 |
"""Move a number of packs which have been obsoleted out of the way.
|
1494 |
||
1495 |
Each pack and its associated indices are moved out of the way.
|
|
1496 |
||
1497 |
Note: for correctness this function should only be called after a new
|
|
1498 |
pack names index has been written without these pack names, and with
|
|
1499 |
the names of packs that contain the data previously available via these
|
|
1500 |
packs.
|
|
1501 |
||
2592.3.187
by Robert Collins
Finish cleaning up the packing logic to take Pack objects - all tests pass. |
1502 |
:param packs: The packs to obsolete.
|
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
1503 |
:param return: None.
|
1504 |
"""
|
|
2592.3.187
by Robert Collins
Finish cleaning up the packing logic to take Pack objects - all tests pass. |
1505 |
for pack in packs: |
2592.3.200
by Robert Collins
Make NewPack reopen the index files, separating out the task of refreshing the index maps in the repository and managing the completion of writing a single pack to disk. |
1506 |
pack.pack_transport.rename(pack.file_name(), |
2592.3.187
by Robert Collins
Finish cleaning up the packing logic to take Pack objects - all tests pass. |
1507 |
'../obsolete_packs/' + pack.file_name()) |
2592.3.226
by Martin Pool
formatting and docstrings |
1508 |
# TODO: Probably needs to know all possible indices for this pack
|
1509 |
# - or maybe list the directory and move all indices matching this
|
|
2592.5.13
by Martin Pool
Clean up duplicate index_transport variables |
1510 |
# name whether we recognize it or not?
|
2592.3.187
by Robert Collins
Finish cleaning up the packing logic to take Pack objects - all tests pass. |
1511 |
for suffix in ('.iix', '.six', '.tix', '.rix'): |
1512 |
self._index_transport.rename(pack.name + suffix, |
|
1513 |
'../obsolete_packs/' + pack.name + suffix) |
|
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
1514 |
|
1515 |
def pack_distribution(self, total_revisions): |
|
1516 |
"""Generate a list of the number of revisions to put in each pack.
|
|
1517 |
||
1518 |
:param total_revisions: The total number of revisions in the
|
|
1519 |
repository.
|
|
1520 |
"""
|
|
1521 |
if total_revisions == 0: |
|
1522 |
return [0] |
|
1523 |
digits = reversed(str(total_revisions)) |
|
1524 |
result = [] |
|
1525 |
for exponent, count in enumerate(digits): |
|
1526 |
size = 10 ** exponent |
|
1527 |
for pos in range(int(count)): |
|
1528 |
result.append(size) |
|
1529 |
return list(reversed(result)) |
|
1530 |
||
2592.5.12
by Martin Pool
Move pack_transport and pack_name onto RepositoryPackCollection |
1531 |
def _pack_tuple(self, name): |
1532 |
"""Return a tuple with the transport and file name for a pack name."""
|
|
1533 |
return self._pack_transport, name + '.pack' |
|
1534 |
||
2592.3.236
by Martin Pool
Make RepositoryPackCollection.remove_pack_from_memory private |
1535 |
def _remove_pack_from_memory(self, pack): |
2592.3.209
by Robert Collins
Revision index management looking sane for packs. |
1536 |
"""Remove pack from the packs accessed by this repository.
|
1537 |
|
|
1538 |
Only affects memory state, until self._save_pack_names() is invoked.
|
|
1539 |
"""
|
|
1540 |
self._names.pop(pack.name) |
|
2592.3.232
by Martin Pool
Disambiguate two member variables called _packs into _packs_by_name and _pack_collection |
1541 |
self._packs_by_name.pop(pack.name) |
2592.3.213
by Robert Collins
Retain packs and indices in memory within a lock, even when write groups are entered and exited. |
1542 |
self._remove_pack_indices(pack) |
3794.3.1
by John Arbash Meinel
In _remove_pack_from_memory, also remove the object from the PackCollection.packs list. |
1543 |
self.packs.remove(pack) |
2592.3.213
by Robert Collins
Retain packs and indices in memory within a lock, even when write groups are entered and exited. |
1544 |
|
1545 |
def _remove_pack_indices(self, pack): |
|
1546 |
"""Remove the indices for pack from the aggregated indices."""
|
|
2592.3.209
by Robert Collins
Revision index management looking sane for packs. |
1547 |
self.revision_index.remove_index(pack.revision_index, pack) |
2592.3.211
by Robert Collins
Pack inventory index management cleaned up. |
1548 |
self.inventory_index.remove_index(pack.inventory_index, pack) |
2592.3.212
by Robert Collins
Cleanup text index management in packs. |
1549 |
self.text_index.remove_index(pack.text_index, pack) |
2592.3.210
by Robert Collins
Signature index management looking sane for packs. |
1550 |
self.signature_index.remove_index(pack.signature_index, pack) |
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
1551 |
|
1552 |
def reset(self): |
|
2592.3.190
by Robert Collins
Move flush and reset operations to the pack collection rather than the thunk layers. |
1553 |
"""Clear all cached data."""
|
1554 |
# cached revision data
|
|
1555 |
self.repo._revision_knit = None |
|
2592.3.209
by Robert Collins
Revision index management looking sane for packs. |
1556 |
self.revision_index.clear() |
2592.3.190
by Robert Collins
Move flush and reset operations to the pack collection rather than the thunk layers. |
1557 |
# cached signature data
|
1558 |
self.repo._signature_knit = None |
|
2592.3.210
by Robert Collins
Signature index management looking sane for packs. |
1559 |
self.signature_index.clear() |
2592.3.212
by Robert Collins
Cleanup text index management in packs. |
1560 |
# cached file text data
|
1561 |
self.text_index.clear() |
|
2592.3.190
by Robert Collins
Move flush and reset operations to the pack collection rather than the thunk layers. |
1562 |
self.repo._text_knit = None |
2592.3.211
by Robert Collins
Pack inventory index management cleaned up. |
1563 |
# cached inventory data
|
1564 |
self.inventory_index.clear() |
|
2592.3.192
by Robert Collins
Move new revision index management to NewPack. |
1565 |
# remove the open pack
|
1566 |
self._new_pack = None |
|
2592.3.190
by Robert Collins
Move flush and reset operations to the pack collection rather than the thunk layers. |
1567 |
# information about packs.
|
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
1568 |
self._names = None |
2592.3.90
by Robert Collins
Slightly broken, but branch and fetch performance is now roughly on par (for bzr.dev) with knits - should be much faster for large repos. |
1569 |
self.packs = [] |
2592.3.232
by Martin Pool
Disambiguate two member variables called _packs into _packs_by_name and _pack_collection |
1570 |
self._packs_by_name = {} |
2592.3.188
by Robert Collins
Allow pack repositories to have multiple writers active at one time, for greater concurrency. |
1571 |
self._packs_at_load = None |
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
1572 |
|
2592.3.207
by Robert Collins
Start removing the dependency on RepositoryPackCollection._make_index_map. |
1573 |
def _make_index_map(self, index_suffix): |
2592.3.226
by Martin Pool
formatting and docstrings |
1574 |
"""Return information on existing indices.
|
2592.3.207
by Robert Collins
Start removing the dependency on RepositoryPackCollection._make_index_map. |
1575 |
|
1576 |
:param suffix: Index suffix added to pack name.
|
|
1577 |
||
1578 |
:returns: (pack_map, indices) where indices is a list of GraphIndex
|
|
1579 |
objects, and pack_map is a mapping from those objects to the
|
|
1580 |
pack tuple they describe.
|
|
1581 |
"""
|
|
1582 |
# TODO: stop using this; it creates new indices unnecessarily.
|
|
2592.3.176
by Robert Collins
Various pack refactorings. |
1583 |
self.ensure_loaded() |
2592.3.226
by Martin Pool
formatting and docstrings |
1584 |
suffix_map = {'.rix': 'revision_index', |
1585 |
'.six': 'signature_index', |
|
1586 |
'.iix': 'inventory_index', |
|
1587 |
'.tix': 'text_index', |
|
2592.3.207
by Robert Collins
Start removing the dependency on RepositoryPackCollection._make_index_map. |
1588 |
}
|
1589 |
return self._packs_list_to_pack_map_and_index_list(self.all_packs(), |
|
1590 |
suffix_map[index_suffix]) |
|
2592.5.15
by Martin Pool
Split out common code for making index maps |
1591 |
|
2592.3.179
by Robert Collins
Generate the revision_index_map for packing during the core operation, from the pack objects. |
1592 |
def _packs_list_to_pack_map_and_index_list(self, packs, index_attribute): |
1593 |
"""Convert a list of packs to an index pack map and index list.
|
|
1594 |
||
1595 |
:param packs: The packs list to process.
|
|
1596 |
:param index_attribute: The attribute that the desired index is found
|
|
1597 |
on.
|
|
1598 |
:return: A tuple (map, list) where map contains the dict from
|
|
1599 |
index:pack_tuple, and lsit contains the indices in the same order
|
|
1600 |
as the packs list.
|
|
1601 |
"""
|
|
1602 |
indices = [] |
|
1603 |
pack_map = {} |
|
1604 |
for pack in packs: |
|
1605 |
index = getattr(pack, index_attribute) |
|
1606 |
indices.append(index) |
|
2592.3.200
by Robert Collins
Make NewPack reopen the index files, separating out the task of refreshing the index maps in the repository and managing the completion of writing a single pack to disk. |
1607 |
pack_map[index] = (pack.pack_transport, pack.file_name()) |
2592.3.179
by Robert Collins
Generate the revision_index_map for packing during the core operation, from the pack objects. |
1608 |
return pack_map, indices |
1609 |
||
2592.3.93
by Robert Collins
Steps toward filtering revisions/inventories/texts during fetch. |
1610 |
def _index_contents(self, pack_map, key_filter=None): |
1611 |
"""Get an iterable of the index contents from a pack_map.
|
|
1612 |
||
1613 |
:param pack_map: A map from indices to pack details.
|
|
1614 |
:param key_filter: An optional filter to limit the
|
|
1615 |
keys returned.
|
|
1616 |
"""
|
|
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
1617 |
indices = [index for index in pack_map.iterkeys()] |
1618 |
all_index = CombinedGraphIndex(indices) |
|
2592.3.93
by Robert Collins
Steps toward filtering revisions/inventories/texts during fetch. |
1619 |
if key_filter is None: |
1620 |
return all_index.iter_all_entries() |
|
1621 |
else: |
|
1622 |
return all_index.iter_entries(key_filter) |
|
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
1623 |
|
2592.3.237
by Martin Pool
Rename RepositoryPackCollection.release_names to _unlock_names |
1624 |
def _unlock_names(self): |
2592.3.188
by Robert Collins
Allow pack repositories to have multiple writers active at one time, for greater concurrency. |
1625 |
"""Release the mutex around the pack-names index."""
|
1626 |
self.repo.control_files.unlock() |
|
1627 |
||
3789.1.2
by John Arbash Meinel
Add RepositoryPackCollection.reload_pack_names() |
1628 |
def _diff_pack_names(self): |
1629 |
"""Read the pack names from disk, and compare it to the one in memory.
|
|
1630 |
||
1631 |
:return: (disk_nodes, deleted_nodes, new_nodes)
|
|
1632 |
disk_nodes The final set of nodes that should be referenced
|
|
1633 |
deleted_nodes Nodes which have been removed from when we started
|
|
1634 |
new_nodes Nodes that are newly introduced
|
|
1635 |
"""
|
|
1636 |
# load the disk nodes across
|
|
1637 |
disk_nodes = set() |
|
1638 |
for index, key, value in self._iter_disk_pack_index(): |
|
1639 |
disk_nodes.add((key, value)) |
|
1640 |
||
1641 |
# do a two-way diff against our original content
|
|
1642 |
current_nodes = set() |
|
1643 |
for name, sizes in self._names.iteritems(): |
|
1644 |
current_nodes.add( |
|
1645 |
((name, ), ' '.join(str(size) for size in sizes))) |
|
1646 |
||
1647 |
# Packs no longer present in the repository, which were present when we
|
|
1648 |
# locked the repository
|
|
1649 |
deleted_nodes = self._packs_at_load - current_nodes |
|
1650 |
# Packs which this process is adding
|
|
1651 |
new_nodes = current_nodes - self._packs_at_load |
|
1652 |
||
1653 |
# Update the disk_nodes set to include the ones we are adding, and
|
|
1654 |
# remove the ones which were removed by someone else
|
|
1655 |
disk_nodes.difference_update(deleted_nodes) |
|
1656 |
disk_nodes.update(new_nodes) |
|
1657 |
||
1658 |
return disk_nodes, deleted_nodes, new_nodes |
|
1659 |
||
1660 |
def _syncronize_pack_names_from_disk_nodes(self, disk_nodes): |
|
1661 |
"""Given the correct set of pack files, update our saved info.
|
|
1662 |
||
1663 |
:return: (removed, added, modified)
|
|
1664 |
removed pack names removed from self._names
|
|
1665 |
added pack names added to self._names
|
|
1666 |
modified pack names that had changed value
|
|
1667 |
"""
|
|
1668 |
removed = [] |
|
1669 |
added = [] |
|
1670 |
modified = [] |
|
1671 |
## self._packs_at_load = disk_nodes
|
|
3221.12.13
by Robert Collins
Implement generic stacking rather than pack-internals based stacking. |
1672 |
new_names = dict(disk_nodes) |
2592.3.209
by Robert Collins
Revision index management looking sane for packs. |
1673 |
# drop no longer present nodes
|
1674 |
for pack in self.all_packs(): |
|
3221.12.13
by Robert Collins
Implement generic stacking rather than pack-internals based stacking. |
1675 |
if (pack.name,) not in new_names: |
3789.1.2
by John Arbash Meinel
Add RepositoryPackCollection.reload_pack_names() |
1676 |
removed.append(pack.name) |
2592.3.236
by Martin Pool
Make RepositoryPackCollection.remove_pack_from_memory private |
1677 |
self._remove_pack_from_memory(pack) |
2592.3.209
by Robert Collins
Revision index management looking sane for packs. |
1678 |
# add new nodes/refresh existing ones
|
3221.12.13
by Robert Collins
Implement generic stacking rather than pack-internals based stacking. |
1679 |
for key, value in disk_nodes: |
1680 |
name = key[0] |
|
1681 |
sizes = self._parse_index_sizes(value) |
|
2592.3.209
by Robert Collins
Revision index management looking sane for packs. |
1682 |
if name in self._names: |
1683 |
# existing
|
|
3221.12.13
by Robert Collins
Implement generic stacking rather than pack-internals based stacking. |
1684 |
if sizes != self._names[name]: |
2592.3.209
by Robert Collins
Revision index management looking sane for packs. |
1685 |
# the pack for name has had its indices replaced - rare but
|
1686 |
# important to handle. XXX: probably can never happen today
|
|
1687 |
# because the three-way merge code above does not handle it
|
|
1688 |
# - you may end up adding the same key twice to the new
|
|
1689 |
# disk index because the set values are the same, unless
|
|
1690 |
# the only index shows up as deleted by the set difference
|
|
1691 |
# - which it may. Until there is a specific test for this,
|
|
1692 |
# assume its broken. RBC 20071017.
|
|
2592.3.236
by Martin Pool
Make RepositoryPackCollection.remove_pack_from_memory private |
1693 |
self._remove_pack_from_memory(self.get_pack_by_name(name)) |
2592.3.209
by Robert Collins
Revision index management looking sane for packs. |
1694 |
self._names[name] = sizes |
1695 |
self.get_pack_by_name(name) |
|
3789.1.2
by John Arbash Meinel
Add RepositoryPackCollection.reload_pack_names() |
1696 |
modified.append(name) |
2592.3.209
by Robert Collins
Revision index management looking sane for packs. |
1697 |
else: |
1698 |
# new
|
|
3221.12.13
by Robert Collins
Implement generic stacking rather than pack-internals based stacking. |
1699 |
self._names[name] = sizes |
2592.3.209
by Robert Collins
Revision index management looking sane for packs. |
1700 |
self.get_pack_by_name(name) |
3789.1.2
by John Arbash Meinel
Add RepositoryPackCollection.reload_pack_names() |
1701 |
added.append(name) |
1702 |
return removed, added, modified |
|
1703 |
||
1704 |
def _save_pack_names(self, clear_obsolete_packs=False): |
|
1705 |
"""Save the list of packs.
|
|
1706 |
||
1707 |
This will take out the mutex around the pack names list for the
|
|
1708 |
duration of the method call. If concurrent updates have been made, a
|
|
1709 |
three-way merge between the current list and the current in memory list
|
|
1710 |
is performed.
|
|
1711 |
||
1712 |
:param clear_obsolete_packs: If True, clear out the contents of the
|
|
1713 |
obsolete_packs directory.
|
|
1714 |
"""
|
|
1715 |
self.lock_names() |
|
1716 |
try: |
|
1717 |
builder = self._index_builder_class() |
|
1718 |
disk_nodes, deleted_nodes, new_nodes = self._diff_pack_names() |
|
1719 |
# TODO: handle same-name, index-size-changes here -
|
|
1720 |
# e.g. use the value from disk, not ours, *unless* we're the one
|
|
1721 |
# changing it.
|
|
1722 |
for key, value in disk_nodes: |
|
1723 |
builder.add_node(key, value) |
|
1724 |
self.transport.put_file('pack-names', builder.finish(), |
|
1725 |
mode=self.repo.bzrdir._get_file_mode()) |
|
1726 |
# move the baseline forward
|
|
1727 |
self._packs_at_load = disk_nodes |
|
1728 |
if clear_obsolete_packs: |
|
1729 |
self._clear_obsolete_packs() |
|
1730 |
finally: |
|
1731 |
self._unlock_names() |
|
1732 |
# synchronise the memory packs list with what we just wrote:
|
|
1733 |
self._syncronize_pack_names_from_disk_nodes(disk_nodes) |
|
1734 |
||
1735 |
def reload_pack_names(self): |
|
1736 |
"""Sync our pack listing with what is present in the repository.
|
|
1737 |
||
1738 |
This should be called when we find out that something we thought was
|
|
1739 |
present is now missing. This happens when another process re-packs the
|
|
1740 |
repository, etc.
|
|
1741 |
"""
|
|
1742 |
# This is functionally similar to _save_pack_names, but we don't write
|
|
1743 |
# out the new value.
|
|
1744 |
disk_nodes, _, _ = self._diff_pack_names() |
|
1745 |
self._packs_at_load = disk_nodes |
|
3789.1.8
by John Arbash Meinel
Change the api of reload_pack_names(). |
1746 |
(removed, added, |
1747 |
modified) = self._syncronize_pack_names_from_disk_nodes(disk_nodes) |
|
1748 |
if removed or added or modified: |
|
1749 |
return True |
|
1750 |
return False |
|
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
1751 |
|
3789.2.20
by John Arbash Meinel
The autopack code can now trigger itself to retry when _copy_revision_texts fails. |
1752 |
def _restart_autopack(self): |
1753 |
"""Reload the pack names list, and restart the autopack code."""
|
|
1754 |
if not self.reload_pack_names(): |
|
1755 |
# Re-raise the original exception, because something went missing
|
|
1756 |
# and a restart didn't find it
|
|
1757 |
raise
|
|
1758 |
raise errors.RetryAutopack(False, sys.exc_info()) |
|
1759 |
||
3446.2.1
by Martin Pool
Failure to delete an obsolete pack file should not be fatal. |
1760 |
def _clear_obsolete_packs(self): |
1761 |
"""Delete everything from the obsolete-packs directory.
|
|
1762 |
"""
|
|
1763 |
obsolete_pack_transport = self.transport.clone('obsolete_packs') |
|
1764 |
for filename in obsolete_pack_transport.list_dir('.'): |
|
1765 |
try: |
|
1766 |
obsolete_pack_transport.delete(filename) |
|
1767 |
except (errors.PathError, errors.TransportError), e: |
|
1768 |
warning("couldn't delete obsolete pack, skipping it:\n%s" % (e,)) |
|
1769 |
||
2592.3.202
by Robert Collins
Move write stream management into NewPack. |
1770 |
def _start_write_group(self): |
2592.3.190
by Robert Collins
Move flush and reset operations to the pack collection rather than the thunk layers. |
1771 |
# Do not permit preparation for writing if we're not in a 'write lock'.
|
2592.3.188
by Robert Collins
Allow pack repositories to have multiple writers active at one time, for greater concurrency. |
1772 |
if not self.repo.is_write_locked(): |
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
1773 |
raise errors.NotWriteLocked(self) |
2592.3.194
by Robert Collins
Output the revision index from NewPack.finish |
1774 |
self._new_pack = NewPack(self._upload_transport, self._index_transport, |
3010.1.11
by Robert Collins
Provide file modes to files created by pack repositories |
1775 |
self._pack_transport, upload_suffix='.pack', |
3735.1.1
by Robert Collins
Add development2 formats using BTree indices. |
1776 |
file_mode=self.repo.bzrdir._get_file_mode(), |
1777 |
index_builder_class=self._index_builder_class, |
|
1778 |
index_class=self._index_class) |
|
2592.3.209
by Robert Collins
Revision index management looking sane for packs. |
1779 |
# allow writing: queue writes to a new index
|
1780 |
self.revision_index.add_writable_index(self._new_pack.revision_index, |
|
1781 |
self._new_pack) |
|
2592.3.211
by Robert Collins
Pack inventory index management cleaned up. |
1782 |
self.inventory_index.add_writable_index(self._new_pack.inventory_index, |
1783 |
self._new_pack) |
|
2592.3.212
by Robert Collins
Cleanup text index management in packs. |
1784 |
self.text_index.add_writable_index(self._new_pack.text_index, |
1785 |
self._new_pack) |
|
2592.3.210
by Robert Collins
Signature index management looking sane for packs. |
1786 |
self.signature_index.add_writable_index(self._new_pack.signature_index, |
1787 |
self._new_pack) |
|
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
1788 |
|
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
1789 |
self.repo.inventories._index._add_callback = self.inventory_index.add_callback |
1790 |
self.repo.revisions._index._add_callback = self.revision_index.add_callback |
|
1791 |
self.repo.signatures._index._add_callback = self.signature_index.add_callback |
|
1792 |
self.repo.texts._index._add_callback = self.text_index.add_callback |
|
2592.5.9
by Martin Pool
Move some more bits that seem to belong in RepositoryPackCollection into there |
1793 |
|
2592.5.8
by Martin Pool
Delegate abort_write_group to RepositoryPackCollection |
1794 |
def _abort_write_group(self): |
1795 |
# FIXME: just drop the transient index.
|
|
1796 |
# forget what names there are
|
|
3163.1.2
by Martin Pool
RepositoryPackCollection._abort_write_group should check it actually has a new pack before aborting (#180208) |
1797 |
if self._new_pack is not None: |
1798 |
self._new_pack.abort() |
|
1799 |
self._remove_pack_indices(self._new_pack) |
|
1800 |
self._new_pack = None |
|
2592.3.213
by Robert Collins
Retain packs and indices in memory within a lock, even when write groups are entered and exited. |
1801 |
self.repo._text_knit = None |
2592.5.6
by Martin Pool
Move pack repository start_write_group to pack collection object |
1802 |
|
2592.5.7
by Martin Pool
move commit_write_group to RepositoryPackCollection |
1803 |
def _commit_write_group(self): |
2592.3.213
by Robert Collins
Retain packs and indices in memory within a lock, even when write groups are entered and exited. |
1804 |
self._remove_pack_indices(self._new_pack) |
2592.3.198
by Robert Collins
Factor out data_inserted to reduce code duplication in detecting empty packs. |
1805 |
if self._new_pack.data_inserted(): |
2592.3.209
by Robert Collins
Revision index management looking sane for packs. |
1806 |
# get all the data to disk and read to use
|
2592.3.194
by Robert Collins
Output the revision index from NewPack.finish |
1807 |
self._new_pack.finish() |
2592.3.201
by Robert Collins
Cleanup RepositoryPackCollection.allocate. |
1808 |
self.allocate(self._new_pack) |
2592.3.194
by Robert Collins
Output the revision index from NewPack.finish |
1809 |
self._new_pack = None |
2592.5.7
by Martin Pool
move commit_write_group to RepositoryPackCollection |
1810 |
if not self.autopack(): |
2592.3.201
by Robert Collins
Cleanup RepositoryPackCollection.allocate. |
1811 |
# when autopack takes no steps, the names list is still
|
1812 |
# unsaved.
|
|
2592.5.10
by Martin Pool
Rename RepositoryPackCollection.save to _save_pack_names |
1813 |
self._save_pack_names() |
2592.5.7
by Martin Pool
move commit_write_group to RepositoryPackCollection |
1814 |
else: |
2592.3.202
by Robert Collins
Move write stream management into NewPack. |
1815 |
self._new_pack.abort() |
2951.1.1
by Robert Collins
(robertc) Fix data-refresh logic for packs not to refresh mid-transaction when a names write lock is held. (Robert Collins) |
1816 |
self._new_pack = None |
2592.3.213
by Robert Collins
Retain packs and indices in memory within a lock, even when write groups are entered and exited. |
1817 |
self.repo._text_knit = None |
2592.5.8
by Martin Pool
Delegate abort_write_group to RepositoryPackCollection |
1818 |
|
1819 |
||
2592.3.224
by Martin Pool
Rename GraphKnitRepository etc to KnitPackRepository |
1820 |
class KnitPackRepository(KnitRepository): |
3350.6.7
by Robert Collins
Review feedback, making things more clear, adding documentation on what is used where. |
1821 |
"""Repository with knit objects stored inside pack containers.
|
1822 |
|
|
1823 |
The layering for a KnitPackRepository is:
|
|
1824 |
||
1825 |
Graph | HPSS | Repository public layer |
|
|
1826 |
===================================================
|
|
1827 |
Tuple based apis below, string based, and key based apis above
|
|
1828 |
---------------------------------------------------
|
|
1829 |
KnitVersionedFiles
|
|
1830 |
Provides .texts, .revisions etc
|
|
1831 |
This adapts the N-tuple keys to physical knit records which only have a
|
|
1832 |
single string identifier (for historical reasons), which in older formats
|
|
1833 |
was always the revision_id, and in the mapped code for packs is always
|
|
1834 |
the last element of key tuples.
|
|
1835 |
---------------------------------------------------
|
|
1836 |
GraphIndex
|
|
1837 |
A separate GraphIndex is used for each of the
|
|
1838 |
texts/inventories/revisions/signatures contained within each individual
|
|
1839 |
pack file. The GraphIndex layer works in N-tuples and is unaware of any
|
|
1840 |
semantic value.
|
|
1841 |
===================================================
|
|
1842 |
|
|
1843 |
"""
|
|
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
1844 |
|
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
1845 |
def __init__(self, _format, a_bzrdir, control_files, _commit_builder_class, |
1846 |
_serializer): |
|
1847 |
KnitRepository.__init__(self, _format, a_bzrdir, control_files, |
|
1848 |
_commit_builder_class, _serializer) |
|
3407.2.13
by Martin Pool
Remove indirection through control_files to get transports |
1849 |
index_transport = self._transport.clone('indices') |
3350.6.5
by Robert Collins
Update to bzr.dev. |
1850 |
self._pack_collection = RepositoryPackCollection(self, self._transport, |
2592.5.11
by Martin Pool
Move upload_transport from pack repositories to the pack collection |
1851 |
index_transport, |
3407.2.13
by Martin Pool
Remove indirection through control_files to get transports |
1852 |
self._transport.clone('upload'), |
3735.1.1
by Robert Collins
Add development2 formats using BTree indices. |
1853 |
self._transport.clone('packs'), |
1854 |
_format.index_builder_class, |
|
1855 |
_format.index_class) |
|
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
1856 |
self.inventories = KnitVersionedFiles( |
1857 |
_KnitGraphIndex(self._pack_collection.inventory_index.combined_index, |
|
1858 |
add_callback=self._pack_collection.inventory_index.add_callback, |
|
1859 |
deltas=True, parents=True, is_locked=self.is_locked), |
|
1860 |
data_access=self._pack_collection.inventory_index.data_access, |
|
1861 |
max_delta_chain=200) |
|
1862 |
self.revisions = KnitVersionedFiles( |
|
1863 |
_KnitGraphIndex(self._pack_collection.revision_index.combined_index, |
|
1864 |
add_callback=self._pack_collection.revision_index.add_callback, |
|
1865 |
deltas=False, parents=True, is_locked=self.is_locked), |
|
1866 |
data_access=self._pack_collection.revision_index.data_access, |
|
1867 |
max_delta_chain=0) |
|
1868 |
self.signatures = KnitVersionedFiles( |
|
1869 |
_KnitGraphIndex(self._pack_collection.signature_index.combined_index, |
|
1870 |
add_callback=self._pack_collection.signature_index.add_callback, |
|
1871 |
deltas=False, parents=False, is_locked=self.is_locked), |
|
1872 |
data_access=self._pack_collection.signature_index.data_access, |
|
1873 |
max_delta_chain=0) |
|
1874 |
self.texts = KnitVersionedFiles( |
|
1875 |
_KnitGraphIndex(self._pack_collection.text_index.combined_index, |
|
1876 |
add_callback=self._pack_collection.text_index.add_callback, |
|
1877 |
deltas=True, parents=True, is_locked=self.is_locked), |
|
1878 |
data_access=self._pack_collection.text_index.data_access, |
|
1879 |
max_delta_chain=200) |
|
2592.3.188
by Robert Collins
Allow pack repositories to have multiple writers active at one time, for greater concurrency. |
1880 |
# True when the repository object is 'write locked' (as opposed to the
|
1881 |
# physical lock only taken out around changes to the pack-names list.)
|
|
1882 |
# Another way to represent this would be a decorator around the control
|
|
1883 |
# files object that presents logical locks as physical ones - if this
|
|
1884 |
# gets ugly consider that alternative design. RBC 20071011
|
|
1885 |
self._write_lock_count = 0 |
|
1886 |
self._transaction = None |
|
2592.3.96
by Robert Collins
Merge index improvements (includes bzr.dev). |
1887 |
# for tests
|
2951.1.3
by Robert Collins
Partial support for native reconcile with packs. |
1888 |
self._reconcile_does_inventory_gc = True |
2951.2.9
by Robert Collins
* ``pack-0.92`` repositories can now be reconciled. |
1889 |
self._reconcile_fixes_text_parents = True |
2951.1.3
by Robert Collins
Partial support for native reconcile with packs. |
1890 |
self._reconcile_backsup_inventory = False |
3606.7.7
by John Arbash Meinel
Add tests for the fetching behavior. |
1891 |
self._fetch_order = 'unordered' |
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
1892 |
|
3575.3.1
by Andrew Bennetts
Deprecate knit repositories. |
1893 |
def _warn_if_deprecated(self): |
3606.10.1
by John Arbash Meinel
Create a new --1.6-rich-root, deprecate the old one. |
1894 |
# This class isn't deprecated, but one sub-format is
|
1895 |
if isinstance(self._format, RepositoryFormatKnitPack5RichRootBroken): |
|
3606.10.3
by John Arbash Meinel
When warning give an exact upgrade request. |
1896 |
from bzrlib import repository |
1897 |
if repository._deprecation_warning_done: |
|
1898 |
return
|
|
1899 |
repository._deprecation_warning_done = True |
|
1900 |
warning("Format %s for %s is deprecated - please use" |
|
1901 |
" 'bzr upgrade --1.6.1-rich-root'"
|
|
1902 |
% (self._format, self.bzrdir.transport.base)) |
|
3575.3.1
by Andrew Bennetts
Deprecate knit repositories. |
1903 |
|
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
1904 |
def _abort_write_group(self): |
2592.3.232
by Martin Pool
Disambiguate two member variables called _packs into _packs_by_name and _pack_collection |
1905 |
self._pack_collection._abort_write_group() |
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
1906 |
|
2951.1.2
by Robert Collins
Partial refactoring of pack_repo to create a Packer object for packing. |
1907 |
def _find_inconsistent_revision_parents(self): |
1908 |
"""Find revisions with incorrectly cached parents.
|
|
1909 |
||
1910 |
:returns: an iterator yielding tuples of (revison-id, parents-in-index,
|
|
1911 |
parents-in-revision).
|
|
1912 |
"""
|
|
3052.1.6
by John Arbash Meinel
Change the lock check to raise ObjectNotLocked. |
1913 |
if not self.is_locked(): |
1914 |
raise errors.ObjectNotLocked(self) |
|
2951.1.2
by Robert Collins
Partial refactoring of pack_repo to create a Packer object for packing. |
1915 |
pb = ui.ui_factory.nested_progress_bar() |
2951.1.11
by Robert Collins
Do not try to use try:finally: around a yield for python 2.4. |
1916 |
result = [] |
2951.1.2
by Robert Collins
Partial refactoring of pack_repo to create a Packer object for packing. |
1917 |
try: |
1918 |
revision_nodes = self._pack_collection.revision_index \ |
|
1919 |
.combined_index.iter_all_entries() |
|
1920 |
index_positions = [] |
|
1921 |
# Get the cached index values for all revisions, and also the location
|
|
1922 |
# in each index of the revision text so we can perform linear IO.
|
|
1923 |
for index, key, value, refs in revision_nodes: |
|
1924 |
pos, length = value[1:].split(' ') |
|
1925 |
index_positions.append((index, int(pos), key[0], |
|
1926 |
tuple(parent[0] for parent in refs[0]))) |
|
1927 |
pb.update("Reading revision index.", 0, 0) |
|
1928 |
index_positions.sort() |
|
2951.1.10
by Robert Collins
Peer review feedback with Ian. |
1929 |
batch_count = len(index_positions) / 1000 + 1 |
1930 |
pb.update("Checking cached revision graph.", 0, batch_count) |
|
1931 |
for offset in xrange(batch_count): |
|
2951.1.2
by Robert Collins
Partial refactoring of pack_repo to create a Packer object for packing. |
1932 |
pb.update("Checking cached revision graph.", offset) |
1933 |
to_query = index_positions[offset * 1000:(offset + 1) * 1000] |
|
1934 |
if not to_query: |
|
1935 |
break
|
|
1936 |
rev_ids = [item[2] for item in to_query] |
|
1937 |
revs = self.get_revisions(rev_ids) |
|
1938 |
for revision, item in zip(revs, to_query): |
|
1939 |
index_parents = item[3] |
|
1940 |
rev_parents = tuple(revision.parent_ids) |
|
1941 |
if index_parents != rev_parents: |
|
2951.1.11
by Robert Collins
Do not try to use try:finally: around a yield for python 2.4. |
1942 |
result.append((revision.revision_id, index_parents, rev_parents)) |
2951.1.2
by Robert Collins
Partial refactoring of pack_repo to create a Packer object for packing. |
1943 |
finally: |
1944 |
pb.finished() |
|
2951.1.11
by Robert Collins
Do not try to use try:finally: around a yield for python 2.4. |
1945 |
return result |
2951.1.2
by Robert Collins
Partial refactoring of pack_repo to create a Packer object for packing. |
1946 |
|
3099.3.3
by John Arbash Meinel
Deprecate get_parents() in favor of get_parent_map() |
1947 |
@symbol_versioning.deprecated_method(symbol_versioning.one_one) |
2592.3.216
by Robert Collins
Implement get_parents and _make_parents_provider for Pack repositories. |
1948 |
def get_parents(self, revision_ids): |
3099.3.1
by John Arbash Meinel
Implement get_parent_map for ParentProviders |
1949 |
"""See graph._StackedParentsProvider.get_parents."""
|
1950 |
parent_map = self.get_parent_map(revision_ids) |
|
1951 |
return [parent_map.get(r, None) for r in revision_ids] |
|
1952 |
||
2592.3.216
by Robert Collins
Implement get_parents and _make_parents_provider for Pack repositories. |
1953 |
def _make_parents_provider(self): |
3099.3.1
by John Arbash Meinel
Implement get_parent_map for ParentProviders |
1954 |
return graph.CachingParentsProvider(self) |
2592.3.216
by Robert Collins
Implement get_parents and _make_parents_provider for Pack repositories. |
1955 |
|
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
1956 |
def _refresh_data(self): |
2951.1.1
by Robert Collins
(robertc) Fix data-refresh logic for packs not to refresh mid-transaction when a names write lock is held. (Robert Collins) |
1957 |
if self._write_lock_count == 1 or ( |
1958 |
self.control_files._lock_count == 1 and |
|
1959 |
self.control_files._lock_mode == 'r'): |
|
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
1960 |
# forget what names there are
|
2592.3.232
by Martin Pool
Disambiguate two member variables called _packs into _packs_by_name and _pack_collection |
1961 |
self._pack_collection.reset() |
2592.3.219
by Robert Collins
Review feedback. |
1962 |
# XXX: Better to do an in-memory merge when acquiring a new lock -
|
1963 |
# factor out code from _save_pack_names.
|
|
2949.1.2
by Robert Collins
* Fetch with pack repositories will no longer read the entire history graph. |
1964 |
self._pack_collection.ensure_loaded() |
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
1965 |
|
1966 |
def _start_write_group(self): |
|
2592.3.232
by Martin Pool
Disambiguate two member variables called _packs into _packs_by_name and _pack_collection |
1967 |
self._pack_collection._start_write_group() |
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
1968 |
|
1969 |
def _commit_write_group(self): |
|
2592.3.232
by Martin Pool
Disambiguate two member variables called _packs into _packs_by_name and _pack_collection |
1970 |
return self._pack_collection._commit_write_group() |
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
1971 |
|
2592.3.188
by Robert Collins
Allow pack repositories to have multiple writers active at one time, for greater concurrency. |
1972 |
def get_transaction(self): |
1973 |
if self._write_lock_count: |
|
1974 |
return self._transaction |
|
1975 |
else: |
|
1976 |
return self.control_files.get_transaction() |
|
1977 |
||
1978 |
def is_locked(self): |
|
1979 |
return self._write_lock_count or self.control_files.is_locked() |
|
1980 |
||
1981 |
def is_write_locked(self): |
|
1982 |
return self._write_lock_count |
|
1983 |
||
1984 |
def lock_write(self, token=None): |
|
1985 |
if not self._write_lock_count and self.is_locked(): |
|
1986 |
raise errors.ReadOnlyError(self) |
|
1987 |
self._write_lock_count += 1 |
|
1988 |
if self._write_lock_count == 1: |
|
1989 |
self._transaction = transactions.WriteTransaction() |
|
3221.12.13
by Robert Collins
Implement generic stacking rather than pack-internals based stacking. |
1990 |
for repo in self._fallback_repositories: |
1991 |
# Writes don't affect fallback repos
|
|
1992 |
repo.lock_read() |
|
2592.3.188
by Robert Collins
Allow pack repositories to have multiple writers active at one time, for greater concurrency. |
1993 |
self._refresh_data() |
1994 |
||
1995 |
def lock_read(self): |
|
1996 |
if self._write_lock_count: |
|
1997 |
self._write_lock_count += 1 |
|
1998 |
else: |
|
1999 |
self.control_files.lock_read() |
|
3221.12.13
by Robert Collins
Implement generic stacking rather than pack-internals based stacking. |
2000 |
for repo in self._fallback_repositories: |
2001 |
# Writes don't affect fallback repos
|
|
2002 |
repo.lock_read() |
|
2592.3.188
by Robert Collins
Allow pack repositories to have multiple writers active at one time, for greater concurrency. |
2003 |
self._refresh_data() |
2004 |
||
2005 |
def leave_lock_in_place(self): |
|
2006 |
# not supported - raise an error
|
|
2007 |
raise NotImplementedError(self.leave_lock_in_place) |
|
2008 |
||
2009 |
def dont_leave_lock_in_place(self): |
|
2010 |
# not supported - raise an error
|
|
2011 |
raise NotImplementedError(self.dont_leave_lock_in_place) |
|
2012 |
||
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
2013 |
@needs_write_lock
|
2014 |
def pack(self): |
|
2015 |
"""Compress the data within the repository.
|
|
2016 |
||
2017 |
This will pack all the data to a single pack. In future it may
|
|
2018 |
recompress deltas or do other such expensive operations.
|
|
2019 |
"""
|
|
2592.3.232
by Martin Pool
Disambiguate two member variables called _packs into _packs_by_name and _pack_collection |
2020 |
self._pack_collection.pack() |
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
2021 |
|
2022 |
@needs_write_lock
|
|
2023 |
def reconcile(self, other=None, thorough=False): |
|
2024 |
"""Reconcile this repository."""
|
|
2025 |
from bzrlib.reconcile import PackReconciler |
|
2026 |
reconciler = PackReconciler(self, thorough=thorough) |
|
2027 |
reconciler.reconcile() |
|
2028 |
return reconciler |
|
2029 |
||
2592.3.188
by Robert Collins
Allow pack repositories to have multiple writers active at one time, for greater concurrency. |
2030 |
def unlock(self): |
2031 |
if self._write_lock_count == 1 and self._write_group is not None: |
|
2592.3.244
by Martin Pool
unlock while in a write group now aborts the write group, unlocks, and errors. |
2032 |
self.abort_write_group() |
2033 |
self._transaction = None |
|
2034 |
self._write_lock_count = 0 |
|
2592.3.188
by Robert Collins
Allow pack repositories to have multiple writers active at one time, for greater concurrency. |
2035 |
raise errors.BzrError( |
2592.3.244
by Martin Pool
unlock while in a write group now aborts the write group, unlocks, and errors. |
2036 |
'Must end write group before releasing write lock on %s' |
2037 |
% self) |
|
2592.3.188
by Robert Collins
Allow pack repositories to have multiple writers active at one time, for greater concurrency. |
2038 |
if self._write_lock_count: |
2039 |
self._write_lock_count -= 1 |
|
2040 |
if not self._write_lock_count: |
|
2041 |
transaction = self._transaction |
|
2042 |
self._transaction = None |
|
2043 |
transaction.finish() |
|
3221.12.13
by Robert Collins
Implement generic stacking rather than pack-internals based stacking. |
2044 |
for repo in self._fallback_repositories: |
2045 |
repo.unlock() |
|
2592.3.188
by Robert Collins
Allow pack repositories to have multiple writers active at one time, for greater concurrency. |
2046 |
else: |
2047 |
self.control_files.unlock() |
|
3221.12.13
by Robert Collins
Implement generic stacking rather than pack-internals based stacking. |
2048 |
for repo in self._fallback_repositories: |
2049 |
repo.unlock() |
|
2050 |
||
2051 |
||
2052 |
class RepositoryFormatPack(MetaDirRepositoryFormat): |
|
2053 |
"""Format logic for pack structured repositories.
|
|
2054 |
||
2055 |
This repository format has:
|
|
2056 |
- a list of packs in pack-names
|
|
2057 |
- packs in packs/NAME.pack
|
|
2058 |
- indices in indices/NAME.{iix,six,tix,rix}
|
|
2059 |
- knit deltas in the packs, knit indices mapped to the indices.
|
|
2060 |
- thunk objects to support the knits programming API.
|
|
2061 |
- a format marker of its own
|
|
2062 |
- an optional 'shared-storage' flag
|
|
2063 |
- an optional 'no-working-trees' flag
|
|
2064 |
- a LockDir lock
|
|
2065 |
"""
|
|
2066 |
||
2067 |
# Set this attribute in derived classes to control the repository class
|
|
2068 |
# created by open and initialize.
|
|
2069 |
repository_class = None |
|
2070 |
# Set this attribute in derived classes to control the
|
|
2071 |
# _commit_builder_class that the repository objects will have passed to
|
|
2072 |
# their constructor.
|
|
2073 |
_commit_builder_class = None |
|
2074 |
# Set this attribute in derived clases to control the _serializer that the
|
|
2075 |
# repository objects will have passed to their constructor.
|
|
2076 |
_serializer = None |
|
2077 |
# External references are not supported in pack repositories yet.
|
|
2078 |
supports_external_lookups = False |
|
3735.1.1
by Robert Collins
Add development2 formats using BTree indices. |
2079 |
# What index classes to use
|
2080 |
index_builder_class = None |
|
2081 |
index_class = None |
|
3221.12.13
by Robert Collins
Implement generic stacking rather than pack-internals based stacking. |
2082 |
|
2083 |
def initialize(self, a_bzrdir, shared=False): |
|
2084 |
"""Create a pack based repository.
|
|
2085 |
||
2086 |
:param a_bzrdir: bzrdir to contain the new repository; must already
|
|
2087 |
be initialized.
|
|
2088 |
:param shared: If true the repository will be initialized as a shared
|
|
2089 |
repository.
|
|
2090 |
"""
|
|
2091 |
mutter('creating repository in %s.', a_bzrdir.transport.base) |
|
2092 |
dirs = ['indices', 'obsolete_packs', 'packs', 'upload'] |
|
3735.1.1
by Robert Collins
Add development2 formats using BTree indices. |
2093 |
builder = self.index_builder_class() |
3221.12.13
by Robert Collins
Implement generic stacking rather than pack-internals based stacking. |
2094 |
files = [('pack-names', builder.finish())] |
2095 |
utf8_files = [('format', self.get_format_string())] |
|
2096 |
||
2097 |
self._upload_blank_content(a_bzrdir, dirs, files, utf8_files, shared) |
|
2098 |
return self.open(a_bzrdir=a_bzrdir, _found=True) |
|
2099 |
||
2100 |
def open(self, a_bzrdir, _found=False, _override_transport=None): |
|
2101 |
"""See RepositoryFormat.open().
|
|
2102 |
|
|
2103 |
:param _override_transport: INTERNAL USE ONLY. Allows opening the
|
|
2104 |
repository at a slightly different url
|
|
2105 |
than normal. I.e. during 'upgrade'.
|
|
2106 |
"""
|
|
2107 |
if not _found: |
|
2108 |
format = RepositoryFormat.find_format(a_bzrdir) |
|
2109 |
if _override_transport is not None: |
|
2110 |
repo_transport = _override_transport |
|
2111 |
else: |
|
2112 |
repo_transport = a_bzrdir.get_repository_transport(None) |
|
2113 |
control_files = lockable_files.LockableFiles(repo_transport, |
|
2114 |
'lock', lockdir.LockDir) |
|
2115 |
return self.repository_class(_format=self, |
|
2116 |
a_bzrdir=a_bzrdir, |
|
2117 |
control_files=control_files, |
|
2118 |
_commit_builder_class=self._commit_builder_class, |
|
2119 |
_serializer=self._serializer) |
|
2120 |
||
2121 |
||
2122 |
class RepositoryFormatKnitPack1(RepositoryFormatPack): |
|
2123 |
"""A no-subtrees parameterized Pack repository.
|
|
2124 |
||
2125 |
This format was introduced in 0.92.
|
|
2126 |
"""
|
|
2127 |
||
2128 |
repository_class = KnitPackRepository |
|
2129 |
_commit_builder_class = PackCommitBuilder |
|
3224.5.1
by Andrew Bennetts
Lots of assorted hackery to reduce the number of imports for common operations. Improves 'rocks', 'st' and 'help' times by ~50ms on my laptop. |
2130 |
@property
|
2131 |
def _serializer(self): |
|
2132 |
return xml5.serializer_v5 |
|
3735.1.1
by Robert Collins
Add development2 formats using BTree indices. |
2133 |
# What index classes to use
|
2134 |
index_builder_class = InMemoryGraphIndex |
|
2135 |
index_class = GraphIndex |
|
3221.12.13
by Robert Collins
Implement generic stacking rather than pack-internals based stacking. |
2136 |
|
2137 |
def _get_matching_bzrdir(self): |
|
2138 |
return bzrdir.format_registry.make_bzrdir('pack-0.92') |
|
2139 |
||
2140 |
def _ignore_setting_bzrdir(self, format): |
|
2141 |
pass
|
|
2142 |
||
2143 |
_matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir) |
|
2144 |
||
2145 |
def get_format_string(self): |
|
2146 |
"""See RepositoryFormat.get_format_string()."""
|
|
2147 |
return "Bazaar pack repository format 1 (needs bzr 0.92)\n" |
|
2148 |
||
2149 |
def get_format_description(self): |
|
2150 |
"""See RepositoryFormat.get_format_description()."""
|
|
2151 |
return "Packs containing knits without subtree support" |
|
2152 |
||
2153 |
def check_conversion_target(self, target_format): |
|
2154 |
pass
|
|
2592.3.188
by Robert Collins
Allow pack repositories to have multiple writers active at one time, for greater concurrency. |
2155 |
|
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
2156 |
|
2592.3.224
by Martin Pool
Rename GraphKnitRepository etc to KnitPackRepository |
2157 |
class RepositoryFormatKnitPack3(RepositoryFormatPack): |
3128.1.3
by Vincent Ladeuil
Since we are there s/parameteris.*/parameteriz&/. |
2158 |
"""A subtrees parameterized Pack repository.
|
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
2159 |
|
2592.3.215
by Robert Collins
Review feedback. |
2160 |
This repository format uses the xml7 serializer to get:
|
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
2161 |
- support for recording full info about the tree root
|
2162 |
- support for recording tree-references
|
|
2592.3.215
by Robert Collins
Review feedback. |
2163 |
|
2939.2.1
by Ian Clatworthy
use 'knitpack' naming instead of 'experimental' for pack formats |
2164 |
This format was introduced in 0.92.
|
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
2165 |
"""
|
2166 |
||
2592.3.224
by Martin Pool
Rename GraphKnitRepository etc to KnitPackRepository |
2167 |
repository_class = KnitPackRepository |
2592.3.166
by Robert Collins
Merge KnitRepository3 removal branch. |
2168 |
_commit_builder_class = PackRootCommitBuilder |
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
2169 |
rich_root_data = True |
2170 |
supports_tree_reference = True |
|
3224.5.1
by Andrew Bennetts
Lots of assorted hackery to reduce the number of imports for common operations. Improves 'rocks', 'st' and 'help' times by ~50ms on my laptop. |
2171 |
@property
|
2172 |
def _serializer(self): |
|
2173 |
return xml7.serializer_v7 |
|
3735.1.1
by Robert Collins
Add development2 formats using BTree indices. |
2174 |
# What index classes to use
|
2175 |
index_builder_class = InMemoryGraphIndex |
|
2176 |
index_class = GraphIndex |
|
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
2177 |
|
2178 |
def _get_matching_bzrdir(self): |
|
2939.2.5
by Ian Clatworthy
review feedback from lifeless |
2179 |
return bzrdir.format_registry.make_bzrdir( |
3010.3.2
by Martin Pool
Rename pack0.92 to pack-0.92 |
2180 |
'pack-0.92-subtree') |
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
2181 |
|
2182 |
def _ignore_setting_bzrdir(self, format): |
|
2183 |
pass
|
|
2184 |
||
2185 |
_matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir) |
|
2186 |
||
2187 |
def check_conversion_target(self, target_format): |
|
2188 |
if not target_format.rich_root_data: |
|
2189 |
raise errors.BadConversionTarget( |
|
2190 |
'Does not support rich root data.', target_format) |
|
2191 |
if not getattr(target_format, 'supports_tree_reference', False): |
|
2192 |
raise errors.BadConversionTarget( |
|
2193 |
'Does not support nested trees', target_format) |
|
2194 |
||
2195 |
def get_format_string(self): |
|
2196 |
"""See RepositoryFormat.get_format_string()."""
|
|
2939.2.6
by Ian Clatworthy
more review feedback from lifeless and poolie |
2197 |
return "Bazaar pack repository format 1 with subtree support (needs bzr 0.92)\n" |
2592.3.88
by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo. |
2198 |
|
2199 |
def get_format_description(self): |
|
2200 |
"""See RepositoryFormat.get_format_description()."""
|
|
2939.2.1
by Ian Clatworthy
use 'knitpack' naming instead of 'experimental' for pack formats |
2201 |
return "Packs containing knits with subtree support\n" |
2996.2.11
by Aaron Bentley
Implement rich-root-pack format ( #164639) |
2202 |
|
2203 |
||
2204 |
class RepositoryFormatKnitPack4(RepositoryFormatPack): |
|
3128.1.3
by Vincent Ladeuil
Since we are there s/parameteris.*/parameteriz&/. |
2205 |
"""A rich-root, no subtrees parameterized Pack repository.
|
2996.2.11
by Aaron Bentley
Implement rich-root-pack format ( #164639) |
2206 |
|
2996.2.12
by Aaron Bentley
Text fixes from review |
2207 |
This repository format uses the xml6 serializer to get:
|
2996.2.11
by Aaron Bentley
Implement rich-root-pack format ( #164639) |
2208 |
- support for recording full info about the tree root
|
2209 |
||
2996.2.12
by Aaron Bentley
Text fixes from review |
2210 |
This format was introduced in 1.0.
|
2996.2.11
by Aaron Bentley
Implement rich-root-pack format ( #164639) |
2211 |
"""
|
2212 |
||
2213 |
repository_class = KnitPackRepository |
|
2214 |
_commit_builder_class = PackRootCommitBuilder |
|
2215 |
rich_root_data = True |
|
2216 |
supports_tree_reference = False |
|
3224.5.1
by Andrew Bennetts
Lots of assorted hackery to reduce the number of imports for common operations. Improves 'rocks', 'st' and 'help' times by ~50ms on my laptop. |
2217 |
@property
|
2218 |
def _serializer(self): |
|
2219 |
return xml6.serializer_v6 |
|
3735.1.1
by Robert Collins
Add development2 formats using BTree indices. |
2220 |
# What index classes to use
|
2221 |
index_builder_class = InMemoryGraphIndex |
|
2222 |
index_class = GraphIndex |
|
2996.2.11
by Aaron Bentley
Implement rich-root-pack format ( #164639) |
2223 |
|
2224 |
def _get_matching_bzrdir(self): |
|
2225 |
return bzrdir.format_registry.make_bzrdir( |
|
2226 |
'rich-root-pack') |
|
2227 |
||
2228 |
def _ignore_setting_bzrdir(self, format): |
|
2229 |
pass
|
|
2230 |
||
2231 |
_matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir) |
|
2232 |
||
2233 |
def check_conversion_target(self, target_format): |
|
2234 |
if not target_format.rich_root_data: |
|
2235 |
raise errors.BadConversionTarget( |
|
2236 |
'Does not support rich root data.', target_format) |
|
2237 |
||
2238 |
def get_format_string(self): |
|
2239 |
"""See RepositoryFormat.get_format_string()."""
|
|
2240 |
return ("Bazaar pack repository format 1 with rich root" |
|
2241 |
" (needs bzr 1.0)\n") |
|
2242 |
||
2243 |
def get_format_description(self): |
|
2244 |
"""See RepositoryFormat.get_format_description()."""
|
|
2245 |
return "Packs containing knits with rich root support\n" |
|
3152.2.1
by Robert Collins
* A new repository format 'development' has been added. This format will |
2246 |
|
2247 |
||
3549.1.5
by Martin Pool
Add stable format names for stacked branches |
2248 |
class RepositoryFormatKnitPack5(RepositoryFormatPack): |
2249 |
"""Repository that supports external references to allow stacking.
|
|
2250 |
||
2251 |
New in release 1.6.
|
|
2252 |
||
2253 |
Supports external lookups, which results in non-truncated ghosts after
|
|
2254 |
reconcile compared to pack-0.92 formats.
|
|
2255 |
"""
|
|
2256 |
||
2257 |
repository_class = KnitPackRepository |
|
2258 |
_commit_builder_class = PackCommitBuilder |
|
2259 |
supports_external_lookups = True |
|
3735.1.1
by Robert Collins
Add development2 formats using BTree indices. |
2260 |
# What index classes to use
|
2261 |
index_builder_class = InMemoryGraphIndex |
|
2262 |
index_class = GraphIndex |
|
3549.1.5
by Martin Pool
Add stable format names for stacked branches |
2263 |
|
3224.5.27
by Andrew Bennetts
Avoid importing bzrlib.xml* as a side-effect of importing bzrlib.repofmt.pack_repo. |
2264 |
@property
|
2265 |
def _serializer(self): |
|
2266 |
return xml5.serializer_v5 |
|
2267 |
||
3549.1.5
by Martin Pool
Add stable format names for stacked branches |
2268 |
def _get_matching_bzrdir(self): |
3735.1.2
by Robert Collins
Remove 1.5 series dev formats and document development2 a little better. |
2269 |
return bzrdir.format_registry.make_bzrdir('1.6') |
3549.1.5
by Martin Pool
Add stable format names for stacked branches |
2270 |
|
2271 |
def _ignore_setting_bzrdir(self, format): |
|
2272 |
pass
|
|
2273 |
||
2274 |
_matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir) |
|
2275 |
||
2276 |
def get_format_string(self): |
|
2277 |
"""See RepositoryFormat.get_format_string()."""
|
|
2278 |
return "Bazaar RepositoryFormatKnitPack5 (bzr 1.6)\n" |
|
2279 |
||
2280 |
def get_format_description(self): |
|
2281 |
"""See RepositoryFormat.get_format_description()."""
|
|
3606.3.1
by Aaron Bentley
Update repo format strings |
2282 |
return "Packs 5 (adds stacking support, requires bzr 1.6)" |
3549.1.5
by Martin Pool
Add stable format names for stacked branches |
2283 |
|
2284 |
def check_conversion_target(self, target_format): |
|
2285 |
pass
|
|
2286 |
||
2287 |
||
3549.1.6
by Martin Pool
Change stacked-subtree to stacked-rich-root |
2288 |
class RepositoryFormatKnitPack5RichRoot(RepositoryFormatPack): |
3606.10.1
by John Arbash Meinel
Create a new --1.6-rich-root, deprecate the old one. |
2289 |
"""A repository with rich roots and stacking.
|
2290 |
||
2291 |
New in release 1.6.1.
|
|
2292 |
||
2293 |
Supports stacking on other repositories, allowing data to be accessed
|
|
2294 |
without being stored locally.
|
|
2295 |
"""
|
|
2296 |
||
2297 |
repository_class = KnitPackRepository |
|
2298 |
_commit_builder_class = PackRootCommitBuilder |
|
2299 |
rich_root_data = True |
|
2300 |
supports_tree_reference = False # no subtrees |
|
2301 |
supports_external_lookups = True |
|
3735.1.1
by Robert Collins
Add development2 formats using BTree indices. |
2302 |
# What index classes to use
|
2303 |
index_builder_class = InMemoryGraphIndex |
|
2304 |
index_class = GraphIndex |
|
3606.10.1
by John Arbash Meinel
Create a new --1.6-rich-root, deprecate the old one. |
2305 |
|
3224.5.27
by Andrew Bennetts
Avoid importing bzrlib.xml* as a side-effect of importing bzrlib.repofmt.pack_repo. |
2306 |
@property
|
2307 |
def _serializer(self): |
|
2308 |
return xml6.serializer_v6 |
|
2309 |
||
3606.10.1
by John Arbash Meinel
Create a new --1.6-rich-root, deprecate the old one. |
2310 |
def _get_matching_bzrdir(self): |
2311 |
return bzrdir.format_registry.make_bzrdir( |
|
3606.10.2
by John Arbash Meinel
Name the new format 1.6.1-rich-root, and NEWS for fixing bug #262333 |
2312 |
'1.6.1-rich-root') |
3606.10.1
by John Arbash Meinel
Create a new --1.6-rich-root, deprecate the old one. |
2313 |
|
2314 |
def _ignore_setting_bzrdir(self, format): |
|
2315 |
pass
|
|
2316 |
||
2317 |
_matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir) |
|
2318 |
||
2319 |
def check_conversion_target(self, target_format): |
|
2320 |
if not target_format.rich_root_data: |
|
2321 |
raise errors.BadConversionTarget( |
|
2322 |
'Does not support rich root data.', target_format) |
|
2323 |
||
2324 |
def get_format_string(self): |
|
2325 |
"""See RepositoryFormat.get_format_string()."""
|
|
2326 |
return "Bazaar RepositoryFormatKnitPack5RichRoot (bzr 1.6.1)\n" |
|
2327 |
||
2328 |
def get_format_description(self): |
|
2329 |
return "Packs 5 rich-root (adds stacking support, requires bzr 1.6.1)" |
|
2330 |
||
2331 |
||
2332 |
class RepositoryFormatKnitPack5RichRootBroken(RepositoryFormatPack): |
|
3606.3.1
by Aaron Bentley
Update repo format strings |
2333 |
"""A repository with rich roots and external references.
|
3549.1.5
by Martin Pool
Add stable format names for stacked branches |
2334 |
|
2335 |
New in release 1.6.
|
|
2336 |
||
2337 |
Supports external lookups, which results in non-truncated ghosts after
|
|
2338 |
reconcile compared to pack-0.92 formats.
|
|
3606.10.1
by John Arbash Meinel
Create a new --1.6-rich-root, deprecate the old one. |
2339 |
|
2340 |
This format was deprecated because the serializer it uses accidentally
|
|
2341 |
supported subtrees, when the format was not intended to. This meant that
|
|
2342 |
someone could accidentally fetch from an incorrect repository.
|
|
3549.1.5
by Martin Pool
Add stable format names for stacked branches |
2343 |
"""
|
2344 |
||
2345 |
repository_class = KnitPackRepository |
|
2346 |
_commit_builder_class = PackRootCommitBuilder |
|
2347 |
rich_root_data = True |
|
3549.1.6
by Martin Pool
Change stacked-subtree to stacked-rich-root |
2348 |
supports_tree_reference = False # no subtrees |
3549.1.5
by Martin Pool
Add stable format names for stacked branches |
2349 |
|
2350 |
supports_external_lookups = True |
|
3735.1.1
by Robert Collins
Add development2 formats using BTree indices. |
2351 |
# What index classes to use
|
2352 |
index_builder_class = InMemoryGraphIndex |
|
2353 |
index_class = GraphIndex |
|
3549.1.5
by Martin Pool
Add stable format names for stacked branches |
2354 |
|
3224.5.27
by Andrew Bennetts
Avoid importing bzrlib.xml* as a side-effect of importing bzrlib.repofmt.pack_repo. |
2355 |
@property
|
2356 |
def _serializer(self): |
|
2357 |
return xml7.serializer_v7 |
|
2358 |
||
3549.1.5
by Martin Pool
Add stable format names for stacked branches |
2359 |
def _get_matching_bzrdir(self): |
2360 |
return bzrdir.format_registry.make_bzrdir( |
|
3735.1.2
by Robert Collins
Remove 1.5 series dev formats and document development2 a little better. |
2361 |
'1.6.1-rich-root') |
3549.1.5
by Martin Pool
Add stable format names for stacked branches |
2362 |
|
2363 |
def _ignore_setting_bzrdir(self, format): |
|
2364 |
pass
|
|
2365 |
||
2366 |
_matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir) |
|
2367 |
||
2368 |
def check_conversion_target(self, target_format): |
|
2369 |
if not target_format.rich_root_data: |
|
2370 |
raise errors.BadConversionTarget( |
|
2371 |
'Does not support rich root data.', target_format) |
|
3606.10.1
by John Arbash Meinel
Create a new --1.6-rich-root, deprecate the old one. |
2372 |
|
3549.1.5
by Martin Pool
Add stable format names for stacked branches |
2373 |
def get_format_string(self): |
2374 |
"""See RepositoryFormat.get_format_string()."""
|
|
3549.1.6
by Martin Pool
Change stacked-subtree to stacked-rich-root |
2375 |
return "Bazaar RepositoryFormatKnitPack5RichRoot (bzr 1.6)\n" |
3549.1.5
by Martin Pool
Add stable format names for stacked branches |
2376 |
|
2377 |
def get_format_description(self): |
|
3606.10.1
by John Arbash Meinel
Create a new --1.6-rich-root, deprecate the old one. |
2378 |
return ("Packs 5 rich-root (adds stacking support, requires bzr 1.6)" |
2379 |
" (deprecated)") |
|
3549.1.5
by Martin Pool
Add stable format names for stacked branches |
2380 |
|
2381 |
||
3735.1.1
by Robert Collins
Add development2 formats using BTree indices. |
2382 |
class RepositoryFormatPackDevelopment2(RepositoryFormatPack): |
2383 |
"""A no-subtrees development repository.
|
|
2384 |
||
2385 |
This format should be retained until the second release after bzr 1.7.
|
|
2386 |
||
2387 |
This is pack-1.6.1 with B+Tree indices.
|
|
2388 |
"""
|
|
2389 |
||
2390 |
repository_class = KnitPackRepository |
|
2391 |
_commit_builder_class = PackCommitBuilder |
|
2392 |
supports_external_lookups = True |
|
2393 |
# What index classes to use
|
|
2394 |
index_builder_class = BTreeBuilder |
|
2395 |
index_class = BTreeGraphIndex |
|
2396 |
||
3224.5.27
by Andrew Bennetts
Avoid importing bzrlib.xml* as a side-effect of importing bzrlib.repofmt.pack_repo. |
2397 |
@property
|
2398 |
def _serializer(self): |
|
2399 |
return xml5.serializer_v5 |
|
2400 |
||
3735.1.1
by Robert Collins
Add development2 formats using BTree indices. |
2401 |
def _get_matching_bzrdir(self): |
2402 |
return bzrdir.format_registry.make_bzrdir('development2') |
|
2403 |
||
2404 |
def _ignore_setting_bzrdir(self, format): |
|
2405 |
pass
|
|
2406 |
||
2407 |
_matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir) |
|
2408 |
||
2409 |
def get_format_string(self): |
|
2410 |
"""See RepositoryFormat.get_format_string()."""
|
|
2411 |
return "Bazaar development format 2 (needs bzr.dev from before 1.8)\n" |
|
2412 |
||
2413 |
def get_format_description(self): |
|
2414 |
"""See RepositoryFormat.get_format_description()."""
|
|
2415 |
return ("Development repository format, currently the same as " |
|
3735.1.2
by Robert Collins
Remove 1.5 series dev formats and document development2 a little better. |
2416 |
"1.6.1 with B+Trees.\n") |
3735.1.1
by Robert Collins
Add development2 formats using BTree indices. |
2417 |
|
2418 |
def check_conversion_target(self, target_format): |
|
2419 |
pass
|
|
2420 |
||
2421 |
||
2422 |
class RepositoryFormatPackDevelopment2Subtree(RepositoryFormatPack): |
|
2423 |
"""A subtrees development repository.
|
|
2424 |
||
2425 |
This format should be retained until the second release after bzr 1.7.
|
|
2426 |
||
3735.1.2
by Robert Collins
Remove 1.5 series dev formats and document development2 a little better. |
2427 |
1.6.1-subtree[as it might have been] with B+Tree indices.
|
3735.1.1
by Robert Collins
Add development2 formats using BTree indices. |
2428 |
"""
|
2429 |
||
2430 |
repository_class = KnitPackRepository |
|
2431 |
_commit_builder_class = PackRootCommitBuilder |
|
2432 |
rich_root_data = True |
|
2433 |
supports_tree_reference = True |
|
2434 |
supports_external_lookups = True |
|
2435 |
# What index classes to use
|
|
2436 |
index_builder_class = BTreeBuilder |
|
2437 |
index_class = BTreeGraphIndex |
|
2438 |
||
3224.5.27
by Andrew Bennetts
Avoid importing bzrlib.xml* as a side-effect of importing bzrlib.repofmt.pack_repo. |
2439 |
@property
|
2440 |
def _serializer(self): |
|
2441 |
return xml7.serializer_v7 |
|
2442 |
||
3735.1.1
by Robert Collins
Add development2 formats using BTree indices. |
2443 |
def _get_matching_bzrdir(self): |
2444 |
return bzrdir.format_registry.make_bzrdir( |
|
2445 |
'development2-subtree') |
|
2446 |
||
2447 |
def _ignore_setting_bzrdir(self, format): |
|
2448 |
pass
|
|
2449 |
||
2450 |
_matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir) |
|
2451 |
||
2452 |
def check_conversion_target(self, target_format): |
|
2453 |
if not target_format.rich_root_data: |
|
2454 |
raise errors.BadConversionTarget( |
|
2455 |
'Does not support rich root data.', target_format) |
|
2456 |
if not getattr(target_format, 'supports_tree_reference', False): |
|
2457 |
raise errors.BadConversionTarget( |
|
2458 |
'Does not support nested trees', target_format) |
|
2459 |
||
2460 |
def get_format_string(self): |
|
2461 |
"""See RepositoryFormat.get_format_string()."""
|
|
2462 |
return ("Bazaar development format 2 with subtree support " |
|
2463 |
"(needs bzr.dev from before 1.8)\n") |
|
2464 |
||
2465 |
def get_format_description(self): |
|
2466 |
"""See RepositoryFormat.get_format_description()."""
|
|
2467 |
return ("Development repository format, currently the same as " |
|
3735.1.2
by Robert Collins
Remove 1.5 series dev formats and document development2 a little better. |
2468 |
"1.6.1-subtree with B+Tree indices.\n") |