« back to all changes in this revision
Viewing changes to bzrlib/add.py
- browse files at revision 1767.2.2
- compare with another revision
- compare with revision 1769
- download diff
- download tarball
- view history from revision 1767.2.2
- Committer: Robert Collins
- Date: 2006-06-13 12:55:45 UTC
- mfrom: (1757.2.16 add)
- mto: This revision was merged to the branch mainline in revision 1769.
- Revision ID: robertc@robertcollins.net-20060613125545-cc77adde61a471bf
Merge add performance improvements.
- files added:
- bzrlib/benchmarks/bench_inventory.py
- files modified:
- bzrlib/add.py
added
removed
bzrlib/add.py
55
55
class AddAction(object):
56
56
"""A class which defines what action to take when adding a file."""
57
57
58
def __init__(self, to_file=None, should_add=None, should_print=None):
58
def __init__(self, to_file=None, should_print=None):
59
59
self._to_file = to_file
60
60
if to_file is None:
61
61
self._to_file = sys.stdout
62
self.should_add = False
63
if should_add is not None:
64
self.should_add = should_add
65
62
self.should_print = False
66
63
if should_print is not None:
67
64
self.should_print = should_print
68
65
69
def __call__(self, inv, parent_ie, path, kind):
66
def __call__(self, inv, parent_ie, path, kind, _quote=bzrlib.osutils.quotefn):
70
67
"""Add path to inventory.
71
68
72
69
The default action does nothing.
73
70
74
71
:param inv: The inventory we are working with.
75
:param path: The path being added
72
:param path: The FastPath being added
76
73
:param kind: The kind of the object being added.
77
74
"""
78
if self.should_add:
79
self._add_to_inv(inv, parent_ie, path, kind)
80
if self.should_print:
81
self._print(inv, parent_ie, path, kind)
82
83
def _print(self, inv, parent_ie, path, kind):
84
"""Print a line to self._to_file for each file that would be added."""
85
self._to_file.write('added ')
86
self._to_file.write(bzrlib.osutils.quotefn(path))
87
self._to_file.write('\n')
88
89
def _add_to_inv(self, inv, parent_ie, path, kind):
90
"""Add each file to the given inventory. Produce no output."""
91
if parent_ie is not None:
92
entry = bzrlib.inventory.make_entry(
93
kind, bzrlib.osutils.basename(path), parent_ie.file_id)
94
inv.add(entry)
95
else:
96
entry = inv.add_path(path, kind=kind)
97
# mutter("added %r kind %r file_id={%s}", path, kind, entry.file_id)
75
if not self.should_print:
76
return
77
self._to_file.write('added %s\n' % _quote(path.raw_path))
98
78
99
79
100
80
# TODO: jam 20050105 These could be used for compatibility
102
82
# one which exists at the time they are called, so they
103
83
# don't work for the test suite.
104
84
# deprecated
105
add_action_null = AddAction()
106
add_action_add = AddAction(should_add=True)
107
add_action_print = AddAction(should_print=True)
108
add_action_add_and_print = AddAction(should_add=True, should_print=True)
109
110
111
def smart_add(file_list, recurse=True, action=None):
85
add_action_add = AddAction()
86
add_action_null = add_action_add
87
add_action_add_and_print = AddAction(should_print=True)
88
add_action_print = add_action_add_and_print
89
90
91
def smart_add(file_list, recurse=True, action=None, save=True):
112
92
"""Add files to version, optionally recursing into directories.
113
93
114
94
This is designed more towards DWIM for humans than API simplicity.
115
95
For the specific behaviour see the help for cmd_add().
116
96
117
97
Returns the number of files added.
98
Please see smart_add_tree for more detail.
118
99
"""
119
100
file_list = _prepare_file_list(file_list)
120
101
tree = WorkingTree.open_containing(file_list[0])[0]
121
102
return smart_add_tree(tree, file_list, recurse, action=action)
122
103
123
104
124
def smart_add_tree(tree, file_list, recurse=True, action=None):
105
class FastPath(object):
106
"""A path object with fast accessors for things like basename."""
107
108
__slots__ = ['raw_path', 'base_path']
109
110
def __init__(self, path, base_path=None):
111
"""Construct a FastPath from path."""
112
if base_path is None:
113
self.base_path = bzrlib.osutils.basename(path)
114
else:
115
self.base_path = base_path
116
self.raw_path = path
117
118
119
def smart_add_tree(tree, file_list, recurse=True, action=None, save=True):
125
120
"""Add files to version, optionally recursing into directories.
126
121
127
122
This is designed more towards DWIM for humans than API simplicity.
130
125
This calls reporter with each (path, kind, file_id) of added files.
131
126
132
127
Returns the number of files added.
128
129
:param save: Save the inventory after completing the adds. If False this
130
provides dry-run functionality by doing the add and not saving the
131
inventory. Note that the modified inventory is left in place, allowing
132
further dry-run tasks to take place. To restore the original inventory
133
call tree.read_working_inventory().
133
134
"""
134
135
import os, errno
135
136
from bzrlib.errors import BadFileKindError, ForbiddenFileError
136
137
assert isinstance(recurse, bool)
137
138
if action is None:
138
action = AddAction(should_add=True)
139
action = AddAction()
139
140
140
141
prepared_list = _prepare_file_list(file_list)
141
142
mutter("smart add of %r, originally %r", prepared_list, file_list)
142
143
inv = tree.read_working_inventory()
143
144
added = []
144
145
ignored = {}
145
user_files = set()
146
files_to_add = []
146
dirs_to_add = []
147
user_dirs = set()
147
148
148
149
# validate user file paths and convert all paths to tree
149
150
# relative : its cheaper to make a tree relative path an abspath
150
151
# than to convert an abspath to tree relative.
151
152
for filepath in prepared_list:
152
rf = tree.relpath(filepath)
153
user_files.add(rf)
154
files_to_add.append((rf, None))
153
rf = FastPath(tree.relpath(filepath))
155
154
# validate user parameters. Our recursive code avoids adding new files
156
155
# that need such validation
157
if tree.is_control_filename(rf):
156
if tree.is_control_filename(rf.raw_path):
158
157
raise ForbiddenFileError('cannot add control file %s' % filepath)
159
160
for filepath, parent_ie in files_to_add:
161
# filepath is tree-relative
162
abspath = tree.abspath(filepath)
163
164
# find the kind of the path being added. This is not
165
# currently determined when we list directories
166
# recursively, but in theory we can determine while
167
# doing the directory listing on *some* platforms.
168
# TODO: a safe, portable, clean interface which will
169
# be faster than os.listdir() + stat. Specifically,
170
# readdir - dirent.d_type supplies the file type when
171
# it is defined. (Apparently Mac OSX has the field but
172
# does not fill it in ?!) Robert C, Martin P.
173
try:
174
kind = bzrlib.osutils.file_kind(abspath)
175
except OSError, e:
176
if hasattr(e, 'errno') and e.errno == errno.ENOENT:
177
raise errors.NoSuchFile(abspath)
178
raise
179
180
# we need to call this to determine the inventory kind to create.
158
159
abspath = tree.abspath(rf.raw_path)
160
kind = bzrlib.osutils.file_kind(abspath)
161
if kind == 'directory':
162
# schedule the dir for scanning
163
user_dirs.add(rf.raw_path)
164
else:
165
if not InventoryEntry.versionable_kind(kind):
166
raise BadFileKindError("cannot add %s of type %s" % (abspath, kind))
167
# ensure the named path is added, so that ignore rules in the later directory
168
# walk dont skip it.
169
# we dont have a parent ie known yet.: use the relatively slower inventory
170
# probing method
171
versioned = inv.has_filename(rf.raw_path)
172
if versioned:
173
continue
174
added.extend(__add_one_and_parent(tree, inv, None, rf, kind, action))
175
176
if not recurse:
177
# no need to walk any directories at all.
178
if len(added) > 0 and save:
179
tree._write_inventory(inv)
180
return added, ignored
181
182
# only walk the minimal parents needed: we have user_dirs to override
183
# ignores.
184
prev_dir = None
185
for path in sorted(user_dirs):
186
if (prev_dir is None or not
187
bzrlib.osutils.is_inside_or_parent_of_any([prev_dir], path)):
188
dirs_to_add.append((rf, None))
189
prev_dir = path
190
191
# this will eventually be *just* directories, right now it starts off with
192
# just directories.
193
for directory, parent_ie in dirs_to_add:
194
# directory is tree-relative
195
abspath = tree.abspath(directory.raw_path)
196
197
# get the contents of this directory.
198
199
# find the kind of the path being added.
200
kind = bzrlib.osutils.file_kind(abspath)
201
181
202
if not InventoryEntry.versionable_kind(kind):
182
if filepath in user_files:
183
raise BadFileKindError("cannot add %s of type %s" % (abspath, kind))
184
else:
185
warning("skipping %s (can't add file of kind '%s')", abspath, kind)
186
continue
203
warning("skipping %s (can't add file of kind '%s')", abspath, kind)
204
continue
187
205
188
206
if parent_ie is not None:
189
versioned = bzrlib.osutils.basename(filepath) in parent_ie.children
207
versioned = directory.base_path in parent_ie.children
190
208
else:
191
209
# without the parent ie, use the relatively slower inventory
192
210
# probing method
193
versioned = inv.has_filename(filepath)
211
versioned = inv.has_filename(directory.raw_path)
194
212
195
213
if kind == 'directory':
196
214
try:
203
221
else:
204
222
sub_tree = False
205
223
206
if filepath == '':
224
if directory.raw_path == '':
207
225
# mutter("tree root doesn't need to be added")
208
226
sub_tree = False
209
227
elif versioned:
212
230
elif sub_tree:
213
231
mutter("%r is a nested bzr tree", abspath)
214
232
else:
215
added.extend(__add_one(tree, inv, parent_ie, filepath, kind, action))
233
__add_one(tree, inv, parent_ie, directory, kind, action)
234
added.append(directory.raw_path)
216
235
217
if kind == 'directory' and recurse and not sub_tree:
218
try:
219
if parent_ie is not None:
220
# must be present:
221
this_ie = parent_ie.children[bzrlib.osutils.basename(filepath)]
236
if kind == 'directory' and not sub_tree:
237
if parent_ie is not None:
238
# must be present:
239
this_ie = parent_ie.children[directory.base_path]
240
else:
241
# without the parent ie, use the relatively slower inventory
242
# probing method
243
this_id = inv.path2id(directory.raw_path)
244
if this_id is None:
245
this_ie = None
222
246
else:
223
# without the parent ie, use the relatively slower inventory
224
# probing method
225
this_id = inv.path2id(filepath)
226
if this_id is None:
227
this_ie = None
228
else:
229
this_ie = inv[this_id]
230
except KeyError:
231
this_ie = None
247
this_ie = inv[this_id]
232
248
233
249
for subf in os.listdir(abspath):
234
250
# here we could use TreeDirectory rather than
235
251
# string concatenation.
236
subp = bzrlib.osutils.pathjoin(filepath, subf)
252
subp = bzrlib.osutils.pathjoin(directory.raw_path, subf)
237
253
# TODO: is_control_filename is very slow. Make it faster.
238
254
# TreeDirectory.is_control_filename could also make this
239
255
# faster - its impossible for a non root dir to have a
240
256
# control file.
241
257
if tree.is_control_filename(subp):
242
258
mutter("skip control directory %r", subp)
259
elif subf in this_ie.children:
260
# recurse into this already versioned subdir.
261
dirs_to_add.append((FastPath(subp, subf), this_ie))
243
262
else:
263
# user selection overrides ignoes
244
264
# ignore while selecting files - if we globbed in the
245
265
# outer loop we would ignore user files.
246
266
ignore_glob = tree.is_ignored(subp)
251
271
ignored[ignore_glob].append(subp)
252
272
else:
253
273
#mutter("queue to add sub-file %r", subp)
254
files_to_add.append((subp, this_ie))
274
dirs_to_add.append((FastPath(subp, subf), this_ie))
255
275
256
if len(added) > 0:
276
if len(added) > 0 and save:
257
277
tree._write_inventory(inv)
258
278
return added, ignored
259
279
260
280
261
def __add_one(tree, inv, parent_ie, path, kind, action):
281
def __add_one_and_parent(tree, inv, parent_ie, path, kind, action):
262
282
"""Add a new entry to the inventory and automatically add unversioned parents.
263
283
264
Actual adding of the entry is delegated to the action callback.
265
266
284
:param inv: Inventory which will receive the new entry.
267
285
:param parent_ie: Parent inventory entry if known, or None. If
268
286
None, the parent is looked up by name and used if present, otherwise
271
289
:param action: callback(inv, parent_ie, path, kind); return ignored.
272
290
:returns: A list of paths which have been added.
273
291
"""
274
275
292
# Nothing to do if path is already versioned.
276
293
# This is safe from infinite recursion because the tree root is
277
294
# always versioned.
280
297
added = []
281
298
else:
282
299
# slower but does not need parent_ie
283
if inv.has_filename(path):
300
if inv.has_filename(path.raw_path):
284
301
return []
285
# add parent
286
added = __add_one(tree, inv, None, dirname(path), 'directory', action)
287
parent_id = inv.path2id(dirname(path))
288
if parent_id is not None:
289
parent_ie = inv[inv.path2id(dirname(path))]
290
else:
291
parent_ie = None
302
# its really not there : add the parent
303
# note that the dirname use leads to some extra str copying etc but as
304
# there are a limited number of dirs we can be nested under, it should
305
# generally find it very fast and not recurse after that.
306
added = __add_one_and_parent(tree, inv, None, FastPath(dirname(path.raw_path)), 'directory', action)
307
parent_id = inv.path2id(dirname(path.raw_path))
308
parent_ie = inv[parent_id]
309
__add_one(tree, inv, parent_ie, path, kind, action)
310
return added + [path.raw_path]
311
312
313
def __add_one(tree, inv, parent_ie, path, kind, action):
314
"""Add a new entry to the inventory.
315
316
:param inv: Inventory which will receive the new entry.
317
:param parent_ie: Parent inventory entry.
318
:param kind: Kind of new entry (file, directory, etc)
319
:param action: callback(inv, parent_ie, path, kind); return ignored.
320
:returns: None
321
"""
292
322
action(inv, parent_ie, path, kind)
293
294
return added + [path]
323
entry = bzrlib.inventory.make_entry(kind, path.base_path, parent_ie.file_id)
324
inv.add(entry)
Loggerhead is a web-based interface for Breezy
Version: 2.0.1