~bzr-pqm/bzr/bzr.dev : revision 2592.3.196

98

ExistingPack and NewPack are used.

99

"""

100

101

def __init__(self, revision_index, inventory_index):

101

def __init__(self, revision_index, inventory_index, text_index):

102

"""Create a pack instance.

103

104

:param revision_index: A GraphIndex for determining what revisions are

105

present in the Pack and accessing the locations of their texts.

106

:param inventory_index: A GraphIndex for determining what inventories are

107

present in the Pack and accessing the locations of their texts.

107

present in the Pack and accessing the locations of their

108

texts/deltas.

109

:param text_index: A GraphIndex for determining what file texts

110

are present in the pack and accessing the locations of their

111

texts/deltas (via (fileid, revisionid) tuples).

108

112

"""

109

113

self.revision_index = revision_index

110

114

self.inventory_index = inventory_index

115

self.text_index = text_index

111

116

112

117

def get_revision_count(self):

113

118

return self.revision_index.key_count()

134

139

135

140

def __init__(self, transport, name, revision_index, inventory_index,

136

141

text_index, signature_index):

137

Pack.__init__(self, revision_index, inventory_index)

142

Pack.__init__(self, revision_index, inventory_index, text_index)

138

143

self.text_index = text_index

139

144

self.signature_index = signature_index

140

145

self.name = name

180

185

# knit code hasn't been updated enough to understand that, so we

181

186

# have a regular 2-list index giving parents and compression

182

187

# source.

183

InMemoryGraphIndex(reference_lists=2))

188

InMemoryGraphIndex(reference_lists=2),

189

# Texts: compression and per file graph, for all fileids - so two

190

# reference lists and two elements in the key tuple.

191

InMemoryGraphIndex(reference_lists=2, key_elements=2),

192

)

184

193

# where should the new pack be opened

185

194

self.upload_transport = upload_transport

186

195

# where are indices written out to

210

219

"""

211

220

new_name = self._hash.hexdigest()

212

221

self.index_sizes = [None, None, None, None]

213

214

revision_index_name = self.revision_index_name(new_name)

215

self.index_sizes[0] = self.index_transport.put_file(

216

revision_index_name, self.revision_index.finish())

217

if 'fetch' in debug.debug_flags:

218

# XXX: size might be interesting?

219

mutter('%s: create_pack: wrote revision index: %s%s t+%6.3fs',

220

time.ctime(), self.upload_transport.base, self.random_name,

221

time.time() - self.start_time)

222

223

inv_index_name = self.inventory_index_name(new_name)

224

self.index_sizes[1] = self.index_transport.put_file(inv_index_name,

225

self.inventory_index.finish())

226

if 'fetch' in debug.debug_flags:

227

# XXX: size might be interesting?

228

mutter('%s: create_pack: wrote inventory index: %s%s t+%6.3fs',

229

time.ctime(), self.upload_transport.base, self.random_name,

230

time.time() - self.start_time)

222

self._write_index(new_name, self.revision_index, 0,

223

self.revision_index_name, 'revision')

224

self._write_index(new_name, self.inventory_index, 1,

225

self.inventory_index_name, 'inventory')

226

self._write_index(new_name, self.text_index, 2,

227

self.text_index_name, 'file texts')

228

229

def _write_index(self, new_name, index, index_offset, name_getter, label):

230

"""Write out an index.

231

232

:param new_name: The basename of the pack.

233

:param index: The index object to serialise.

234

:param index_offset: Where in self.index_sizes to remember this.

235

:param name_getter: What to use to get the name of the index on disk.

236

:param label: What label to give the index e.g. 'revision'.

237

"""

238

index_name = name_getter(new_name)

239

self.index_sizes[index_offset] = self.index_transport.put_file(

240

index_name, index.finish())

241

if 'fetch' in debug.debug_flags:

242

# XXX: size might be interesting?

243

mutter('%s: create_pack: wrote %s index: %s%s t+%6.3fs',

244

time.ctime(), label, self.upload_transport.base,

245

self.random_name, time.time() - self.start_time)

231

246

232

247

233

248

class RepositoryPackCollection(object):

293

308

file_id_index = GraphIndexPrefixAdapter(

294

309

self.repo._text_all_indices,

295

310

(file_id, ), 1,

296

add_nodes_callback=self.repo._text_write_index.add_nodes)

311

add_nodes_callback=self.repo._packs._new_pack.text_index.add_nodes)

297

312

self.repo._text_knit._index._graph_index = file_id_index

298

313

self.repo._text_knit._index._add_callback = file_id_index.add_nodes

299

314

return self.repo._text_knit.add_lines_with_ghosts(

422

437

self.repo._inv_pack_map = None

423

438

return inv_index

424

439

425

def flush_text_index(self, new_name):

440

def flush_text_index(self, new_name, text_index_length):

426

441

"""Write the index out to new_name."""

427

442

# write a revision index (might be empty)

428

443

new_index_name = self._new_pack.text_index_name(new_name)

429

text_index_length = self._index_transport.put_file(

430

new_index_name, self.repo._text_write_index.finish())

431

444

txt_index = GraphIndex(self._index_transport, new_index_name,

432

445

text_index_length)

433

self.repo._text_write_index = None

434

446

self.repo.weave_store._setup_knit(False)

435

447

if self.repo._text_all_indices is not None:

436

448

# text 'knits' have been used, replace the mutated memory index

440

452

# remove the write buffering index. XXX: API break

441

453

# - clearly we need a remove_index call too.

442

454

del self.repo._text_all_indices._indices[1]

443

return txt_index, text_index_length

455

return txt_index

444

456

445

457

def create_pack_from_packs(self, packs, suffix, revision_ids=None):

446

458

"""Create a new pack by reading data from other packs.

501

513

writer = pack.ContainerWriter(write_data)

502

514

writer.begin()

503

515

# open new indices

504

text_index = InMemoryGraphIndex(reference_lists=2, key_elements=2)

505

516

signature_index = InMemoryGraphIndex(reference_lists=0)

506

517

# select revisions

507

518

if revision_ids:

572

583

a_missing_key[0])

573

584

# copy text keys and adjust values

574

585

list(self._copy_nodes_graph(text_nodes, text_index_map, writer,

575

text_index))

586

new_pack.text_index))

576

587

if 'fetch' in debug.debug_flags:

577

588

mutter('%s: create_pack: file texts copied: %s%s %d items t+%6.3fs',

578

589

time.ctime(), self._upload_transport.base, random_name,

579

text_index.key_count(),

590

new_pack.text_index.key_count(),

580

591

time.time() - new_pack.start_time)

581

592

# select signature keys

582

593

signature_filter = revision_keys # same keyspace

601

612

# if nothing has been written, discard the new pack.

602

613

if 0 == sum((new_pack.get_revision_count(),

603

614

new_pack.inventory_index.key_count(),

604

text_index.key_count(),

615

new_pack.text_index.key_count(),

605

616

signature_index.key_count(),

606

617

)):

607

618

self._upload_transport.delete(random_name)

611

622

new_pack.finish()

612

623

revision_index_length = new_pack.index_sizes[0]

613

624

inventory_index_length = new_pack.index_sizes[1]

625

text_index_length = new_pack.index_sizes[2]

614

626

615

text_index_name = new_pack.text_index_name(new_name)

616

text_index_length = index_transport.put_file(text_index_name,

617

text_index.finish())

618

if 'fetch' in debug.debug_flags:

619

# XXX: size might be interesting?

620

mutter('%s: create_pack: wrote file texts index: %s%s t+%6.3fs',

621

time.ctime(), self._upload_transport.base, random_name,

622

time.time() - new_pack.start_time)

623

627

signature_index_name = new_pack.signature_index_name(new_name)

624

628

signature_index_length = index_transport.put_file(signature_index_name,

625

629

signature_index.finish())

636

640

write_stream.close()

637

641

self._upload_transport.rename(random_name, '../packs/' + new_name + '.pack')

638

642

result = ExistingPack(self._upload_transport.clone('../packs/'), new_name,

639

new_pack.revision_index, new_pack.inventory_index, text_index,

643

new_pack.revision_index, new_pack.inventory_index, new_pack.text_index,

640

644

signature_index)

641

645

if 'fetch' in debug.debug_flags:

642

646

# XXX: size might be interesting?

996

1000

self.repo._signature_write_index = None

997

1001

self.repo._signature_all_indices = None

998

1002

self.repo._signature_knit_access = None

999

# remove any accumlating index of text data

1000

self.repo._text_write_index = None

1001

1003

# no access object.

1002

1004

self.repo._text_knit_access = None

1003

1005

# no write-knit

1137

1139

self._open_pack_writer.end()

1138

1140

new_name = self._new_pack._hash.hexdigest()

1139

1141

self._new_pack.finish()

1140

txt_index, text_index_length = self.flush_text_index(new_name)

1142

txt_index = \

1143

self.flush_text_index(new_name, self._new_pack.index_sizes[2])

1141

1144

inv_index = \

1142

1145

self.flush_inventory_index(new_name, self._new_pack.index_sizes[1])

1143

1146

rev_index, \

1156

1159

# its a deliberate attack or data corruption has

1157

1160

# occuring during the write of that file.

1158

1161

self.allocate(new_name, self._new_pack.index_sizes[0],

1159

self._new_pack.index_sizes[1], text_index_length,

1162

self._new_pack.index_sizes[1], self._new_pack.index_sizes[2],

1160

1163

signature_index_length)

1161

1164

self.repo._open_pack_tuple = None

1162

1165

self._new_pack = None

1313

1316

self._transport = weavestore._transport

1314

1317

1315

1318

def data_inserted(self):

1316

# XXX: Should we define __len__ for indices?

1317

if (getattr(self.repo, '_text_write_index', None) and

1318

self.repo._text_write_index.key_count()):

1319

if (self.repo._packs._new_pack is not None and

1320

self.repo._packs._new_pack.text_index.key_count()):

1319

1321

return True

1320

1322

1321

1323

def _ensure_all_index(self, for_write=None):

1326

1328

self.repo._text_pack_map = pack_map

1327

1329

if for_write or self.repo.is_in_write_group():

1328

1330

# allow writing: queue writes to a new index

1329

indices.insert(0, self.repo._text_write_index)

1331

indices.insert(0, self.repo._packs._new_pack.text_index)

1330

1332

self._setup_knit(self.repo.is_in_write_group())

1331

1333

self.repo._text_all_indices = CombinedGraphIndex(indices)

1332

1334

1337

1339

"""

1338

1340

self._ensure_all_index()

1339

1341

if force_write or self.repo.is_in_write_group():

1340

add_callback = self.repo._text_write_index.add_nodes

1341

self.repo._text_pack_map[self.repo._text_write_index] = self.repo._open_pack_tuple

1342

add_callback = self.repo._packs._new_pack.text_index.add_nodes

1343

self.repo._text_pack_map[self.repo._packs._new_pack.text_index] = self.repo._open_pack_tuple

1342

1344

else:

1343

1345

add_callback = None # no data-adding permitted.

1344

1346

1365

1367

return iter(ids)

1366

1368

1367

1369

def setup(self):

1368

# setup in-memory indices to accumulate data.

1369

self.repo._text_write_index = InMemoryGraphIndex(reference_lists=2,

1370

key_elements=2)

1371

1370

# we require that text 'knits' be accessed from within the write

1372

1371

# group to be able to be written to, simply because it makes this

1373

1372

# code cleaner - we don't need to track all 'open' knits and

1378

1377

1379

1378

def _setup_knit(self, for_write):

1380

1379

if for_write:

1381

writer = (self.repo._packs._open_pack_writer, self.repo._text_write_index)

1380

writer = (self.repo._packs._open_pack_writer, self.repo._packs._new_pack.text_index)

1382

1381

else:

1383

1382

writer = None

1384

1383

self.repo._text_knit_access = _PackAccess(