Fennel: /home/pub/open/dev/fennel/test/BackupRestoreTest.cpp Source File (original) (raw)
00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022
00023
00024 #include "fennel/common/CommonPreamble.h"
00025 #include "fennel/common/FileSystem.h"
00026 #include "fennel/test/SnapshotSegmentTestBase.h"
00027 #include "fennel/cache/PagePredicate.h"
00028 #include "fennel/txn/LogicalTxn.h"
00029 #include "fennel/txn/LogicalTxnLog.h"
00030 #include "fennel/segment/VersionedRandomAllocationSegment.h"
00031 #include "fennel/segment/SnapshotRandomAllocationSegment.h"
00032 #include "fennel/segment/SegPageBackupRestoreDevice.h"
00033 #include "fennel/db/Database.h"
00034
00035 #include <boost/test/test_tools.hpp>
00036
00037 using namespace fennel;
00038
00043 class BackupRestoreTest : virtual public SnapshotSegmentTestBase
00044 {
00045 struct TestNode : public StoredNode
00046 {
00047 static const MagicNumber MAGIC_NUMBER = 0xa496c71bff0d41bdLL;
00048
00049 uint x;
00050 };
00051
00052 typedef SegNodeLock TestPageLock;
00053
00054 SharedDatabase pDatabase;
00055 PageId persistentPageId;
00056
00057 void createSnapshotData();
00058 void executeSnapshotTxn(int i);
00059 void verifySnapshotData(uint x);
00060
00061 void testBackupRestore(bool isCompressed);
00062 void backup(
00063 std::string backupFileName,
00064 TxnId lowerBoundCsn,
00065 TxnId upperBoundCsn,
00066 bool isCompressed);
00067 void restore(
00068 std::string backupFileName,
00069 TxnId lowerBoundCsn,
00070 TxnId upperBoundCsn,
00071 bool isCompressed);
00072 std::string getCompressionProgram(bool isCompressed);
00073 void verifyData();
00074
00075 public:
00076 explicit BackupRestoreTest()
00077 {
00078 FENNEL_UNIT_TEST_CASE(BackupRestoreTest, testHeaderBackupRestore);
00079 FENNEL_UNIT_TEST_CASE(BackupRestoreTest, testBackupCleanup);
00080 FENNEL_UNIT_TEST_CASE(BackupRestoreTest, testBackupRestoreUncompressed);
00081 FENNEL_UNIT_TEST_CASE(BackupRestoreTest, testBackupRestoreCompressed);
00082 }
00083
00087 void testHeaderBackupRestore();
00088
00092 void testBackupCleanup();
00093
00097 void testBackupRestoreUncompressed();
00098
00102 void testBackupRestoreCompressed();
00103 };
00104
00105 void BackupRestoreTest::testBackupRestoreUncompressed()
00106 {
00107 testBackupRestore(false);
00108 }
00109
00110 void BackupRestoreTest::testBackupRestoreCompressed()
00111 {
00112 testBackupRestore(true);
00113 }
00114
00115 void BackupRestoreTest::createSnapshotData()
00116 {
00117
00118 pDatabase = Database::newDatabase(
00119 pCache,
00120 configMap,
00121 DeviceMode::createNew,
00122 shared_from_this());
00123
00124 SharedLogicalTxn pTxn = pDatabase->getTxnLog()->newLogicalTxn(pCache);
00125 SharedSegment pSegment =
00126 pDatabase->getSegmentFactory()->newSnapshotRandomAllocationSegment(
00127 pDatabase->getDataSegment(),
00128 pDatabase->getDataSegment(),
00129 pTxn->getTxnId());
00130 SnapshotRandomAllocationSegment *pSnapshotSegment =
00131 SegmentFactory::dynamicCast<SnapshotRandomAllocationSegment *>(
00132 pSegment);
00133 SegmentAccessor segmentAccessor(pSegment,pCache);
00134
00135 TestPageLock pageLock(segmentAccessor);
00136 persistentPageId = pageLock.allocatePage();
00137 pageLock.getNodeForWrite().x = 0;
00138 pageLock.unlock();
00139 pTxn->commit();
00140 pTxn = pDatabase->getTxnLog()->newLogicalTxn(pCache);
00141 pSnapshotSegment->commitChanges(pTxn->getTxnId());
00142 pSnapshotSegment->checkpoint(CHECKPOINT_FLUSH_ALL);
00143 pTxn->commit();
00144 pDatabase->checkpointImpl();
00145
00146
00147 executeSnapshotTxn(5);
00148 }
00149
00150 void BackupRestoreTest::testHeaderBackupRestore()
00151 {
00152 configMap.setStringParam(
00153 Database::paramDatabaseDir,".");
00154 configMap.setStringParam(
00155 "databaseInitSize","1000");
00156 configMap.setStringParam(
00157 "tempInitSize","1000");
00158 configMap.setStringParam(
00159 "databaseShadowLogInitSize","1000");
00160 configMap.setStringParam(
00161 "databaseTxnLogInitSize","1000");
00162 configMap.setStringParam(
00163 "forceTxns","true");
00164 configMap.setStringParam(
00165 "disableSnapshots","false");
00166
00167 CacheParams cacheParams;
00168 cacheParams.readConfig(configMap);
00169 pCache = Cache::newCache(cacheParams);
00170
00171
00172
00173
00174 createSnapshotData();
00175 pDatabase->checkpointImpl();
00176 verifySnapshotData(5);
00177
00178
00179 std::string fullBackup = "fullBackup.dat";
00180 FileSize dataDeviceSize;
00181 bool aborted = false;
00182 TxnId fullTxnId =
00183 pDatabase->initiateBackup(
00184 fullBackup,
00185 false,
00186 0,
00187 NULL_TXN_ID,
00188 "",
00189 dataDeviceSize,
00190 aborted);
00191 pDatabase->completeBackup(NULL_TXN_ID, fullTxnId, aborted);
00192
00193
00194 executeSnapshotTxn(10);
00195 verifySnapshotData(15);
00196
00197
00198 std::string incrBackup1 = "incrBackup1.dat";
00199 TxnId incrTxnId1 =
00200 pDatabase->initiateBackup(
00201 incrBackup1,
00202 true,
00203 0,
00204 fullTxnId,
00205 "",
00206 dataDeviceSize,
00207 aborted);
00208
00209
00210 uint nPagesBefore =
00211 pDatabase->getDataSegment()->getAllocatedSizeInPages();
00212 pDatabase->deallocateOldPages(incrTxnId1);
00213 uint nPagesAfter =
00214 pDatabase->getDataSegment()->getAllocatedSizeInPages();
00215 BOOST_REQUIRE(nPagesBefore == nPagesAfter);
00216
00217 pDatabase->completeBackup(fullTxnId, incrTxnId1, aborted);
00218
00219
00220 pDatabase->deallocateOldPages(incrTxnId1);
00221 nPagesAfter = pDatabase->getDataSegment()->getAllocatedSizeInPages();
00222 BOOST_REQUIRE(nPagesBefore > nPagesAfter);
00223
00224
00225 executeSnapshotTxn(20);
00226 verifySnapshotData(35);
00227
00228
00229 std::string incrBackup2 = "incrBackup2.dat";
00230 TxnId incrTxnId2 =
00231 pDatabase->initiateBackup(
00232 incrBackup2,
00233 true,
00234 4096,
00235 incrTxnId1,
00236 "",
00237 dataDeviceSize,
00238 aborted);
00239 pDatabase->completeBackup(incrTxnId1, incrTxnId2, aborted);
00240
00241
00242 pDatabase->restoreFromBackup(
00243 fullBackup,
00244 1002 * pCache->getPageSize(),
00245 "",
00246 NULL_TXN_ID,
00247 fullTxnId,
00248 aborted);
00249 verifySnapshotData(5);
00250
00251
00252 pDatabase->restoreFromBackup(
00253 incrBackup1,
00254 1002 * pCache->getPageSize(),
00255 "",
00256 fullTxnId,
00257 incrTxnId1,
00258 aborted);
00259 verifySnapshotData(15);
00260
00261
00262 pDatabase->restoreFromBackup(
00263 incrBackup2,
00264 1002 * pCache->getPageSize(),
00265 "",
00266 incrTxnId1,
00267 incrTxnId2,
00268 aborted);
00269 verifySnapshotData(35);
00270
00271 pDatabase.reset();
00272 }
00273
00274 void BackupRestoreTest::testBackupCleanup()
00275 {
00276 configMap.setStringParam(
00277 Database::paramDatabaseDir,".");
00278 configMap.setStringParam(
00279 "databaseInitSize","1000");
00280 configMap.setStringParam(
00281 "tempInitSize","1000");
00282 configMap.setStringParam(
00283 "databaseShadowLogInitSize","1000");
00284 configMap.setStringParam(
00285 "databaseTxnLogInitSize","1000");
00286 configMap.setStringParam(
00287 "forceTxns","true");
00288 configMap.setStringParam(
00289 "disableSnapshots","false");
00290
00291 CacheParams cacheParams;
00292 cacheParams.readConfig(configMap);
00293 pCache = Cache::newCache(cacheParams);
00294
00295
00296
00297
00298 createSnapshotData();
00299 pDatabase->checkpointImpl();
00300 verifySnapshotData(5);
00301
00302
00303 executeSnapshotTxn(10);
00304 verifySnapshotData(15);
00305
00306 std::string fullBackup = "fullBackup.dat";
00307
00308
00309
00310
00311
00312
00313
00314
00315 FileSystem::remove(fullBackup.c_str());
00316 FileSize spaceAvailable;
00317 FileSystem::getDiskFreeSpace(".", spaceAvailable);
00318 FileSize dataDeviceSize;
00319 bool aborted = false;
00320 try {
00321 pDatabase->initiateBackup(
00322 fullBackup,
00323 true,
00324 spaceAvailable * 1000,
00325 NULL_TXN_ID,
00326 "",
00327 dataDeviceSize,
00328 aborted);
00329 BOOST_FAIL("Out of space exception not returned");
00330 } catch (FennelExcn &ex) {
00331 std::string errMsg = ex.getMessage();
00332 if (errMsg.find("Insufficient space") != 0) {
00333 BOOST_FAIL("Wrong exception returned");
00334 }
00335 }
00336
00337
00338
00339 uint nPagesBefore =
00340 pDatabase->getDataSegment()->getAllocatedSizeInPages();
00341 pDatabase->deallocateOldPages(pDatabase->getLastCommittedTxnId());
00342 uint nPagesAfter =
00343 pDatabase->getDataSegment()->getAllocatedSizeInPages();
00344 BOOST_REQUIRE(nPagesBefore > nPagesAfter);
00345
00346
00347 executeSnapshotTxn(20);
00348 verifySnapshotData(35);
00349
00350
00351 pDatabase->initiateBackup(
00352 fullBackup,
00353 false,
00354 0,
00355 NULL_TXN_ID,
00356 getCompressionProgram(true),
00357 dataDeviceSize,
00358 aborted);
00359 pDatabase->abortBackup();
00360
00361
00362
00363 nPagesBefore =
00364 pDatabase->getDataSegment()->getAllocatedSizeInPages();
00365 pDatabase->deallocateOldPages(pDatabase->getLastCommittedTxnId());
00366 nPagesAfter =
00367 pDatabase->getDataSegment()->getAllocatedSizeInPages();
00368 BOOST_REQUIRE(nPagesBefore > nPagesAfter);
00369
00370
00371 pDatabase->abortBackup();
00372
00373 pDatabase.reset();
00374 }
00375
00376 void BackupRestoreTest::executeSnapshotTxn(int i)
00377 {
00378 SharedLogicalTxn pTxn = pDatabase->getTxnLog()->newLogicalTxn(pCache);
00379 SharedSegment pSegment =
00380 pDatabase->getSegmentFactory()->newSnapshotRandomAllocationSegment(
00381 pDatabase->getDataSegment(),
00382 pDatabase->getDataSegment(),
00383 pTxn->getTxnId());
00384 SnapshotRandomAllocationSegment *pSnapshotSegment =
00385 SegmentFactory::dynamicCast<SnapshotRandomAllocationSegment *>(
00386 pSegment);
00387
00388
00389 SegmentAccessor segmentAccessor(pSegment, pCache);
00390 TestPageLock pageLock(segmentAccessor);
00391 pageLock.lockExclusive(persistentPageId);
00392 pageLock.getNodeForWrite().x += i;
00393 pageLock.unlock();
00394 pTxn->commit();
00395
00396
00397 pTxn = pDatabase->getTxnLog()->newLogicalTxn(pCache);
00398 pSnapshotSegment->commitChanges(pTxn->getTxnId());
00399 pSnapshotSegment->checkpoint(CHECKPOINT_FLUSH_ALL);
00400 pTxn->commit();
00401 pDatabase->checkpointImpl();
00402 }
00403
00404 void BackupRestoreTest::verifySnapshotData(uint x)
00405 {
00406
00407
00408
00409 SharedSegment pSegment =
00410 pDatabase->getSegmentFactory()->newSnapshotRandomAllocationSegment(
00411 pDatabase->getDataSegment(),
00412 pDatabase->getDataSegment(),
00413 pDatabase->getLastCommittedTxnId());
00414 SegmentAccessor segmentAccessor(pSegment,pCache);
00415 TestPageLock pageLock(segmentAccessor);
00416 pageLock.lockShared(persistentPageId);
00417 BOOST_CHECK_EQUAL(pageLock.getNodeForRead().x,x);
00418 }
00419
00420 void BackupRestoreTest::testBackupRestore(bool isCompressed)
00421 {
00422
00423
00424 currCsn = TxnId(0);
00425 openStorage(DeviceMode::createNew);
00426 testAllocateAll();
00427 closeStorage();
00428
00429
00430 currCsn = TxnId(5);
00431 updatedCsns.push_back(currCsn);
00432 openStorage(DeviceMode::load);
00433 testSkipWrite(5);
00434 closeStorage();
00435
00436
00437 std::string backup5 = "backupTxnId5.dat";
00438 backup(backup5, NULL_TXN_ID, TxnId(5), isCompressed);
00439
00440
00441 currCsn = TxnId(7);
00442 updatedCsns.push_back(currCsn);
00443 openStorage(DeviceMode::load);
00444 testSkipWrite(7);
00445 closeStorage();
00446
00447
00448 std::string backup5to7 = "backupTxnId5to7.dat";
00449 backup(backup5to7, TxnId(5), TxnId(7), isCompressed);
00450
00451
00452 currCsn = TxnId(11);
00453 updatedCsns.push_back(currCsn);
00454 openStorage(DeviceMode::load);
00455 testSkipWrite(11);
00456 closeStorage();
00457
00458
00459 std::string backup7to11 = "backupTxnId7to11.dat";
00460 backup(backup7to11, TxnId(7), TxnId(11), isCompressed);
00461
00462
00463 std::string backup5to11 = "backupTxnId5to11.dat";
00464 backup(backup5to11, TxnId(5), TxnId(11), isCompressed);
00465
00466
00467 restore(backup5, NULL_TXN_ID, TxnId(5), isCompressed);
00468
00469
00470
00471
00472 updatedCsns.clear();
00473 updatedCsns.push_back(TxnId(5));
00474 verifyData();
00475
00476
00477
00478 restore(backup5to7, TxnId(5), TxnId(7), isCompressed);
00479 updatedCsns.push_back(TxnId(7));
00480 verifyData();
00481
00482
00483
00484 restore(backup7to11, TxnId(7), TxnId(11), isCompressed);
00485 updatedCsns.push_back(TxnId(11));
00486 verifyData();
00487
00488
00489
00490 restore(backup5, NULL_TXN_ID, TxnId(5), isCompressed);
00491 updatedCsns.clear();
00492 updatedCsns.push_back(TxnId(5));
00493 verifyData();
00494 restore(backup5to11, TxnId(5), TxnId(11), isCompressed);
00495 updatedCsns.push_back(TxnId(7));
00496 updatedCsns.push_back(TxnId(11));
00497 verifyData();
00498 }
00499
00500 void BackupRestoreTest::backup(
00501 std::string backupFileName,
00502 TxnId lowerBoundCsn,
00503 TxnId upperBoundCsn,
00504 bool isCompressed)
00505 {
00506 openStorage(DeviceMode::load);
00507 VersionedRandomAllocationSegment pVRSegment =
00508 SegmentFactory::dynamicCast<VersionedRandomAllocationSegment *>(
00509 pVersionedRandomSegment);
00510 SegmentAccessor scratchAccessor =
00511 pSegmentFactory->newScratchSegment(pCache, 12);
00512
00513 if (isCompressed) {
00514 backupFileName.append(".gz");
00515 }
00516 SharedSegPageBackupRestoreDevice pBackupDevice =
00517 SegPageBackupRestoreDevice::newSegPageBackupRestoreDevice(
00518 backupFileName,
00519 "w",
00520 getCompressionProgram(isCompressed),
00521 10,
00522 2,
00523 scratchAccessor,
00524 pCache->getDeviceAccessScheduler(pRandomAccessDevice),
00525 pRandomAccessDevice);
00526 bool abortFlag = false;
00527 pVRSegment->backupAllocationNodes(
00528 pBackupDevice,
00529 false,
00530 lowerBoundCsn,
00531 upperBoundCsn,
00532 abortFlag);
00533 pVRSegment->backupDataPages(
00534 pBackupDevice,
00535 lowerBoundCsn,
00536 upperBoundCsn,
00537 abortFlag);
00538
00539 assert(pBackupDevice.unique());
00540 pBackupDevice.reset();
00541
00542 scratchAccessor.reset();
00543 closeStorage();
00544 }
00545
00546 void BackupRestoreTest::restore(
00547 std::string backupFileName,
00548 TxnId lowerBoundCsn,
00549 TxnId upperBoundCsn,
00550 bool isCompressed)
00551 {
00552 openStorage(DeviceMode::load);
00553 VersionedRandomAllocationSegment *pVRSegment =
00554 SegmentFactory::dynamicCast<VersionedRandomAllocationSegment *>(
00555 pVersionedRandomSegment);
00556 SegmentAccessor scratchAccessor =
00557 pSegmentFactory->newScratchSegment(pCache, 10);
00558
00559
00560
00561 MappedPageListenerPredicate pagePredicate(pVRSegment);
00562 pCache->checkpointPages(pagePredicate, CHECKPOINT_FLUSH_AND_UNMAP);
00563
00564 if (isCompressed) {
00565 backupFileName.append(".gz");
00566 }
00567 SharedSegPageBackupRestoreDevice pBackupDevice =
00568 SegPageBackupRestoreDevice::newSegPageBackupRestoreDevice(
00569 backupFileName,
00570 "r",
00571 getCompressionProgram(isCompressed),
00572 10,
00573 0,
00574 scratchAccessor,
00575 pCache->getDeviceAccessScheduler(pRandomAccessDevice),
00576 pRandomAccessDevice);
00577 bool abortFlag = false;
00578 pVRSegment->restoreFromBackup(
00579 pBackupDevice,
00580 lowerBoundCsn,
00581 upperBoundCsn,
00582 abortFlag);
00583
00584 assert(pBackupDevice.unique());
00585 pBackupDevice.reset();
00586
00587 scratchAccessor.reset();
00588 closeStorage();
00589 }
00590
00591 std::string BackupRestoreTest::getCompressionProgram(bool isCompressed)
00592 {
00593 if (!isCompressed) {
00594 return "";
00595 } else {
00596 return "gzip";
00597 }
00598 }
00599
00600 void BackupRestoreTest::verifyData()
00601 {
00602 openStorage(DeviceMode::load);
00603 testSequentialRead();
00604 closeStorage();
00605 }
00606
00607 FENNEL_UNIT_TEST_SUITE(BackupRestoreTest);
00608
00609