Fennel: /home/pub/open/dev/fennel/lucidera/test/LbmMinusExecStreamTest.cpp Source File (original) (raw)
00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022 #include "fennel/common/CommonPreamble.h"
00023 #include "fennel/lucidera/test/LbmExecStreamTestBase.h"
00024 #include "fennel/lucidera/bitmap/LbmMinusExecStream.h"
00025 #include <stdarg.h>
00026
00027 #include <boost/test/test_tools.hpp>
00028
00029 using namespace fennel;
00030
00034 class LbmMinusExecStreamTest : public LbmExecStreamTestBase
00035 {
00036 void testMinus(
00037 uint nInputs, uint nRows, std::vector const &inputData);
00038
00043 void testRestartingMinus(
00044 uint nRows,
00045 std::vector const &repeatSeqValues,
00046 uint subtrahendInterval);
00047
00048 void newMinusStream(
00049 LbmMinusExecStreamParams ¶ms,
00050 ExecStreamEmbryo &embryo,
00051 TupleDescriptor const &outputDesc);
00052
00053 public:
00054 explicit LbmMinusExecStreamTest()
00055 {
00056 FENNEL_UNIT_TEST_CASE(LbmMinusExecStreamTest, test2Inputs);
00057 FENNEL_UNIT_TEST_CASE(LbmMinusExecStreamTest, test3Inputs);
00058 FENNEL_UNIT_TEST_CASE(LbmMinusExecStreamTest, testLargeOutput);
00059 FENNEL_UNIT_TEST_CASE(LbmMinusExecStreamTest, testAnchorLarger1);
00060 FENNEL_UNIT_TEST_CASE(LbmMinusExecStreamTest, testAnchorLarger2);
00061 FENNEL_UNIT_TEST_CASE(LbmMinusExecStreamTest, testChildrenLarger);
00062
00063
00064 FENNEL_UNIT_TEST_CASE(LbmMinusExecStreamTest, testEvens);
00065 FENNEL_UNIT_TEST_CASE(LbmMinusExecStreamTest, testNines);
00066 FENNEL_UNIT_TEST_CASE(LbmMinusExecStreamTest, testClose);
00067 FENNEL_UNIT_TEST_CASE(
00068 LbmMinusExecStreamTest, testLowCardinalityRestart);
00069 }
00070
00071 void test2Inputs();
00072 void test3Inputs();
00073 void testLargeOutput();
00074 void testAnchorLarger1();
00075 void testAnchorLarger2();
00076 void testChildrenLarger();
00077
00078 void testEvens();
00079 void testNines();
00080 void testClose();
00081 void testLowCardinalityRestart();
00082 };
00083
00093 class RestartingMinusExecStreamGenerator
00094 : public MockProducerExecStreamGenerator
00095 {
00096 protected:
00097 uint nKeys;
00098 std::vector repeatSeqValues;
00099 uint subtrahendInterval;
00100 uint interval;
00101 boost::shared_array changeIndexes;
00102 uint current;
00103 uint lastRow;
00104
00105 public:
00106 RestartingMinusExecStreamGenerator(
00107 uint nRows, std::vector repeatSeqValues, uint subtrahendInterval)
00108 {
00109 this->nKeys = repeatSeqValues.size();
00110 this->repeatSeqValues = repeatSeqValues;
00111 interval = LbmExecStreamTestBase::getTupleInterval(repeatSeqValues);
00112
00113
00114 boost::scoped_array valueCounts;
00115 valueCounts.reset(new uint[interval]);
00116 for (uint i = 0; i < interval; i++) {
00117 valueCounts[i] = getValueCount(nRows, i);
00118 }
00119
00120 for (uint iRow = 0; iRow < nRows; iRow += subtrahendInterval) {
00121 valueCounts[iRow % interval]--;
00122 }
00123
00124
00125 changeIndexes.reset(new uint[interval]);
00126 changeIndexes[0] = valueCounts[0];
00127 for (uint i = 1; i < interval; i++) {
00128 changeIndexes[i] = changeIndexes[i - 1] + valueCounts[i];
00129 }
00130 current = 0;
00131 lastRow = 0;
00132 }
00133
00134 virtual int64_t generateValue(uint iRow, uint iCol)
00135 {
00136
00137 assert (lastRow <= iRow);
00138 lastRow = iRow;
00139
00140 if (iRow >= changeIndexes[current]) {
00141 current++;
00142 assert (current < interval);
00143 }
00144 assert (iCol < nKeys);
00145 return current % repeatSeqValues[iCol];
00146 }
00147
00151 uint getValueCount(uint nRows, uint iValue)
00152 {
00153 uint nCopies = nRows / interval;
00154 if (iValue < nRows % interval) {
00155 nCopies++;
00156 }
00157 return nCopies;
00158 }
00159
00163 uint getRowCount()
00164 {
00165 return changeIndexes[interval - 1];
00166 }
00167 };
00168
00169 void LbmMinusExecStreamTest::test2Inputs()
00170 {
00171 uint nInputs = 2;
00172 uint nRows = 1000;
00173 std::vector inputData;
00174 InputData input;
00175
00176
00177 input.bitmapSize = 4;
00178 input.startRid = LcsRid(10);
00179 input.skipRows = 1;
00180 inputData.push_back(input);
00181
00182
00183 input.bitmapSize = 8;
00184 input.startRid = LcsRid(0);
00185 input.skipRows = 2;
00186 inputData.push_back(input);
00187
00188
00189 input.bitmapSize = resultBitmapSize(11, nRows);
00190 input.startRid = LcsRid(11);
00191 input.skipRows = 2;
00192 inputData.push_back(input);
00193
00194 testMinus(nInputs, nRows, inputData);
00195 }
00196
00197 void LbmMinusExecStreamTest::test3Inputs()
00198 {
00199 uint nInputs = 3;
00200 uint nRows = 1000;
00201 std::vector inputData;
00202 InputData input;
00203
00204
00205 input.bitmapSize = 9;
00206 input.startRid = LcsRid(22);
00207 input.skipRows = 1;
00208 inputData.push_back(input);
00209
00210
00211 input.bitmapSize = 8;
00212 input.startRid = LcsRid(2);
00213 input.skipRows = 3;
00214 inputData.push_back(input);
00215
00216
00217 input.bitmapSize = 10;
00218 input.startRid = LcsRid(3);
00219 input.skipRows = 3;
00220 inputData.push_back(input);
00221
00222
00223 input.bitmapSize = resultBitmapSize(22, nRows);
00224 input.startRid = LcsRid(22);
00225 input.skipRows = 3;
00226 inputData.push_back(input);
00227
00228 testMinus(nInputs, nRows, inputData);
00229 }
00230
00235 void LbmMinusExecStreamTest::testLargeOutput()
00236 {
00237 uint nInputs = 16;
00238 uint nRows = 100000;
00239 std::vector inputData;
00240 InputData input;
00241
00242
00243 input.bitmapSize = 1;
00244 input.startRid = LcsRid(0);
00245 input.skipRows = 1;
00246 inputData.push_back(input);
00247
00248
00249 input.bitmapSize = 1;
00250 input.startRid = LcsRid(1);
00251 input.skipRows = 16;
00252 inputData.push_back(input);
00253
00254
00255 input.bitmapSize = 1;
00256 input.startRid = LcsRid(2);
00257 input.skipRows = 16;
00258 inputData.push_back(input);
00259
00260
00261 input.bitmapSize = 1;
00262 input.startRid = LcsRid(3);
00263 input.skipRows = 16;
00264 inputData.push_back(input);
00265
00266
00267 input.bitmapSize = 1;
00268 input.startRid = LcsRid(4);
00269 input.skipRows = 16;
00270 inputData.push_back(input);
00271
00272
00273 input.bitmapSize = 1;
00274 input.startRid = LcsRid(5);
00275 input.skipRows = 16;
00276 inputData.push_back(input);
00277
00278
00279 input.bitmapSize = 1;
00280 input.startRid = LcsRid(6);
00281 input.skipRows = 16;
00282 inputData.push_back(input);
00283
00284
00285 input.bitmapSize = 1;
00286 input.startRid = LcsRid(7);
00287 input.skipRows = 16;
00288 inputData.push_back(input);
00289
00290
00291 input.bitmapSize = 1;
00292 input.startRid = LcsRid(8);
00293 input.skipRows = 16;
00294 inputData.push_back(input);
00295
00296
00297 input.bitmapSize = 1;
00298 input.startRid = LcsRid(9);
00299 input.skipRows = 16;
00300 inputData.push_back(input);
00301
00302
00303 input.bitmapSize = 1;
00304 input.startRid = LcsRid(10);
00305 input.skipRows = 16;
00306 inputData.push_back(input);
00307
00308
00309 input.bitmapSize = 1;
00310 input.startRid = LcsRid(11);
00311 input.skipRows = 16;
00312 inputData.push_back(input);
00313
00314
00315 input.bitmapSize = 1;
00316 input.startRid = LcsRid(12);
00317 input.skipRows = 16;
00318 inputData.push_back(input);
00319
00320
00321 input.bitmapSize = 1;
00322 input.startRid = LcsRid(13);
00323 input.skipRows = 16;
00324 inputData.push_back(input);
00325
00326
00327 input.bitmapSize = 1;
00328 input.startRid = LcsRid(14);
00329 input.skipRows = 16;
00330 inputData.push_back(input);
00331
00332
00333 input.bitmapSize = 1;
00334 input.startRid = LcsRid(15);
00335 input.skipRows = 16;
00336 inputData.push_back(input);
00337
00338
00339 input.bitmapSize = resultBitmapSize(0, nRows);
00340 input.startRid = LcsRid(0);
00341 input.skipRows = 16;
00342 inputData.push_back(input);
00343
00344 testMinus(nInputs, nRows, inputData);
00345 }
00346
00351 void LbmMinusExecStreamTest::testAnchorLarger1()
00352 {
00353 uint nInputs = 3;
00354 uint nRows = 100;
00355 std::vector inputData;
00356 InputData input;
00357
00358
00359 input.bitmapSize = 10;
00360 input.startRid = LcsRid(0);
00361 input.skipRows = 17;
00362 inputData.push_back(input);
00363
00364
00365 input.bitmapSize = 10;
00366 input.startRid = LcsRid(0);
00367 input.skipRows = 60;
00368 inputData.push_back(input);
00369
00370
00371 input.bitmapSize = 10;
00372 input.startRid = LcsRid(0);
00373 input.skipRows = 70;
00374 inputData.push_back(input);
00375
00376
00377 input.bitmapSize = resultBitmapSize(17, nRows);
00378 input.startRid = LcsRid(17);
00379 input.skipRows = 17;
00380 inputData.push_back(input);
00381
00382 testMinus(nInputs, nRows, inputData);
00383 }
00384
00389 void LbmMinusExecStreamTest::testAnchorLarger2()
00390 {
00391 uint nInputs = 3;
00392 uint nRows = 100;
00393 std::vector inputData;
00394 InputData input;
00395
00396
00397 input.bitmapSize = 10;
00398 input.startRid = LcsRid(0);
00399 input.skipRows = 17;
00400 inputData.push_back(input);
00401
00402
00403 input.bitmapSize = 10;
00404 input.startRid = LcsRid(0);
00405 input.skipRows = 60;
00406 inputData.push_back(input);
00407
00408
00409 input.bitmapSize = 10;
00410 input.startRid = LcsRid(0);
00411 input.skipRows = 61;
00412 inputData.push_back(input);
00413
00414
00415 input.bitmapSize = resultBitmapSize(17, nRows);
00416 input.startRid = LcsRid(17);
00417 input.skipRows = 17;
00418 inputData.push_back(input);
00419
00420 testMinus(nInputs, nRows, inputData);
00421 }
00422
00427 void LbmMinusExecStreamTest::testChildrenLarger()
00428 {
00429 uint nInputs = 3;
00430 uint nRows = 100;
00431 std::vector inputData;
00432 InputData input;
00433
00434
00435 input.bitmapSize = 10;
00436 input.startRid = LcsRid(0);
00437 input.skipRows = 60;
00438 inputData.push_back(input);
00439
00440
00441 input.bitmapSize = 10;
00442 input.startRid = LcsRid(70);
00443 input.skipRows = 70;
00444 inputData.push_back(input);
00445
00446
00447 input.bitmapSize = 10;
00448 input.startRid = LcsRid(80);
00449 input.skipRows = 80;
00450 inputData.push_back(input);
00451
00452
00453 input.bitmapSize = resultBitmapSize(0, nRows);
00454 input.startRid = LcsRid(0);
00455 input.skipRows = 60;
00456 inputData.push_back(input);
00457
00458 testMinus(nInputs, nRows, inputData);
00459 }
00460
00461 void LbmMinusExecStreamTest::testEvens()
00462 {
00463 uint nRows = 100000;
00464 std::vector repeatSeqValues;
00465 repeatSeqValues.push_back(1);
00466 repeatSeqValues.push_back(5);
00467 repeatSeqValues.push_back(9);
00468 uint subtrahendInterval = 2;
00469
00470 testRestartingMinus(nRows, repeatSeqValues, subtrahendInterval);
00471 }
00472
00473 void LbmMinusExecStreamTest::testNines()
00474 {
00475 uint nRows = 1000;
00476 std::vector repeatSeqValues;
00477 repeatSeqValues.push_back(1);
00478 repeatSeqValues.push_back(5);
00479 repeatSeqValues.push_back(9);
00480 uint subtrahendInterval = 9;
00481
00482 testRestartingMinus(nRows, repeatSeqValues, subtrahendInterval);
00483 }
00484
00485 void LbmMinusExecStreamTest::testClose()
00486 {
00487 uint nRows = 1000;
00488 std::vector repeatSeqValues;
00489 repeatSeqValues.push_back(2);
00490 repeatSeqValues.push_back(3);
00491 uint subtrahendInterval = 4;
00492
00493 testRestartingMinus(nRows, repeatSeqValues, subtrahendInterval);
00494 }
00495
00496 void LbmMinusExecStreamTest::testLowCardinalityRestart()
00497 {
00498 uint nRows = 10000;
00499 std::vector repeatSeqValues;
00500 repeatSeqValues.push_back(2);
00501 uint subtrahendInterval = 21;
00502
00503 testRestartingMinus(nRows, repeatSeqValues, subtrahendInterval);
00504 }
00505
00506 void LbmMinusExecStreamTest::testMinus(
00507 uint nInputs, uint nRows, std::vector const &inputData)
00508 {
00509 boost::scoped_array bmInputs;
00510 boost::scoped_array valuesParams;
00511 std::vector valuesStreamEmbryoList;
00512 ExecStreamEmbryo valuesStreamEmbryo;
00513
00514
00515 assert(inputData.size() == nInputs + 1);
00516
00517 bmInputs.reset(new BitmapInput[nInputs + 1]);
00518 valuesParams.reset(new ValuesExecStreamParams[nInputs]);
00519
00520
00521 for (uint i = 0; i < nInputs; i++) {
00522 initBitmapInput(bmInputs[i], nRows, inputData[i]);
00523 initValuesExecStream(
00524 i, valuesParams[i], valuesStreamEmbryo, bmInputs[i]);
00525 valuesStreamEmbryoList.push_back(valuesStreamEmbryo);
00526 }
00527
00528 if (inputData[nInputs].bitmapSize > 0) {
00529 initBitmapInput(bmInputs[nInputs], nRows, inputData[nInputs]);
00530 } else {
00531 bmInputs[nInputs].bufArray.reset();
00532 bmInputs[nInputs].nBitmaps = 0;
00533 }
00534
00535 LbmMinusExecStreamParams minusParams;
00536 ExecStreamEmbryo minusEmbryo;
00537 newMinusStream(minusParams, minusEmbryo, bitmapTupleDesc);
00538
00539 SharedExecStream pOutputStream = prepareConfluenceGraph(
00540 valuesStreamEmbryoList, minusEmbryo);
00541
00542 verifyBufferedOutput(
00543 *pOutputStream, bitmapTupleDesc, bmInputs[nInputs].nBitmaps,
00544 bmInputs[nInputs].bufArray.get());
00545 }
00546
00547 void LbmMinusExecStreamTest::testRestartingMinus(
00548 uint nRows,
00549 std::vector const &repeatSeqValues,
00550 uint subtrahendInterval)
00551 {
00552 uint nKeys = repeatSeqValues.size();
00553
00554
00555 initKeyBitmap(nRows, repeatSeqValues);
00556 ValuesExecStreamParams valuesParams;
00557 valuesParams.outputTupleDesc = keyBitmapTupleDesc;
00558 valuesParams.pTupleBuffer = keyBitmapBuf;
00559 valuesParams.bufSize = keyBitmapBufSize;
00560
00561 ExecStreamEmbryo minuendEmbryo;
00562 minuendEmbryo.init(new ValuesExecStream(), valuesParams);
00563 minuendEmbryo.getStream()->setName("MinuendValuesExecStream");
00564
00565
00566 InputData subtrahendData;
00567 subtrahendData.startRid = LcsRid(0);
00568 subtrahendData.skipRows = subtrahendInterval;
00569 subtrahendData.bitmapSize = 8;
00570
00571 BitmapInput bmInput;
00572 initBitmapInput(bmInput, nRows, subtrahendData);
00573
00574 ValuesExecStreamParams subtrahendParams;
00575 ExecStreamEmbryo subtrahendEmbryo;
00576 initValuesExecStream(0, subtrahendParams, subtrahendEmbryo, bmInput);
00577
00578
00579 LbmMinusExecStreamParams minusParams;
00580 ExecStreamEmbryo minusEmbryo;
00581 newMinusStream(minusParams, minusEmbryo, keyBitmapTupleDesc);
00582
00583
00584 ExecStreamEmbryo normalizerEmbryo;
00585 LbmNormalizerExecStreamParams normalizerParams;
00586 initNormalizerExecStream(normalizerParams, normalizerEmbryo, nKeys);
00587
00588
00589 SharedExecStream pOutputStream = prepareConfluenceTransformGraph(
00590 minuendEmbryo, subtrahendEmbryo, minusEmbryo, normalizerEmbryo);
00591 RestartingMinusExecStreamGenerator
00592 verifier(nRows, repeatSeqValues, subtrahendInterval);
00593 verifyOutput(
00594 *pOutputStream,
00595 verifier.getRowCount(),
00596 verifier);
00597 }
00598
00599 void LbmMinusExecStreamTest::newMinusStream(
00600 LbmMinusExecStreamParams ¶ms,
00601 ExecStreamEmbryo &embryo,
00602 TupleDescriptor const &outputDesc)
00603 {
00604 params.rowLimitParamId = DynamicParamId(1);
00605 params.startRidParamId = DynamicParamId(2);
00606 params.outputTupleDesc = outputDesc;
00607 params.scratchAccessor =
00608 pSegmentFactory->newScratchSegment(pCache, 10);
00609
00610 embryo.init(new LbmMinusExecStream(), params);
00611 embryo.getStream()->setName("MinusExecStream");
00612 }
00613
00614 FENNEL_UNIT_TEST_SUITE(LbmMinusExecStreamTest);
00615
00616
00617