Fennel: /home/pub/open/dev/fennel/lucidera/test/LbmExecStreamTestBase.cpp Source File (original) (raw)
00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022 #include "fennel/common/CommonPreamble.h"
00023 #include "fennel/lucidera/test/LbmExecStreamTestBase.h"
00024
00025 FENNEL_BEGIN_CPPFILE("$Id: //open/dev/fennel/lucidera/test/LbmExecStreamTestBase.cpp#10 $");
00026
00027 void LbmExecStreamTestBase::initBitmapInput(
00028 BitmapInput &bmInput, uint nRows, InputData const &inputData)
00029 {
00030 LbmNumberStreamInput input;
00031 SharedNumberStream pNumberStream(
00032 new SkipNumberStream(
00033 opaqueToInt(inputData.startRid),
00034 NumberStream::BIG_NUMBER, inputData.skipRows));
00035 input.pStream = pNumberStream;
00036 input.bitmapSize = inputData.bitmapSize;
00037
00038 initBitmapInput(bmInput, nRows, input);
00039 }
00040
00041 void LbmExecStreamTestBase::initBitmapInput(
00042 BitmapInput &bmInput, uint nRows, LbmNumberStreamInput input)
00043 {
00044 bmInput.fullBufSize = input.pStream->getMaxRowCount(nRows) * 16;
00045 bmInput.bufArray.reset(new FixedBuffer[bmInput.fullBufSize]);
00046 bmInput.nBitmaps = 0;
00047 bmInput.currBufSize = 0;
00048 generateBitmaps(nRows, input, bmInput);
00049 }
00050
00051 void LbmExecStreamTestBase::generateBitmaps(
00052 uint nRows, LbmNumberStreamInput input, BitmapInput &bmInput)
00053 {
00054 LbmEntry lbmEntry;
00055 boost::scoped_array entryBuf;
00056 LcsRid rid = LcsRid(input.pStream->getNext());
00057
00058
00059 uint scratchBufSize = LbmEntry::getScratchBufferSize(bitmapColSize);
00060 entryBuf.reset(new FixedBuffer[scratchBufSize]);
00061 lbmEntry.init(entryBuf.get(), NULL, scratchBufSize, bitmapTupleDesc);
00062 bitmapTupleData[0].pData = (PConstBuffer) &rid;
00063 lbmEntry.setEntryTuple(bitmapTupleData);
00064 TraceLevel level = getSourceTraceLevel(getTraceName());
00065 if (level <= TRACE_FINER) {
00066 std::cout << "Set root: " << rid << std::endl;
00067 }
00068
00069
00070 while (input.pStream->hasNext()) {
00071 rid = LcsRid(input.pStream->getNext());
00072 if (rid >= LcsRid(nRows)) {
00073 break;
00074 }
00075 if (level <= TRACE_FINER) {
00076 std::cout << "Set value: " << rid << std::endl;
00077 }
00078 if ((rid > LcsRid(0) &&
00079 opaqueToInt(rid % (input.bitmapSize*8)) == 0) ||
00080 !lbmEntry.setRID(LcsRid(rid)))
00081 {
00082
00083
00084
00085 produceEntry(lbmEntry, bitmapTupleAccessor, bmInput);
00086 lbmEntry.setEntryTuple(bitmapTupleData);
00087 }
00088 }
00089
00090 produceEntry(lbmEntry, bitmapTupleAccessor, bmInput);
00091
00092 assert(bmInput.currBufSize <= bmInput.fullBufSize);
00093 }
00094
00095 void LbmExecStreamTestBase::produceEntry(
00096 LbmEntry &lbmEntry, TupleAccessor &bitmapTupleAccessor,
00097 BitmapInput &bmInput)
00098 {
00099 TupleData bitmapTuple = lbmEntry.produceEntryTuple();
00100 bitmapTupleAccessor.marshal(
00101 bitmapTuple, bmInput.bufArray.get() + bmInput.currBufSize);
00102 bmInput.currBufSize += bitmapTupleAccessor.getCurrentByteCount();
00103 ++bmInput.nBitmaps;
00104 }
00105
00106 void LbmExecStreamTestBase::initValuesExecStream(
00107 uint idx, ValuesExecStreamParams &valuesParams,
00108 ExecStreamEmbryo &valuesStreamEmbryo, BitmapInput &bmInput)
00109 {
00110 valuesParams.outputTupleDesc = bitmapTupleDesc;
00111 valuesParams.pTupleBuffer = bmInput.bufArray;
00112 valuesParams.bufSize = bmInput.currBufSize;
00113
00114 valuesStreamEmbryo.init(new ValuesExecStream(), valuesParams);
00115 std::ostringstream oss;
00116 oss << "InputValuesExecStream" << "#" << idx;
00117 valuesStreamEmbryo.getStream()->setName(oss.str());
00118 }
00119
00120 void LbmExecStreamTestBase::initSorterExecStream(
00121 ExternalSortExecStreamParams ¶ms,
00122 ExecStreamEmbryo &embryo,
00123 TupleDescriptor const &outputDesc,
00124 uint nKeys)
00125 {
00126 params.outputTupleDesc = outputDesc;
00127 params.distinctness = DUP_ALLOW;
00128 params.pTempSegment = pRandomSegment;
00129 params.pCacheAccessor = pCache;
00130 params.scratchAccessor =
00131 pSegmentFactory->newScratchSegment(pCache, 10);
00132 params.keyProj.clear();
00133 for (uint i = 0; i < nKeys; i++) {
00134 params.keyProj.push_back(i);
00135 }
00136 params.storeFinalRun = false;
00137 params.estimatedNumRows = MAXU;
00138 params.earlyClose = false;
00139
00140 embryo.init(ExternalSortExecStream::newExternalSortExecStream(), params);
00141 embryo.getStream()->setName("SorterExecStream");
00142 }
00143
00144 void LbmExecStreamTestBase::initNormalizerExecStream(
00145 LbmNormalizerExecStreamParams ¶ms,
00146 ExecStreamEmbryo &embryo,
00147 uint nKeys)
00148 {
00149 TupleProjection keyProj;
00150 for (int i = 0; i < nKeys; i++) {
00151 keyProj.push_back(i);
00152 }
00153 params.keyProj = keyProj;
00154
00155 TupleDescriptor keyDesc;
00156 keyDesc.projectFrom(keyBitmapTupleDesc, keyProj);
00157 params.outputTupleDesc = keyDesc;
00158
00159 embryo.init(new LbmNormalizerExecStream(), params);
00160 embryo.getStream()->setName("Normalizer");
00161 }
00162
00163 void LbmExecStreamTestBase::generateBitmaps(
00164 uint nRows, uint start, uint skipRows, PBuffer pBuf, uint &bufSize,
00165 uint fullBufSize, uint &nBitmaps, bool includeKeys)
00166 {
00167 LbmEntry lbmEntry;
00168 boost::scoped_array entryBuf;
00169 LcsRid rid = LcsRid(start);
00170
00171
00172 uint scratchBufSize = LbmEntry::getScratchBufferSize(bitmapColSize);
00173 entryBuf.reset(new FixedBuffer[scratchBufSize]);
00174 lbmEntry.init(entryBuf.get(), NULL, scratchBufSize, bitmapTupleDesc);
00175 bitmapTupleData[0].pData = (PConstBuffer) &rid;
00176 lbmEntry.setEntryTuple(bitmapTupleData);
00177
00178
00179 for (rid = LcsRid(start + skipRows); rid < LcsRid(nRows); rid += skipRows) {
00180 if (!lbmEntry.setRID(LcsRid(rid))) {
00181
00182
00183 produceEntry(
00184 lbmEntry, bitmapTupleAccessor,
00185 pBuf, bufSize, nBitmaps, includeKeys);
00186 lbmEntry.setEntryTuple(bitmapTupleData);
00187 }
00188 }
00189
00190 produceEntry(
00191 lbmEntry, bitmapTupleAccessor, pBuf, bufSize, nBitmaps, includeKeys);
00192
00193 assert(bufSize <= fullBufSize);
00194 }
00195
00196 void LbmExecStreamTestBase::produceEntry(
00197 LbmEntry &lbmEntry, TupleAccessor &bitmapTupleAccessor, PBuffer pBuf,
00198 uint &bufSize, uint &nBitmaps, bool includeKeys)
00199 {
00200 TupleData bitmapTuple = lbmEntry.produceEntryTuple();
00201 if (includeKeys) {
00202 int nKeys = keyBitmapTupleData.size() - bitmapTuple.size();
00203 assert(nKeys > 0);
00204 for (uint i = 0; i < bitmapTupleData.size(); i++) {
00205 keyBitmapTupleData[nKeys + i] = bitmapTuple[i];
00206 }
00207 keyBitmapTupleAccessor.marshal(keyBitmapTupleData, pBuf + bufSize);
00208 bufSize += keyBitmapTupleAccessor.getCurrentByteCount();
00209 } else {
00210 bitmapTupleAccessor.marshal(bitmapTuple, pBuf + bufSize);
00211 bufSize += bitmapTupleAccessor.getCurrentByteCount();
00212 }
00213 ++nBitmaps;
00214 }
00215
00216 void LbmExecStreamTestBase::initKeyBitmap(
00217 uint nRows,
00218 std::vector const &repeatSeqValues)
00219 {
00220
00221 uint skipRows = getTupleInterval(repeatSeqValues);
00222
00223
00224
00225 uint nInputKeys = repeatSeqValues.size();
00226 keyBitmapTupleDesc.clear();
00227 for (uint i = 0; i < nInputKeys; i++) {
00228 keyBitmapTupleDesc.push_back(attrDesc_int64);
00229 }
00230 for (uint i = 0; i < bitmapTupleDesc.size(); i++) {
00231 keyBitmapTupleDesc.push_back(bitmapTupleDesc[i]);
00232 }
00233
00234
00235 keyBitmapTupleAccessor.compute(keyBitmapTupleDesc);
00236 keyBitmapTupleData.compute(keyBitmapTupleDesc);
00237 boost::scoped_array vals(new uint64_t[nInputKeys]);
00238 for (uint i = 0; i < nInputKeys; i++) {
00239 keyBitmapTupleData[i].pData = (PConstBuffer) &vals[i];
00240 }
00241
00242 uint fullBufSize = nRows * keyBitmapTupleAccessor.getMaxByteCount();
00243 keyBitmapBuf.reset(new FixedBuffer[fullBufSize]);
00244 PBuffer pBuf = keyBitmapBuf.get();
00245 keyBitmapBufSize = 0;
00246 uint nBitmaps = 0;
00247 for (uint i = 0; i < skipRows; i++) {
00248
00249 for (uint j = 0; j < nInputKeys; j++) {
00250 vals[j] = i % repeatSeqValues[j];
00251 }
00252 generateBitmaps(
00253 nRows, i, skipRows, pBuf, keyBitmapBufSize,
00254 fullBufSize, nBitmaps, true);
00255 }
00256 }
00257
00261 uint LbmExecStreamTestBase::getTupleInterval(
00262 std::vector const &repeatSeqValues, uint nKeys)
00263 {
00264 if (nKeys == 0) {
00265 nKeys = repeatSeqValues.size();
00266 }
00267 uint interval = 1;
00268 for (uint i = 0; i < nKeys; i++) {
00269 interval *= repeatSeqValues[i];
00270 }
00271 return interval;
00272 }
00273
00274 void LbmExecStreamTestBase::testCaseSetUp()
00275 {
00276 ExecStreamUnitTestBase::testCaseSetUp();
00277
00278 bitmapColSize = pRandomSegment->getUsablePageSize() / 8;
00279 attrDesc_bitmap = TupleAttributeDescriptor(
00280 stdTypeFactory.newDataType(STANDARD_TYPE_VARBINARY),
00281 true, bitmapColSize);
00282 attrDesc_int64 = TupleAttributeDescriptor(
00283 stdTypeFactory.newDataType(STANDARD_TYPE_INT_64));
00284
00285 bitmapTupleDesc.clear();
00286 bitmapTupleDesc.push_back(attrDesc_int64);
00287 bitmapTupleDesc.push_back(attrDesc_bitmap);
00288 bitmapTupleDesc.push_back(attrDesc_bitmap);
00289
00290 bitmapTupleData.compute(bitmapTupleDesc);
00291 bitmapTupleData[1].pData = NULL;
00292 bitmapTupleData[1].cbData = 0;
00293 bitmapTupleData[2].pData = NULL;
00294 bitmapTupleData[2].cbData = 0;
00295
00296 bitmapTupleAccessor.compute(bitmapTupleDesc);
00297 }
00298
00299 FENNEL_END_CPPFILE("$Id: //open/dev/fennel/lucidera/test/LbmExecStreamTestBase.cpp#10 $");
00300
00301