41 #include <llvm/Analysis/ValueTracking.h> 42 #include <llvm/IR/BasicBlock.h> 43 #include <llvm/IR/Instructions.h> 44 #include <llvm/IR/Module.h> 120 llvm::VectorType::get(llvm::Type::getInt1Ty(*ctx), target.
getVectorWidth());
124 llvm::VectorType::get(llvm::Type::getInt8Ty(*ctx), target.
getVectorWidth());
128 llvm::VectorType::get(llvm::Type::getInt16Ty(*ctx), target.
getVectorWidth());
132 llvm::VectorType::get(llvm::Type::getInt32Ty(*ctx), target.
getVectorWidth());
136 llvm::VectorType::get(llvm::Type::getInt64Ty(*ctx), target.
getVectorWidth());
139 FATAL(
"Unhandled mask width for initializing MaskType");
160 LLVMTrue = llvm::ConstantInt::getTrue(*ctx);
161 LLVMFalse = llvm::ConstantInt::getFalse(*ctx);
165 std::vector<llvm::Constant *> maskOnes;
166 llvm::Constant *onMask = NULL;
169 onMask = llvm::ConstantInt::get(llvm::Type::getInt1Ty(*ctx), 1,
false );
172 onMask = llvm::ConstantInt::get(llvm::Type::getInt8Ty(*ctx), -1,
true );
175 onMask = llvm::ConstantInt::get(llvm::Type::getInt16Ty(*ctx), -1,
true );
178 onMask = llvm::ConstantInt::get(llvm::Type::getInt32Ty(*ctx), -1,
true );
181 onMask = llvm::ConstantInt::get(llvm::Type::getInt64Ty(*ctx), -1,
true );
184 FATAL(
"Unhandled mask width for onMask");
188 maskOnes.push_back(onMask);
191 std::vector<llvm::Constant *> maskZeros;
192 llvm::Constant *offMask = NULL;
195 offMask = llvm::ConstantInt::get(llvm::Type::getInt1Ty(*ctx), 0,
true );
198 offMask = llvm::ConstantInt::get(llvm::Type::getInt8Ty(*ctx), 0,
true );
201 offMask = llvm::ConstantInt::get(llvm::Type::getInt16Ty(*ctx), 0,
true );
204 offMask = llvm::ConstantInt::get(llvm::Type::getInt32Ty(*ctx), 0,
true );
207 offMask = llvm::ConstantInt::get(llvm::Type::getInt64Ty(*ctx), 0,
true );
210 FATAL(
"Unhandled mask width for offMask");
213 maskZeros.push_back(offMask);
218 return llvm::ConstantInt::get(llvm::Type::getInt8Ty(*
g->
ctx), ival,
true );
222 return llvm::ConstantInt::get(llvm::Type::getInt8Ty(*
g->
ctx), ival,
false );
226 return llvm::ConstantInt::get(llvm::Type::getInt16Ty(*
g->
ctx), ival,
true );
230 return llvm::ConstantInt::get(llvm::Type::getInt16Ty(*
g->
ctx), ival,
false );
234 return llvm::ConstantInt::get(llvm::Type::getInt32Ty(*
g->
ctx), ival,
true );
238 return llvm::ConstantInt::get(llvm::Type::getInt32Ty(*
g->
ctx), ival,
false );
242 return llvm::ConstantInt::get(llvm::Type::getInt64Ty(*
g->
ctx), ival,
true );
246 return llvm::ConstantInt::get(llvm::Type::getInt64Ty(*
g->
ctx), ival,
false );
249 llvm::Constant *
LLVMFloat(
float fval) {
return llvm::ConstantFP::get(llvm::Type::getFloatTy(*
g->
ctx), fval); }
251 llvm::Constant *
LLVMDouble(
double dval) {
return llvm::ConstantFP::get(llvm::Type::getDoubleTy(*
g->
ctx), dval); }
255 std::vector<llvm::Constant *> vals;
258 return llvm::ConstantVector::get(vals);
262 std::vector<llvm::Constant *> vals;
265 return llvm::ConstantVector::get(vals);
270 std::vector<llvm::Constant *> vals;
273 return llvm::ConstantVector::get(vals);
277 std::vector<llvm::Constant *> vals;
280 return llvm::ConstantVector::get(vals);
285 std::vector<llvm::Constant *> vals;
288 return llvm::ConstantVector::get(vals);
292 std::vector<llvm::Constant *> vals;
295 return llvm::ConstantVector::get(vals);
300 std::vector<llvm::Constant *> vals;
303 return llvm::ConstantVector::get(vals);
307 std::vector<llvm::Constant *> vals;
310 return llvm::ConstantVector::get(vals);
315 std::vector<llvm::Constant *> vals;
318 return llvm::ConstantVector::get(vals);
322 std::vector<llvm::Constant *> vals;
325 return llvm::ConstantVector::get(vals);
330 std::vector<llvm::Constant *> vals;
333 return llvm::ConstantVector::get(vals);
337 std::vector<llvm::Constant *> vals;
340 return llvm::ConstantVector::get(vals);
345 std::vector<llvm::Constant *> vals;
348 return llvm::ConstantVector::get(vals);
352 std::vector<llvm::Constant *> vals;
355 return llvm::ConstantVector::get(vals);
360 std::vector<llvm::Constant *> vals;
363 return llvm::ConstantVector::get(vals);
367 std::vector<llvm::Constant *> vals;
370 return llvm::ConstantVector::get(vals);
375 std::vector<llvm::Constant *> vals;
378 return llvm::ConstantVector::get(vals);
382 std::vector<llvm::Constant *> vals;
385 return llvm::ConstantVector::get(vals);
390 std::vector<llvm::Constant *> vals;
393 return llvm::ConstantVector::get(vals);
397 std::vector<llvm::Constant *> vals;
400 return llvm::ConstantVector::get(vals);
418 std::vector<llvm::Constant *> vals;
421 return llvm::ConstantVector::get(vals);
425 std::vector<llvm::Constant *> vals;
443 return llvm::ConstantVector::get(vals);
448 std::vector<llvm::Constant *> vals;
451 return llvm::ConstantVector::get(vals);
455 std::vector<llvm::Constant *> vals;
457 llvm::Constant *v = llvm::ConstantInt::get(
LLVMTypes::Int8Type, bvec[i] ? 0xff : 0,
false );
460 return llvm::ConstantVector::get(vals);
464 llvm::VectorType *vecType = llvm::dyn_cast<llvm::VectorType>(type);
466 if (vecType != NULL) {
467 llvm::Constant *v = llvm::ConstantInt::get(vecType->getElementType(), val,
true );
468 std::vector<llvm::Constant *> vals;
469 for (
int i = 0; i < (int)vecType->getNumElements(); ++i)
471 return llvm::ConstantVector::get(vals);
473 return llvm::ConstantInt::get(type, val,
true );
477 llvm::VectorType *vecType = llvm::dyn_cast<llvm::VectorType>(type);
479 if (vecType != NULL) {
480 llvm::Constant *v = llvm::ConstantInt::get(vecType->getElementType(), val,
false );
481 std::vector<llvm::Constant *> vals;
482 for (
int i = 0; i < (int)vecType->getNumElements(); ++i)
484 return llvm::ConstantVector::get(vals);
486 return llvm::ConstantInt::get(type, val,
false );
494 static bool lValuesAreEqual(llvm::Value *v0, llvm::Value *v1, std::vector<llvm::PHINode *> &seenPhi0,
495 std::vector<llvm::PHINode *> &seenPhi1) {
503 Assert(seenPhi0.size() == seenPhi1.size());
504 for (
unsigned int i = 0; i < seenPhi0.size(); ++i)
505 if (v0 == seenPhi0[i] && v1 == seenPhi1[i])
508 llvm::BinaryOperator *bo0 = llvm::dyn_cast<llvm::BinaryOperator>(v0);
509 llvm::BinaryOperator *bo1 = llvm::dyn_cast<llvm::BinaryOperator>(v1);
510 if (bo0 != NULL && bo1 != NULL) {
511 if (bo0->getOpcode() != bo1->getOpcode())
513 return (
lValuesAreEqual(bo0->getOperand(0), bo1->getOperand(0), seenPhi0, seenPhi1) &&
514 lValuesAreEqual(bo0->getOperand(1), bo1->getOperand(1), seenPhi0, seenPhi1));
517 llvm::CastInst *cast0 = llvm::dyn_cast<llvm::CastInst>(v0);
518 llvm::CastInst *cast1 = llvm::dyn_cast<llvm::CastInst>(v1);
519 if (cast0 != NULL && cast1 != NULL) {
520 if (cast0->getOpcode() != cast1->getOpcode())
522 return lValuesAreEqual(cast0->getOperand(0), cast1->getOperand(0), seenPhi0, seenPhi1);
525 llvm::PHINode *phi0 = llvm::dyn_cast<llvm::PHINode>(v0);
526 llvm::PHINode *phi1 = llvm::dyn_cast<llvm::PHINode>(v1);
527 if (phi0 != NULL && phi1 != NULL) {
528 if (phi0->getNumIncomingValues() != phi1->getNumIncomingValues())
531 seenPhi0.push_back(phi0);
532 seenPhi1.push_back(phi1);
534 unsigned int numIncoming = phi0->getNumIncomingValues();
537 bool anyFailure =
false;
538 for (
unsigned int i = 0; i < numIncoming; ++i) {
541 Assert(phi0->getIncomingBlock(i) == phi1->getIncomingBlock(i));
542 if (!
lValuesAreEqual(phi0->getIncomingValue(i), phi1->getIncomingValue(i), seenPhi0, seenPhi1)) {
561 llvm::ConstantInt *intOffset = llvm::dyn_cast<llvm::ConstantInt>(offset);
562 Assert(intOffset && (intOffset->getBitWidth() == 32 || intOffset->getBitWidth() == 64));
563 return intOffset->getSExtValue();
570 llvm::ConstantVector *cv = llvm::dyn_cast<llvm::ConstantVector>(v);
572 llvm::Constant *c = llvm::dyn_cast<llvm::Constant>(cv->getOperand(0));
577 for (
int i = 1; i < (int)cv->getNumOperands(); ++i) {
578 if (!llvm::isa<llvm::UndefValue>(cv->getOperand(i))) {
588 bool searchFirstUndef) {
589 std::vector<llvm::Value *> elements(vectorWidth,
nullptr);
592 if (llvm::InsertElementInst *ie = llvm::dyn_cast<llvm::InsertElementInst>(inst)) {
596 Assert(iOffset >= 0 && iOffset < vectorWidth);
599 if (elements[iOffset] == NULL) {
600 elements[iOffset] = ie->getOperand(1);
604 llvm::Value *insertBase = ie->getOperand(0);
605 ie = llvm::dyn_cast<llvm::InsertElementInst>(insertBase);
610 if (llvm::isa<llvm::UndefValue>(insertBase)) {
614 if (llvm::isa<llvm::ConstantVector>(insertBase) || llvm::isa<llvm::ConstantAggregateZero>(insertBase)) {
615 llvm::Constant *cv = llvm::dyn_cast<llvm::Constant>(insertBase);
616 Assert(vectorWidth == (
int)(cv->getNumOperands()));
617 for (
int i = 0; i < vectorWidth; i++) {
618 if (elements[i] == NULL) {
619 elements[i] = cv->getOperand(i);
627 Assert(compare ==
true || elements[0] != NULL);
637 if (compare ==
false) {
644 for (
int i = 0; i < vectorWidth; i++) {
645 if (elements[i] == NULL) {
651 if (null_number == vectorWidth) {
655 if ((undef ==
false) && (null_number != 0)) {
661 for (
int i = 0; i < vectorWidth; i++) {
662 if (elements[i] == NULL) {
666 std::vector<llvm::PHINode *> seenPhi0;
667 std::vector<llvm::PHINode *> seenPhi1;
668 if (
lValuesAreEqual(elements[NonNull], elements[i], seenPhi0, seenPhi1) ==
false) {
672 return elements[NonNull];
684 else if (llvm::ShuffleVectorInst *shuf = llvm::dyn_cast<llvm::ShuffleVectorInst>(inst)) {
685 llvm::Value *indices = shuf->getOperand(2);
686 if (llvm::isa<llvm::ConstantAggregateZero>(indices)) {
687 llvm::Value *op = shuf->getOperand(0);
688 llvm::InsertElementInst *ie = llvm::dyn_cast<llvm::InsertElementInst>(op);
689 if (ie == NULL && searchFirstUndef) {
691 llvm::BinaryOperator *bop = llvm::dyn_cast<llvm::BinaryOperator>(op);
692 if (bop != NULL && ((bop->getOpcode() == llvm::Instruction::Add) ||
IsOrEquivalentToAdd(bop))) {
694 ie = llvm::dyn_cast<llvm::InsertElementInst>(bop->getOperand(0));
698 if (ie != NULL && llvm::isa<llvm::UndefValue>(ie->getOperand(0))) {
699 llvm::ConstantInt *ci = llvm::dyn_cast<llvm::ConstantInt>(ie->getOperand(2));
701 return ie->getOperand(1);
711 llvm::VectorType *vt = llvm::dyn_cast<llvm::VectorType>(v->getType());
713 Assert(llvm::isa<llvm::IntegerType>(vt->getElementType()));
715 *nElts = (int)vt->getNumElements();
717 if (llvm::isa<llvm::ConstantAggregateZero>(v)) {
718 for (
int i = 0; i < (int)vt->getNumElements(); ++i)
723 llvm::ConstantDataVector *cv = llvm::dyn_cast<llvm::ConstantDataVector>(v);
727 for (
int i = 0; i < (int)cv->getNumElements(); ++i)
728 ret[i] = cv->getElementAsInteger(i);
732 static bool lVectorValuesAllEqual(llvm::Value *v,
int vectorLength, std::vector<llvm::PHINode *> &seenPhis,
733 llvm::Value **splatValue = NULL);
742 std::vector<llvm::PHINode *> &seenPhis) {
743 if (llvm::isa<llvm::VectorType>(val->getType()) ==
false) {
746 llvm::ConstantInt *ci = llvm::dyn_cast<llvm::ConstantInt>(val);
748 return (ci->getZExtValue() % baseValue) == 0;
752 if (llvm::isa<llvm::InsertElementInst>(val) || llvm::isa<llvm::ShuffleVectorInst>(val)) {
756 return element ?
lIsExactMultiple(element, baseValue, vectorLength, seenPhis) :
false;
759 llvm::PHINode *phi = llvm::dyn_cast<llvm::PHINode>(val);
761 for (
unsigned int i = 0; i < seenPhis.size(); ++i)
762 if (phi == seenPhis[i])
765 seenPhis.push_back(phi);
766 unsigned int numIncoming = phi->getNumIncomingValues();
770 for (
unsigned int i = 0; i < numIncoming; ++i) {
771 llvm::Value *incoming = phi->getIncomingValue(i);
782 llvm::BinaryOperator *bop = llvm::dyn_cast<llvm::BinaryOperator>(val);
783 if (bop != NULL && ((bop->getOpcode() == llvm::Instruction::Add) ||
IsOrEquivalentToAdd(bop))) {
784 llvm::Value *op0 = bop->getOperand(0);
785 llvm::Value *op1 = bop->getOperand(1);
816 std::vector<llvm::PHINode *> &seenPhis,
bool &canAdd) {
817 Assert(llvm::isa<llvm::VectorType>(val->getType()));
819 Assert(baseValue > 0 && (baseValue & (baseValue - 1)) == 0);
830 int64_t firstDiv = vecVals[0] / baseValue;
831 for (
int i = 1; i < nElts; ++i)
832 if ((vecVals[i] / baseValue) != firstDiv)
838 llvm::PHINode *phi = llvm::dyn_cast<llvm::PHINode>(val);
840 for (
unsigned int i = 0; i < seenPhis.size(); ++i)
841 if (phi == seenPhis[i])
844 seenPhis.push_back(phi);
845 unsigned int numIncoming = phi->getNumIncomingValues();
849 for (
unsigned int i = 0; i < numIncoming; ++i) {
850 llvm::Value *incoming = phi->getIncomingValue(i);
852 bool mult =
lAllDivBaseEqual(incoming, baseValue, vectorLength, seenPhis, ca);
862 llvm::BinaryOperator *bop = llvm::dyn_cast<llvm::BinaryOperator>(val);
863 if (bop != NULL && ((bop->getOpcode() == llvm::Instruction::Add) ||
IsOrEquivalentToAdd(bop)) && canAdd ==
true) {
864 llvm::Value *op0 = bop->getOperand(0);
865 llvm::Value *op1 = bop->getOperand(1);
879 Assert(nElts == vectorLength);
882 int64_t firstConstDiv = addConstants[0] / baseValue;
883 for (
int i = 1; i < vectorLength; ++i)
884 if ((addConstants[i] / baseValue) != firstConstDiv)
914 int maxMod = int(addConstants[0] % baseValue);
915 for (
int i = 1; i < vectorLength; ++i)
916 maxMod = std::max(maxMod,
int(addConstants[i] % baseValue));
919 std::vector<llvm::PHINode *> seenPhisEEM;
941 Assert(nElts == vectorLength);
944 for (
int i = 0; i < vectorLength; ++i)
945 if (shiftAmount[i] != shiftAmount[0])
950 int pow2 = 1 << shiftAmount[0];
952 std::vector<llvm::PHINode *> seenPhis;
955 fprintf(stderr,
"check all div base equal:\n");
958 fprintf(stderr,
"----> %s\n\n", eq ?
"true" :
"false");
964 llvm::Value **splatValue) {
965 if (vectorLength == 1)
968 if (llvm::isa<llvm::ConstantAggregateZero>(v)) {
970 llvm::ConstantAggregateZero *caz = llvm::dyn_cast<llvm::ConstantAggregateZero>(v);
971 *splatValue = caz->getSequentialElement();
976 llvm::ConstantVector *cv = llvm::dyn_cast<llvm::ConstantVector>(v);
978 llvm::Value *splat = cv->getSplatValue();
979 if (splat != NULL && splatValue) {
982 return (splat != NULL);
985 llvm::ConstantDataVector *cdv = llvm::dyn_cast<llvm::ConstantDataVector>(v);
987 llvm::Value *splat = cdv->getSplatValue();
988 if (splat != NULL && splatValue) {
991 return (splat != NULL);
994 llvm::BinaryOperator *bop = llvm::dyn_cast<llvm::BinaryOperator>(v);
1003 if (bop->getOpcode() == llvm::Instruction::AShr || bop->getOpcode() == llvm::Instruction::LShr)
1009 llvm::CastInst *cast = llvm::dyn_cast<llvm::CastInst>(v);
1013 llvm::InsertElementInst *ie = llvm::dyn_cast<llvm::InsertElementInst>(v);
1018 llvm::PHINode *phi = llvm::dyn_cast<llvm::PHINode>(v);
1020 for (
unsigned int i = 0; i < seenPhis.size(); ++i)
1021 if (seenPhis[i] == phi)
1024 seenPhis.push_back(phi);
1026 unsigned int numIncoming = phi->getNumIncomingValues();
1029 for (
unsigned int i = 0; i < numIncoming; ++i) {
1031 seenPhis.pop_back();
1036 seenPhis.pop_back();
1040 if (llvm::isa<llvm::UndefValue>(v))
1044 Assert(!llvm::isa<llvm::Constant>(v));
1046 if (llvm::isa<llvm::CallInst>(v) || llvm::isa<llvm::LoadInst>(v) || !llvm::isa<llvm::Instruction>(v))
1049 llvm::ShuffleVectorInst *shuffle = llvm::dyn_cast<llvm::ShuffleVectorInst>(v);
1050 if (shuffle != NULL) {
1051 llvm::Value *indices = shuffle->getOperand(2);
1062 fprintf(stderr,
"all equal: ");
1064 fprintf(stderr,
"\n");
1065 llvm::Instruction *inst = llvm::dyn_cast<llvm::Instruction>(v);
1067 inst->getParent()->dump();
1068 fprintf(stderr,
"\n");
1069 fprintf(stderr,
"\n");
1081 llvm::VectorType *vt = llvm::dyn_cast<llvm::VectorType>(v->getType());
1083 int vectorLength = vt->getNumElements();
1085 std::vector<llvm::PHINode *> seenPhis;
1088 Debug(
SourcePos(),
"LLVMVectorValuesAllEqual(%s) -> %s.", v->getName().str().c_str(), equal ?
"true" :
"false");
1089 #ifndef ISPC_NO_DUMPS 1100 llvm::BinaryOperator *bop = llvm::dyn_cast<llvm::BinaryOperator>(op);
1101 if (bop != NULL && bop->getOpcode() == llvm::Instruction::Or) {
1104 llvm::Module *module = bop->getParent()->getParent()->getParent();
1105 llvm::Value *op0 = bop->getOperand(0), *op1 = bop->getOperand(1);
1106 if (!haveNoCommonBitsSet(op0, op1, module->getDataLayout()) ==
false) {
1114 static bool lVectorIsLinear(llvm::Value *v,
int vectorLength,
int stride, std::vector<llvm::PHINode *> &seenPhis);
1123 llvm::SmallVector<llvm::Constant *, ISPC_MAX_NVEC> elements;
1124 for (
int i = 0; i < (int)cv->getNumElements(); ++i)
1125 elements.push_back(cv->getElementAsConstant(i));
1126 Assert((
int)elements.size() == vectorLength);
1128 llvm::ConstantInt *ci = llvm::dyn_cast<llvm::ConstantInt>(elements[0]);
1133 int64_t prevVal = ci->getSExtValue();
1138 for (
int i = 1; i < vectorLength; ++i) {
1139 ci = llvm::dyn_cast<llvm::ConstantInt>(elements[i]);
1143 int64_t nextVal = ci->getSExtValue();
1144 if (prevVal + stride != nextVal)
1156 std::vector<llvm::PHINode *> &seenPhis) {
1159 llvm::ConstantDataVector *cv = llvm::dyn_cast<llvm::ConstantDataVector>(op0);
1163 llvm::Constant *csplat = cv->getSplatValue();
1167 llvm::ConstantInt *splat = llvm::dyn_cast<llvm::ConstantInt>(csplat);
1174 int64_t splatVal = splat->getSExtValue();
1175 if (splatVal == 0 || splatVal > stride || (stride % splatVal) != 0)
1180 return lVectorIsLinear(op1, vectorLength, (
int)(stride / splatVal), seenPhis);
1187 std::vector<llvm::PHINode *> &seenPhis) {
1190 llvm::ConstantDataVector *cv = llvm::dyn_cast<llvm::ConstantDataVector>(op1);
1194 llvm::Constant *csplat = cv->getSplatValue();
1198 llvm::ConstantInt *splat = llvm::dyn_cast<llvm::ConstantInt>(csplat);
1205 int64_t equivalentMul = (1LL << splat->getSExtValue());
1206 if (equivalentMul > stride || (stride % equivalentMul) != 0)
1211 return lVectorIsLinear(op0, vectorLength, (
int)(stride / equivalentMul), seenPhis);
1220 std::vector<llvm::PHINode *> &seenPhis) {
1226 Assert(nElts == vectorLength);
1229 for (
int i = 1; i < vectorLength; ++i)
1230 if (maskValue[i] != maskValue[0])
1234 int64_t maskPlusOne = maskValue[0] + 1;
1235 bool isPowTwo = (maskPlusOne & (maskPlusOne - 1)) == 0;
1236 if (isPowTwo ==
false)
1246 bool isMult =
lAllDivBaseEqual(op0, maskPlusOne, vectorLength, seenPhis, canAdd);
1250 static bool lVectorIsLinear(llvm::Value *v,
int vectorLength,
int stride, std::vector<llvm::PHINode *> &seenPhis) {
1253 llvm::ConstantDataVector *cv = llvm::dyn_cast<llvm::ConstantDataVector>(v);
1257 llvm::BinaryOperator *bop = llvm::dyn_cast<llvm::BinaryOperator>(v);
1260 llvm::Value *op0 = bop->getOperand(0), *op1 = bop->getOperand(1);
1274 }
else if (bop->getOpcode() == llvm::Instruction::Sub)
1277 return (
lVectorIsLinear(bop->getOperand(0), vectorLength, stride, seenPhis) &&
1279 else if (bop->getOpcode() == llvm::Instruction::Mul) {
1287 }
else if (bop->getOpcode() == llvm::Instruction::Shl) {
1292 }
else if (bop->getOpcode() == llvm::Instruction::And) {
1301 llvm::CastInst *ci = llvm::dyn_cast<llvm::CastInst>(v);
1303 return lVectorIsLinear(ci->getOperand(0), vectorLength, stride, seenPhis);
1305 if (llvm::isa<llvm::CallInst>(v) || llvm::isa<llvm::LoadInst>(v))
1308 llvm::PHINode *phi = llvm::dyn_cast<llvm::PHINode>(v);
1310 for (
unsigned int i = 0; i < seenPhis.size(); ++i)
1311 if (seenPhis[i] == phi)
1314 seenPhis.push_back(phi);
1316 unsigned int numIncoming = phi->getNumIncomingValues();
1319 for (
unsigned int i = 0; i < numIncoming; ++i) {
1320 if (!
lVectorIsLinear(phi->getIncomingValue(i), vectorLength, stride, seenPhis)) {
1321 seenPhis.pop_back();
1326 seenPhis.pop_back();
1331 if (llvm::isa<llvm::InsertElementInst>(v))
1337 llvm::ShuffleVectorInst *shuffle = llvm::dyn_cast<llvm::ShuffleVectorInst>(v);
1338 if (shuffle != NULL)
1342 fprintf(stderr,
"linear check: ");
1344 fprintf(stderr,
"\n");
1345 llvm::Instruction *inst = llvm::dyn_cast<llvm::Instruction>(v);
1347 inst->getParent()->dump();
1348 fprintf(stderr,
"\n");
1349 fprintf(stderr,
"\n");
1362 llvm::VectorType *vt = llvm::dyn_cast<llvm::VectorType>(v->getType());
1364 int vectorLength = vt->getNumElements();
1366 std::vector<llvm::PHINode *> seenPhis;
1368 Debug(
SourcePos(),
"LLVMVectorIsLinear(%s) -> %s.", v->getName().str().c_str(), linear ?
"true" :
"false");
1369 #ifndef ISPC_NO_DUMPS 1377 #ifndef ISPC_NO_DUMPS 1378 static void lDumpValue(llvm::Value *v, std::set<llvm::Value *> &done) {
1379 if (done.find(v) != done.end())
1382 llvm::Instruction *inst = llvm::dyn_cast<llvm::Instruction>(v);
1383 if (done.size() > 0 && inst == NULL)
1386 fprintf(stderr,
" ");
1393 for (
unsigned i = 0; i < inst->getNumOperands(); ++i)
1398 std::set<llvm::Value *> done;
1400 fprintf(stderr,
"----\n");
1405 llvm::VectorType *vt = llvm::dyn_cast<llvm::VectorType>(v->getType());
1410 if (llvm::isa<llvm::ConstantAggregateZero>(v) ==
true) {
1411 return llvm::Constant::getNullValue(vt->getElementType());
1413 if (llvm::ConstantVector *cv = llvm::dyn_cast<llvm::ConstantVector>(v)) {
1414 return cv->getOperand(0);
1416 if (llvm::ConstantDataVector *cdv = llvm::dyn_cast<llvm::ConstantDataVector>(v))
1417 return cdv->getElementAsConstant(0);
1421 Assert(llvm::isa<llvm::Constant>(v) ==
false);
1422 Assert(llvm::isa<llvm::Instruction>(v) ==
true);
1424 std::string newName = v->getName().str() + std::string(
".elt0");
1428 llvm::BinaryOperator *bop = llvm::dyn_cast<llvm::BinaryOperator>(v);
1436 return llvm::BinaryOperator::Create(bop->getOpcode(), v0, v1, newName, bop);
1439 llvm::CastInst *cast = llvm::dyn_cast<llvm::CastInst>(v);
1444 return llvm::CastInst::Create(cast->getOpcode(), v, vt->getElementType(), newName, cast);
1447 llvm::PHINode *phi = llvm::dyn_cast<llvm::PHINode>(v);
1450 if (phiMap.find(phi) != phiMap.end())
1461 llvm::Instruction *phiInsertPos = &*(phi->getParent()->begin());
1462 llvm::PHINode *scalarPhi =
1463 llvm::PHINode::Create(vt->getElementType(), phi->getNumIncomingValues(), newName, phiInsertPos);
1464 phiMap[phi] = scalarPhi;
1466 for (
unsigned i = 0; i < phi->getNumIncomingValues(); ++i) {
1468 scalarPhi->addIncoming(v, phi->getIncomingBlock(i));
1479 if (llvm::isa<llvm::ShuffleVectorInst>(v)) {
1480 llvm::ShuffleVectorInst *shuf = llvm::dyn_cast<llvm::ShuffleVectorInst>(v);
1481 llvm::Value *indices = shuf->getOperand(2);
1482 if (llvm::isa<llvm::ConstantAggregateZero>(indices)) {
1489 if (llvm::isa<llvm::InsertElementInst>(v)) {
1496 llvm::Instruction *insertAfter = llvm::dyn_cast<llvm::Instruction>(v);
1497 Assert(insertAfter != NULL);
1498 llvm::Instruction *ee = llvm::ExtractElementInst::Create(v,
LLVMInt32(0),
"first_elt", (llvm::Instruction *)NULL);
1499 ee->insertAfter(insertAfter);
1504 std::map<llvm::PHINode *, llvm::PHINode *> phiMap;
1515 Assert(v1->getType() == v2->getType());
1517 llvm::VectorType *vt = llvm::dyn_cast<llvm::VectorType>(v1->getType());
1521 int resultSize = 2 * vt->getNumElements();
1523 for (
int i = 0; i < resultSize; ++i)
1534 llvm::Instruction *insertBefore) {
1535 std::vector<llvm::Constant *> shufVec;
1536 for (
int i = 0; i < shufSize; ++i) {
1543 llvm::ArrayRef<llvm::Constant *> aref(&shufVec[0], &shufVec[shufSize]);
1544 llvm::Value *vec = llvm::ConstantVector::get(aref);
1546 return new llvm::ShuffleVectorInst(v1, v2, vec,
"shuffle", insertBefore);
1552 std::string ret = std::string(v->getName());
1554 return strdup(ret.c_str());
1557 const char *
LLVMGetName(
const char *op, llvm::Value *v1, llvm::Value *v2) {
1560 r += v1->getName().str();
1562 r += v2->getName().str();
1563 return strdup(r.c_str());
llvm::Constant * LLVMIntAsType(int64_t val, llvm::Type *type)
static llvm::Type * FloatType
static llvm::Type * Int32VectorPointerType
llvm::Constant * LLVMUInt64Vector(uint64_t ival)
static int lRoundUpPow2(int v)
static bool lIsFirstElementConstVector(llvm::Value *v)
llvm::Constant * LLVMUInt32Vector(uint32_t ival)
llvm::ConstantInt * LLVMUInt32(uint32_t ival)
static bool lVectorIsLinear(llvm::Value *v, int vectorLength, int stride, std::vector< llvm::PHINode *> &seenPhis)
static void lDumpValue(llvm::Value *v, std::set< llvm::Value *> &done)
llvm::Constant * LLVMBoolVector(bool b)
static llvm::Type * DoubleType
static bool lVectorShiftRightAllEqual(llvm::Value *val, llvm::Value *shift, int vectorLength)
llvm::ConstantInt * LLVMInt8(int8_t ival)
llvm::Constant * LLVMInt64Vector(int64_t ival)
llvm::ConstantInt * LLVMUInt8(uint8_t ival)
Structure that defines a compilation target.
static llvm::VectorType * VoidPointerVectorType
static llvm::VectorType * BoolVectorType
static bool lIsExactMultiple(llvm::Value *val, int baseValue, int vectorLength, std::vector< llvm::PHINode *> &seenPhis)
bool LLVMExtractVectorInts(llvm::Value *v, int64_t ret[], int *nElts)
const char * LLVMGetName(llvm::Value *v, const char *s)
llvm::Constant * LLVMMaskAllOn
static llvm::Type * BoolType
llvm::ConstantInt * LLVMInt16(int16_t ival)
static llvm::VectorType * Int32VectorType
int getMaskBitCount() const
void InitLLVMUtil(llvm::LLVMContext *ctx, Target &target)
static llvm::VectorType * BoolVectorStorageType
llvm::Constant * LLVMFalseInStorage
static llvm::Type * FloatVectorPointerType
static llvm::Type * Int8PointerType
static llvm::Type * Int32PointerType
llvm::Constant * LLVMUIntAsType(uint64_t val, llvm::Type *type)
static llvm::Type * Int16VectorPointerType
static bool lValuesAreEqual(llvm::Value *v0, llvm::Value *v1, std::vector< llvm::PHINode *> &seenPhi0, std::vector< llvm::PHINode *> &seenPhi1)
static llvm::Type * Int16Type
static llvm::Type * DoubleVectorPointerType
static llvm::VectorType * Int1VectorType
llvm::Constant * LLVMFloatVector(float fval)
static llvm::Type * VoidType
llvm::Constant * LLVMDoubleVector(double dval)
static llvm::Type * Int8VectorPointerType
static llvm::VectorType * Int8VectorType
llvm::Constant * LLVMInt32Vector(int32_t ival)
static bool lAllDivBaseEqual(llvm::Value *val, int64_t baseValue, int vectorLength, std::vector< llvm::PHINode *> &seenPhis, bool &canAdd)
llvm::Constant * LLVMTrue
static llvm::VectorType * FloatVectorType
static llvm::Type * Int64Type
static llvm::Type * Int8Type
bool IsOrEquivalentToAdd(llvm::Value *op)
static llvm::VectorType * Int64VectorType
Header file with declarations for various LLVM utility stuff.
static llvm::Type * Int64PointerType
static bool lCheckShlForLinear(llvm::Value *op0, llvm::Value *op1, int vectorLength, int stride, std::vector< llvm::PHINode *> &seenPhis)
llvm::ConstantInt * LLVMUInt64(uint64_t ival)
llvm::Constant * LLVMBoolVectorInStorage(bool b)
static bool lCheckMulForLinear(llvm::Value *op0, llvm::Value *op1, int vectorLength, int stride, std::vector< llvm::PHINode *> &seenPhis)
static llvm::Type * FloatPointerType
llvm::Constant * LLVMMaskAllOff
Representation of a range of positions in a source file.
static llvm::Type * Int16PointerType
llvm::Value * LLVMConcatVectors(llvm::Value *v1, llvm::Value *v2, llvm::Instruction *insertBefore)
llvm::ConstantInt * LLVMInt32(int32_t ival)
bool LLVMVectorIsLinear(llvm::Value *v, int stride)
llvm::Constant * LLVMUInt8Vector(uint8_t ival)
static llvm::Type * PointerIntType
static llvm::PointerType * VoidPointerType
llvm::Constant * LLVMFloat(float fval)
int getVectorWidth() const
llvm::Constant * LLVMDouble(double dval)
void LLVMDumpValue(llvm::Value *v)
static llvm::Type * Int64VectorPointerType
static llvm::Type * Int32Type
static llvm::Type * DoublePointerType
static llvm::Type * BoolStorageType
static llvm::Value * lExtractFirstVectorElement(llvm::Value *v, std::map< llvm::PHINode *, llvm::PHINode *> &phiMap)
llvm::Constant * LLVMFalse
llvm::Value * LLVMExtractFirstVectorElement(llvm::Value *v)
llvm::Constant * LLVMInt16Vector(int16_t ival)
static llvm::VectorType * MaskType
void Debug(SourcePos p, const char *fmt,...)
llvm::ConstantInt * LLVMUInt16(uint16_t ival)
bool LLVMVectorValuesAllEqual(llvm::Value *v, llvm::Value **splat)
static llvm::VectorType * DoubleVectorType
llvm::Constant * LLVMInt8Vector(int8_t ival)
llvm::Constant * LLVMUInt16Vector(uint16_t ival)
static int64_t lGetIntValue(llvm::Value *offset)
llvm::ConstantInt * LLVMInt64(int64_t ival)
static llvm::VectorType * Int16VectorType
static bool lVectorValuesAllEqual(llvm::Value *v, int vectorLength, std::vector< llvm::PHINode *> &seenPhis, llvm::Value **splatValue=NULL)
Main ispc.header file. Defines Target, Globals and Opt classes.
llvm::Value * LLVMShuffleVectors(llvm::Value *v1, llvm::Value *v2, int32_t shuf[], int shufSize, llvm::Instruction *insertBefore)
static bool lCheckAndForLinear(llvm::Value *op0, llvm::Value *op1, int vectorLength, int stride, std::vector< llvm::PHINode *> &seenPhis)
static bool lVectorIsLinearConstantInts(llvm::ConstantDataVector *cv, int vectorLength, int stride)
File with declarations for classes related to type representation.
llvm::Value * LLVMFlattenInsertChain(llvm::Value *inst, int vectorWidth, bool compare, bool undef, bool searchFirstUndef)
llvm::Constant * LLVMTrueInStorage