47 #include <llvm/BinaryFormat/Dwarf.h> 50 #include <llvm/IR/DerivedTypes.h> 51 #include <llvm/IR/Instructions.h> 52 #include <llvm/IR/Metadata.h> 53 #include <llvm/IR/Module.h> 66 static CFInfo *
GetLoop(
bool isUniform, llvm::BasicBlock *breakTarget, llvm::BasicBlock *continueTarget,
71 llvm::BasicBlock *continueTarget, llvm::Value *savedBreakLanesPtr,
72 llvm::Value *savedContinueLanesPtr, llvm::Value *savedMask,
73 llvm::Value *savedBlockEntryMask);
75 static CFInfo *
GetSwitch(
bool isUniform, llvm::BasicBlock *breakTarget, llvm::BasicBlock *continueTarget,
76 llvm::Value *savedBreakLanesPtr, llvm::Value *savedContinueLanesPtr,
77 llvm::Value *savedMask, llvm::Value *savedBlockEntryMask, llvm::Value *switchExpr,
78 llvm::BasicBlock *bbDefault,
79 const std::vector<std::pair<int, llvm::BasicBlock *>> *bbCases,
80 const std::map<llvm::BasicBlock *, llvm::BasicBlock *> *bbNext,
bool scUniform);
105 isUniform = uniformIf;
106 savedBreakTarget = savedContinueTarget = NULL;
107 savedBreakLanesPtr = savedContinueLanesPtr = NULL;
108 savedMask = savedBlockEntryMask = sm;
109 savedSwitchExpr = NULL;
110 savedDefaultBlock = NULL;
111 savedCaseBlocks = NULL;
112 savedNextBlocks = NULL;
113 savedSwitchConditionWasUniform =
false;
115 CFInfo(
CFType t,
bool iu, llvm::BasicBlock *bt, llvm::BasicBlock *ct, llvm::Value *sb, llvm::Value *sc,
116 llvm::Value *sm, llvm::Value *lm, llvm::Value *sse = NULL, llvm::BasicBlock *bbd = NULL,
117 const std::vector<std::pair<int, llvm::BasicBlock *>> *bbc = NULL,
118 const std::map<llvm::BasicBlock *, llvm::BasicBlock *> *bbn = NULL,
bool scu =
false) {
122 savedBreakTarget = bt;
123 savedContinueTarget = ct;
124 savedBreakLanesPtr = sb;
125 savedContinueLanesPtr = sc;
127 savedBlockEntryMask = lm;
128 savedSwitchExpr = sse;
129 savedDefaultBlock = bbd;
130 savedCaseBlocks = bbc;
131 savedNextBlocks = bbn;
132 savedSwitchConditionWasUniform = scu;
134 CFInfo(
CFType t, llvm::BasicBlock *bt, llvm::BasicBlock *ct, llvm::Value *sb, llvm::Value *sc, llvm::Value *sm,
139 savedBreakTarget = bt;
140 savedContinueTarget = ct;
141 savedBreakLanesPtr = sb;
142 savedContinueLanesPtr = sc;
144 savedBlockEntryMask = lm;
145 savedSwitchExpr = NULL;
146 savedDefaultBlock = NULL;
147 savedCaseBlocks = NULL;
148 savedNextBlocks = NULL;
149 savedSwitchConditionWasUniform =
false;
158 return new CFInfo(
Loop, isUniform, breakTarget, continueTarget, savedBreakLanesPtr, savedContinueLanesPtr,
159 savedMask, savedBlockEntryMask);
177 FATAL(
"Unhandled foreach type");
181 return new CFInfo(cfType, breakTarget, continueTarget, savedBreakLanesPtr, savedContinueLanesPtr, savedMask,
189 const std::vector<std::pair<int, llvm::BasicBlock *>> *savedCases,
190 const std::map<llvm::BasicBlock *, llvm::BasicBlock *> *savedNext,
191 bool savedSwitchConditionUniform) {
192 return new CFInfo(
Switch, isUniform, breakTarget, continueTarget, savedBreakLanesPtr, savedContinueLanesPtr,
193 savedMask, savedBlockEntryMask, savedSwitchExpr, savedDefaultBlock, savedCases, savedNext,
194 savedSwitchConditionUniform);
202 switchConditionWasUniform =
false;
205 allocaBlock = llvm::BasicBlock::Create(*
g->
ctx,
"allocas", llvmFunction, 0);
206 bblock = llvm::BasicBlock::Create(*
g->
ctx,
"entry", llvmFunction, 0);
208 llvm::BranchInst::Create(bblock, allocaBlock);
210 funcStartPos = funSym->
pos;
220 blockEntryMask = NULL;
221 breakLanesPtr = continueLanesPtr = NULL;
222 breakTarget = continueTarget = NULL;
232 launchedTasks =
false;
236 disableGSWarningCount = 0;
238 const Type *returnType =
function->GetReturnType();
240 returnValuePtr = NULL;
242 returnValuePtr = AllocaInst(returnType,
"return_value_memory");
261 llvm::Value *globalAllOnMaskPtr =
m->
module->getNamedGlobal(
"__all_on_mask");
262 if (globalAllOnMaskPtr == NULL) {
270 #if ISPC_LLVM_VERSION <= ISPC_LLVM_8_0 274 llvm::Constant *offFunc = llvm::cast<llvm::Constant>(offFuncCallee.getCallee());
276 AssertPos(currentPos, llvm::isa<llvm::Function>(offFunc));
277 llvm::BasicBlock *offBB = llvm::BasicBlock::Create(*
g->
ctx,
"entry", (llvm::Function *)offFunc, 0);
278 llvm::StoreInst *inst =
new llvm::StoreInst(
LLVMMaskAllOff, globalAllOnMaskPtr, offBB);
280 #if ISPC_LLVM_VERSION <= ISPC_LLVM_9_0 286 llvm::ReturnInst::Create(*
g->
ctx, offBB);
289 llvm::Value *allOnMask = LoadInst(globalAllOnMaskPtr, NULL,
"all_on_mask");
294 currentPos = funSym->
pos;
300 llvm::DIType *diSubprogramType = NULL;
303 if (functionType == NULL)
306 diSubprogramType = functionType->
GetDIType(scope);
314 Assert(llvm::isa<llvm::DISubroutineType>(diSubprogramType));
315 llvm::DISubroutineType *diSubprogramType_n = llvm::cast<llvm::DISubroutineType>(diSubprogramType);
316 llvm::DINode::DIFlags flags = llvm::DINode::FlagPrototyped;
318 std::string mangledName = std::string(llvmFunction->getName());
319 if (mangledName == funSym->
name)
324 int firstLine = funcStartPos.first_line;
326 #if ISPC_LLVM_VERSION < ISPC_LLVM_8_0 327 diSubprogram =
m->
diBuilder->createFunction(diFile , funSym->
name, mangledName, diFile, firstLine,
328 diSubprogramType_n, isStatic,
true,
329 firstLine, flags, isOptimized);
330 llvmFunction->setSubprogram(diSubprogram);
333 llvm::DISubprogram::DISPFlags SPFlags = llvm::DISubprogram::SPFlagDefinition;
335 SPFlags |= llvm::DISubprogram::SPFlagOptimized;
337 SPFlags |= llvm::DISubprogram::SPFlagLocalToUnit;
339 diSubprogram =
m->
diBuilder->createFunction(diFile , funSym->
name, mangledName, diFile, firstLine,
340 diSubprogramType_n, firstLine, flags, SPFlags);
341 llvmFunction->setSubprogram(diSubprogram);
353 AssertPos(currentPos, controlFlowInfo.size() == 0);
368 return BinaryOperator(llvm::Instruction::And, GetInternalMask(), functionMaskValue,
"internal_mask&function_mask");
374 functionMaskValue = value;
376 StoreInst(GetFullMask(), fullMaskPointer);
382 StoreInst(value, internalMaskPointer);
384 StoreInst(GetFullMask(), fullMaskPointer);
388 llvm::Value *mask = BinaryOperator(llvm::Instruction::And, oldMask, test,
"oldMask&test");
389 SetInternalMask(mask);
393 llvm::Value *notTest = BinaryOperator(llvm::Instruction::Xor, test,
LLVMMaskAllOn,
"~test");
394 llvm::Value *mask = BinaryOperator(llvm::Instruction::And, oldMask, notTest,
"oldMask&~test");
395 SetInternalMask(mask);
400 llvm::Value *any = Any(GetFullMask());
401 BranchInst(btrue, bfalse, any);
409 llvm::Value *all = All(GetFullMask());
410 BranchInst(btrue, bfalse, all);
419 BranchIfMaskAny(bfalse, btrue);
432 CFInfo *ci = popCFState();
462 if (continueLanesPtr != NULL || breakLanesPtr != NULL) {
467 llvm::Value *bcLanes = NULL;
469 if (continueLanesPtr != NULL)
470 bcLanes = LoadInst(continueLanesPtr, NULL,
"continue_lanes");
474 if (breakLanesPtr != NULL) {
475 llvm::Value *breakLanes = LoadInst(breakLanesPtr, NULL,
"break_lanes");
476 bcLanes = BinaryOperator(llvm::Instruction::Or, bcLanes, breakLanes,
"|break_lanes");
479 llvm::Value *notBreakOrContinue =
480 BinaryOperator(llvm::Instruction::Xor, bcLanes,
LLVMMaskAllOn,
"!(break|continue)_lanes");
481 llvm::Value *oldMask = GetInternalMask();
482 llvm::Value *newMask = BinaryOperator(llvm::Instruction::And, oldMask, notBreakOrContinue,
"new_mask");
483 SetInternalMask(newMask);
490 llvm::Value *oldMask = GetInternalMask();
491 controlFlowInfo.push_back(
CFInfo::GetLoop(uniformCF, breakTarget, continueTarget, breakLanesPtr, continueLanesPtr,
492 oldMask, blockEntryMask));
497 breakLanesPtr = continueLanesPtr = NULL;
509 blockEntryMask = NULL;
513 CFInfo *ci = popCFState();
527 if (ft == FOREACH_REGULAR) {
528 for (
int i = 0; i < (int)controlFlowInfo.size(); ++i) {
530 Error(currentPos,
"Nested \"foreach\" statements are currently " 543 llvm::Value *oldMask = GetInternalMask();
544 controlFlowInfo.push_back(
545 CFInfo::GetForeach(ft, breakTarget, continueTarget, breakLanesPtr, continueLanesPtr, oldMask, blockEntryMask));
546 breakLanesPtr = NULL;
551 continueTarget = NULL;
553 blockEntryMask = NULL;
557 CFInfo *ci = popCFState();
568 llvm::Value *returnedLanes = LoadInst(returnedLanesPtr, NULL,
"returned_lanes");
569 llvm::Value *notReturned = BinaryOperator(llvm::Instruction::Xor, returnedLanes,
LLVMMaskAllOn,
"~returned_lanes");
570 llvm::Value *newMask = BinaryOperator(llvm::Instruction::And, oldMask, notReturned,
"new_mask");
571 SetInternalMask(newMask);
580 int i = controlFlowInfo.size() - 1;
581 while (i >= 0 && controlFlowInfo[i]->
IsIf())
586 return controlFlowInfo[i]->IsSwitch();
590 if (breakTarget == NULL) {
591 Error(currentPos,
"\"break\" statement is illegal outside of " 592 "for/while/do loops and \"switch\" statements.");
595 AssertPos(currentPos, controlFlowInfo.size() > 0);
600 if (inSwitchStatement() ==
true && switchConditionWasUniform ==
true && ifsInCFAllUniform(
CFInfo::Switch)) {
603 AssertPos(currentPos, breakTarget != NULL);
604 BranchInst(breakTarget);
612 if (inSwitchStatement() ==
false && ifsInCFAllUniform(
CFInfo::Loop)) {
613 BranchInst(breakTarget);
622 AssertPos(currentPos, breakLanesPtr != NULL);
624 llvm::Value *mask = GetInternalMask();
625 llvm::Value *breakMask = LoadInst(breakLanesPtr, NULL,
"break_mask");
626 llvm::Value *newMask = BinaryOperator(llvm::Instruction::Or, mask, breakMask,
"mask|break_mask");
627 StoreInst(newMask, breakLanesPtr);
635 if (doCoherenceCheck) {
636 if (continueTarget != NULL)
643 jumpIfAllLoopLanesAreDone(continueTarget);
644 else if (breakTarget != NULL)
647 jumpIfAllLoopLanesAreDone(breakTarget);
653 for (
int i = (
int)controlFlowInfo.size() - 1; i >= 0; --i) {
661 if (!continueTarget) {
662 Error(currentPos,
"\"continue\" statement illegal outside of " 663 "for/while/do/foreach loops.");
666 AssertPos(currentPos, controlFlowInfo.size() > 0);
675 AddInstrumentationPoint(
"continue: uniform CF, jumped");
676 BranchInst(continueTarget);
682 llvm::Value *mask = GetInternalMask();
683 llvm::Value *continueMask = LoadInst(continueLanesPtr, NULL,
"continue_mask");
684 llvm::Value *newMask = BinaryOperator(llvm::Instruction::Or, mask, continueMask,
"mask|continueMask");
685 StoreInst(newMask, continueLanesPtr);
691 if (doCoherenceCheck)
695 jumpIfAllLoopLanesAreDone(continueTarget);
704 AssertPos(currentPos, controlFlowInfo.size() > 0);
708 int i = controlFlowInfo.size() - 1;
709 while (i >= 0 && controlFlowInfo[i]->type != type) {
710 if (controlFlowInfo[i]->
isUniform ==
false)
720 llvm::Value *allDone = NULL;
722 if (breakLanesPtr == NULL) {
723 llvm::Value *continued = LoadInst(continueLanesPtr, NULL,
"continue_lanes");
724 continued = BinaryOperator(llvm::Instruction::And, continued, GetFunctionMask(),
"continued&func");
725 allDone = MasksAllEqual(continued, blockEntryMask);
730 llvm::Value *returned = LoadInst(returnedLanesPtr, NULL,
"returned_lanes");
731 llvm::Value *breaked = LoadInst(breakLanesPtr, NULL,
"break_lanes");
732 llvm::Value *finishedLanes = BinaryOperator(llvm::Instruction::Or, returned, breaked,
"returned|breaked");
733 if (continueLanesPtr != NULL) {
735 llvm::Value *continued = LoadInst(continueLanesPtr, NULL,
"continue_lanes");
737 BinaryOperator(llvm::Instruction::Or, finishedLanes, continued,
"returned|breaked|continued");
740 finishedLanes = BinaryOperator(llvm::Instruction::And, finishedLanes, GetFunctionMask(),
"finished&func");
743 allDone = MasksAllEqual(finishedLanes, blockEntryMask);
746 llvm::BasicBlock *bAll = CreateBasicBlock(
"all_continued_or_breaked");
747 llvm::BasicBlock *bNotAll = CreateBasicBlock(
"not_all_continued_or_breaked");
748 BranchInst(bAll, bNotAll, allDone);
753 AddInstrumentationPoint(
"break/continue: all dynamically went");
759 AddInstrumentationPoint(
"break/continue: not all went");
763 if (continueLanesPtr == NULL)
767 llvm::Value *mask = GetInternalMask();
768 llvm::Value *continueMask = LoadInst(continueLanesPtr, NULL,
"continue_mask");
769 llvm::Value *orMask = BinaryOperator(llvm::Instruction::Or, mask, continueMask,
"mask|continue_mask");
770 SetInternalMask(orMask);
777 if (breakLanesPtr == NULL)
785 llvm::Value *oldMask = GetInternalMask();
786 controlFlowInfo.push_back(
CFInfo::GetSwitch(cfIsUniform, breakTarget, continueTarget, breakLanesPtr,
787 continueLanesPtr, oldMask, blockEntryMask, switchExpr, defaultBlock,
788 caseBlocks, nextBlocks, switchConditionWasUniform));
792 breakTarget = bbBreak;
794 continueLanesPtr = NULL;
795 continueTarget = NULL;
796 blockEntryMask = NULL;
808 CFInfo *ci = popCFState();
817 llvm::Value *allOff = None(mask);
818 llvm::BasicBlock *bbSome = CreateBasicBlock(
"case_default_on");
823 AssertPos(currentPos, nextBlocks->find(bblock) != nextBlocks->end());
824 llvm::BasicBlock *bbNext = nextBlocks->find(bblock)->second;
828 BranchInst(bbNext, bbSome, allOff);
829 SetCurrentBasicBlock(bbSome);
835 AssertPos(currentPos, controlFlowInfo.size() > 0);
836 int i = controlFlowInfo.size() - 1;
840 return controlFlowInfo[i]->savedMask;
844 if (inSwitchStatement() ==
false) {
845 Error(pos,
"\"default\" label illegal outside of \"switch\" " 852 AssertPos(currentPos, defaultBlock != NULL);
858 BranchInst(defaultBlock);
859 SetCurrentBasicBlock(defaultBlock);
861 if (switchConditionWasUniform)
874 llvm::Value *matchesDefault = getMaskAtSwitchEntry();
875 for (
int i = 0; i < (int)caseBlocks->size(); ++i) {
876 int value = (*caseBlocks)[i].first;
877 llvm::Value *valueVec =
882 llvm::Value *matchesCaseValue =
883 CmpInst(llvm::Instruction::ICmp, llvm::CmpInst::ICMP_EQ, switchExpr, valueVec,
"cmp_case_value");
884 matchesCaseValue = I1VecToBoolVec(matchesCaseValue);
886 llvm::Value *notMatchesCaseValue = NotOperator(matchesCaseValue);
888 BinaryOperator(llvm::Instruction::And, matchesDefault, notMatchesCaseValue,
"default&~case_match");
894 llvm::Value *oldMask = GetInternalMask();
895 llvm::Value *newMask = BinaryOperator(llvm::Instruction::Or, oldMask, matchesDefault,
"old_mask|matches_default");
896 SetInternalMask(newMask);
899 addSwitchMaskCheck(newMask);
903 if (inSwitchStatement() ==
false) {
904 Error(pos,
"\"case\" label illegal outside of \"switch\" statement.");
909 llvm::BasicBlock *bbCase = NULL;
910 AssertPos(currentPos, caseBlocks != NULL);
911 for (
int i = 0; i < (int)caseBlocks->size(); ++i)
912 if ((*caseBlocks)[i].first == value) {
913 bbCase = (*caseBlocks)[i].second;
921 SetCurrentBasicBlock(bbCase);
923 if (switchConditionWasUniform)
929 llvm::Value *valueVec =
931 llvm::Value *matchesCaseValue =
932 CmpInst(llvm::Instruction::ICmp, llvm::CmpInst::ICMP_EQ, switchExpr, valueVec,
"cmp_case_value");
933 matchesCaseValue = I1VecToBoolVec(matchesCaseValue);
937 llvm::Value *entryMask = getMaskAtSwitchEntry();
938 matchesCaseValue = BinaryOperator(llvm::Instruction::And, entryMask, matchesCaseValue,
"entry_mask&case_match");
941 llvm::Value *oldMask = GetInternalMask();
942 llvm::Value *newMask = BinaryOperator(llvm::Instruction::Or, oldMask, matchesCaseValue,
"mask|case_match");
943 SetInternalMask(newMask);
946 addSwitchMaskCheck(newMask);
950 const std::vector<std::pair<int, llvm::BasicBlock *>> &bbCases,
951 const std::map<llvm::BasicBlock *, llvm::BasicBlock *> &bbNext) {
954 AssertPos(currentPos, controlFlowInfo.size() && controlFlowInfo.back()->IsSwitch());
957 defaultBlock = bbDefault;
958 caseBlocks =
new std::vector<std::pair<int, llvm::BasicBlock *>>(bbCases);
959 nextBlocks =
new std::map<llvm::BasicBlock *, llvm::BasicBlock *>(bbNext);
960 switchConditionWasUniform = (llvm::isa<llvm::VectorType>(expr->getType()) ==
false);
962 if (switchConditionWasUniform ==
true) {
965 llvm::SwitchInst *s = llvm::SwitchInst::Create(expr, bbDefault, bbCases.size(), bblock);
966 for (
int i = 0; i < (int)bbCases.size(); ++i) {
968 s->addCase(
LLVMInt32(bbCases[i].first), bbCases[i].second);
971 s->addCase(
LLVMInt64(bbCases[i].first), bbCases[i].second);
982 if (nextBlocks->size() > 0) {
986 std::map<llvm::BasicBlock *, llvm::BasicBlock *>::const_iterator iter;
987 iter = nextBlocks->find(NULL);
988 AssertPos(currentPos, iter != nextBlocks->end());
989 llvm::BasicBlock *bbFirst = iter->second;
998 for (
unsigned int i = 0; i < controlFlowInfo.size(); ++i)
1005 for (
unsigned int i = 0; i < controlFlowInfo.size(); ++i)
1023 Error(ls->
pos,
"Multiple labels named \"%s\" in function.", ls->
name.c_str());
1032 labelMap.erase(labelMap.begin(), labelMap.end());
1033 WalkAST(code, initLabelBBlocks, NULL,
this);
1037 if (labelMap.find(label) != labelMap.end())
1038 return labelMap[label];
1045 std::vector<std::string> labels(labelMap.size());
1048 std::map<std::string, llvm::BasicBlock *>::iterator iter;
1049 for (iter = labelMap.begin(); iter != labelMap.end(); iter++)
1050 labels.push_back(iter->first);
1056 const Type *returnType =
function->GetReturnType();
1059 Error(expr->
pos,
"Can't return non-void type \"%s\" from void function.",
1063 Error(funcStartPos,
"Must provide return value for return " 1064 "statement for non-void function.");
1070 llvm::Value *retVal = expr->
GetValue(
this);
1071 if (retVal != NULL) {
1072 if (returnType->
IsUniformType() || CastType<ReferenceType>(returnType) != NULL)
1073 StoreInst(retVal, returnValuePtr, returnType);
1079 StoreInst(retVal, returnValuePtr, GetInternalMask(), returnType,
1086 if (VaryingCFDepth() == 0) {
1090 AddInstrumentationPoint(
"return: uniform control flow");
1095 llvm::Value *oldReturnedLanes = LoadInst(returnedLanesPtr, NULL,
"old_returned_lanes");
1096 llvm::Value *newReturnedLanes =
1097 BinaryOperator(llvm::Instruction::Or, oldReturnedLanes, GetFullMask(),
"old_mask|returned_lanes");
1101 if (doCoherenceCheck) {
1103 llvm::Value *cmp = MasksAllEqual(functionMaskValue, newReturnedLanes);
1104 llvm::BasicBlock *bDoReturn = CreateBasicBlock(
"do_return");
1105 llvm::BasicBlock *bNoReturn = CreateBasicBlock(
"no_return");
1106 BranchInst(bDoReturn, bNoReturn, cmp);
1109 AddInstrumentationPoint(
"return: all lanes have returned");
1117 StoreInst(newReturnedLanes, returnedLanesPtr);
1118 AddInstrumentationPoint(
"return: some but not all lanes have returned");
1125 std::vector<Symbol *> mm;
1134 llvm::Function *fmm = mm[0]->function;
1135 return CallInst(fmm, NULL, mask,
LLVMGetName(mask,
"_any"));
1141 std::vector<Symbol *> mm;
1150 llvm::Function *fmm = mm[0]->function;
1151 return CallInst(fmm, NULL, mask,
LLVMGetName(mask,
"_all"));
1157 std::vector<Symbol *> mm;
1166 llvm::Function *fmm = mm[0]->function;
1167 return CallInst(fmm, NULL, mask,
LLVMGetName(mask,
"_none"));
1171 const char *__movmsk =
"__movmsk";
1175 std::vector<Symbol *> mm;
1184 llvm::Function *fmm = mm[0]->function;
1185 return CallInst(fmm, NULL, v,
LLVMGetName(v,
"_movmsk"));
1191 llvm::Value *cmp = CmpInst(llvm::Instruction::ICmp, llvm::CmpInst::ICMP_EQ,
1194 cmp = I1VecToBoolVec(cmp);
1199 llvm::Function *fmm =
m->
module->getFunction(
"__wasm_cmp_msk_eq");
1200 return CallInst(fmm, NULL, {v1, v2},
LLVMGetName(
"wasm_cmp_msk_eq", v1, v2));
1202 llvm::Value *mm1 = LaneMask(v1);
1203 llvm::Value *mm2 = LaneMask(v2);
1204 return CmpInst(llvm::Instruction::ICmp, llvm::CmpInst::ICMP_EQ, mm1, mm2,
LLVMGetName(
"equal", v1, v2));
1209 llvm::SmallVector<llvm::Constant *, 16> array;
1215 llvm::Constant *index = llvm::ConstantVector::get(array);
1221 llvm::Constant *lstr = llvm::ConstantDataArray::getString(*
g->
ctx, str);
1222 llvm::GlobalValue::LinkageTypes linkage = llvm::GlobalValue::InternalLinkage;
1223 llvm::Value *lstrPtr =
1224 new llvm::GlobalVariable(*
m->
module, lstr->getType(),
true , linkage, lstr,
"__str");
1229 return llvm::BasicBlock::Create(*
g->
ctx, name, llvmFunction);
1241 llvm::ArrayType *at = llvm::dyn_cast<llvm::ArrayType>(b->getType());
1246 llvm::Value *ret = llvm::UndefValue::get(boolArrayType);
1248 for (
unsigned int i = 0; i < at->getNumElements(); ++i) {
1249 llvm::Value *elt = ExtractInst(b, i);
1251 ret = InsertInst(ret, sext, i);
1259 llvm::Constant *sConstant = llvm::ConstantDataArray::getString(*
g->
ctx, s,
true);
1260 std::string var_name =
"_";
1261 var_name = var_name + s;
1262 llvm::Value *sPtr =
new llvm::GlobalVariable(*
m->
module, sConstant->getType(),
true ,
1263 llvm::GlobalValue::InternalLinkage, sConstant, var_name.c_str());
1265 llvm::ArrayRef<llvm::Value *> arrayRef(&indices[0], &indices[2]);
1266 return llvm::GetElementPtrInst::Create(
PTYPE(sPtr), sPtr, arrayRef,
"sptr", bblock);
1274 std::vector<llvm::Value *> args;
1280 args.push_back(
LLVMInt32(currentPos.first_line));
1282 args.push_back(LaneMask(GetFullMask()));
1284 llvm::Function *finst =
m->
module->getFunction(
"ISPCInstrument");
1285 CallInst(finst, NULL, args,
"");
1293 llvm::Instruction *inst = llvm::dyn_cast<llvm::Instruction>(value);
1306 llvm::DIScope *parentScope;
1307 llvm::DILexicalBlock *lexicalBlock;
1308 if (debugScopes.size() > 0)
1309 parentScope = debugScopes.back();
1311 parentScope = diSubprogram;
1313 lexicalBlock =
m->
diBuilder->createLexicalBlock(parentScope, diFile, currentPos.first_line,
1316 currentPos.first_column);
1317 debugScopes.push_back(llvm::cast<llvm::DILexicalBlockBase>(lexicalBlock));
1323 AssertPos(currentPos, debugScopes.size() > 0);
1324 debugScopes.pop_back();
1329 AssertPos(currentPos, debugScopes.size() > 0);
1330 return debugScopes.back();
1337 llvm::DIScope *scope = GetDIScope();
1339 llvm::DILocalVariable *var =
m->
diBuilder->createAutoVariable(
1342 llvm::Instruction *declareInst =
1345 AddDebugPos(declareInst, &sym->
pos, scope);
1352 llvm::DINode::DIFlags flags = llvm::DINode::FlagZero;
1353 llvm::DIScope *scope = diSubprogram;
1355 llvm::DILocalVariable *var =
1357 diType,
true , flags);
1359 llvm::Instruction *declareInst =
1362 AddDebugPos(declareInst, &sym->
pos, scope);
1371 llvm::ArrayType *arrayType = llvm::dyn_cast<llvm::ArrayType>(t);
1372 if (arrayType == NULL)
1377 llvm::VectorType *vectorElementType = llvm::dyn_cast<llvm::VectorType>(arrayType->getElementType());
1380 return (
int)arrayType->getNumElements();
1385 if (v0 == NULL || v1 == NULL) {
1390 AssertPos(currentPos, v0->getType() == v1->getType());
1391 llvm::Type *
type = v0->getType();
1393 if (arraySize == 0) {
1394 llvm::Instruction *bop = llvm::BinaryOperator::Create(inst, v0, v1, name ? name :
"", bblock);
1401 llvm::Value *ret = llvm::UndefValue::get(type);
1402 for (
int i = 0; i < arraySize; ++i) {
1403 llvm::Value *a = ExtractInst(v0, i);
1404 llvm::Value *b = ExtractInst(v1, i);
1405 llvm::Value *op = BinaryOperator(inst, a, b);
1406 ret = InsertInst(ret, op, i);
1421 llvm::Type *
type = v->getType();
1423 if (arraySize == 0) {
1424 llvm::Instruction *binst = llvm::BinaryOperator::CreateNot(v, name ? name :
"not", bblock);
1428 llvm::Value *ret = llvm::UndefValue::get(type);
1429 for (
int i = 0; i < arraySize; ++i) {
1430 llvm::Value *a = ExtractInst(v, i);
1431 llvm::Value *op = llvm::BinaryOperator::CreateNot(a, name ? name :
"not", bblock);
1433 ret = InsertInst(ret, op, i);
1443 llvm::ArrayType *arrayType = llvm::dyn_cast<llvm::ArrayType>(
type);
1444 Assert(arrayType != NULL);
1446 llvm::VectorType *vectorElementType = llvm::dyn_cast<llvm::VectorType>(arrayType->getElementType());
1447 Assert(vectorElementType != NULL);
1451 return llvm::ArrayType::get(base, arrayType->getNumElements());
1455 llvm::Value *v0, llvm::Value *v1,
const char *name) {
1456 if (v0 == NULL || v1 == NULL) {
1461 AssertPos(currentPos, v0->getType() == v1->getType());
1462 llvm::Type *
type = v0->getType();
1464 if (arraySize == 0) {
1465 llvm::Instruction *ci = llvm::CmpInst::Create(inst, pred, v0, v1, name ? name :
"cmp", bblock);
1470 llvm::Value *ret = llvm::UndefValue::get(boolType);
1471 for (
int i = 0; i < arraySize; ++i) {
1472 llvm::Value *a = ExtractInst(v0, i);
1473 llvm::Value *b = ExtractInst(v1, i);
1474 llvm::Value *op = CmpInst(inst, pred, a, b, name);
1475 ret = InsertInst(ret, op, i);
1482 if (value == NULL) {
1487 llvm::Value *ret = NULL;
1488 llvm::Type *eltType = value->getType();
1489 llvm::Type *vecType = NULL;
1491 llvm::PointerType *pt = llvm::dyn_cast<llvm::PointerType>(eltType);
1495 value = PtrToIntInst(value);
1503 if (llvm::Constant *const_val = llvm::dyn_cast<llvm::Constant>(value)) {
1504 #if ISPC_LLVM_VERSION < ISPC_LLVM_11_0 1508 llvm::ConstantVector::getSplat({
static_cast<unsigned int>(
g->
target->
getVectorWidth()),
false}, const_val);
1513 ret = BroadcastValue(value, vecType, name);
1519 if (value == NULL) {
1527 llvm::Instruction *inst =
new llvm::BitCastInst(value, type, name, bblock);
1533 if (value == NULL) {
1538 if (llvm::isa<llvm::VectorType>(value->getType()))
1545 llvm::Instruction *inst =
new llvm::PtrToIntInst(value, type, name, bblock);
1551 if (value == NULL) {
1559 llvm::Type *fromType = value->getType();
1560 if (llvm::isa<llvm::VectorType>(fromType)) {
1562 if (fromType == toType)
1565 else if (fromType->getScalarSizeInBits() > toType->getScalarSizeInBits())
1566 return TruncInst(value, toType, name);
1568 AssertPos(currentPos, fromType->getScalarSizeInBits() < toType->getScalarSizeInBits());
1569 return ZExtInst(value, toType, name);
1573 llvm::Instruction *inst =
new llvm::PtrToIntInst(value, toType, name, bblock);
1579 if (value == NULL) {
1587 llvm::Type *fromType = value->getType();
1588 if (llvm::isa<llvm::VectorType>(fromType)) {
1590 if (fromType == toType)
1593 else if (fromType->getScalarSizeInBits() > toType->getScalarSizeInBits())
1594 return TruncInst(value, toType, name);
1596 AssertPos(currentPos, fromType->getScalarSizeInBits() < toType->getScalarSizeInBits());
1597 return ZExtInst(value, toType, name);
1601 llvm::Instruction *inst =
new llvm::IntToPtrInst(value, toType, name, bblock);
1607 if (value == NULL) {
1617 llvm::Instruction *inst =
new llvm::TruncInst(value, type, name, bblock);
1624 if (value == NULL) {
1634 llvm::Instruction *inst = llvm::CastInst::Create(op, value, type, name, bblock);
1640 if (value == NULL) {
1650 llvm::Instruction *inst = llvm::CastInst::CreateFPCast(value, type, name, bblock);
1656 if (value == NULL) {
1666 llvm::Instruction *inst =
new llvm::SExtInst(value, type, name, bblock);
1672 if (value == NULL) {
1682 llvm::Instruction *inst =
new llvm::ZExtInst(value, type, name, bblock);
1699 bool indexIsVarying = llvm::isa<llvm::VectorType>(index->getType());
1700 llvm::Value *offset = NULL;
1701 if (indexIsVarying ==
false) {
1713 offset = BinaryOperator(llvm::Instruction::Mul, scale, index);
1714 offset = SmearUniform(offset);
1724 scale = SmearUniform(scale);
1727 offset = BinaryOperator(llvm::Instruction::Mul, scale, index,
LLVMGetName(
"mul", scale, index));
1738 bool baseIsUniform = (llvm::isa<llvm::PointerType>(basePtr->getType()));
1739 AssertPos(currentPos, baseIsUniform ==
false || indexIsVarying ==
true);
1740 llvm::Value *varyingPtr = baseIsUniform ? SmearUniform(basePtr) : basePtr;
1743 return BinaryOperator(llvm::Instruction::Add, varyingPtr, offset,
LLVMGetName(basePtr,
"_offset"));
1747 llvm::Type *type0 = (*v0)->getType();
1748 llvm::Type *type1 = (*v1)->getType();
1752 if (llvm::isa<llvm::VectorType>(type0) && !llvm::isa<llvm::VectorType>(type1)) {
1753 *v1 = SmearUniform(*v1,
"smear_v1");
1754 type1 = (*v1)->getType();
1756 if (!llvm::isa<llvm::VectorType>(type0) && llvm::isa<llvm::VectorType>(type1)) {
1757 *v0 = SmearUniform(*v0,
"smear_v0");
1758 type0 = (*v0)->getType();
1780 llvm::Value *ptrSliceOffset, llvm::Value **newSliceOffset) {
1783 int logWidth = 0, sw = soaWidth;
1788 Assert((1 << logWidth) == soaWidth);
1791 Assert(indexValue != NULL);
1792 llvm::Type *indexType = indexValue->getType();
1796 llvm::Value *indexSum = ctx->
BinaryOperator(llvm::Instruction::Add, indexValue, ptrSliceOffset,
"index_sum");
1799 *newSliceOffset = ctx->
BinaryOperator(llvm::Instruction::And, indexSum, mask,
"slice_index_minor");
1807 return ctx->
BinaryOperator(llvm::Instruction::AShr, indexSum, shift,
"slice_index_major");
1814 std::vector<llvm::Type *> eltTypes;
1815 eltTypes.push_back(ptr->getType());
1816 eltTypes.push_back(offset->getType());
1817 llvm::StructType *st = llvm::StructType::get(*
g->
ctx, eltTypes);
1819 llvm::Value *ret = llvm::UndefValue::get(st);
1820 ret = InsertInst(ret, ptr, 0,
LLVMGetName(ret,
"_slice_ptr"));
1821 ret = InsertInst(ret, offset, 1,
LLVMGetName(ret,
"_slice_offset"));
1827 if (basePtr == NULL || index == NULL) {
1834 if (CastType<ReferenceType>(ptrRefType) != NULL)
1837 ptrType = CastType<PointerType>(ptrRefType);
1842 AssertPos(currentPos, llvm::isa<llvm::StructType>(basePtr->getType()));
1844 llvm::Value *ptrSliceOffset = ExtractInst(basePtr, 1);
1851 llvm::Value *newSliceOffset;
1854 ptrSliceOffset = newSliceOffset;
1859 llvm::Value *p = GetElementPtrInst(ExtractInst(basePtr, 0), index, ptrType->
GetAsNonSlice(), name);
1862 return MakeSlicePointer(p, ptrSliceOffset);
1868 AssertPos(currentPos, llvm::isa<llvm::PointerType>(basePtr->getType()));
1870 AssertPos(currentPos, llvm::isa<llvm::VectorType>(basePtr->getType()));
1872 bool indexIsVaryingType = llvm::isa<llvm::VectorType>(index->getType());
1874 if (indexIsVaryingType ==
false && ptrType->
IsUniformType() ==
true) {
1877 llvm::Value *ind[1] = {index};
1878 llvm::ArrayRef<llvm::Value *> arrayRef(&ind[0], &ind[1]);
1879 llvm::Instruction *inst =
1880 llvm::GetElementPtrInst::Create(
PTYPE(basePtr), basePtr, arrayRef, name ? name :
"gep", bblock);
1884 return applyVaryingGEP(basePtr, index, ptrType);
1888 const Type *ptrRefType,
const char *name) {
1889 if (basePtr == NULL || index0 == NULL || index1 == NULL) {
1896 if (CastType<ReferenceType>(ptrRefType) != NULL)
1899 ptrType = CastType<PointerType>(ptrRefType);
1907 AssertPos(currentPos, llvm::isa<llvm::StructType>(basePtr->getType()));
1908 llvm::Value *ptrSliceOffset = ExtractInst(basePtr, 1);
1910 llvm::Value *newSliceOffset;
1912 index1 =
lComputeSliceIndex(
this, soaWidth, index1, ptrSliceOffset, &newSliceOffset);
1913 ptrSliceOffset = newSliceOffset;
1916 llvm::Value *p = GetElementPtrInst(ExtractInst(basePtr, 0), index0, index1, ptrType->
GetAsNonSlice(), name);
1917 return MakeSlicePointer(p, ptrSliceOffset);
1920 bool index0IsVaryingType = llvm::isa<llvm::VectorType>(index0->getType());
1921 bool index1IsVaryingType = llvm::isa<llvm::VectorType>(index1->getType());
1923 if (index0IsVaryingType ==
false && index1IsVaryingType ==
false && ptrType->
IsUniformType() ==
true) {
1926 llvm::Value *indices[2] = {index0, index1};
1927 llvm::ArrayRef<llvm::Value *> arrayRef(&indices[0], &indices[2]);
1928 llvm::Instruction *inst =
1929 llvm::GetElementPtrInst::Create(
PTYPE(basePtr), basePtr, arrayRef, name ? name :
"gep", bblock);
1934 llvm::Value *ptr0 = GetElementPtrInst(basePtr, index0, ptrType);
1942 bool ptr0IsUniform = llvm::isa<llvm::PointerType>(ptr0->getType());
1944 const Type *ptr0Type =
1947 return applyVaryingGEP(ptr0, index1, ptr0Type);
1952 const char *name,
const PointerType **resultPtrType) {
1953 if (resultPtrType != NULL)
1954 AssertPos(currentPos, ptrRefType != NULL);
1956 llvm::PointerType *llvmPtrType = llvm::dyn_cast<llvm::PointerType>(fullBasePtr->getType());
1957 if (llvmPtrType != NULL) {
1958 llvm::StructType *llvmStructType = llvm::dyn_cast<llvm::StructType>(llvmPtrType->getElementType());
1959 if (llvmStructType != NULL && llvmStructType->isSized() ==
false) {
1968 if (ptrRefType != NULL) {
1970 if (CastType<ReferenceType>(ptrRefType) != NULL)
1973 ptrType = CastType<PointerType>(ptrRefType);
1980 llvm::Value *basePtr = fullBasePtr;
1981 bool baseIsSlicePtr = llvm::isa<llvm::StructType>(fullBasePtr->getType());
1983 if (baseIsSlicePtr) {
1989 basePtr = ExtractInst(fullBasePtr, 0);
1990 if (resultPtrType == NULL)
1991 resultPtrType = &rpt;
1996 if (resultPtrType != NULL) {
2004 llvm::Value *resultPtr = NULL;
2008 llvm::ArrayRef<llvm::Value *> arrayRef(&offsets[0], &offsets[2]);
2010 llvm::GetElementPtrInst::Create(
PTYPE(basePtr), basePtr, arrayRef, name ? name :
"struct_offset", bblock);
2015 llvm::Value *offset = NULL;
2027 llvm::Value *scale =
2029 offset = BinaryOperator(llvm::Instruction::Mul, size, scale);
2032 offset = SmearUniform(offset,
"offset_smear");
2040 resultPtr = BinaryOperator(llvm::Instruction::Add, basePtr, offset,
"struct_ptr_offset");
2046 return MakeSlicePointer(resultPtr, ExtractInst(fullBasePtr, 1));
2053 if ((value == NULL) || (fromType == NULL) || (toType == NULL)) {
2061 llvm::Value *newBool = value;
2063 newBool = TruncInst(value, toType);
2066 newBool = SExtInst(value, toType);
2077 llvm::PointerType *pt = llvm::dyn_cast<llvm::PointerType>(ptr->getType());
2083 llvm::LoadInst *inst =
new llvm::LoadInst(ptr, name, bblock);
2086 #if ISPC_LLVM_VERSION <= ISPC_LLVM_9_0 2095 llvm::Value *loadVal = inst;
2097 if ((type != NULL) && (type->
IsBoolType())) {
2098 if (CastType<AtomicType>(type) != NULL) {
2099 loadVal = SwitchBoolSize(loadVal, inst->getType(), type->
LLVMType(
g->
ctx));
2100 }
else if ((CastType<VectorType>(type) != NULL)) {
2103 loadVal = SwitchBoolSize(loadVal, inst->getType(), type->
LLVMType(
g->
ctx));
2115 Assert(CastType<PointerType>(*ptrType) != NULL);
2133 if ((*ptrType)->IsUniformType())
2152 llvm::Type *llvmReturnType = unifType->
LLVMType(
g->
ctx);
2153 llvm::Value *retValue = llvm::UndefValue::get(llvmReturnType);
2157 llvm::Value *eltPtr = AddElementOffset(ptr, i, ptrType,
"elt_offset", &eltPtrType);
2158 llvm::Value *eltValue = LoadInst(eltPtr, mask, eltPtrType, name);
2159 retValue = InsertInst(retValue, eltValue, i,
"set_value");
2168 return LoadInst(ptr, mask, ptrType, name);
2173 const char *name,
bool one_elem) {
2179 AssertPos(currentPos, ptrRefType != NULL && mask != NULL);
2186 if (CastType<ReferenceType>(ptrRefType) != NULL) {
2190 ptrType = CastType<PointerType>(ptrRefType);
2195 if (CastType<UndefinedStructType>(ptrType->
GetBaseType())) {
2196 Error(currentPos,
"Unable to load to undefined struct type \"%s\".",
2203 return loadUniformFromSOA(ptr, mask, ptrType, name);
2220 #if ISPC_LLVM_VERSION <= ISPC_LLVM_9_0 2221 llvm::Instruction *inst =
new llvm::LoadInst(ptr, name,
false , align, bblock);
2223 llvm::Instruction *inst =
2224 new llvm::LoadInst(ptr, name,
false , llvm::MaybeAlign(align), bblock);
2227 llvm::Value *loadVal = inst;
2229 if (elType->
IsBoolType() && (CastType<AtomicType>(elType) != NULL)) {
2230 loadVal = SwitchBoolSize(loadVal, inst->getType(), elType->
LLVMType(
g->
ctx));
2237 llvm::Value *gather_result = gather(ptr, ptrType, GetFullMask(), name);
2239 return gather_result;
2243 Warning(currentPos,
"Dereferencing varying pointer to uniform struct with 'bound uniform' member,\n" 2244 " only one value will survive. Possible loss of data.");
2247 std::vector<Symbol *> mm;
2256 llvm::Function *fmm = mm[0]->function;
2257 llvm::Value *int_mask = CallInst(fmm, NULL, mask,
LLVMGetName(mask,
"_movmsk"));
2258 std::vector<Symbol *> lz;
2260 llvm::Function *flz = lz[0]->function;
2261 llvm::Value *elem_idx = CallInst(flz, NULL, int_mask,
LLVMGetName(mask,
"_clz"));
2262 llvm::Value *elem = llvm::ExtractElementInst::Create(gather_result, elem_idx,
2263 LLVMGetName(gather_result,
"_umasked_elem"), bblock);
2274 llvm::Type *llvmReturnType = returnType->
LLVMType(
g->
ctx);
2277 if (collectionType != NULL) {
2280 llvm::Value *retValue = llvm::UndefValue::get(llvmReturnType);
2282 const CollectionType *returnCollectionType = CastType<CollectionType>(returnType->GetBaseType());
2286 llvm::Value *eltPtr = AddElementOffset(ptr, i, ptrType,
"gather_elt_ptr", &eltPtrType);
2288 eltPtr = addVaryingOffsetsIfNeeded(eltPtr, eltPtrType);
2292 int need_one_elem = CastType<StructType>(ptrType->
GetBaseType()) &&
2295 llvm::Value *eltValues = LoadInst(eltPtr, mask, eltPtrType, name, need_one_elem);
2297 retValue = InsertInst(retValue, eltValues, i,
"set_value");
2300 }
else if (ptrType->
IsSlice()) {
2311 AddInstrumentationPoint(
"gather");
2315 const PointerType *pt = CastType<PointerType>(returnType);
2316 const char *funcName = NULL;
2318 funcName =
g->
target->
is32Bit() ?
"__pseudo_gather32_i32" :
"__pseudo_gather64_i64";
2320 else if (returnType->IsBoolType())
2321 funcName =
g->
target->
is32Bit() ?
"__pseudo_gather32_i8" :
"__pseudo_gather64_i8";
2323 funcName =
g->
target->
is32Bit() ?
"__pseudo_gather32_double" :
"__pseudo_gather64_double";
2325 funcName =
g->
target->
is32Bit() ?
"__pseudo_gather32_i64" :
"__pseudo_gather64_i64";
2327 funcName =
g->
target->
is32Bit() ?
"__pseudo_gather32_float" :
"__pseudo_gather64_float";
2329 funcName =
g->
target->
is32Bit() ?
"__pseudo_gather32_i32" :
"__pseudo_gather64_i32";
2331 funcName =
g->
target->
is32Bit() ?
"__pseudo_gather32_i16" :
"__pseudo_gather64_i16";
2334 funcName =
g->
target->
is32Bit() ?
"__pseudo_gather32_i8" :
"__pseudo_gather64_i8";
2337 llvm::Function *gatherFunc =
m->
module->getFunction(funcName);
2338 AssertPos(currentPos, gatherFunc != NULL);
2340 llvm::Value *gatherCall = CallInst(gatherFunc, NULL, ptr, mask, name);
2345 if (disableGSWarningCount == 0)
2346 addGSMetadata(gatherCall, currentPos);
2349 if (returnType->IsBoolType()) {
2360 gatherCall = SExtInst(gatherCall, llvmReturnType);
2362 gatherCall = SExtInst(gatherCall, llvmReturnType);
2366 gatherCall = TruncInst(gatherCall, llvmReturnType);
2377 llvm::Instruction *inst = llvm::dyn_cast<llvm::Instruction>(v);
2380 llvm::MDString *str = llvm::MDString::get(*
g->
ctx, pos.
name);
2381 llvm::MDNode *md = llvm::MDNode::get(*
g->
ctx, str);
2382 inst->setMetadata(
"filename", md);
2385 md = llvm::MDNode::get(*
g->
ctx, first_line);
2386 inst->setMetadata(
"first_line", md);
2389 md = llvm::MDNode::get(*
g->
ctx, first_column);
2390 inst->setMetadata(
"first_column", md);
2393 md = llvm::MDNode::get(*
g->
ctx, last_line);
2394 inst->setMetadata(
"last_line", md);
2397 md = llvm::MDNode::get(*
g->
ctx, last_column);
2398 inst->setMetadata(
"last_column", md);
2402 if (llvmType == NULL) {
2407 llvm::AllocaInst *inst = NULL;
2411 llvm::Instruction *retInst = allocaBlock->getTerminator();
2413 unsigned AS = llvmFunction->getParent()->getDataLayout().getAllocaAddrSpace();
2414 inst =
new llvm::AllocaInst(llvmType, AS, name ? name :
"", retInst);
2418 unsigned AS = llvmFunction->getParent()->getDataLayout().getAllocaAddrSpace();
2419 inst =
new llvm::AllocaInst(llvmType, AS, name ? name :
"", bblock);
2427 llvm::ArrayType *arrayType = llvm::dyn_cast<llvm::ArrayType>(llvmType);
2428 if (align == 0 && arrayType != NULL && !llvm::isa<llvm::VectorType>(arrayType->getElementType()))
2432 #if ISPC_LLVM_VERSION <= ISPC_LLVM_9_0 2433 inst->setAlignment(align);
2435 inst->setAlignment(llvm::MaybeAlign(align));
2443 if (ptrType == NULL) {
2448 llvm::Type *llvmStorageType = ptrType->
LLVMType(
g->
ctx);
2449 if ((((CastType<AtomicType>(ptrType) != NULL) || (CastType<VectorType>(ptrType) != NULL)) &&
2455 return AllocaInst(llvmStorageType, name, align, atEntryBlock);
2465 if (value == NULL || ptr == NULL) {
2470 AssertPos(currentPos, CastType<PointerType>(ptrType) != NULL);
2474 const CollectionType *collectionType = CastType<CollectionType>(valueType);
2475 if (collectionType != NULL) {
2481 if (eltType == NULL) {
2485 llvm::Value *eltValue = ExtractInst(value, i,
"value_member");
2486 llvm::Value *eltPtr = AddElementOffset(ptr, i, ptrType,
"struct_ptr_ptr");
2488 StoreInst(eltValue, eltPtr, mask, eltType, eltPtrType);
2496 valueType = valueType->GetAsNonConstType();
2499 llvm::Function *maskedStoreFunc = NULL;
2500 llvm::Type *llvmValueType = value->getType();
2501 llvm::Type *llvmValueStorageType = llvmValueType;
2503 const PointerType *pt = CastType<PointerType>(valueType);
2505 if ((pt == NULL) && (valueType->IsBoolType())) {
2515 llvm::Value *v0 = ExtractInst(value, 0);
2516 llvm::Value *p0 = AddElementOffset(ptr, 0, ptrType);
2520 llvm::Value *v1 = ExtractInst(value, 1);
2521 llvm::Value *p1 = AddElementOffset(ptr, 1, ptrType);
2529 maskedStoreFunc =
m->
module->getFunction(
"__pseudo_masked_store_i32");
2531 maskedStoreFunc =
m->
module->getFunction(
"__pseudo_masked_store_i64");
2533 llvm::Value *notMask = BinaryOperator(llvm::Instruction::Xor, mask,
LLVMMaskAllOn,
"~mask");
2534 llvm::Value *old = LoadInst(ptr, valueType);
2535 llvm::Value *maskedOld = BinaryOperator(llvm::Instruction::And, old, notMask,
"old&~mask");
2536 llvm::Value *maskedNew = BinaryOperator(llvm::Instruction::And, value, mask,
"new&mask");
2537 llvm::Value *
final = BinaryOperator(llvm::Instruction::Or, maskedOld, maskedNew,
"old_new_result");
2538 StoreInst(
final, ptr, valueType);
2541 maskedStoreFunc =
m->
module->getFunction(
"__pseudo_masked_store_double");
2543 maskedStoreFunc =
m->
module->getFunction(
"__pseudo_masked_store_i64");
2545 maskedStoreFunc =
m->
module->getFunction(
"__pseudo_masked_store_float");
2547 maskedStoreFunc =
m->
module->getFunction(
"__pseudo_masked_store_i32");
2549 maskedStoreFunc =
m->
module->getFunction(
"__pseudo_masked_store_i16");
2551 maskedStoreFunc =
m->
module->getFunction(
"__pseudo_masked_store_i8");
2552 value = SwitchBoolSize(value, llvmValueType, llvmValueStorageType);
2554 AssertPos(currentPos, maskedStoreFunc != NULL);
2556 std::vector<llvm::Value *> args;
2557 args.push_back(ptr);
2558 args.push_back(value);
2559 args.push_back(mask);
2560 CallInst(maskedStoreFunc, NULL, args);
2570 llvm::Value *mask) {
2571 const PointerType *ptrType = CastType<PointerType>(origPt);
2575 const CollectionType *srcCollectionType = CastType<CollectionType>(valueType);
2576 if (srcCollectionType != NULL) {
2586 AssertPos(currentPos, dstCollectionType != NULL);
2589 for (
int i = 0; i < srcCollectionType->GetElementCount(); ++i) {
2592 llvm::Value *eltValue = ExtractInst(value, i);
2593 const Type *srcEltType = srcCollectionType->GetElementType(i);
2599 eltValue = SmearUniform(eltValue,
"to_varying");
2605 llvm::Value *eltPtr = AddElementOffset(ptr, i, ptrType);
2616 eltPtr = addVaryingOffsetsIfNeeded(eltPtr, dstEltPtrType);
2620 scatter(eltValue, eltPtr, srcEltType, dstEltPtrType, mask);
2623 }
else if (ptrType->
IsSlice()) {
2629 const PointerType *pt = CastType<PointerType>(valueType);
2633 pt != NULL || CastType<AtomicType>(valueType) != NULL || CastType<EnumType>(valueType) != NULL);
2635 llvm::Type *
type = value->getType();
2636 llvm::Type *llvmStorageType =
type;
2638 if ((pt != NULL) && (valueType->
IsBoolType())) {
2640 value = SwitchBoolSize(value, type, llvmStorageType);
2642 const char *funcName = NULL;
2644 funcName =
g->
target->
is32Bit() ?
"__pseudo_scatter32_i32" :
"__pseudo_scatter64_i64";
2646 funcName =
g->
target->
is32Bit() ?
"__pseudo_scatter32_double" :
"__pseudo_scatter64_double";
2648 funcName =
g->
target->
is32Bit() ?
"__pseudo_scatter32_i64" :
"__pseudo_scatter64_i64";
2650 funcName =
g->
target->
is32Bit() ?
"__pseudo_scatter32_float" :
"__pseudo_scatter64_float";
2652 funcName =
g->
target->
is32Bit() ?
"__pseudo_scatter32_i32" :
"__pseudo_scatter64_i32";
2654 funcName =
g->
target->
is32Bit() ?
"__pseudo_scatter32_i16" :
"__pseudo_scatter64_i16";
2656 funcName =
g->
target->
is32Bit() ?
"__pseudo_scatter32_i8" :
"__pseudo_scatter64_i8";
2659 llvm::Function *scatterFunc =
m->
module->getFunction(funcName);
2660 AssertPos(currentPos, scatterFunc != NULL);
2662 AddInstrumentationPoint(
"scatter");
2664 std::vector<llvm::Value *> args;
2665 args.push_back(ptr);
2666 args.push_back(value);
2667 args.push_back(mask);
2668 llvm::Value *inst = CallInst(scatterFunc, NULL, args);
2670 if (disableGSWarningCount == 0)
2671 addGSMetadata(inst, currentPos);
2675 if (value == NULL || ptr == NULL) {
2681 llvm::PointerType *pt = llvm::dyn_cast<llvm::PointerType>(ptr->getType());
2684 if ((ptrType != NULL) && (ptrType->
IsBoolType())) {
2685 if ((CastType<AtomicType>(ptrType) != NULL)) {
2687 }
else if (CastType<VectorType>(ptrType) != NULL) {
2688 const VectorType *vType = CastType<VectorType>(ptrType);
2695 llvm::StoreInst *inst =
new llvm::StoreInst(value, ptr, bblock);
2698 #if ISPC_LLVM_VERSION <= ISPC_LLVM_9_0 2709 const Type *ptrRefType) {
2710 if (value == NULL || ptr == NULL) {
2717 if (CastType<ReferenceType>(ptrRefType) != NULL)
2720 ptrType = CastType<PointerType>(ptrRefType);
2724 if (CastType<UndefinedStructType>(ptrType->
GetBaseType())) {
2725 Error(currentPos,
"Unable to store to undefined struct type \"%s\".",
2734 storeUniformToSOA(value, ptr, mask, valueType, ptrType);
2737 StoreInst(value, ptr, valueType);
2741 StoreInst(value, ptr, valueType);
2743 maskedStore(value, ptr, ptrType, mask);
2748 scatter(value, ptr, valueType, ptrType, GetFullMask());
2762 llvm::Value *eltValue = ExtractInst(value, i);
2765 llvm::Value *dstEltPtr = AddElementOffset(ptr, i, ptrType,
"slice_offset", &dstEltPtrType);
2766 StoreInst(eltValue, dstEltPtr, mask, eltType, dstEltPtrType);
2773 StoreInst(value, ptr, valueType);
2786 #if ISPC_LLVM_VERSION <= ISPC_LLVM_8_0 2787 llvm::Constant *mcFunc =
2788 #if ISPC_LLVM_VERSION == ISPC_LLVM_6_0 // LLVM 6.0 2799 llvm::FunctionCallee mcFuncCallee =
2802 llvm::Constant *mcFunc = llvm::cast<llvm::Constant>(mcFuncCallee.getCallee());
2805 AssertPos(currentPos, llvm::isa<llvm::Function>(mcFunc));
2807 std::vector<llvm::Value *> args;
2808 args.push_back(dest);
2809 args.push_back(src);
2810 args.push_back(count);
2811 #if ISPC_LLVM_VERSION < ISPC_LLVM_7_0 2813 args.push_back(align);
2816 CallInst(mcFunc, NULL, args,
"");
2820 llvm::Instruction *b = llvm::BranchInst::Create(dest, bblock);
2830 llvm::Instruction *b = llvm::BranchInst::Create(trueBlock, falseBlock, test, bblock);
2842 snprintf(buf,
sizeof(buf),
"_extract_%d", elt);
2845 llvm::Instruction *ei = NULL;
2846 if (llvm::isa<llvm::VectorType>(v->getType()))
2847 ei = llvm::ExtractElementInst::Create(v,
LLVMInt32(elt), name, bblock);
2849 ei = llvm::ExtractValueInst::Create(v, elt, name, bblock);
2855 if (v == NULL || eltVal == NULL) {
2862 snprintf(buf,
sizeof(buf),
"_insert_%d", elt);
2866 llvm::Instruction *ii = NULL;
2867 if (llvm::isa<llvm::VectorType>(v->getType()))
2868 ii = llvm::InsertElementInst::Create(v, eltVal,
LLVMInt32(elt), name, bblock);
2870 ii = llvm::InsertValueInst::Create(v, eltVal, elt, name, bblock);
2876 if (v1 == NULL || v2 == NULL || mask == NULL) {
2883 snprintf(buf,
sizeof(buf),
"_shuffle");
2887 llvm::Instruction *ii =
new llvm::ShuffleVectorInst(v1, v2, mask, name, bblock);
2894 if (v == NULL || vecType == NULL) {
2899 llvm::VectorType *ty = llvm::dyn_cast<llvm::VectorType>(vecType);
2900 Assert(ty && ty->getVectorElementType() == v->getType());
2904 snprintf(buf,
sizeof(buf),
"_broadcast");
2913 llvm::Value *undef1 = llvm::UndefValue::get(vecType);
2914 llvm::Value *undef2 = llvm::UndefValue::get(vecType);
2917 llvm::Twine tw = llvm::Twine(name) + llvm::Twine(
"_init");
2918 llvm::Value *insert = InsertInst(undef1, v, 0, tw.str().c_str());
2921 #if ISPC_LLVM_VERSION < ISPC_LLVM_11_0 2922 llvm::Constant *zeroVec = llvm::ConstantVector::getSplat(
2923 vecType->getVectorNumElements(), llvm::Constant::getNullValue(llvm::Type::getInt32Ty(*
g->
ctx)));
2925 llvm::Constant *zeroVec =
2926 llvm::ConstantVector::getSplat({
static_cast<unsigned int>(vecType->getVectorNumElements()),
false},
2927 llvm::Constant::getNullValue(llvm::Type::getInt32Ty(*
g->
ctx)));
2929 llvm::Value *ret = ShuffleInst(insert, undef2, zeroVec, name);
2935 llvm::PHINode *pn = llvm::PHINode::Create(type, count, name ? name :
"phi", bblock);
2942 if (test == NULL || val0 == NULL || val1 == NULL) {
2950 llvm::Instruction *inst = llvm::SelectInst::Create(test, val0, val1, name, bblock);
2959 llvm::FunctionType *ft = llvm::dyn_cast<llvm::FunctionType>(callee->getType());
2962 llvm::PointerType *pt = llvm::dyn_cast<llvm::PointerType>(callee->getType());
2968 ft = llvm::dyn_cast<llvm::FunctionType>(pt->getElementType());
2972 return ft->getNumParams();
2976 const std::vector<llvm::Value *> &args,
const char *name) {
2982 std::vector<llvm::Value *> argVals = args;
2987 AssertPos(currentPos, argVals.size() + 1 == calleeArgCount || argVals.size() == calleeArgCount);
2988 if (argVals.size() + 1 == calleeArgCount)
2989 argVals.push_back(GetFullMask());
2991 if (llvm::isa<llvm::VectorType>(func->getType()) ==
false) {
2994 llvm::Instruction *ci = llvm::CallInst::Create(func, argVals, name ? name :
"", bblock);
3000 llvm::CallInst *cc = llvm::dyn_cast<llvm::CallInst>(ci);
3001 if (cc && cc->getCalledFunction() && cc->getCalledFunction()->returnDoesNotAlias()) {
3002 cc->addAttribute(llvm::AttributeList::ReturnIndex, llvm::Attribute::NoAlias);
3017 llvm::BasicBlock *bbTest = CreateBasicBlock(
"varying_funcall_test");
3018 llvm::BasicBlock *bbCall = CreateBasicBlock(
"varying_funcall_call");
3019 llvm::BasicBlock *bbDone = CreateBasicBlock(
"varying_funcall_done");
3022 llvm::Value *origMask = GetInternalMask();
3027 llvm::Type *llvmReturnType = returnType->
LLVMType(
g->
ctx);
3028 llvm::Value *resultPtr = NULL;
3029 if (llvmReturnType->isVoidTy() ==
false)
3030 resultPtr = AllocaInst(returnType);
3037 StoreInst(GetFullMask(), maskPtr);
3045 SetCurrentBasicBlock(bbTest);
3047 llvm::Value *maskLoad = LoadInst(maskPtr);
3048 llvm::Value *any = Any(maskLoad);
3049 BranchInst(bbCall, bbDone, any);
3054 SetCurrentBasicBlock(bbCall);
3058 llvm::Value *currentMask = LoadInst(maskPtr);
3059 llvm::Function *cttz =
m->
module->getFunction(
"__count_trailing_zeros_i64");
3061 llvm::Value *firstLane64 = CallInst(cttz, NULL, LaneMask(currentMask),
"first_lane64");
3066 llvm::Value *fptr = llvm::ExtractElementInst::Create(func, firstLane,
"extract_fptr", bblock);
3069 llvm::Value *fptrSmear = SmearUniform(fptr,
"func_ptr");
3074 llvm::Value *fpOverlap = CmpInst(llvm::Instruction::ICmp, llvm::CmpInst::ICMP_EQ, fptrSmear, func);
3075 fpOverlap = I1VecToBoolVec(fpOverlap);
3082 llvm::Value *callMask = BinaryOperator(llvm::Instruction::And, currentMask, fpOverlap,
"call_mask");
3085 SetInternalMask(callMask);
3090 llvm::Type *llvmFPtrType = llvm::PointerType::get(llvmFuncType, 0);
3091 llvm::Value *fptrCast = IntToPtrInst(fptr, llvmFPtrType);
3094 llvm::Value *callResult = CallInst(fptrCast, funcType, args, name);
3099 AssertPos(currentPos, resultPtr != NULL);
3102 AssertPos(currentPos, resultPtr == NULL);
3107 llvm::Value *notCallMask = BinaryOperator(llvm::Instruction::Xor, callMask,
LLVMMaskAllOn,
"~callMask");
3108 currentMask = BinaryOperator(llvm::Instruction::And, currentMask, notCallMask,
"currentMask&~callMask");
3109 StoreInst(currentMask, maskPtr);
3118 SetCurrentBasicBlock(bbDone);
3119 SetInternalMask(origMask);
3120 return resultPtr ? LoadInst(resultPtr, funcType->
GetReturnType()) : NULL;
3126 std::vector<llvm::Value *> args;
3127 args.push_back(arg);
3128 return CallInst(func, funcType, args, name);
3132 llvm::Value *arg1,
const char *name) {
3133 std::vector<llvm::Value *> args;
3134 args.push_back(arg0);
3135 args.push_back(arg1);
3136 return CallInst(func, funcType, args, name);
3144 llvm::Instruction *rinst = NULL;
3145 if (returnValuePtr != NULL) {
3148 llvm::Value *retVal = LoadInst(returnValuePtr, function->GetReturnType(),
"return_value");
3149 rinst = llvm::ReturnInst::Create(*
g->
ctx, retVal, bblock);
3151 AssertPos(currentPos, function->GetReturnType()->IsVoidType());
3152 rinst = llvm::ReturnInst::Create(*
g->
ctx, bblock);
3161 llvm::Value *launchCount[3]) {
3162 if (callee == NULL) {
3167 launchedTasks =
true;
3169 AssertPos(currentPos, llvm::isa<llvm::Function>(callee));
3170 llvm::Type *argType = (llvm::dyn_cast<llvm::Function>(callee))->arg_begin()->getType();
3171 AssertPos(currentPos, llvm::PointerType::classof(argType));
3172 llvm::PointerType *pt = llvm::dyn_cast<llvm::PointerType>(argType);
3174 AssertPos(currentPos, llvm::StructType::classof(pt->getElementType()));
3175 llvm::StructType *argStructType =
static_cast<llvm::StructType *
>(pt->getElementType());
3177 llvm::Function *falloc =
m->
module->getFunction(
"ISPCAlloc");
3179 llvm::Value *structSize =
g->
target->
SizeOf(argStructType, bblock);
3186 std::vector<llvm::Value *> allocArgs;
3187 allocArgs.push_back(launchGroupHandlePtr);
3188 allocArgs.push_back(structSize);
3190 llvm::Value *voidmem = CallInst(falloc, NULL, allocArgs,
"args_ptr");
3191 llvm::Value *argmem = BitCastInst(voidmem, pt);
3195 for (
unsigned int i = 0; i < argVals.size(); ++i) {
3196 llvm::Value *ptr = AddElementOffset(argmem, i, NULL,
"funarg");
3198 StoreInst(argVals[i], ptr);
3201 if (argStructType->getNumElements() == argVals.size() + 1) {
3203 llvm::Value *mask = GetFullMask();
3204 llvm::Value *ptr = AddElementOffset(argmem, argVals.size(), NULL,
"funarg_mask");
3205 StoreInst(mask, ptr);
3212 llvm::Function *flaunch =
m->
module->getFunction(
"ISPCLaunch");
3214 std::vector<llvm::Value *> args;
3215 args.push_back(launchGroupHandlePtr);
3216 args.push_back(fptr);
3217 args.push_back(voidmem);
3218 args.push_back(launchCount[0]);
3219 args.push_back(launchCount[1]);
3220 args.push_back(launchCount[2]);
3221 return CallInst(flaunch, NULL, args,
"");
3225 llvm::Value *launchGroupHandle = LoadInst(launchGroupHandlePtr);
3227 llvm::Value *nonNull = CmpInst(llvm::Instruction::ICmp, llvm::CmpInst::ICMP_NE, launchGroupHandle, nullPtrValue);
3228 llvm::BasicBlock *bSync = CreateBasicBlock(
"call_sync");
3229 llvm::BasicBlock *bPostSync = CreateBasicBlock(
"post_sync");
3230 BranchInst(bSync, bPostSync, nonNull);
3232 SetCurrentBasicBlock(bSync);
3233 llvm::Function *fsync =
m->
module->getFunction(
"ISPCSync");
3235 FATAL(
"Couldn't find ISPCSync declaration?!");
3236 CallInst(fsync, NULL, launchGroupHandle,
"");
3240 StoreInst(nullPtrValue, launchGroupHandlePtr);
3242 BranchInst(bPostSync);
3244 SetCurrentBasicBlock(bPostSync);
3255 const PointerType *pt = CastType<PointerType>(ptrType);
3262 if (baseType->IsVaryingType() ==
false)
3267 llvm::Value *unifSize =
g->
target->
SizeOf(llvmBaseUniformType, bblock);
3268 unifSize = SmearUniform(unifSize);
3272 llvm::Value *varyingOffsets = ProgramIndexVector(is32bits);
3274 llvm::Value *offset = BinaryOperator(llvm::Instruction::Mul, unifSize, varyingOffsets);
3282 return BinaryOperator(llvm::Instruction::Add, ptr, offset);
3286 AssertPos(currentPos, controlFlowInfo.size() > 0);
3287 CFInfo *ci = controlFlowInfo.back();
3288 controlFlowInfo.pop_back();
3290 if (ci->IsSwitch()) {
3292 continueTarget = ci->savedContinueTarget;
3293 breakLanesPtr = ci->savedBreakLanesPtr;
3294 continueLanesPtr = ci->savedContinueLanesPtr;
3295 blockEntryMask = ci->savedBlockEntryMask;
3296 switchExpr = ci->savedSwitchExpr;
3297 defaultBlock = ci->savedDefaultBlock;
3298 caseBlocks = ci->savedCaseBlocks;
3299 nextBlocks = ci->savedNextBlocks;
3300 switchConditionWasUniform = ci->savedSwitchConditionWasUniform;
3301 }
else if (ci->IsLoop() || ci->IsForeach()) {
3302 breakTarget = ci->savedBreakTarget;
3303 continueTarget = ci->savedContinueTarget;
3304 breakLanesPtr = ci->savedBreakLanesPtr;
3305 continueLanesPtr = ci->savedContinueLanesPtr;
3306 blockEntryMask = ci->savedBlockEntryMask;
const Function * GetFunction() const
llvm::Constant * LLVMIntAsType(int64_t val, llvm::Type *type)
static const AtomicType * VaryingInt32
static CFInfo * GetSwitch(bool isUniform, llvm::BasicBlock *breakTarget, llvm::BasicBlock *continueTarget, llvm::Value *savedBreakLanesPtr, llvm::Value *savedContinueLanesPtr, llvm::Value *savedMask, llvm::Value *savedBlockEntryMask, llvm::Value *switchExpr, llvm::BasicBlock *bbDefault, const std::vector< std::pair< int, llvm::BasicBlock *>> *bbCases, const std::map< llvm::BasicBlock *, llvm::BasicBlock *> *bbNext, bool scUniform)
llvm::Value * Any(llvm::Value *mask)
virtual const Type * GetAsVaryingType() const =0
bool IsUniformType() const
llvm::Value * savedBreakLanesPtr
void InitializeLabelMap(Stmt *code)
llvm::Value * PtrToIntInst(llvm::Value *value, const char *name=NULL)
void jumpIfAllLoopLanesAreDone(llvm::BasicBlock *target)
llvm::Value * AddElementOffset(llvm::Value *basePtr, int elementNum, const Type *ptrType, const char *name=NULL, const PointerType **resultPtrType=NULL)
void BranchIfMaskNone(llvm::BasicBlock *btrue, llvm::BasicBlock *bfalse)
llvm::DIFile * GetDIFile() const
bool InForeachLoop() const
void StartSwitch(bool isUniform, llvm::BasicBlock *bbAfterSwitch)
llvm::Value * ShuffleInst(llvm::Value *v1, llvm::Value *v2, llvm::Value *mask, const char *name=NULL)
llvm::Value * ProgramIndexVector(bool is32bits=true)
ASTNode * WalkAST(ASTNode *node, ASTPreCallBackFunc preFunc, ASTPostCallBackFunc postFunc, void *data)
void SetInternalMask(llvm::Value *val)
void StartLoop(llvm::BasicBlock *breakTarget, llvm::BasicBlock *continueTarget, bool uniformControlFlow)
llvm::Instruction * FPCastInst(llvm::Value *value, llvm::Type *type, const char *name=NULL)
const Type * GetReturnType() const
Declaration of the FunctionEmitContext class
void EmitVariableDebugInfo(Symbol *sym)
static llvm::Type * lGetMatchingBoolVectorType(llvm::Type *type)
CFInfo(CFType t, bool iu, llvm::BasicBlock *bt, llvm::BasicBlock *ct, llvm::Value *sb, llvm::Value *sc, llvm::Value *sm, llvm::Value *lm, llvm::Value *sse=NULL, llvm::BasicBlock *bbd=NULL, const std::vector< std::pair< int, llvm::BasicBlock *>> *bbc=NULL, const std::map< llvm::BasicBlock *, llvm::BasicBlock *> *bbn=NULL, bool scu=false)
CFInfo(CFType t, llvm::BasicBlock *bt, llvm::BasicBlock *ct, llvm::Value *sb, llvm::Value *sc, llvm::Value *sm, llvm::Value *lm)
llvm::BasicBlock * savedBreakTarget
bool IsVaryingType() const
void SetInternalMaskAnd(llvm::Value *oldMask, llvm::Value *val)
CFInfo(CFType t, bool uniformIf, llvm::Value *sm)
void BranchInst(llvm::BasicBlock *block)
const std::vector< std::pair< int, llvm::BasicBlock * > > * savedCaseBlocks
void maskedStore(llvm::Value *value, llvm::Value *ptr, const Type *ptrType, llvm::Value *mask)
llvm::Instruction * ZExtInst(llvm::Value *value, llvm::Type *type, const char *name=NULL)
#define AssertPos(pos, expr)
bool IsFrozenSlice() const
Interface class for statements in the ispc language.
llvm::Constant * LLVMInt64Vector(int64_t ival)
FunctionEmitContext(Function *function, Symbol *funSym, llvm::Function *llvmFunction, SourcePos firstStmtPos)
llvm::Value * NotOperator(llvm::Value *v, const char *name=NULL)
llvm::Value * SizeOf(llvm::Type *type, llvm::BasicBlock *insertAtEnd)
llvm::Value * LoadInst(llvm::Value *ptr, llvm::Value *mask, const Type *ptrType, const char *name=NULL, bool one_elem=false)
static llvm::VectorType * VoidPointerVectorType
static llvm::VectorType * BoolVectorType
llvm::DIType * GetDIType(llvm::DIScope *scope) const
void BranchIfMaskAll(llvm::BasicBlock *btrue, llvm::BasicBlock *bfalse)
std::vector< std::string > GetLabels()
int getNativeVectorAlignment() const
const char * LLVMGetName(llvm::Value *v, const char *s)
llvm::Instruction * TruncInst(llvm::Value *value, llvm::Type *type, const char *name=NULL)
llvm::Constant * LLVMMaskAllOn
llvm::Value * AllocaInst(llvm::Type *llvmType, const char *name=NULL, int align=0, bool atEntryBlock=true)
Abstract base class for types that represent sequences.
llvm::Value * CmpInst(llvm::Instruction::OtherOps inst, llvm::CmpInst::Predicate pred, llvm::Value *v0, llvm::Value *v1, const char *name=NULL)
void StartVaryingIf(llvm::Value *oldMask)
static llvm::Type * BoolType
llvm::Value * LaunchInst(llvm::Value *callee, std::vector< llvm::Value *> &argVals, llvm::Value *launchCount[3])
bool ifsInCFAllUniform(int cfType) const
void addSwitchMaskCheck(llvm::Value *mask)
void StartForeach(ForeachType ft)
static llvm::VectorType * Int32VectorType
int getMaskBitCount() const
Expr * TypeConvertExpr(Expr *expr, const Type *toType, const char *errorMsgBase)
static llvm::VectorType * BoolVectorStorageType
static PointerType * GetVarying(const Type *t)
void Continue(bool doCoherenceCheck)
llvm::Value * GetFullMask()
int VaryingCFDepth() const
void AddInstrumentationPoint(const char *note)
llvm::Value * MakeSlicePointer(llvm::Value *ptr, llvm::Value *offset)
int GetNumParameters() const
llvm::Value * gather(llvm::Value *ptr, const PointerType *ptrType, llvm::Value *mask, const char *name)
llvm::Value * SwitchBoolSize(llvm::Value *value, llvm::Type *fromType, llvm::Type *toType, const char *name=NULL)
void restoreMaskGivenReturns(llvm::Value *oldMask)
Type implementation for pointers to other types.
void BranchIfMaskAny(llvm::BasicBlock *btrue, llvm::BasicBlock *bfalse)
void RestoreContinuedLanes()
llvm::Value * loadUniformFromSOA(llvm::Value *ptr, llvm::Value *mask, const PointerType *ptrType, const char *name)
int getNativeVectorWidth() const
llvm::BasicBlock * GetCurrentBasicBlock()
static PointerType * GetUniform(const Type *t, bool isSlice=false)
virtual llvm::Value * GetValue(FunctionEmitContext *ctx) const =0
void Break(bool doCoherenceCheck)
static llvm::Value * lComputeSliceIndex(FunctionEmitContext *ctx, int soaWidth, llvm::Value *indexValue, llvm::Value *ptrSliceOffset, llvm::Value **newSliceOffset)
static llvm::VectorType * Int1VectorType
llvm::Value * CallInst(llvm::Value *func, const FunctionType *funcType, const std::vector< llvm::Value *> &args, const char *name=NULL)
llvm::BasicBlock * CreateBasicBlock(const char *name)
llvm::Value * savedSwitchExpr
static CFInfo * GetLoop(bool isUniform, llvm::BasicBlock *breakTarget, llvm::BasicBlock *continueTarget, llvm::Value *savedBreakLanesPtr, llvm::Value *savedContinueLanesPtr, llvm::Value *savedMask, llvm::Value *savedBlockEntryMask)
virtual llvm::Type * LLVMType(llvm::LLVMContext *ctx) const =0
header file with declarations for symbol and symbol table classes.
llvm::Value * BroadcastValue(llvm::Value *v, llvm::Type *vecType, const char *name=NULL)
static void addGSMetadata(llvm::Value *inst, SourcePos pos)
virtual const Type * GetReferenceTarget() const
static llvm::Value * lFinalSliceOffset(FunctionEmitContext *ctx, llvm::Value *ptr, const PointerType **ptrType)
bool disableMaskAllOnOptimizations
virtual llvm::DIType * GetDIType(llvm::DIScope *scope) const =0
static llvm::Type * VoidType
File with declarations for classes related to statements in the language.
llvm::Type * LLVMType(llvm::LLVMContext *ctx) const
void StoreInst(llvm::Value *value, llvm::Value *ptr, const Type *ptrType=NULL)
void EmitCaseLabel(int value, bool checkMask, SourcePos pos)
llvm::BasicBlock * savedContinueTarget
llvm::Value * GetFunctionMask()
static llvm::VectorType * Int8VectorType
static CFInfo * GetIf(bool isUniform, llvm::Value *savedMask)
void AddDebugPos(llvm::Value *instruction, const SourcePos *pos=NULL, llvm::DIScope *scope=NULL)
llvm::Constant * LLVMInt32Vector(int32_t ival)
SourcePos GetDebugPos() const
Abstract base class for nodes in the abstract syntax tree (AST).
llvm::Value * GetElementPtrInst(llvm::Value *basePtr, llvm::Value *index, const Type *ptrType, const char *name=NULL)
virtual int GetElementCount() const =0
static unsigned int lCalleeArgCount(llvm::Value *callee, const FunctionType *funcType)
void CurrentLanesReturned(Expr *value, bool doCoherenceCheck)
bool LookupFunction(const char *name, std::vector< Symbol *> *matches=NULL)
llvm::BasicBlock * savedDefaultBlock
static llvm::VectorType * FloatVectorType
llvm::Value * LaneMask(llvm::Value *mask)
virtual bool IsBoolType() const =0
static llvm::Type * Int64Type
void MemcpyInst(llvm::Value *dest, llvm::Value *src, llvm::Value *count, llvm::Value *align=NULL)
llvm::PHINode * PhiNode(llvm::Type *type, int count, const char *name=NULL)
Representation of a structure holding a number of members.
llvm::Value * addVaryingOffsetsIfNeeded(llvm::Value *ptr, const Type *ptrType)
static llvm::VectorType * Int64VectorType
Header file with declarations for various LLVM utility stuff.
llvm::Value * getMaskAtSwitchEntry()
void MatchIntegerTypes(llvm::Value **v0, llvm::Value **v1)
static bool IsBasicType(const Type *type)
virtual const Type * GetType() const =0
uint32_t RoundUpPow2(uint32_t v)
AtomicType represents basic types like floats, ints, etc.
void SetBlockEntryMask(llvm::Value *mask)
llvm::Constant * LLVMMaskAllOff
StorageClass storageClass
Representation of a range of positions in a source file.
llvm::Value * StructOffset(llvm::Type *type, int element, llvm::BasicBlock *insertAtEnd)
llvm::ConstantInt * LLVMInt32(int32_t ival)
virtual std::string GetString() const =0
Abstract base class for types that represent collections of other types.
llvm::Value * None(llvm::Value *mask)
bool force32BitAddressing
llvm::Instruction * SExtInst(llvm::Value *value, llvm::Type *type, const char *name=NULL)
static llvm::Type * PointerIntType
static llvm::PointerType * VoidPointerType
void Error(SourcePos p, const char *fmt,...)
int getVectorWidth() const
void SwitchInst(llvm::Value *expr, llvm::BasicBlock *defaultBlock, const std::vector< std::pair< int, llvm::BasicBlock *>> &caseBlocks, const std::map< llvm::BasicBlock *, llvm::BasicBlock *> &nextBlocks)
const PointerType * GetAsSlice() const
static bool lEnclosingLoopIsForeachActive(const std::vector< CFInfo *> &controlFlowInfo)
llvm::Value * GetStringPtr(const std::string &str)
void storeUniformToSOA(llvm::Value *value, llvm::Value *ptr, llvm::Value *mask, const Type *valueType, const PointerType *ptrType)
const AtomicType * GetElementType() const
bool savedSwitchConditionWasUniform
A (short) vector of atomic types.
llvm::Value * InsertInst(llvm::Value *v, llvm::Value *eltVal, int elt, const char *name=NULL)
void DisableGatherScatterWarnings()
static llvm::Type * Int32Type
const llvm::DataLayout * getDataLayout() const
void SetDebugPos(SourcePos pos)
virtual const Type * GetAsUniformType() const =0
Representation of a function in a source file.
virtual const Type * GetElementType() const =0
llvm::Value * GetFullMaskPointer()
llvm::Value * GetInternalMask()
const char * GetISAString() const
llvm::Value * BitCastInst(llvm::Value *value, llvm::Type *type, const char *name=NULL)
void SetInternalMaskAndNot(llvm::Value *oldMask, llvm::Value *test)
llvm::Constant * LLVMFalse
Type representing a function (return type + argument types)
Representation of a program symbol.
llvm::Value * ExtractInst(llvm::Value *v, int elt, const char *name=NULL)
const PointerType * GetAsNonSlice() const
void EnableGatherScatterWarnings()
Interface class that defines the type abstraction.
static bool initLabelBBlocks(ASTNode *node, void *data)
Expr abstract base class and expression implementations.
void SetCurrentBasicBlock(llvm::BasicBlock *bblock)
static llvm::VectorType * MaskType
llvm::Value * savedContinueLanesPtr
llvm::DICompileUnit * diCompileUnit
void EmitDefaultLabel(bool checkMask, SourcePos pos)
llvm::Instruction * SelectInst(llvm::Value *test, llvm::Value *val0, llvm::Value *val1, const char *name=NULL)
static int lArrayVectorWidth(llvm::Type *t)
void EmitFunctionParameterDebugInfo(Symbol *sym, int parameterNum)
Expr is the abstract base class that defines the interface that all expression types must implement...
llvm::Value * IntToPtrInst(llvm::Value *value, llvm::Type *type, const char *name=NULL)
static llvm::VectorType * DoubleVectorType
llvm::Value * MasksAllEqual(llvm::Value *mask1, llvm::Value *mask2)
llvm::Value * All(llvm::Value *mask)
bool inSwitchStatement() const
llvm::ConstantInt * LLVMInt64(int64_t ival)
virtual const Type * GetBaseType() const =0
llvm::DIScope * GetDIScope() const
llvm::Value * SmearUniform(llvm::Value *value, const char *name=NULL)
static llvm::VectorType * Int16VectorType
const std::map< llvm::BasicBlock *, llvm::BasicBlock * > * savedNextBlocks
Variability GetVariability() const
llvm::Value * savedBlockEntryMask
const Type * GetBaseType() const
Declaration of the Module class, which is the ispc-side representation of the results of compiling a ...
llvm::DIBuilder * diBuilder
void Warning(SourcePos p, const char *fmt,...)
static CFInfo * GetForeach(FunctionEmitContext::ForeachType ft, llvm::BasicBlock *breakTarget, llvm::BasicBlock *continueTarget, llvm::Value *savedBreakLanesPtr, llvm::Value *savedContinueLanesPtr, llvm::Value *savedMask, llvm::Value *savedBlockEntryMask)
static bool EqualIgnoringConst(const Type *a, const Type *b)
void scatter(llvm::Value *value, llvm::Value *ptr, const Type *valueType, const Type *ptrType, llvm::Value *mask)
llvm::Instruction * CastInst(llvm::Instruction::CastOps op, llvm::Value *value, llvm::Type *type, const char *name=NULL)
llvm::Instruction * ReturnInst()
llvm::Value * applyVaryingGEP(llvm::Value *basePtr, llvm::Value *index, const Type *ptrType)
llvm::Value * BinaryOperator(llvm::Instruction::BinaryOps inst, llvm::Value *v0, llvm::Value *v1, const char *name=NULL)
llvm::BasicBlock * GetLabeledBasicBlock(const std::string &label)
void SetFunctionMask(llvm::Value *val)
std::map< std::string, llvm::BasicBlock * > labelMap
virtual llvm::Type * LLVMStorageType(llvm::LLVMContext *ctx) const
SymbolTable * symbolTable
File with declarations for classes related to type representation.
llvm::Value * I1VecToBoolVec(llvm::Value *b)
llvm::FunctionType * LLVMFunctionType(llvm::LLVMContext *ctx, bool disableMask=false) const
static llvm::Value * lGetStringAsValue(llvm::BasicBlock *bblock, const char *s)
virtual const Type * GetElementType(int index) const =0