Intel SPMD Program Compiler  1.12.0
llvmutil.cpp
Go to the documentation of this file.
1 /*
2  Copyright (c) 2010-2019, Intel Corporation
3  All rights reserved.
4 
5  Redistribution and use in source and binary forms, with or without
6  modification, are permitted provided that the following conditions are
7  met:
8 
9  * Redistributions of source code must retain the above copyright
10  notice, this list of conditions and the following disclaimer.
11 
12  * Redistributions in binary form must reproduce the above copyright
13  notice, this list of conditions and the following disclaimer in the
14  documentation and/or other materials provided with the distribution.
15 
16  * Neither the name of Intel Corporation nor the names of its
17  contributors may be used to endorse or promote products derived from
18  this software without specific prior written permission.
19 
20 
21  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
22  IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
23  TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
24  PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
25  OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
26  EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
27  PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
28  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
29  LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
30  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
31  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 */
33 
34 /** @file llvmutil.cpp
35  @brief Implementations of various LLVM utility types and classes.
36 */
37 
38 #include "llvmutil.h"
39 #include "ispc.h"
40 #include "type.h"
41 #if ISPC_LLVM_VERSION == ISPC_LLVM_3_2
42 #include <llvm/BasicBlock.h>
43 #include <llvm/Instructions.h>
44 #else
45 #include <llvm/IR/BasicBlock.h>
46 #include <llvm/IR/Instructions.h>
47 #endif
48 #include <map>
49 #include <set>
50 
51 llvm::Type *LLVMTypes::VoidType = NULL;
52 llvm::PointerType *LLVMTypes::VoidPointerType = NULL;
53 llvm::Type *LLVMTypes::PointerIntType = NULL;
54 llvm::Type *LLVMTypes::BoolType = NULL;
55 
56 llvm::Type *LLVMTypes::Int8Type = NULL;
57 llvm::Type *LLVMTypes::Int16Type = NULL;
58 llvm::Type *LLVMTypes::Int32Type = NULL;
59 llvm::Type *LLVMTypes::Int64Type = NULL;
60 llvm::Type *LLVMTypes::FloatType = NULL;
61 llvm::Type *LLVMTypes::DoubleType = NULL;
62 
63 llvm::Type *LLVMTypes::Int8PointerType = NULL;
64 llvm::Type *LLVMTypes::Int16PointerType = NULL;
65 llvm::Type *LLVMTypes::Int32PointerType = NULL;
66 llvm::Type *LLVMTypes::Int64PointerType = NULL;
67 llvm::Type *LLVMTypes::FloatPointerType = NULL;
68 llvm::Type *LLVMTypes::DoublePointerType = NULL;
69 
70 llvm::VectorType *LLVMTypes::MaskType = NULL;
71 llvm::VectorType *LLVMTypes::BoolVectorType = NULL;
72 
73 llvm::VectorType *LLVMTypes::Int1VectorType = NULL;
74 llvm::VectorType *LLVMTypes::Int8VectorType = NULL;
75 llvm::VectorType *LLVMTypes::Int16VectorType = NULL;
76 llvm::VectorType *LLVMTypes::Int32VectorType = NULL;
77 llvm::VectorType *LLVMTypes::Int64VectorType = NULL;
78 llvm::VectorType *LLVMTypes::FloatVectorType = NULL;
79 llvm::VectorType *LLVMTypes::DoubleVectorType = NULL;
80 
81 llvm::Type *LLVMTypes::Int8VectorPointerType = NULL;
82 llvm::Type *LLVMTypes::Int16VectorPointerType = NULL;
83 llvm::Type *LLVMTypes::Int32VectorPointerType = NULL;
84 llvm::Type *LLVMTypes::Int64VectorPointerType = NULL;
85 llvm::Type *LLVMTypes::FloatVectorPointerType = NULL;
86 llvm::Type *LLVMTypes::DoubleVectorPointerType = NULL;
87 
88 llvm::VectorType *LLVMTypes::VoidPointerVectorType = NULL;
89 
90 llvm::Constant *LLVMTrue = NULL;
91 llvm::Constant *LLVMFalse = NULL;
92 llvm::Constant *LLVMMaskAllOn = NULL;
93 llvm::Constant *LLVMMaskAllOff = NULL;
94 
95 void InitLLVMUtil(llvm::LLVMContext *ctx, Target &target) {
96  LLVMTypes::VoidType = llvm::Type::getVoidTy(*ctx);
97  LLVMTypes::VoidPointerType = llvm::PointerType::get(llvm::Type::getInt8Ty(*ctx), 0);
98  LLVMTypes::PointerIntType = target.is32Bit() ? llvm::Type::getInt32Ty(*ctx) : llvm::Type::getInt64Ty(*ctx);
99 
100  LLVMTypes::BoolType = llvm::Type::getInt1Ty(*ctx);
101  LLVMTypes::Int8Type = llvm::Type::getInt8Ty(*ctx);
102  LLVMTypes::Int16Type = llvm::Type::getInt16Ty(*ctx);
103  LLVMTypes::Int32Type = llvm::Type::getInt32Ty(*ctx);
104  LLVMTypes::Int64Type = llvm::Type::getInt64Ty(*ctx);
105  LLVMTypes::FloatType = llvm::Type::getFloatTy(*ctx);
106  LLVMTypes::DoubleType = llvm::Type::getDoubleTy(*ctx);
107 
108  LLVMTypes::Int8PointerType = llvm::PointerType::get(LLVMTypes::Int8Type, 0);
109  LLVMTypes::Int16PointerType = llvm::PointerType::get(LLVMTypes::Int16Type, 0);
110  LLVMTypes::Int32PointerType = llvm::PointerType::get(LLVMTypes::Int32Type, 0);
111  LLVMTypes::Int64PointerType = llvm::PointerType::get(LLVMTypes::Int64Type, 0);
112  LLVMTypes::FloatPointerType = llvm::PointerType::get(LLVMTypes::FloatType, 0);
113  LLVMTypes::DoublePointerType = llvm::PointerType::get(LLVMTypes::DoubleType, 0);
114 
115  switch (target.getMaskBitCount()) {
116  case 1:
118  llvm::VectorType::get(llvm::Type::getInt1Ty(*ctx), target.getVectorWidth());
119  break;
120  case 8:
122  llvm::VectorType::get(llvm::Type::getInt8Ty(*ctx), target.getVectorWidth());
123  break;
124  case 16:
126  llvm::VectorType::get(llvm::Type::getInt16Ty(*ctx), target.getVectorWidth());
127  break;
128  case 32:
130  llvm::VectorType::get(llvm::Type::getInt32Ty(*ctx), target.getVectorWidth());
131  break;
132  case 64:
134  llvm::VectorType::get(llvm::Type::getInt64Ty(*ctx), target.getVectorWidth());
135  break;
136  default:
137  FATAL("Unhandled mask width for initializing MaskType");
138  }
139 
140  LLVMTypes::Int1VectorType = llvm::VectorType::get(llvm::Type::getInt1Ty(*ctx), target.getVectorWidth());
141  LLVMTypes::Int8VectorType = llvm::VectorType::get(LLVMTypes::Int8Type, target.getVectorWidth());
142  LLVMTypes::Int16VectorType = llvm::VectorType::get(LLVMTypes::Int16Type, target.getVectorWidth());
143  LLVMTypes::Int32VectorType = llvm::VectorType::get(LLVMTypes::Int32Type, target.getVectorWidth());
144  LLVMTypes::Int64VectorType = llvm::VectorType::get(LLVMTypes::Int64Type, target.getVectorWidth());
145  LLVMTypes::FloatVectorType = llvm::VectorType::get(LLVMTypes::FloatType, target.getVectorWidth());
146  LLVMTypes::DoubleVectorType = llvm::VectorType::get(LLVMTypes::DoubleType, target.getVectorWidth());
147 
148  LLVMTypes::Int8VectorPointerType = llvm::PointerType::get(LLVMTypes::Int8VectorType, 0);
149  LLVMTypes::Int16VectorPointerType = llvm::PointerType::get(LLVMTypes::Int16VectorType, 0);
150  LLVMTypes::Int32VectorPointerType = llvm::PointerType::get(LLVMTypes::Int32VectorType, 0);
151  LLVMTypes::Int64VectorPointerType = llvm::PointerType::get(LLVMTypes::Int64VectorType, 0);
152  LLVMTypes::FloatVectorPointerType = llvm::PointerType::get(LLVMTypes::FloatVectorType, 0);
153  LLVMTypes::DoubleVectorPointerType = llvm::PointerType::get(LLVMTypes::DoubleVectorType, 0);
154 
156 
157  LLVMTrue = llvm::ConstantInt::getTrue(*ctx);
158  LLVMFalse = llvm::ConstantInt::getFalse(*ctx);
159 
160  std::vector<llvm::Constant *> maskOnes;
161  llvm::Constant *onMask = NULL;
162  switch (target.getMaskBitCount()) {
163  case 1:
164  onMask = llvm::ConstantInt::get(llvm::Type::getInt1Ty(*ctx), 1, false /*unsigned*/); // 0x1
165  break;
166  case 8:
167  onMask = llvm::ConstantInt::get(llvm::Type::getInt8Ty(*ctx), -1, true /*signed*/); // 0xff
168  break;
169  case 16:
170  onMask = llvm::ConstantInt::get(llvm::Type::getInt16Ty(*ctx), -1, true /*signed*/); // 0xffff
171  break;
172  case 32:
173  onMask = llvm::ConstantInt::get(llvm::Type::getInt32Ty(*ctx), -1, true /*signed*/); // 0xffffffff
174  break;
175  case 64:
176  onMask = llvm::ConstantInt::get(llvm::Type::getInt64Ty(*ctx), -1, true /*signed*/); // 0xffffffffffffffffull
177  break;
178  default:
179  FATAL("Unhandled mask width for onMask");
180  }
181 
182  for (int i = 0; i < target.getVectorWidth(); ++i)
183  maskOnes.push_back(onMask);
184  LLVMMaskAllOn = llvm::ConstantVector::get(maskOnes);
185 
186  std::vector<llvm::Constant *> maskZeros;
187  llvm::Constant *offMask = NULL;
188  switch (target.getMaskBitCount()) {
189  case 1:
190  offMask = llvm::ConstantInt::get(llvm::Type::getInt1Ty(*ctx), 0, true /*signed*/);
191  break;
192  case 8:
193  offMask = llvm::ConstantInt::get(llvm::Type::getInt8Ty(*ctx), 0, true /*signed*/);
194  break;
195  case 16:
196  offMask = llvm::ConstantInt::get(llvm::Type::getInt16Ty(*ctx), 0, true /*signed*/);
197  break;
198  case 32:
199  offMask = llvm::ConstantInt::get(llvm::Type::getInt32Ty(*ctx), 0, true /*signed*/);
200  break;
201  case 64:
202  offMask = llvm::ConstantInt::get(llvm::Type::getInt64Ty(*ctx), 0, true /*signed*/);
203  break;
204  default:
205  FATAL("Unhandled mask width for offMask");
206  }
207  for (int i = 0; i < target.getVectorWidth(); ++i)
208  maskZeros.push_back(offMask);
209  LLVMMaskAllOff = llvm::ConstantVector::get(maskZeros);
210 }
211 
212 llvm::ConstantInt *LLVMInt8(int8_t ival) {
213  return llvm::ConstantInt::get(llvm::Type::getInt8Ty(*g->ctx), ival, true /*signed*/);
214 }
215 
216 llvm::ConstantInt *LLVMUInt8(uint8_t ival) {
217  return llvm::ConstantInt::get(llvm::Type::getInt8Ty(*g->ctx), ival, false /*unsigned*/);
218 }
219 
220 llvm::ConstantInt *LLVMInt16(int16_t ival) {
221  return llvm::ConstantInt::get(llvm::Type::getInt16Ty(*g->ctx), ival, true /*signed*/);
222 }
223 
224 llvm::ConstantInt *LLVMUInt16(uint16_t ival) {
225  return llvm::ConstantInt::get(llvm::Type::getInt16Ty(*g->ctx), ival, false /*unsigned*/);
226 }
227 
228 llvm::ConstantInt *LLVMInt32(int32_t ival) {
229  return llvm::ConstantInt::get(llvm::Type::getInt32Ty(*g->ctx), ival, true /*signed*/);
230 }
231 
232 llvm::ConstantInt *LLVMUInt32(uint32_t ival) {
233  return llvm::ConstantInt::get(llvm::Type::getInt32Ty(*g->ctx), ival, false /*unsigned*/);
234 }
235 
236 llvm::ConstantInt *LLVMInt64(int64_t ival) {
237  return llvm::ConstantInt::get(llvm::Type::getInt64Ty(*g->ctx), ival, true /*signed*/);
238 }
239 
240 llvm::ConstantInt *LLVMUInt64(uint64_t ival) {
241  return llvm::ConstantInt::get(llvm::Type::getInt64Ty(*g->ctx), ival, false /*unsigned*/);
242 }
243 
244 llvm::Constant *LLVMFloat(float fval) { return llvm::ConstantFP::get(llvm::Type::getFloatTy(*g->ctx), fval); }
245 
246 llvm::Constant *LLVMDouble(double dval) { return llvm::ConstantFP::get(llvm::Type::getDoubleTy(*g->ctx), dval); }
247 
248 llvm::Constant *LLVMInt8Vector(int8_t ival) {
249  llvm::Constant *v = LLVMInt8(ival);
250  std::vector<llvm::Constant *> vals;
251  for (int i = 0; i < g->target->getVectorWidth(); ++i)
252  vals.push_back(v);
253  return llvm::ConstantVector::get(vals);
254 }
255 
256 llvm::Constant *LLVMInt8Vector(const int8_t *ivec) {
257  std::vector<llvm::Constant *> vals;
258  for (int i = 0; i < g->target->getVectorWidth(); ++i)
259  vals.push_back(LLVMInt8(ivec[i]));
260  return llvm::ConstantVector::get(vals);
261 }
262 
263 llvm::Constant *LLVMUInt8Vector(uint8_t ival) {
264  llvm::Constant *v = LLVMUInt8(ival);
265  std::vector<llvm::Constant *> vals;
266  for (int i = 0; i < g->target->getVectorWidth(); ++i)
267  vals.push_back(v);
268  return llvm::ConstantVector::get(vals);
269 }
270 
271 llvm::Constant *LLVMUInt8Vector(const uint8_t *ivec) {
272  std::vector<llvm::Constant *> vals;
273  for (int i = 0; i < g->target->getVectorWidth(); ++i)
274  vals.push_back(LLVMUInt8(ivec[i]));
275  return llvm::ConstantVector::get(vals);
276 }
277 
278 llvm::Constant *LLVMInt16Vector(int16_t ival) {
279  llvm::Constant *v = LLVMInt16(ival);
280  std::vector<llvm::Constant *> vals;
281  for (int i = 0; i < g->target->getVectorWidth(); ++i)
282  vals.push_back(v);
283  return llvm::ConstantVector::get(vals);
284 }
285 
286 llvm::Constant *LLVMInt16Vector(const int16_t *ivec) {
287  std::vector<llvm::Constant *> vals;
288  for (int i = 0; i < g->target->getVectorWidth(); ++i)
289  vals.push_back(LLVMInt16(ivec[i]));
290  return llvm::ConstantVector::get(vals);
291 }
292 
293 llvm::Constant *LLVMUInt16Vector(uint16_t ival) {
294  llvm::Constant *v = LLVMUInt16(ival);
295  std::vector<llvm::Constant *> vals;
296  for (int i = 0; i < g->target->getVectorWidth(); ++i)
297  vals.push_back(v);
298  return llvm::ConstantVector::get(vals);
299 }
300 
301 llvm::Constant *LLVMUInt16Vector(const uint16_t *ivec) {
302  std::vector<llvm::Constant *> vals;
303  for (int i = 0; i < g->target->getVectorWidth(); ++i)
304  vals.push_back(LLVMUInt16(ivec[i]));
305  return llvm::ConstantVector::get(vals);
306 }
307 
308 llvm::Constant *LLVMInt32Vector(int32_t ival) {
309  llvm::Constant *v = LLVMInt32(ival);
310  std::vector<llvm::Constant *> vals;
311  for (int i = 0; i < g->target->getVectorWidth(); ++i)
312  vals.push_back(v);
313  return llvm::ConstantVector::get(vals);
314 }
315 
316 llvm::Constant *LLVMInt32Vector(const int32_t *ivec) {
317  std::vector<llvm::Constant *> vals;
318  for (int i = 0; i < g->target->getVectorWidth(); ++i)
319  vals.push_back(LLVMInt32(ivec[i]));
320  return llvm::ConstantVector::get(vals);
321 }
322 
323 llvm::Constant *LLVMUInt32Vector(uint32_t ival) {
324  llvm::Constant *v = LLVMUInt32(ival);
325  std::vector<llvm::Constant *> vals;
326  for (int i = 0; i < g->target->getVectorWidth(); ++i)
327  vals.push_back(v);
328  return llvm::ConstantVector::get(vals);
329 }
330 
331 llvm::Constant *LLVMUInt32Vector(const uint32_t *ivec) {
332  std::vector<llvm::Constant *> vals;
333  for (int i = 0; i < g->target->getVectorWidth(); ++i)
334  vals.push_back(LLVMUInt32(ivec[i]));
335  return llvm::ConstantVector::get(vals);
336 }
337 
338 llvm::Constant *LLVMFloatVector(float fval) {
339  llvm::Constant *v = LLVMFloat(fval);
340  std::vector<llvm::Constant *> vals;
341  for (int i = 0; i < g->target->getVectorWidth(); ++i)
342  vals.push_back(v);
343  return llvm::ConstantVector::get(vals);
344 }
345 
346 llvm::Constant *LLVMFloatVector(const float *fvec) {
347  std::vector<llvm::Constant *> vals;
348  for (int i = 0; i < g->target->getVectorWidth(); ++i)
349  vals.push_back(LLVMFloat(fvec[i]));
350  return llvm::ConstantVector::get(vals);
351 }
352 
353 llvm::Constant *LLVMDoubleVector(double dval) {
354  llvm::Constant *v = LLVMDouble(dval);
355  std::vector<llvm::Constant *> vals;
356  for (int i = 0; i < g->target->getVectorWidth(); ++i)
357  vals.push_back(v);
358  return llvm::ConstantVector::get(vals);
359 }
360 
361 llvm::Constant *LLVMDoubleVector(const double *dvec) {
362  std::vector<llvm::Constant *> vals;
363  for (int i = 0; i < g->target->getVectorWidth(); ++i)
364  vals.push_back(LLVMDouble(dvec[i]));
365  return llvm::ConstantVector::get(vals);
366 }
367 
368 llvm::Constant *LLVMInt64Vector(int64_t ival) {
369  llvm::Constant *v = LLVMInt64(ival);
370  std::vector<llvm::Constant *> vals;
371  for (int i = 0; i < g->target->getVectorWidth(); ++i)
372  vals.push_back(v);
373  return llvm::ConstantVector::get(vals);
374 }
375 
376 llvm::Constant *LLVMInt64Vector(const int64_t *ivec) {
377  std::vector<llvm::Constant *> vals;
378  for (int i = 0; i < g->target->getVectorWidth(); ++i)
379  vals.push_back(LLVMInt64(ivec[i]));
380  return llvm::ConstantVector::get(vals);
381 }
382 
383 llvm::Constant *LLVMUInt64Vector(uint64_t ival) {
384  llvm::Constant *v = LLVMUInt64(ival);
385  std::vector<llvm::Constant *> vals;
386  for (int i = 0; i < g->target->getVectorWidth(); ++i)
387  vals.push_back(v);
388  return llvm::ConstantVector::get(vals);
389 }
390 
391 llvm::Constant *LLVMUInt64Vector(const uint64_t *ivec) {
392  std::vector<llvm::Constant *> vals;
393  for (int i = 0; i < g->target->getVectorWidth(); ++i)
394  vals.push_back(LLVMUInt64(ivec[i]));
395  return llvm::ConstantVector::get(vals);
396 }
397 
398 llvm::Constant *LLVMBoolVector(bool b) {
399  llvm::Constant *v;
401  v = llvm::ConstantInt::get(LLVMTypes::Int64Type, b ? 0xffffffffffffffffull : 0, false /*unsigned*/);
403  v = llvm::ConstantInt::get(LLVMTypes::Int32Type, b ? 0xffffffff : 0, false /*unsigned*/);
405  v = llvm::ConstantInt::get(LLVMTypes::Int16Type, b ? 0xffff : 0, false /*unsigned*/);
407  v = llvm::ConstantInt::get(LLVMTypes::Int8Type, b ? 0xff : 0, false /*unsigned*/);
408  else {
410  v = b ? LLVMTrue : LLVMFalse;
411  }
412 
413  std::vector<llvm::Constant *> vals;
414  for (int i = 0; i < g->target->getVectorWidth(); ++i)
415  vals.push_back(v);
416  return llvm::ConstantVector::get(vals);
417 }
418 
419 llvm::Constant *LLVMBoolVector(const bool *bvec) {
420  std::vector<llvm::Constant *> vals;
421  for (int i = 0; i < g->target->getVectorWidth(); ++i) {
422  llvm::Constant *v;
424  v = llvm::ConstantInt::get(LLVMTypes::Int64Type, bvec[i] ? 0xffffffffffffffffull : 0, false /*unsigned*/);
426  v = llvm::ConstantInt::get(LLVMTypes::Int32Type, bvec[i] ? 0xffffffff : 0, false /*unsigned*/);
428  v = llvm::ConstantInt::get(LLVMTypes::Int16Type, bvec[i] ? 0xffff : 0, false /*unsigned*/);
430  v = llvm::ConstantInt::get(LLVMTypes::Int8Type, bvec[i] ? 0xff : 0, false /*unsigned*/);
431  else {
433  v = bvec[i] ? LLVMTrue : LLVMFalse;
434  }
435 
436  vals.push_back(v);
437  }
438  return llvm::ConstantVector::get(vals);
439 }
440 
441 llvm::Constant *LLVMIntAsType(int64_t val, llvm::Type *type) {
442  llvm::VectorType *vecType = llvm::dyn_cast<llvm::VectorType>(type);
443 
444  if (vecType != NULL) {
445  llvm::Constant *v = llvm::ConstantInt::get(vecType->getElementType(), val, true /* signed */);
446  std::vector<llvm::Constant *> vals;
447  for (int i = 0; i < (int)vecType->getNumElements(); ++i)
448  vals.push_back(v);
449  return llvm::ConstantVector::get(vals);
450  } else
451  return llvm::ConstantInt::get(type, val, true /* signed */);
452 }
453 
454 llvm::Constant *LLVMUIntAsType(uint64_t val, llvm::Type *type) {
455  llvm::VectorType *vecType = llvm::dyn_cast<llvm::VectorType>(type);
456 
457  if (vecType != NULL) {
458  llvm::Constant *v = llvm::ConstantInt::get(vecType->getElementType(), val, false /* unsigned */);
459  std::vector<llvm::Constant *> vals;
460  for (int i = 0; i < (int)vecType->getNumElements(); ++i)
461  vals.push_back(v);
462  return llvm::ConstantVector::get(vals);
463  } else
464  return llvm::ConstantInt::get(type, val, false /* unsigned */);
465 }
466 
467 /** Conservative test to see if two llvm::Values are equal. There are
468  (potentially many) cases where the two values actually are equal but
469  this will return false. However, if it does return true, the two
470  vectors definitely are equal.
471 */
472 static bool lValuesAreEqual(llvm::Value *v0, llvm::Value *v1, std::vector<llvm::PHINode *> &seenPhi0,
473  std::vector<llvm::PHINode *> &seenPhi1) {
474  // Thanks to the fact that LLVM hashes and returns the same pointer for
475  // constants (of all sorts, even constant expressions), this first test
476  // actually catches a lot of cases. LLVM's SSA form also helps a lot
477  // with this..
478  if (v0 == v1)
479  return true;
480 
481  Assert(seenPhi0.size() == seenPhi1.size());
482  for (unsigned int i = 0; i < seenPhi0.size(); ++i)
483  if (v0 == seenPhi0[i] && v1 == seenPhi1[i])
484  return true;
485 
486  llvm::BinaryOperator *bo0 = llvm::dyn_cast<llvm::BinaryOperator>(v0);
487  llvm::BinaryOperator *bo1 = llvm::dyn_cast<llvm::BinaryOperator>(v1);
488  if (bo0 != NULL && bo1 != NULL) {
489  if (bo0->getOpcode() != bo1->getOpcode())
490  return false;
491  return (lValuesAreEqual(bo0->getOperand(0), bo1->getOperand(0), seenPhi0, seenPhi1) &&
492  lValuesAreEqual(bo0->getOperand(1), bo1->getOperand(1), seenPhi0, seenPhi1));
493  }
494 
495  llvm::CastInst *cast0 = llvm::dyn_cast<llvm::CastInst>(v0);
496  llvm::CastInst *cast1 = llvm::dyn_cast<llvm::CastInst>(v1);
497  if (cast0 != NULL && cast1 != NULL) {
498  if (cast0->getOpcode() != cast1->getOpcode())
499  return false;
500  return lValuesAreEqual(cast0->getOperand(0), cast1->getOperand(0), seenPhi0, seenPhi1);
501  }
502 
503  llvm::PHINode *phi0 = llvm::dyn_cast<llvm::PHINode>(v0);
504  llvm::PHINode *phi1 = llvm::dyn_cast<llvm::PHINode>(v1);
505  if (phi0 != NULL && phi1 != NULL) {
506  if (phi0->getNumIncomingValues() != phi1->getNumIncomingValues())
507  return false;
508 
509  seenPhi0.push_back(phi0);
510  seenPhi1.push_back(phi1);
511 
512  unsigned int numIncoming = phi0->getNumIncomingValues();
513  // Check all of the incoming values: if all of them are all equal,
514  // then we're good.
515  bool anyFailure = false;
516  for (unsigned int i = 0; i < numIncoming; ++i) {
517  // FIXME: should it be ok if the incoming blocks are different,
518  // where we just return faliure in this case?
519  Assert(phi0->getIncomingBlock(i) == phi1->getIncomingBlock(i));
520  if (!lValuesAreEqual(phi0->getIncomingValue(i), phi1->getIncomingValue(i), seenPhi0, seenPhi1)) {
521  anyFailure = true;
522  break;
523  }
524  }
525 
526  seenPhi0.pop_back();
527  seenPhi1.pop_back();
528 
529  return !anyFailure;
530  }
531 
532  return false;
533 }
534 
535 /** Given an llvm::Value known to be an integer, return its value as
536  an int64_t.
537 */
538 static int64_t lGetIntValue(llvm::Value *offset) {
539  llvm::ConstantInt *intOffset = llvm::dyn_cast<llvm::ConstantInt>(offset);
540  Assert(intOffset && (intOffset->getBitWidth() == 32 || intOffset->getBitWidth() == 64));
541  return intOffset->getSExtValue();
542 }
543 
544 /** Recognizes constant vector with undef operands except the first one:
545  * <i64 4, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef>
546  */
547 static bool lIsFirstElementConstVector(llvm::Value *v) {
548  llvm::ConstantVector *cv = llvm::dyn_cast<llvm::ConstantVector>(v);
549  if (cv != NULL) {
550  llvm::Constant *c = llvm::dyn_cast<llvm::Constant>(cv->getOperand(0));
551  if (c == NULL) {
552  return false;
553  }
554 
555  for (int i = 1; i < (int)cv->getNumOperands(); ++i) {
556  if (!llvm::isa<llvm::UndefValue>(cv->getOperand(i))) {
557  return false;
558  }
559  }
560  return true;
561  }
562  return false;
563 }
564 
565 llvm::Value *LLVMFlattenInsertChain(llvm::Value *inst, int vectorWidth, bool compare, bool undef,
566  bool searchFirstUndef) {
567  llvm::Value **elements = new llvm::Value *[vectorWidth];
568  for (int i = 0; i < vectorWidth; ++i) {
569  elements[i] = NULL;
570  }
571 
572  // Catch a pattern of InsertElement chain.
573  if (llvm::InsertElementInst *ie = llvm::dyn_cast<llvm::InsertElementInst>(inst)) {
574  // Gather elements of vector
575  while (ie != NULL) {
576  int64_t iOffset = lGetIntValue(ie->getOperand(2));
577  Assert(iOffset >= 0 && iOffset < vectorWidth);
578 
579  // Get the scalar value from this insert
580  if (elements[iOffset] == NULL) {
581  elements[iOffset] = ie->getOperand(1);
582  }
583 
584  // Do we have another insert?
585  llvm::Value *insertBase = ie->getOperand(0);
586  ie = llvm::dyn_cast<llvm::InsertElementInst>(insertBase);
587  if (ie != NULL) {
588  continue;
589  }
590 
591  if (llvm::isa<llvm::UndefValue>(insertBase)) {
592  break;
593  }
594 
595  if (llvm::isa<llvm::ConstantVector>(insertBase) || llvm::isa<llvm::ConstantAggregateZero>(insertBase)) {
596  llvm::Constant *cv = llvm::dyn_cast<llvm::Constant>(insertBase);
597  Assert(vectorWidth == (int)(cv->getNumOperands()));
598  for (int i = 0; i < vectorWidth; i++) {
599  if (elements[i] == NULL) {
600  elements[i] = cv->getOperand(i);
601  }
602  }
603  break;
604  } else {
605  // Here chain ends in llvm::LoadInst or some other.
606  // They are not equal to each other so we should return NULL if compare
607  // and first element if we have it.
608  Assert(compare == true || elements[0] != NULL);
609  if (compare) {
610  return NULL;
611  } else {
612  return elements[0];
613  }
614  }
615  // TODO: Also, should we handle some other values like
616  // ConstantDataVectors.
617  }
618  if (compare == false) {
619  // We simply want first element
620  return elements[0];
621  }
622 
623  int null_number = 0;
624  int NonNull = 0;
625  for (int i = 0; i < vectorWidth; i++) {
626  if (elements[i] == NULL) {
627  null_number++;
628  } else {
629  NonNull = i;
630  }
631  }
632  if (null_number == vectorWidth) {
633  // All of elements are NULLs
634  return NULL;
635  }
636  if ((undef == false) && (null_number != 0)) {
637  // We don't want NULLs in chain, but we have them
638  return NULL;
639  }
640 
641  // Compare elements of vector
642  for (int i = 0; i < vectorWidth; i++) {
643  if (elements[i] == NULL) {
644  continue;
645  }
646 
647  std::vector<llvm::PHINode *> seenPhi0;
648  std::vector<llvm::PHINode *> seenPhi1;
649  if (lValuesAreEqual(elements[NonNull], elements[i], seenPhi0, seenPhi1) == false) {
650  return NULL;
651  }
652  }
653  return elements[NonNull];
654  }
655 
656  // Catch a pattern of broadcast implemented as InsertElement + Shuffle:
657  // %broadcast_init.0 = insertelement <4 x i32> undef, i32 %val, i32 0
658  // %broadcast.1 = shufflevector <4 x i32> %smear.0, <4 x i32> undef,
659  // <4 x i32> zeroinitializer
660  // Or:
661  // %gep_ptr2int_broadcast_init = insertelement <8 x i64> undef, i64 %gep_ptr2int, i32 0
662  // %0 = add <8 x i64> %gep_ptr2int_broadcast_init,
663  // <i64 4, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef>
664  // %gep_offset = shufflevector <8 x i64> %0, <8 x i64> undef, <8 x i32> zeroinitializer
665  else if (llvm::ShuffleVectorInst *shuf = llvm::dyn_cast<llvm::ShuffleVectorInst>(inst)) {
666  llvm::Value *indices = shuf->getOperand(2);
667  if (llvm::isa<llvm::ConstantAggregateZero>(indices)) {
668  llvm::Value *op = shuf->getOperand(0);
669  llvm::InsertElementInst *ie = llvm::dyn_cast<llvm::InsertElementInst>(op);
670  if (ie == NULL && searchFirstUndef) {
671  // Trying to recognize 2nd pattern
672  llvm::BinaryOperator *bop = llvm::dyn_cast<llvm::BinaryOperator>(op);
673  if (bop != NULL && bop->getOpcode() == llvm::Instruction::Add) {
674  if (lIsFirstElementConstVector(bop->getOperand(1))) {
675  ie = llvm::dyn_cast<llvm::InsertElementInst>(bop->getOperand(0));
676  }
677  }
678  }
679  if (ie != NULL && llvm::isa<llvm::UndefValue>(ie->getOperand(0))) {
680  llvm::ConstantInt *ci = llvm::dyn_cast<llvm::ConstantInt>(ie->getOperand(2));
681  if (ci->isZero()) {
682  return ie->getOperand(1);
683  }
684  }
685  }
686  }
687  return NULL;
688 }
689 
690 bool LLVMExtractVectorInts(llvm::Value *v, int64_t ret[], int *nElts) {
691  // Make sure we do in fact have a vector of integer values here
692  llvm::VectorType *vt = llvm::dyn_cast<llvm::VectorType>(v->getType());
693  Assert(vt != NULL);
694  Assert(llvm::isa<llvm::IntegerType>(vt->getElementType()));
695 
696  *nElts = (int)vt->getNumElements();
697 
698  if (llvm::isa<llvm::ConstantAggregateZero>(v)) {
699  for (int i = 0; i < (int)vt->getNumElements(); ++i)
700  ret[i] = 0;
701  return true;
702  }
703 
704  llvm::ConstantDataVector *cv = llvm::dyn_cast<llvm::ConstantDataVector>(v);
705  if (cv == NULL)
706  return false;
707 
708  for (int i = 0; i < (int)cv->getNumElements(); ++i)
709  ret[i] = cv->getElementAsInteger(i);
710  return true;
711 }
712 
713 static bool lVectorValuesAllEqual(llvm::Value *v, int vectorLength, std::vector<llvm::PHINode *> &seenPhis,
714  llvm::Value **splatValue = NULL);
715 
716 /** This function checks to see if the given (scalar or vector) value is an
717  exact multiple of baseValue. It returns true if so, and false if not
718  (or if it's not able to determine if it is). Any vector value passed
719  in is required to have the same value in all elements (so that we can
720  just check the first element to be a multiple of the given value.)
721  */
722 static bool lIsExactMultiple(llvm::Value *val, int baseValue, int vectorLength,
723  std::vector<llvm::PHINode *> &seenPhis) {
724  if (llvm::isa<llvm::VectorType>(val->getType()) == false) {
725  // If we've worked down to a constant int, then the moment of truth
726  // has arrived...
727  llvm::ConstantInt *ci = llvm::dyn_cast<llvm::ConstantInt>(val);
728  if (ci != NULL)
729  return (ci->getZExtValue() % baseValue) == 0;
730  } else
732 
733  if (llvm::isa<llvm::InsertElementInst>(val) || llvm::isa<llvm::ShuffleVectorInst>(val)) {
734  llvm::Value *element = LLVMFlattenInsertChain(val, g->target->getVectorWidth());
735  // We just need to check the scalar first value, since we know that
736  // all elements are equal
737  return lIsExactMultiple(element, baseValue, vectorLength, seenPhis);
738  }
739 
740  llvm::PHINode *phi = llvm::dyn_cast<llvm::PHINode>(val);
741  if (phi != NULL) {
742  for (unsigned int i = 0; i < seenPhis.size(); ++i)
743  if (phi == seenPhis[i])
744  return true;
745 
746  seenPhis.push_back(phi);
747  unsigned int numIncoming = phi->getNumIncomingValues();
748 
749  // Check all of the incoming values: if all of them pass, then
750  // we're good.
751  for (unsigned int i = 0; i < numIncoming; ++i) {
752  llvm::Value *incoming = phi->getIncomingValue(i);
753  bool mult = lIsExactMultiple(incoming, baseValue, vectorLength, seenPhis);
754  if (mult == false) {
755  seenPhis.pop_back();
756  return false;
757  }
758  }
759  seenPhis.pop_back();
760  return true;
761  }
762 
763  llvm::BinaryOperator *bop = llvm::dyn_cast<llvm::BinaryOperator>(val);
764  if (bop != NULL && bop->getOpcode() == llvm::Instruction::Add) {
765  llvm::Value *op0 = bop->getOperand(0);
766  llvm::Value *op1 = bop->getOperand(1);
767 
768  bool be0 = lIsExactMultiple(op0, baseValue, vectorLength, seenPhis);
769  bool be1 = lIsExactMultiple(op1, baseValue, vectorLength, seenPhis);
770  return (be0 && be1);
771  }
772  // FIXME: mul? casts? ... ?
773 
774  return false;
775 }
776 
777 /** Returns the next power of two greater than or equal to the given
778  value. */
779 static int lRoundUpPow2(int v) {
780  v--;
781  v |= v >> 1;
782  v |= v >> 2;
783  v |= v >> 4;
784  v |= v >> 8;
785  v |= v >> 16;
786  return v + 1;
787 }
788 
789 /** Try to determine if all of the elements of the given vector value have
790  the same value when divided by the given baseValue. The function
791  returns true if this can be determined to be the case, and false
792  otherwise. (This function may fail to identify some cases where it
793  does in fact have this property, but should never report a given value
794  as being a multiple if it isn't!)
795  */
796 static bool lAllDivBaseEqual(llvm::Value *val, int64_t baseValue, int vectorLength,
797  std::vector<llvm::PHINode *> &seenPhis, bool &canAdd) {
798  Assert(llvm::isa<llvm::VectorType>(val->getType()));
799  // Make sure the base value is a positive power of 2
800  Assert(baseValue > 0 && (baseValue & (baseValue - 1)) == 0);
801 
802  // The easy case
803  if (lVectorValuesAllEqual(val, vectorLength, seenPhis))
804  return true;
805 
806  int64_t vecVals[ISPC_MAX_NVEC];
807  int nElts;
808  if (llvm::isa<llvm::VectorType>(val->getType()) && LLVMExtractVectorInts(val, vecVals, &nElts)) {
809  // If we have a vector of compile-time constant integer values,
810  // then go ahead and check them directly..
811  int64_t firstDiv = vecVals[0] / baseValue;
812  for (int i = 1; i < nElts; ++i)
813  if ((vecVals[i] / baseValue) != firstDiv)
814  return false;
815 
816  return true;
817  }
818 
819  llvm::PHINode *phi = llvm::dyn_cast<llvm::PHINode>(val);
820  if (phi != NULL) {
821  for (unsigned int i = 0; i < seenPhis.size(); ++i)
822  if (phi == seenPhis[i])
823  return true;
824 
825  seenPhis.push_back(phi);
826  unsigned int numIncoming = phi->getNumIncomingValues();
827 
828  // Check all of the incoming values: if all of them pass, then
829  // we're good.
830  for (unsigned int i = 0; i < numIncoming; ++i) {
831  llvm::Value *incoming = phi->getIncomingValue(i);
832  bool ca = canAdd;
833  bool mult = lAllDivBaseEqual(incoming, baseValue, vectorLength, seenPhis, ca);
834  if (mult == false) {
835  seenPhis.pop_back();
836  return false;
837  }
838  }
839  seenPhis.pop_back();
840  return true;
841  }
842 
843  llvm::BinaryOperator *bop = llvm::dyn_cast<llvm::BinaryOperator>(val);
844  if (bop != NULL && bop->getOpcode() == llvm::Instruction::Add && canAdd == true) {
845  llvm::Value *op0 = bop->getOperand(0);
846  llvm::Value *op1 = bop->getOperand(1);
847 
848  // Otherwise we're only going to worry about the following case,
849  // which comes up often when looping over SOA data:
850  // ashr %val, <constant shift>
851  // where %val = add %smear, <0,1,2,3...>
852  // and where the maximum of the <0,...> vector in the add is less than
853  // 1<<(constant shift),
854  // and where %smear is a smear of a value that is a multiple of
855  // baseValue.
856 
857  int64_t addConstants[ISPC_MAX_NVEC];
858  if (LLVMExtractVectorInts(op1, addConstants, &nElts) == false)
859  return false;
860  Assert(nElts == vectorLength);
861 
862  // Do all of them give the same value when divided by baseValue?
863  int64_t firstConstDiv = addConstants[0] / baseValue;
864  for (int i = 1; i < vectorLength; ++i)
865  if ((addConstants[i] / baseValue) != firstConstDiv)
866  return false;
867 
868  if (lVectorValuesAllEqual(op0, vectorLength, seenPhis) == false)
869  return false;
870 
871  // Note that canAdd is a reference parameter; setting this ensures
872  // that we don't allow multiple adds in other parts of the chain of
873  // dependent values from here.
874  canAdd = false;
875 
876  // Now we need to figure out the required alignment (in numbers of
877  // elements of the underlying type being indexed) of the value to
878  // which these integer addConstant[] values are being added to. We
879  // know that we have addConstant[] values that all give the same
880  // value when divided by baseValue, but we may need a less-strict
881  // alignment than baseValue depending on the actual values.
882  //
883  // As an example, consider a case where the baseValue alignment is
884  // 16, but the addConstants here are <0,1,2,3>. In that case, the
885  // value to which addConstants is added to only needs to be a
886  // multiple of 4. Conversely, if addConstants are <4,5,6,7>, then
887  // we need a multiple of 8 to ensure that the final added result
888  // will still have the same value for all vector elements when
889  // divided by baseValue.
890  //
891  // All that said, here we need to find the maximum value of any of
892  // the addConstants[], mod baseValue. If we round that up to the
893  // next power of 2, we'll have a value that will be no greater than
894  // baseValue and sometimes less.
895  int maxMod = int(addConstants[0] % baseValue);
896  for (int i = 1; i < vectorLength; ++i)
897  maxMod = std::max(maxMod, int(addConstants[i] % baseValue));
898  int requiredAlignment = lRoundUpPow2(maxMod);
899 
900  std::vector<llvm::PHINode *> seenPhisEEM;
901  return lIsExactMultiple(op0, requiredAlignment, vectorLength, seenPhisEEM);
902  }
903  // TODO: could handle mul by a vector of equal constant integer values
904  // and the like here and adjust the 'baseValue' value when it evenly
905  // divides, but unclear if it's worthwhile...
906 
907  return false;
908 }
909 
910 /** Given a vector shift right of some value by some amount, try to
911  determine if all of the elements of the final result have the same
912  value (i.e. whether the high bits are all equal, disregarding the low
913  bits that are shifted out.) Returns true if so, and false otherwise.
914  */
915 static bool lVectorShiftRightAllEqual(llvm::Value *val, llvm::Value *shift, int vectorLength) {
916  // Are we shifting all elements by a compile-time constant amount? If
917  // not, give up.
918  int64_t shiftAmount[ISPC_MAX_NVEC];
919  int nElts;
920  if (LLVMExtractVectorInts(shift, shiftAmount, &nElts) == false)
921  return false;
922  Assert(nElts == vectorLength);
923 
924  // Is it the same amount for all elements?
925  for (int i = 0; i < vectorLength; ++i)
926  if (shiftAmount[i] != shiftAmount[0])
927  return false;
928 
929  // Now see if the value divided by (1 << shift) can be determined to
930  // have the same value for all vector elements.
931  int pow2 = 1 << shiftAmount[0];
932  bool canAdd = true;
933  std::vector<llvm::PHINode *> seenPhis;
934  bool eq = lAllDivBaseEqual(val, pow2, vectorLength, seenPhis, canAdd);
935 #if 0
936  fprintf(stderr, "check all div base equal:\n");
937  LLVMDumpValue(shift);
938  LLVMDumpValue(val);
939  fprintf(stderr, "----> %s\n\n", eq ? "true" : "false");
940 #endif
941  return eq;
942 }
943 
944 static bool lVectorValuesAllEqual(llvm::Value *v, int vectorLength, std::vector<llvm::PHINode *> &seenPhis,
945  llvm::Value **splatValue) {
946  if (vectorLength == 1)
947  return true;
948 
949  if (llvm::isa<llvm::ConstantAggregateZero>(v)) {
950  if (splatValue) {
951  llvm::ConstantAggregateZero *caz = llvm::dyn_cast<llvm::ConstantAggregateZero>(v);
952  *splatValue = caz->getSequentialElement();
953  }
954  return true;
955  }
956 
957  llvm::ConstantVector *cv = llvm::dyn_cast<llvm::ConstantVector>(v);
958  if (cv != NULL) {
959  llvm::Value *splat = cv->getSplatValue();
960  if (splat != NULL && splatValue) {
961  *splatValue = splat;
962  }
963  return (splat != NULL);
964  }
965 
966  llvm::ConstantDataVector *cdv = llvm::dyn_cast<llvm::ConstantDataVector>(v);
967  if (cdv != NULL) {
968  llvm::Value *splat = cdv->getSplatValue();
969  if (splat != NULL && splatValue) {
970  *splatValue = splat;
971  }
972  return (splat != NULL);
973  }
974 
975  llvm::BinaryOperator *bop = llvm::dyn_cast<llvm::BinaryOperator>(v);
976  if (bop != NULL) {
977  // Easy case: both operands are all equal -> return true
978  if (lVectorValuesAllEqual(bop->getOperand(0), vectorLength, seenPhis) &&
979  lVectorValuesAllEqual(bop->getOperand(1), vectorLength, seenPhis))
980  return true;
981 
982  // If it's a shift, take a special path that tries to check if the
983  // high (surviving) bits of the values are equal.
984  if (bop->getOpcode() == llvm::Instruction::AShr || bop->getOpcode() == llvm::Instruction::LShr)
985  return lVectorShiftRightAllEqual(bop->getOperand(0), bop->getOperand(1), vectorLength);
986 
987  return false;
988  }
989 
990  llvm::CastInst *cast = llvm::dyn_cast<llvm::CastInst>(v);
991  if (cast != NULL)
992  return lVectorValuesAllEqual(cast->getOperand(0), vectorLength, seenPhis);
993 
994  llvm::InsertElementInst *ie = llvm::dyn_cast<llvm::InsertElementInst>(v);
995  if (ie != NULL) {
996  return (LLVMFlattenInsertChain(ie, vectorLength) != NULL);
997  }
998 
999  llvm::PHINode *phi = llvm::dyn_cast<llvm::PHINode>(v);
1000  if (phi) {
1001  for (unsigned int i = 0; i < seenPhis.size(); ++i)
1002  if (seenPhis[i] == phi)
1003  return true;
1004 
1005  seenPhis.push_back(phi);
1006 
1007  unsigned int numIncoming = phi->getNumIncomingValues();
1008  // Check all of the incoming values: if all of them are all equal,
1009  // then we're good.
1010  for (unsigned int i = 0; i < numIncoming; ++i) {
1011  if (!lVectorValuesAllEqual(phi->getIncomingValue(i), vectorLength, seenPhis)) {
1012  seenPhis.pop_back();
1013  return false;
1014  }
1015  }
1016 
1017  seenPhis.pop_back();
1018  return true;
1019  }
1020 
1021  if (llvm::isa<llvm::UndefValue>(v))
1022  // ?
1023  return false;
1024 
1025  Assert(!llvm::isa<llvm::Constant>(v));
1026 
1027  if (llvm::isa<llvm::CallInst>(v) || llvm::isa<llvm::LoadInst>(v) || !llvm::isa<llvm::Instruction>(v))
1028  return false;
1029 
1030  llvm::ShuffleVectorInst *shuffle = llvm::dyn_cast<llvm::ShuffleVectorInst>(v);
1031  if (shuffle != NULL) {
1032  llvm::Value *indices = shuffle->getOperand(2);
1033  if (lVectorValuesAllEqual(indices, vectorLength, seenPhis))
1034  // The easy case--just a smear of the same element across the
1035  // whole vector.
1036  return true;
1037 
1038  // TODO: handle more general cases?
1039  return false;
1040  }
1041 
1042 #if 0
1043  fprintf(stderr, "all equal: ");
1044  v->dump();
1045  fprintf(stderr, "\n");
1046  llvm::Instruction *inst = llvm::dyn_cast<llvm::Instruction>(v);
1047  if (inst) {
1048  inst->getParent()->dump();
1049  fprintf(stderr, "\n");
1050  fprintf(stderr, "\n");
1051  }
1052 #endif
1053 
1054  return false;
1055 }
1056 
1057 /** Tests to see if all of the elements of the vector in the 'v' parameter
1058  are equal. This is a conservative test and may return false for arrays
1059  where the values are actually all equal.
1060 */
1061 bool LLVMVectorValuesAllEqual(llvm::Value *v, llvm::Value **splat) {
1062  llvm::VectorType *vt = llvm::dyn_cast<llvm::VectorType>(v->getType());
1063  Assert(vt != NULL);
1064  int vectorLength = vt->getNumElements();
1065 
1066  std::vector<llvm::PHINode *> seenPhis;
1067  bool equal = lVectorValuesAllEqual(v, vectorLength, seenPhis, splat);
1068 
1069  Debug(SourcePos(), "LLVMVectorValuesAllEqual(%s) -> %s.", v->getName().str().c_str(), equal ? "true" : "false");
1070 #ifndef ISPC_NO_DUMPS
1071  if (g->debugPrint)
1072  LLVMDumpValue(v);
1073 #endif
1074 
1075  return equal;
1076 }
1077 
1078 static bool lVectorIsLinear(llvm::Value *v, int vectorLength, int stride, std::vector<llvm::PHINode *> &seenPhis);
1079 
1080 /** Given a vector of compile-time constant integer values, test to see if
1081  they are a linear sequence of constant integers starting from an
1082  arbirary value but then having a step of value "stride" between
1083  elements.
1084  */
1085 static bool lVectorIsLinearConstantInts(llvm::ConstantDataVector *cv, int vectorLength, int stride) {
1086  // Flatten the vector out into the elements array
1087  llvm::SmallVector<llvm::Constant *, ISPC_MAX_NVEC> elements;
1088  for (int i = 0; i < (int)cv->getNumElements(); ++i)
1089  elements.push_back(cv->getElementAsConstant(i));
1090  Assert((int)elements.size() == vectorLength);
1091 
1092  llvm::ConstantInt *ci = llvm::dyn_cast<llvm::ConstantInt>(elements[0]);
1093  if (ci == NULL)
1094  // Not a vector of integers
1095  return false;
1096 
1097  int64_t prevVal = ci->getSExtValue();
1098 
1099  // For each element in the array, see if it is both a ConstantInt and
1100  // if the difference between it and the value of the previous element
1101  // is stride. If not, fail.
1102  for (int i = 1; i < vectorLength; ++i) {
1103  ci = llvm::dyn_cast<llvm::ConstantInt>(elements[i]);
1104  if (ci == NULL)
1105  return false;
1106 
1107  int64_t nextVal = ci->getSExtValue();
1108  if (prevVal + stride != nextVal)
1109  return false;
1110 
1111  prevVal = nextVal;
1112  }
1113  return true;
1114 }
1115 
1116 /** Checks to see if (op0 * op1) is a linear vector where the result is a
1117  vector with values that increase by stride.
1118  */
1119 static bool lCheckMulForLinear(llvm::Value *op0, llvm::Value *op1, int vectorLength, int stride,
1120  std::vector<llvm::PHINode *> &seenPhis) {
1121  // Is the first operand a constant integer value splatted across all of
1122  // the lanes?
1123  llvm::ConstantDataVector *cv = llvm::dyn_cast<llvm::ConstantDataVector>(op0);
1124  if (cv == NULL)
1125  return false;
1126 
1127  llvm::Constant *csplat = cv->getSplatValue();
1128  if (csplat == NULL)
1129  return false;
1130 
1131  llvm::ConstantInt *splat = llvm::dyn_cast<llvm::ConstantInt>(csplat);
1132  if (splat == NULL)
1133  return false;
1134 
1135  // If the splat value doesn't evenly divide the stride we're looking
1136  // for, there's no way that we can get the linear sequence we're
1137  // looking or.
1138  int64_t splatVal = splat->getSExtValue();
1139  if (splatVal == 0 || splatVal > stride || (stride % splatVal) != 0)
1140  return false;
1141 
1142  // Check to see if the other operand is a linear vector with stride
1143  // given by stride/splatVal.
1144  return lVectorIsLinear(op1, vectorLength, (int)(stride / splatVal), seenPhis);
1145 }
1146 
1147 /** Checks to see if (op0 << op1) is a linear vector where the result is a
1148  vector with values that increase by stride.
1149  */
1150 static bool lCheckShlForLinear(llvm::Value *op0, llvm::Value *op1, int vectorLength, int stride,
1151  std::vector<llvm::PHINode *> &seenPhis) {
1152  // Is the second operand a constant integer value splatted across all of
1153  // the lanes?
1154  llvm::ConstantDataVector *cv = llvm::dyn_cast<llvm::ConstantDataVector>(op1);
1155  if (cv == NULL)
1156  return false;
1157 
1158  llvm::Constant *csplat = cv->getSplatValue();
1159  if (csplat == NULL)
1160  return false;
1161 
1162  llvm::ConstantInt *splat = llvm::dyn_cast<llvm::ConstantInt>(csplat);
1163  if (splat == NULL)
1164  return false;
1165 
1166  // If (1 << the splat value) doesn't evenly divide the stride we're
1167  // looking for, there's no way that we can get the linear sequence
1168  // we're looking or.
1169  int64_t equivalentMul = (1LL << splat->getSExtValue());
1170  if (equivalentMul > stride || (stride % equivalentMul) != 0)
1171  return false;
1172 
1173  // Check to see if the first operand is a linear vector with stride
1174  // given by stride/splatVal.
1175  return lVectorIsLinear(op0, vectorLength, (int)(stride / equivalentMul), seenPhis);
1176 }
1177 
1178 /** Given (op0 AND op1), try and see if we can determine if the result is a
1179  linear sequence with a step of "stride" between values. Returns true
1180  if so and false otherwise. This pattern comes up when accessing SOA
1181  data.
1182  */
1183 static bool lCheckAndForLinear(llvm::Value *op0, llvm::Value *op1, int vectorLength, int stride,
1184  std::vector<llvm::PHINode *> &seenPhis) {
1185  // Require op1 to be a compile-time constant
1186  int64_t maskValue[ISPC_MAX_NVEC];
1187  int nElts;
1188  if (LLVMExtractVectorInts(op1, maskValue, &nElts) == false)
1189  return false;
1190  Assert(nElts == vectorLength);
1191 
1192  // Is op1 a smear of the same value across all lanes? Give up if not.
1193  for (int i = 1; i < vectorLength; ++i)
1194  if (maskValue[i] != maskValue[0])
1195  return false;
1196 
1197  // If the op1 value isn't power of 2 minus one, then also give up.
1198  int64_t maskPlusOne = maskValue[0] + 1;
1199  bool isPowTwo = (maskPlusOne & (maskPlusOne - 1)) == 0;
1200  if (isPowTwo == false)
1201  return false;
1202 
1203  // The case we'll covert here is op0 being a linear vector with desired
1204  // stride, and where all of the values of op0, when divided by
1205  // maskPlusOne, have the same value.
1206  if (lVectorIsLinear(op0, vectorLength, stride, seenPhis) == false)
1207  return false;
1208 
1209  bool canAdd = true;
1210  bool isMult = lAllDivBaseEqual(op0, maskPlusOne, vectorLength, seenPhis, canAdd);
1211  return isMult;
1212 }
1213 
1214 static bool lVectorIsLinear(llvm::Value *v, int vectorLength, int stride, std::vector<llvm::PHINode *> &seenPhis) {
1215  // First try the easy case: if the values are all just constant
1216  // integers and have the expected stride between them, then we're done.
1217  llvm::ConstantDataVector *cv = llvm::dyn_cast<llvm::ConstantDataVector>(v);
1218  if (cv != NULL)
1219  return lVectorIsLinearConstantInts(cv, vectorLength, stride);
1220 
1221  llvm::BinaryOperator *bop = llvm::dyn_cast<llvm::BinaryOperator>(v);
1222  if (bop != NULL) {
1223  // FIXME: is it right to pass the seenPhis to the all equal check as well??
1224  llvm::Value *op0 = bop->getOperand(0), *op1 = bop->getOperand(1);
1225 
1226  if (bop->getOpcode() == llvm::Instruction::Add) {
1227  // There are two cases to check if we have an add:
1228  //
1229  // programIndex + unif -> ascending linear seqeuence
1230  // unif + programIndex -> ascending linear sequence
1231  bool l0 = lVectorIsLinear(op0, vectorLength, stride, seenPhis);
1232  bool e1 = lVectorValuesAllEqual(op1, vectorLength, seenPhis);
1233  if (l0 && e1)
1234  return true;
1235 
1236  bool e0 = lVectorValuesAllEqual(op0, vectorLength, seenPhis);
1237  bool l1 = lVectorIsLinear(op1, vectorLength, stride, seenPhis);
1238  return (e0 && l1);
1239  } else if (bop->getOpcode() == llvm::Instruction::Sub)
1240  // For subtraction, we only match:
1241  // programIndex - unif -> ascending linear seqeuence
1242  return (lVectorIsLinear(bop->getOperand(0), vectorLength, stride, seenPhis) &&
1243  lVectorValuesAllEqual(bop->getOperand(1), vectorLength, seenPhis));
1244  else if (bop->getOpcode() == llvm::Instruction::Mul) {
1245  // Multiplies are a bit trickier, so are handled in a separate
1246  // function.
1247  bool m0 = lCheckMulForLinear(op0, op1, vectorLength, stride, seenPhis);
1248  if (m0)
1249  return true;
1250  bool m1 = lCheckMulForLinear(op1, op0, vectorLength, stride, seenPhis);
1251  return m1;
1252  } else if (bop->getOpcode() == llvm::Instruction::Shl) {
1253  // Sometimes multiplies come in as shift lefts (especially in
1254  // LLVM 3.4+).
1255  bool linear = lCheckShlForLinear(op0, op1, vectorLength, stride, seenPhis);
1256  return linear;
1257  } else if (bop->getOpcode() == llvm::Instruction::And) {
1258  // Special case for some AND-related patterns that come up when
1259  // looping over SOA data
1260  bool linear = lCheckAndForLinear(op0, op1, vectorLength, stride, seenPhis);
1261  return linear;
1262  } else
1263  return false;
1264  }
1265 
1266  llvm::CastInst *ci = llvm::dyn_cast<llvm::CastInst>(v);
1267  if (ci != NULL)
1268  return lVectorIsLinear(ci->getOperand(0), vectorLength, stride, seenPhis);
1269 
1270  if (llvm::isa<llvm::CallInst>(v) || llvm::isa<llvm::LoadInst>(v))
1271  return false;
1272 
1273  llvm::PHINode *phi = llvm::dyn_cast<llvm::PHINode>(v);
1274  if (phi != NULL) {
1275  for (unsigned int i = 0; i < seenPhis.size(); ++i)
1276  if (seenPhis[i] == phi)
1277  return true;
1278 
1279  seenPhis.push_back(phi);
1280 
1281  unsigned int numIncoming = phi->getNumIncomingValues();
1282  // Check all of the incoming values: if all of them are all equal,
1283  // then we're good.
1284  for (unsigned int i = 0; i < numIncoming; ++i) {
1285  if (!lVectorIsLinear(phi->getIncomingValue(i), vectorLength, stride, seenPhis)) {
1286  seenPhis.pop_back();
1287  return false;
1288  }
1289  }
1290 
1291  seenPhis.pop_back();
1292  return true;
1293  }
1294 
1295  // TODO: is any reason to worry about these?
1296  if (llvm::isa<llvm::InsertElementInst>(v))
1297  return false;
1298 
1299  // TODO: we could also handle shuffles, but we haven't yet seen any
1300  // cases where doing so would detect cases where actually have a linear
1301  // vector.
1302  llvm::ShuffleVectorInst *shuffle = llvm::dyn_cast<llvm::ShuffleVectorInst>(v);
1303  if (shuffle != NULL)
1304  return false;
1305 
1306 #if 0
1307  fprintf(stderr, "linear check: ");
1308  v->dump();
1309  fprintf(stderr, "\n");
1310  llvm::Instruction *inst = llvm::dyn_cast<llvm::Instruction>(v);
1311  if (inst) {
1312  inst->getParent()->dump();
1313  fprintf(stderr, "\n");
1314  fprintf(stderr, "\n");
1315  }
1316 #endif
1317 
1318  return false;
1319 }
1320 
1321 /** Given vector of integer-typed values, see if the elements of the array
1322  have a step of 'stride' between their values. This function tries to
1323  handle as many possibilities as possible, including things like all
1324  elements equal to some non-constant value plus an integer offset, etc.
1325 */
1326 bool LLVMVectorIsLinear(llvm::Value *v, int stride) {
1327  llvm::VectorType *vt = llvm::dyn_cast<llvm::VectorType>(v->getType());
1328  Assert(vt != NULL);
1329  int vectorLength = vt->getNumElements();
1330 
1331  std::vector<llvm::PHINode *> seenPhis;
1332  bool linear = lVectorIsLinear(v, vectorLength, stride, seenPhis);
1333  Debug(SourcePos(), "LLVMVectorIsLinear(%s) -> %s.", v->getName().str().c_str(), linear ? "true" : "false");
1334 #ifndef ISPC_NO_DUMPS
1335  if (g->debugPrint)
1336  LLVMDumpValue(v);
1337 #endif
1338 
1339  return linear;
1340 }
1341 
1342 #ifndef ISPC_NO_DUMPS
1343 static void lDumpValue(llvm::Value *v, std::set<llvm::Value *> &done) {
1344  if (done.find(v) != done.end())
1345  return;
1346 
1347  llvm::Instruction *inst = llvm::dyn_cast<llvm::Instruction>(v);
1348  if (done.size() > 0 && inst == NULL)
1349  return;
1350 
1351  fprintf(stderr, " ");
1352  v->dump();
1353  done.insert(v);
1354 
1355  if (inst == NULL)
1356  return;
1357 
1358  for (unsigned i = 0; i < inst->getNumOperands(); ++i)
1359  lDumpValue(inst->getOperand(i), done);
1360 }
1361 
1362 void LLVMDumpValue(llvm::Value *v) {
1363  std::set<llvm::Value *> done;
1364  lDumpValue(v, done);
1365  fprintf(stderr, "----\n");
1366 }
1367 #endif
1368 
1369 static llvm::Value *lExtractFirstVectorElement(llvm::Value *v, std::map<llvm::PHINode *, llvm::PHINode *> &phiMap) {
1370  llvm::VectorType *vt = llvm::dyn_cast<llvm::VectorType>(v->getType());
1371  Assert(vt != NULL);
1372 
1373  // First, handle various constant types; do the extraction manually, as
1374  // appropriate.
1375  if (llvm::isa<llvm::ConstantAggregateZero>(v) == true) {
1376  return llvm::Constant::getNullValue(vt->getElementType());
1377  }
1378  if (llvm::ConstantVector *cv = llvm::dyn_cast<llvm::ConstantVector>(v)) {
1379  return cv->getOperand(0);
1380  }
1381  if (llvm::ConstantDataVector *cdv = llvm::dyn_cast<llvm::ConstantDataVector>(v))
1382  return cdv->getElementAsConstant(0);
1383 
1384  // Otherwise, all that we should have at this point is an instruction
1385  // of some sort
1386  Assert(llvm::isa<llvm::Constant>(v) == false);
1387  Assert(llvm::isa<llvm::Instruction>(v) == true);
1388 
1389  std::string newName = v->getName().str() + std::string(".elt0");
1390 
1391  // Rewrite regular binary operators and casts to the scalarized
1392  // equivalent.
1393  llvm::BinaryOperator *bop = llvm::dyn_cast<llvm::BinaryOperator>(v);
1394  if (bop != NULL) {
1395  llvm::Value *v0 = lExtractFirstVectorElement(bop->getOperand(0), phiMap);
1396  llvm::Value *v1 = lExtractFirstVectorElement(bop->getOperand(1), phiMap);
1397  Assert(v0 != NULL);
1398  Assert(v1 != NULL);
1399  // Note that the new binary operator is inserted immediately before
1400  // the previous vector one
1401  return llvm::BinaryOperator::Create(bop->getOpcode(), v0, v1, newName, bop);
1402  }
1403 
1404  llvm::CastInst *cast = llvm::dyn_cast<llvm::CastInst>(v);
1405  if (cast != NULL) {
1406  llvm::Value *v = lExtractFirstVectorElement(cast->getOperand(0), phiMap);
1407  // Similarly, the equivalent scalar cast instruction goes right
1408  // before the vector cast
1409  return llvm::CastInst::Create(cast->getOpcode(), v, vt->getElementType(), newName, cast);
1410  }
1411 
1412  llvm::PHINode *phi = llvm::dyn_cast<llvm::PHINode>(v);
1413  if (phi != NULL) {
1414  // For PHI notes, recursively scalarize them.
1415  if (phiMap.find(phi) != phiMap.end())
1416  return phiMap[phi];
1417 
1418  // We need to create the new scalar PHI node immediately, though,
1419  // and put it in the map<>, so that if we come back to this node
1420  // via a recursive lExtractFirstVectorElement() call, then we can
1421  // return the pointer and not get stuck in an infinite loop.
1422  //
1423  // The insertion point for the new phi node also has to be the
1424  // start of the bblock of the original phi node.
1425 
1426 #if ISPC_LLVM_VERSION <= ISPC_LLVM_3_7 /* 3.2, 3.3, 3.4, 3.5, 3.6, 3.7 */
1427  llvm::Instruction *phiInsertPos = phi->getParent()->begin();
1428 #else /* LLVM 3.8+ */
1429  llvm::Instruction *phiInsertPos = &*(phi->getParent()->begin());
1430 #endif
1431  llvm::PHINode *scalarPhi =
1432  llvm::PHINode::Create(vt->getElementType(), phi->getNumIncomingValues(), newName, phiInsertPos);
1433  phiMap[phi] = scalarPhi;
1434 
1435  for (unsigned i = 0; i < phi->getNumIncomingValues(); ++i) {
1436  llvm::Value *v = lExtractFirstVectorElement(phi->getIncomingValue(i), phiMap);
1437  scalarPhi->addIncoming(v, phi->getIncomingBlock(i));
1438  }
1439 
1440  return scalarPhi;
1441  }
1442 
1443  // We should consider "shuffle" case and "insertElement" case separately.
1444  // For example we can have shuffle(mul, undef, zero) but function
1445  // "LLVMFlattenInsertChain" can handle only case shuffle(insertElement, undef, zero).
1446  // Also if we have insertElement under shuffle we will handle it the next call of
1447  // "lExtractFirstVectorElement" function.
1448  if (llvm::isa<llvm::ShuffleVectorInst>(v)) {
1449  llvm::ShuffleVectorInst *shuf = llvm::dyn_cast<llvm::ShuffleVectorInst>(v);
1450  llvm::Value *indices = shuf->getOperand(2);
1451  if (llvm::isa<llvm::ConstantAggregateZero>(indices)) {
1452  return lExtractFirstVectorElement(shuf->getOperand(0), phiMap);
1453  }
1454  }
1455 
1456  // If we have a chain of insertelement instructions, then we can just
1457  // flatten them out and grab the value for the first one.
1458  if (llvm::isa<llvm::InsertElementInst>(v)) {
1459  return LLVMFlattenInsertChain(v, vt->getNumElements(), false);
1460  }
1461 
1462  // Worst case, for everything else, just do a regular extract element
1463  // instruction, which we insert immediately after the instruction we
1464  // have here.
1465  llvm::Instruction *insertAfter = llvm::dyn_cast<llvm::Instruction>(v);
1466  Assert(insertAfter != NULL);
1467  llvm::Instruction *ee = llvm::ExtractElementInst::Create(v, LLVMInt32(0), "first_elt", (llvm::Instruction *)NULL);
1468  ee->insertAfter(insertAfter);
1469  return ee;
1470 }
1471 
1472 llvm::Value *LLVMExtractFirstVectorElement(llvm::Value *v) {
1473  std::map<llvm::PHINode *, llvm::PHINode *> phiMap;
1474  llvm::Value *ret = lExtractFirstVectorElement(v, phiMap);
1475  return ret;
1476 }
1477 
1478 /** Given two vectors of the same type, concatenate them into a vector that
1479  has twice as many elements, where the first half has the elements from
1480  the first vector and the second half has the elements from the second
1481  vector.
1482  */
1483 llvm::Value *LLVMConcatVectors(llvm::Value *v1, llvm::Value *v2, llvm::Instruction *insertBefore) {
1484  Assert(v1->getType() == v2->getType());
1485 
1486  llvm::VectorType *vt = llvm::dyn_cast<llvm::VectorType>(v1->getType());
1487  Assert(vt != NULL);
1488 
1489  int32_t identity[ISPC_MAX_NVEC];
1490  int resultSize = 2 * vt->getNumElements();
1491  Assert(resultSize <= ISPC_MAX_NVEC);
1492  for (int i = 0; i < resultSize; ++i)
1493  identity[i] = i;
1494 
1495  return LLVMShuffleVectors(v1, v2, identity, resultSize, insertBefore);
1496 }
1497 
1498 /** Shuffle two vectors together with a ShuffleVectorInst, returning a
1499  vector with shufSize elements, where the shuf[] array offsets are used
1500  to determine which element from the two given vectors is used for each
1501  result element. */
1502 llvm::Value *LLVMShuffleVectors(llvm::Value *v1, llvm::Value *v2, int32_t shuf[], int shufSize,
1503  llvm::Instruction *insertBefore) {
1504  std::vector<llvm::Constant *> shufVec;
1505  for (int i = 0; i < shufSize; ++i) {
1506  if (shuf[i] == -1)
1507  shufVec.push_back(llvm::UndefValue::get(LLVMTypes::Int32Type));
1508  else
1509  shufVec.push_back(LLVMInt32(shuf[i]));
1510  }
1511 
1512  llvm::ArrayRef<llvm::Constant *> aref(&shufVec[0], &shufVec[shufSize]);
1513  llvm::Value *vec = llvm::ConstantVector::get(aref);
1514 
1515  return new llvm::ShuffleVectorInst(v1, v2, vec, "shuffle", insertBefore);
1516 }
1517 
1518 const char *LLVMGetName(llvm::Value *v, const char *s) {
1519  if (v == NULL)
1520  return s;
1521  std::string ret = v->getName();
1522  ret += s;
1523  return strdup(ret.c_str());
1524 }
1525 
1526 const char *LLVMGetName(const char *op, llvm::Value *v1, llvm::Value *v2) {
1527  std::string r = op;
1528  r += "_";
1529  r += v1->getName().str();
1530  r += "_";
1531  r += v2->getName().str();
1532  return strdup(r.c_str());
1533 }
llvm::Constant * LLVMIntAsType(int64_t val, llvm::Type *type)
Definition: llvmutil.cpp:441
static llvm::Type * FloatType
Definition: llvmutil.h:79
static llvm::Type * Int32VectorPointerType
Definition: llvmutil.h:102
llvm::Constant * LLVMUInt64Vector(uint64_t ival)
Definition: llvmutil.cpp:383
static int lRoundUpPow2(int v)
Definition: llvmutil.cpp:779
static bool lIsFirstElementConstVector(llvm::Value *v)
Definition: llvmutil.cpp:547
llvm::Constant * LLVMUInt32Vector(uint32_t ival)
Definition: llvmutil.cpp:323
llvm::ConstantInt * LLVMUInt32(uint32_t ival)
Definition: llvmutil.cpp:232
static bool lVectorIsLinear(llvm::Value *v, int vectorLength, int stride, std::vector< llvm::PHINode *> &seenPhis)
Definition: llvmutil.cpp:1214
static void lDumpValue(llvm::Value *v, std::set< llvm::Value *> &done)
Definition: llvmutil.cpp:1343
llvm::Constant * LLVMBoolVector(bool b)
Definition: llvmutil.cpp:398
static llvm::Type * DoubleType
Definition: llvmutil.h:80
static bool lVectorShiftRightAllEqual(llvm::Value *val, llvm::Value *shift, int vectorLength)
Definition: llvmutil.cpp:915
llvm::ConstantInt * LLVMInt8(int8_t ival)
Definition: llvmutil.cpp:212
llvm::Constant * LLVMInt64Vector(int64_t ival)
Definition: llvmutil.cpp:368
llvm::ConstantInt * LLVMUInt8(uint8_t ival)
Definition: llvmutil.cpp:216
Structure that defines a compilation target.
Definition: ispc.h:171
Target * target
Definition: ispc.h:544
static llvm::VectorType * VoidPointerVectorType
Definition: llvmutil.h:107
static llvm::VectorType * BoolVectorType
Definition: llvmutil.h:91
static bool lIsExactMultiple(llvm::Value *val, int baseValue, int vectorLength, std::vector< llvm::PHINode *> &seenPhis)
Definition: llvmutil.cpp:722
bool LLVMExtractVectorInts(llvm::Value *v, int64_t ret[], int *nElts)
Definition: llvmutil.cpp:690
const char * LLVMGetName(llvm::Value *v, const char *s)
Definition: llvmutil.cpp:1518
llvm::Constant * LLVMMaskAllOn
Definition: llvmutil.cpp:92
static llvm::Type * BoolType
Definition: llvmutil.h:73
llvm::ConstantInt * LLVMInt16(int16_t ival)
Definition: llvmutil.cpp:220
static llvm::VectorType * Int32VectorType
Definition: llvmutil.h:95
int getMaskBitCount() const
Definition: ispc.h:285
void InitLLVMUtil(llvm::LLVMContext *ctx, Target &target)
Definition: llvmutil.cpp:95
static llvm::Type * FloatVectorPointerType
Definition: llvmutil.h:104
static llvm::Type * Int8PointerType
Definition: llvmutil.h:82
static llvm::Type * Int32PointerType
Definition: llvmutil.h:84
llvm::Constant * LLVMUIntAsType(uint64_t val, llvm::Type *type)
Definition: llvmutil.cpp:454
static llvm::Type * Int16VectorPointerType
Definition: llvmutil.h:101
static bool lValuesAreEqual(llvm::Value *v0, llvm::Value *v1, std::vector< llvm::PHINode *> &seenPhi0, std::vector< llvm::PHINode *> &seenPhi1)
Definition: llvmutil.cpp:472
static llvm::Type * Int16Type
Definition: llvmutil.h:76
static llvm::Type * DoubleVectorPointerType
Definition: llvmutil.h:105
#define Assert(expr)
Definition: ispc.h:163
static llvm::VectorType * Int1VectorType
Definition: llvmutil.h:92
llvm::Constant * LLVMFloatVector(float fval)
Definition: llvmutil.cpp:338
static llvm::Type * VoidType
Definition: llvmutil.h:70
llvm::Constant * LLVMDoubleVector(double dval)
Definition: llvmutil.cpp:353
static llvm::Type * Int8VectorPointerType
Definition: llvmutil.h:100
Globals * g
Definition: ispc.cpp:101
bool debugPrint
Definition: ispc.h:568
static llvm::VectorType * Int8VectorType
Definition: llvmutil.h:93
llvm::Constant * LLVMInt32Vector(int32_t ival)
Definition: llvmutil.cpp:308
static bool lAllDivBaseEqual(llvm::Value *val, int64_t baseValue, int vectorLength, std::vector< llvm::PHINode *> &seenPhis, bool &canAdd)
Definition: llvmutil.cpp:796
llvm::Constant * LLVMTrue
Definition: llvmutil.cpp:90
static llvm::VectorType * FloatVectorType
Definition: llvmutil.h:97
static llvm::Type * Int64Type
Definition: llvmutil.h:78
static llvm::Type * Int8Type
Definition: llvmutil.h:75
static llvm::VectorType * Int64VectorType
Definition: llvmutil.h:96
Header file with declarations for various LLVM utility stuff.
static llvm::Type * Int64PointerType
Definition: llvmutil.h:85
static bool lCheckShlForLinear(llvm::Value *op0, llvm::Value *op1, int vectorLength, int stride, std::vector< llvm::PHINode *> &seenPhis)
Definition: llvmutil.cpp:1150
llvm::ConstantInt * LLVMUInt64(uint64_t ival)
Definition: llvmutil.cpp:240
static bool lCheckMulForLinear(llvm::Value *op0, llvm::Value *op1, int vectorLength, int stride, std::vector< llvm::PHINode *> &seenPhis)
Definition: llvmutil.cpp:1119
static llvm::Type * FloatPointerType
Definition: llvmutil.h:86
llvm::Constant * LLVMMaskAllOff
Definition: llvmutil.cpp:93
Representation of a range of positions in a source file.
Definition: ispc.h:131
static llvm::Type * Int16PointerType
Definition: llvmutil.h:83
llvm::Value * LLVMConcatVectors(llvm::Value *v1, llvm::Value *v2, llvm::Instruction *insertBefore)
Definition: llvmutil.cpp:1483
llvm::ConstantInt * LLVMInt32(int32_t ival)
Definition: llvmutil.cpp:228
bool LLVMVectorIsLinear(llvm::Value *v, int stride)
Definition: llvmutil.cpp:1326
llvm::Constant * LLVMUInt8Vector(uint8_t ival)
Definition: llvmutil.cpp:263
static llvm::Type * PointerIntType
Definition: llvmutil.h:72
static llvm::PointerType * VoidPointerType
Definition: llvmutil.h:71
llvm::Constant * LLVMFloat(float fval)
Definition: llvmutil.cpp:244
int getVectorWidth() const
Definition: ispc.h:279
#define FATAL(message)
Definition: util.h:112
llvm::Constant * LLVMDouble(double dval)
Definition: llvmutil.cpp:246
void LLVMDumpValue(llvm::Value *v)
Definition: llvmutil.cpp:1362
static llvm::Type * Int64VectorPointerType
Definition: llvmutil.h:103
static llvm::Type * Int32Type
Definition: llvmutil.h:77
static llvm::Type * DoublePointerType
Definition: llvmutil.h:87
#define ISPC_MAX_NVEC
Definition: ispc.h:66
static llvm::Value * lExtractFirstVectorElement(llvm::Value *v, std::map< llvm::PHINode *, llvm::PHINode *> &phiMap)
Definition: llvmutil.cpp:1369
llvm::Constant * LLVMFalse
Definition: llvmutil.cpp:91
llvm::Value * LLVMExtractFirstVectorElement(llvm::Value *v)
Definition: llvmutil.cpp:1472
llvm::Constant * LLVMInt16Vector(int16_t ival)
Definition: llvmutil.cpp:278
static llvm::VectorType * MaskType
Definition: llvmutil.h:89
llvm::ConstantInt * LLVMUInt16(uint16_t ival)
Definition: llvmutil.cpp:224
bool LLVMVectorValuesAllEqual(llvm::Value *v, llvm::Value **splat)
Definition: llvmutil.cpp:1061
static llvm::VectorType * DoubleVectorType
Definition: llvmutil.h:98
llvm::Constant * LLVMInt8Vector(int8_t ival)
Definition: llvmutil.cpp:248
llvm::Constant * LLVMUInt16Vector(uint16_t ival)
Definition: llvmutil.cpp:293
static int64_t lGetIntValue(llvm::Value *offset)
Definition: llvmutil.cpp:538
llvm::ConstantInt * LLVMInt64(int64_t ival)
Definition: llvmutil.cpp:236
static llvm::VectorType * Int16VectorType
Definition: llvmutil.h:94
static bool lVectorValuesAllEqual(llvm::Value *v, int vectorLength, std::vector< llvm::PHINode *> &seenPhis, llvm::Value **splatValue=NULL)
Definition: llvmutil.cpp:944
bool is32Bit() const
Definition: ispc.h:269
llvm::LLVMContext * ctx
Definition: ispc.h:645
Main ispc.header file. Defines Target, Globals and Opt classes.
llvm::Value * LLVMShuffleVectors(llvm::Value *v1, llvm::Value *v2, int32_t shuf[], int shufSize, llvm::Instruction *insertBefore)
Definition: llvmutil.cpp:1502
static bool lCheckAndForLinear(llvm::Value *op0, llvm::Value *op1, int vectorLength, int stride, std::vector< llvm::PHINode *> &seenPhis)
Definition: llvmutil.cpp:1183
void Debug(SourcePos p, const char *format,...) PRINTF_FUNC
Definition: util.cpp:363
static bool lVectorIsLinearConstantInts(llvm::ConstantDataVector *cv, int vectorLength, int stride)
Definition: llvmutil.cpp:1085
File with declarations for classes related to type representation.
llvm::Value * LLVMFlattenInsertChain(llvm::Value *inst, int vectorWidth, bool compare, bool undef, bool searchFirstUndef)
Definition: llvmutil.cpp:565