Intel SPMD Program Compiler  1.10.0
llvmutil.cpp
Go to the documentation of this file.
1 /*
2  Copyright (c) 2010-2013, Intel Corporation
3  All rights reserved.
4 
5  Redistribution and use in source and binary forms, with or without
6  modification, are permitted provided that the following conditions are
7  met:
8 
9  * Redistributions of source code must retain the above copyright
10  notice, this list of conditions and the following disclaimer.
11 
12  * Redistributions in binary form must reproduce the above copyright
13  notice, this list of conditions and the following disclaimer in the
14  documentation and/or other materials provided with the distribution.
15 
16  * Neither the name of Intel Corporation nor the names of its
17  contributors may be used to endorse or promote products derived from
18  this software without specific prior written permission.
19 
20 
21  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
22  IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
23  TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
24  PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
25  OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
26  EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
27  PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
28  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
29  LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
30  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
31  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 */
33 
34 /** @file llvmutil.cpp
35  @brief Implementations of various LLVM utility types and classes.
36 */
37 
38 #include "llvmutil.h"
39 #include "ispc.h"
40 #include "type.h"
41 #if ISPC_LLVM_VERSION == ISPC_LLVM_3_2
42  #include <llvm/Instructions.h>
43  #include <llvm/BasicBlock.h>
44 #else
45  #include <llvm/IR/Instructions.h>
46  #include <llvm/IR/BasicBlock.h>
47 #endif
48 #include <set>
49 #include <map>
50 
51 llvm::Type *LLVMTypes::VoidType = NULL;
52 llvm::PointerType *LLVMTypes::VoidPointerType = NULL;
53 llvm::Type *LLVMTypes::PointerIntType = NULL;
54 llvm::Type *LLVMTypes::BoolType = NULL;
55 
56 llvm::Type *LLVMTypes::Int8Type = NULL;
57 llvm::Type *LLVMTypes::Int16Type = NULL;
58 llvm::Type *LLVMTypes::Int32Type = NULL;
59 llvm::Type *LLVMTypes::Int64Type = NULL;
60 llvm::Type *LLVMTypes::FloatType = NULL;
61 llvm::Type *LLVMTypes::DoubleType = NULL;
62 
63 llvm::Type *LLVMTypes::Int8PointerType = NULL;
64 llvm::Type *LLVMTypes::Int16PointerType = NULL;
65 llvm::Type *LLVMTypes::Int32PointerType = NULL;
66 llvm::Type *LLVMTypes::Int64PointerType = NULL;
67 llvm::Type *LLVMTypes::FloatPointerType = NULL;
68 llvm::Type *LLVMTypes::DoublePointerType = NULL;
69 
70 llvm::VectorType *LLVMTypes::MaskType = NULL;
71 llvm::VectorType *LLVMTypes::BoolVectorType = NULL;
72 
73 llvm::VectorType *LLVMTypes::Int1VectorType = NULL;
74 llvm::VectorType *LLVMTypes::Int8VectorType = NULL;
75 llvm::VectorType *LLVMTypes::Int16VectorType = NULL;
76 llvm::VectorType *LLVMTypes::Int32VectorType = NULL;
77 llvm::VectorType *LLVMTypes::Int64VectorType = NULL;
78 llvm::VectorType *LLVMTypes::FloatVectorType = NULL;
79 llvm::VectorType *LLVMTypes::DoubleVectorType = NULL;
80 
81 llvm::Type *LLVMTypes::Int8VectorPointerType = NULL;
82 llvm::Type *LLVMTypes::Int16VectorPointerType = NULL;
83 llvm::Type *LLVMTypes::Int32VectorPointerType = NULL;
84 llvm::Type *LLVMTypes::Int64VectorPointerType = NULL;
85 llvm::Type *LLVMTypes::FloatVectorPointerType = NULL;
86 llvm::Type *LLVMTypes::DoubleVectorPointerType = NULL;
87 
88 llvm::VectorType *LLVMTypes::VoidPointerVectorType = NULL;
89 
90 llvm::Constant *LLVMTrue = NULL;
91 llvm::Constant *LLVMFalse = NULL;
92 llvm::Constant *LLVMMaskAllOn = NULL;
93 llvm::Constant *LLVMMaskAllOff = NULL;
94 
95 
96 void
97 InitLLVMUtil(llvm::LLVMContext *ctx, Target& target) {
98  LLVMTypes::VoidType = llvm::Type::getVoidTy(*ctx);
99  LLVMTypes::VoidPointerType = llvm::PointerType::get(llvm::Type::getInt8Ty(*ctx), 0);
100  LLVMTypes::PointerIntType = target.is32Bit() ? llvm::Type::getInt32Ty(*ctx) :
101  llvm::Type::getInt64Ty(*ctx);
102 
103  LLVMTypes::BoolType = llvm::Type::getInt1Ty(*ctx);
104  LLVMTypes::Int8Type = llvm::Type::getInt8Ty(*ctx);
105  LLVMTypes::Int16Type = llvm::Type::getInt16Ty(*ctx);
106  LLVMTypes::Int32Type = llvm::Type::getInt32Ty(*ctx);
107  LLVMTypes::Int64Type = llvm::Type::getInt64Ty(*ctx);
108  LLVMTypes::FloatType = llvm::Type::getFloatTy(*ctx);
109  LLVMTypes::DoubleType = llvm::Type::getDoubleTy(*ctx);
110 
111  LLVMTypes::Int8PointerType = llvm::PointerType::get(LLVMTypes::Int8Type, 0);
112  LLVMTypes::Int16PointerType = llvm::PointerType::get(LLVMTypes::Int16Type, 0);
113  LLVMTypes::Int32PointerType = llvm::PointerType::get(LLVMTypes::Int32Type, 0);
114  LLVMTypes::Int64PointerType = llvm::PointerType::get(LLVMTypes::Int64Type, 0);
115  LLVMTypes::FloatPointerType = llvm::PointerType::get(LLVMTypes::FloatType, 0);
116  LLVMTypes::DoublePointerType = llvm::PointerType::get(LLVMTypes::DoubleType, 0);
117 
118  switch (target.getMaskBitCount()) {
119  case 1:
121  llvm::VectorType::get(llvm::Type::getInt1Ty(*ctx), target.getVectorWidth());
122  break;
123  case 8:
125  llvm::VectorType::get(llvm::Type::getInt8Ty(*ctx), target.getVectorWidth());
126  break;
127  case 16:
129  llvm::VectorType::get(llvm::Type::getInt16Ty(*ctx), target.getVectorWidth());
130  break;
131  case 32:
133  llvm::VectorType::get(llvm::Type::getInt32Ty(*ctx), target.getVectorWidth());
134  break;
135  case 64:
137  llvm::VectorType::get(llvm::Type::getInt64Ty(*ctx), target.getVectorWidth());
138  break;
139  default:
140  FATAL("Unhandled mask width for initializing MaskType");
141  }
142 
144  llvm::VectorType::get(llvm::Type::getInt1Ty(*ctx), target.getVectorWidth());
146  llvm::VectorType::get(LLVMTypes::Int8Type, target.getVectorWidth());
148  llvm::VectorType::get(LLVMTypes::Int16Type, target.getVectorWidth());
150  llvm::VectorType::get(LLVMTypes::Int32Type, target.getVectorWidth());
152  llvm::VectorType::get(LLVMTypes::Int64Type, target.getVectorWidth());
154  llvm::VectorType::get(LLVMTypes::FloatType, target.getVectorWidth());
156  llvm::VectorType::get(LLVMTypes::DoubleType, target.getVectorWidth());
157 
158  LLVMTypes::Int8VectorPointerType = llvm::PointerType::get(LLVMTypes::Int8VectorType, 0);
159  LLVMTypes::Int16VectorPointerType = llvm::PointerType::get(LLVMTypes::Int16VectorType, 0);
160  LLVMTypes::Int32VectorPointerType = llvm::PointerType::get(LLVMTypes::Int32VectorType, 0);
161  LLVMTypes::Int64VectorPointerType = llvm::PointerType::get(LLVMTypes::Int64VectorType, 0);
162  LLVMTypes::FloatVectorPointerType = llvm::PointerType::get(LLVMTypes::FloatVectorType, 0);
163  LLVMTypes::DoubleVectorPointerType = llvm::PointerType::get(LLVMTypes::DoubleVectorType, 0);
164 
165  LLVMTypes::VoidPointerVectorType = g->target->is32Bit() ? LLVMTypes::Int32VectorType :
167 
168  LLVMTrue = llvm::ConstantInt::getTrue(*ctx);
169  LLVMFalse = llvm::ConstantInt::getFalse(*ctx);
170 
171  std::vector<llvm::Constant *> maskOnes;
172  llvm::Constant *onMask = NULL;
173  switch (target.getMaskBitCount()) {
174  case 1:
175  onMask = llvm::ConstantInt::get(llvm::Type::getInt1Ty(*ctx), 1,
176  false /*unsigned*/); // 0x1
177  break;
178  case 8:
179  onMask = llvm::ConstantInt::get(llvm::Type::getInt8Ty(*ctx), -1,
180  true /*signed*/); // 0xff
181  break;
182  case 16:
183  onMask = llvm::ConstantInt::get(llvm::Type::getInt16Ty(*ctx), -1,
184  true /*signed*/); // 0xffff
185  break;
186  case 32:
187  onMask = llvm::ConstantInt::get(llvm::Type::getInt32Ty(*ctx), -1,
188  true /*signed*/); // 0xffffffff
189  break;
190  case 64:
191  onMask = llvm::ConstantInt::get(llvm::Type::getInt64Ty(*ctx), -1,
192  true /*signed*/); // 0xffffffffffffffffull
193  break;
194  default:
195  FATAL("Unhandled mask width for onMask");
196  }
197 
198  for (int i = 0; i < target.getVectorWidth(); ++i)
199  maskOnes.push_back(onMask);
200  LLVMMaskAllOn = llvm::ConstantVector::get(maskOnes);
201 
202  std::vector<llvm::Constant *> maskZeros;
203  llvm::Constant *offMask = NULL;
204  switch (target.getMaskBitCount()) {
205  case 1:
206  offMask = llvm::ConstantInt::get(llvm::Type::getInt1Ty(*ctx), 0,
207  true /*signed*/);
208  break;
209  case 8:
210  offMask = llvm::ConstantInt::get(llvm::Type::getInt8Ty(*ctx), 0,
211  true /*signed*/);
212  break;
213  case 16:
214  offMask = llvm::ConstantInt::get(llvm::Type::getInt16Ty(*ctx), 0,
215  true /*signed*/);
216  break;
217  case 32:
218  offMask = llvm::ConstantInt::get(llvm::Type::getInt32Ty(*ctx), 0,
219  true /*signed*/);
220  break;
221  case 64:
222  offMask = llvm::ConstantInt::get(llvm::Type::getInt64Ty(*ctx), 0,
223  true /*signed*/);
224  break;
225  default:
226  FATAL("Unhandled mask width for offMask");
227  }
228  for (int i = 0; i < target.getVectorWidth(); ++i)
229  maskZeros.push_back(offMask);
230  LLVMMaskAllOff = llvm::ConstantVector::get(maskZeros);
231 }
232 
233 
234 llvm::ConstantInt *
235 LLVMInt8(int8_t ival) {
236  return llvm::ConstantInt::get(llvm::Type::getInt8Ty(*g->ctx), ival,
237  true /*signed*/);
238 }
239 
240 
241 llvm::ConstantInt *
242 LLVMUInt8(uint8_t ival) {
243  return llvm::ConstantInt::get(llvm::Type::getInt8Ty(*g->ctx), ival,
244  false /*unsigned*/);
245 }
246 
247 
248 llvm::ConstantInt *
249 LLVMInt16(int16_t ival) {
250  return llvm::ConstantInt::get(llvm::Type::getInt16Ty(*g->ctx), ival,
251  true /*signed*/);
252 }
253 
254 
255 llvm::ConstantInt *
256 LLVMUInt16(uint16_t ival) {
257  return llvm::ConstantInt::get(llvm::Type::getInt16Ty(*g->ctx), ival,
258  false /*unsigned*/);
259 }
260 
261 
262 llvm::ConstantInt *
263 LLVMInt32(int32_t ival) {
264  return llvm::ConstantInt::get(llvm::Type::getInt32Ty(*g->ctx), ival,
265  true /*signed*/);
266 }
267 
268 
269 llvm::ConstantInt *
270 LLVMUInt32(uint32_t ival) {
271  return llvm::ConstantInt::get(llvm::Type::getInt32Ty(*g->ctx), ival,
272  false /*unsigned*/);
273 }
274 
275 
276 llvm::ConstantInt *
277 LLVMInt64(int64_t ival) {
278  return llvm::ConstantInt::get(llvm::Type::getInt64Ty(*g->ctx), ival,
279  true /*signed*/);
280 }
281 
282 
283 llvm::ConstantInt *
284 LLVMUInt64(uint64_t ival) {
285  return llvm::ConstantInt::get(llvm::Type::getInt64Ty(*g->ctx), ival,
286  false /*unsigned*/);
287 }
288 
289 
290 llvm::Constant *
291 LLVMFloat(float fval) {
292  return llvm::ConstantFP::get(llvm::Type::getFloatTy(*g->ctx), fval);
293 }
294 
295 
296 llvm::Constant *
297 LLVMDouble(double dval) {
298  return llvm::ConstantFP::get(llvm::Type::getDoubleTy(*g->ctx), dval);
299 }
300 
301 
302 llvm::Constant *
303 LLVMInt8Vector(int8_t ival) {
304  llvm::Constant *v = LLVMInt8(ival);
305  std::vector<llvm::Constant *> vals;
306  for (int i = 0; i < g->target->getVectorWidth(); ++i)
307  vals.push_back(v);
308  return llvm::ConstantVector::get(vals);
309 }
310 
311 
312 llvm::Constant *
313 LLVMInt8Vector(const int8_t *ivec) {
314  std::vector<llvm::Constant *> vals;
315  for (int i = 0; i < g->target->getVectorWidth(); ++i)
316  vals.push_back(LLVMInt8(ivec[i]));
317  return llvm::ConstantVector::get(vals);
318 }
319 
320 
321 llvm::Constant *
322 LLVMUInt8Vector(uint8_t ival) {
323  llvm::Constant *v = LLVMUInt8(ival);
324  std::vector<llvm::Constant *> vals;
325  for (int i = 0; i < g->target->getVectorWidth(); ++i)
326  vals.push_back(v);
327  return llvm::ConstantVector::get(vals);
328 }
329 
330 
331 llvm::Constant *
332 LLVMUInt8Vector(const uint8_t *ivec) {
333  std::vector<llvm::Constant *> vals;
334  for (int i = 0; i < g->target->getVectorWidth(); ++i)
335  vals.push_back(LLVMUInt8(ivec[i]));
336  return llvm::ConstantVector::get(vals);
337 }
338 
339 
340 llvm::Constant *
341 LLVMInt16Vector(int16_t ival) {
342  llvm::Constant *v = LLVMInt16(ival);
343  std::vector<llvm::Constant *> vals;
344  for (int i = 0; i < g->target->getVectorWidth(); ++i)
345  vals.push_back(v);
346  return llvm::ConstantVector::get(vals);
347 }
348 
349 
350 llvm::Constant *
351 LLVMInt16Vector(const int16_t *ivec) {
352  std::vector<llvm::Constant *> vals;
353  for (int i = 0; i < g->target->getVectorWidth(); ++i)
354  vals.push_back(LLVMInt16(ivec[i]));
355  return llvm::ConstantVector::get(vals);
356 }
357 
358 
359 llvm::Constant *
360 LLVMUInt16Vector(uint16_t ival) {
361  llvm::Constant *v = LLVMUInt16(ival);
362  std::vector<llvm::Constant *> vals;
363  for (int i = 0; i < g->target->getVectorWidth(); ++i)
364  vals.push_back(v);
365  return llvm::ConstantVector::get(vals);
366 }
367 
368 
369 llvm::Constant *
370 LLVMUInt16Vector(const uint16_t *ivec) {
371  std::vector<llvm::Constant *> vals;
372  for (int i = 0; i < g->target->getVectorWidth(); ++i)
373  vals.push_back(LLVMUInt16(ivec[i]));
374  return llvm::ConstantVector::get(vals);
375 }
376 
377 
378 llvm::Constant *
379 LLVMInt32Vector(int32_t ival) {
380  llvm::Constant *v = LLVMInt32(ival);
381  std::vector<llvm::Constant *> vals;
382  for (int i = 0; i < g->target->getVectorWidth(); ++i)
383  vals.push_back(v);
384  return llvm::ConstantVector::get(vals);
385 }
386 
387 
388 llvm::Constant *
389 LLVMInt32Vector(const int32_t *ivec) {
390  std::vector<llvm::Constant *> vals;
391  for (int i = 0; i < g->target->getVectorWidth(); ++i)
392  vals.push_back(LLVMInt32(ivec[i]));
393  return llvm::ConstantVector::get(vals);
394 }
395 
396 
397 llvm::Constant *
398 LLVMUInt32Vector(uint32_t ival) {
399  llvm::Constant *v = LLVMUInt32(ival);
400  std::vector<llvm::Constant *> vals;
401  for (int i = 0; i < g->target->getVectorWidth(); ++i)
402  vals.push_back(v);
403  return llvm::ConstantVector::get(vals);
404 }
405 
406 
407 llvm::Constant *
408 LLVMUInt32Vector(const uint32_t *ivec) {
409  std::vector<llvm::Constant *> vals;
410  for (int i = 0; i < g->target->getVectorWidth(); ++i)
411  vals.push_back(LLVMUInt32(ivec[i]));
412  return llvm::ConstantVector::get(vals);
413 }
414 
415 
416 llvm::Constant *
417 LLVMFloatVector(float fval) {
418  llvm::Constant *v = LLVMFloat(fval);
419  std::vector<llvm::Constant *> vals;
420  for (int i = 0; i < g->target->getVectorWidth(); ++i)
421  vals.push_back(v);
422  return llvm::ConstantVector::get(vals);
423 }
424 
425 
426 llvm::Constant *
427 LLVMFloatVector(const float *fvec) {
428  std::vector<llvm::Constant *> vals;
429  for (int i = 0; i < g->target->getVectorWidth(); ++i)
430  vals.push_back(LLVMFloat(fvec[i]));
431  return llvm::ConstantVector::get(vals);
432 }
433 
434 
435 llvm::Constant *
436 LLVMDoubleVector(double dval) {
437  llvm::Constant *v = LLVMDouble(dval);
438  std::vector<llvm::Constant *> vals;
439  for (int i = 0; i < g->target->getVectorWidth(); ++i)
440  vals.push_back(v);
441  return llvm::ConstantVector::get(vals);
442 }
443 
444 
445 llvm::Constant *
446 LLVMDoubleVector(const double *dvec) {
447  std::vector<llvm::Constant *> vals;
448  for (int i = 0; i < g->target->getVectorWidth(); ++i)
449  vals.push_back(LLVMDouble(dvec[i]));
450  return llvm::ConstantVector::get(vals);
451 }
452 
453 
454 llvm::Constant *
455 LLVMInt64Vector(int64_t ival) {
456  llvm::Constant *v = LLVMInt64(ival);
457  std::vector<llvm::Constant *> vals;
458  for (int i = 0; i < g->target->getVectorWidth(); ++i)
459  vals.push_back(v);
460  return llvm::ConstantVector::get(vals);
461 }
462 
463 
464 llvm::Constant *
465 LLVMInt64Vector(const int64_t *ivec) {
466  std::vector<llvm::Constant *> vals;
467  for (int i = 0; i < g->target->getVectorWidth(); ++i)
468  vals.push_back(LLVMInt64(ivec[i]));
469  return llvm::ConstantVector::get(vals);
470 }
471 
472 
473 llvm::Constant *
474 LLVMUInt64Vector(uint64_t ival) {
475  llvm::Constant *v = LLVMUInt64(ival);
476  std::vector<llvm::Constant *> vals;
477  for (int i = 0; i < g->target->getVectorWidth(); ++i)
478  vals.push_back(v);
479  return llvm::ConstantVector::get(vals);
480 }
481 
482 
483 llvm::Constant *
484 LLVMUInt64Vector(const uint64_t *ivec) {
485  std::vector<llvm::Constant *> vals;
486  for (int i = 0; i < g->target->getVectorWidth(); ++i)
487  vals.push_back(LLVMUInt64(ivec[i]));
488  return llvm::ConstantVector::get(vals);
489 }
490 
491 
492 llvm::Constant *
493 LLVMBoolVector(bool b) {
494  llvm::Constant *v;
496  v = llvm::ConstantInt::get(LLVMTypes::Int64Type, b ? 0xffffffffffffffffull : 0,
497  false /*unsigned*/);
499  v = llvm::ConstantInt::get(LLVMTypes::Int32Type, b ? 0xffffffff : 0,
500  false /*unsigned*/);
502  v = llvm::ConstantInt::get(LLVMTypes::Int16Type, b ? 0xffff : 0,
503  false /*unsigned*/);
505  v = llvm::ConstantInt::get(LLVMTypes::Int8Type, b ? 0xff : 0,
506  false /*unsigned*/);
507  else {
509  v = b ? LLVMTrue : LLVMFalse;
510  }
511 
512  std::vector<llvm::Constant *> vals;
513  for (int i = 0; i < g->target->getVectorWidth(); ++i)
514  vals.push_back(v);
515  return llvm::ConstantVector::get(vals);
516 }
517 
518 
519 llvm::Constant *
520 LLVMBoolVector(const bool *bvec) {
521  std::vector<llvm::Constant *> vals;
522  for (int i = 0; i < g->target->getVectorWidth(); ++i) {
523  llvm::Constant *v;
525  v = llvm::ConstantInt::get(LLVMTypes::Int64Type, bvec[i] ? 0xffffffffffffffffull : 0,
526  false /*unsigned*/);
528  v = llvm::ConstantInt::get(LLVMTypes::Int32Type, bvec[i] ? 0xffffffff : 0,
529  false /*unsigned*/);
531  v = llvm::ConstantInt::get(LLVMTypes::Int16Type, bvec[i] ? 0xffff : 0,
532  false /*unsigned*/);
534  v = llvm::ConstantInt::get(LLVMTypes::Int8Type, bvec[i] ? 0xff : 0,
535  false /*unsigned*/);
536  else {
538  v = bvec[i] ? LLVMTrue : LLVMFalse;
539  }
540 
541  vals.push_back(v);
542  }
543  return llvm::ConstantVector::get(vals);
544 }
545 
546 
547 llvm::Constant *
548 LLVMIntAsType(int64_t val, llvm::Type *type) {
549  llvm::VectorType *vecType =
550  llvm::dyn_cast<llvm::VectorType>(type);
551 
552  if (vecType != NULL) {
553  llvm::Constant *v = llvm::ConstantInt::get(vecType->getElementType(),
554  val, true /* signed */);
555  std::vector<llvm::Constant *> vals;
556  for (int i = 0; i < (int)vecType->getNumElements(); ++i)
557  vals.push_back(v);
558  return llvm::ConstantVector::get(vals);
559  }
560  else
561  return llvm::ConstantInt::get(type, val, true /* signed */);
562 }
563 
564 
565 llvm::Constant *
566 LLVMUIntAsType(uint64_t val, llvm::Type *type) {
567  llvm::VectorType *vecType =
568  llvm::dyn_cast<llvm::VectorType>(type);
569 
570  if (vecType != NULL) {
571  llvm::Constant *v = llvm::ConstantInt::get(vecType->getElementType(),
572  val, false /* unsigned */);
573  std::vector<llvm::Constant *> vals;
574  for (int i = 0; i < (int)vecType->getNumElements(); ++i)
575  vals.push_back(v);
576  return llvm::ConstantVector::get(vals);
577  }
578  else
579  return llvm::ConstantInt::get(type, val, false /* unsigned */);
580 }
581 
582 
583 /** Conservative test to see if two llvm::Values are equal. There are
584  (potentially many) cases where the two values actually are equal but
585  this will return false. However, if it does return true, the two
586  vectors definitely are equal.
587 */
588 static bool
589 lValuesAreEqual(llvm::Value *v0, llvm::Value *v1,
590  std::vector<llvm::PHINode *> &seenPhi0,
591  std::vector<llvm::PHINode *> &seenPhi1) {
592  // Thanks to the fact that LLVM hashes and returns the same pointer for
593  // constants (of all sorts, even constant expressions), this first test
594  // actually catches a lot of cases. LLVM's SSA form also helps a lot
595  // with this..
596  if (v0 == v1)
597  return true;
598 
599  Assert(seenPhi0.size() == seenPhi1.size());
600  for (unsigned int i = 0; i < seenPhi0.size(); ++i)
601  if (v0 == seenPhi0[i] && v1 == seenPhi1[i])
602  return true;
603 
604  llvm::BinaryOperator *bo0 = llvm::dyn_cast<llvm::BinaryOperator>(v0);
605  llvm::BinaryOperator *bo1 = llvm::dyn_cast<llvm::BinaryOperator>(v1);
606  if (bo0 != NULL && bo1 != NULL) {
607  if (bo0->getOpcode() != bo1->getOpcode())
608  return false;
609  return (lValuesAreEqual(bo0->getOperand(0), bo1->getOperand(0),
610  seenPhi0, seenPhi1) &&
611  lValuesAreEqual(bo0->getOperand(1), bo1->getOperand(1),
612  seenPhi0, seenPhi1));
613  }
614 
615  llvm::CastInst *cast0 = llvm::dyn_cast<llvm::CastInst>(v0);
616  llvm::CastInst *cast1 = llvm::dyn_cast<llvm::CastInst>(v1);
617  if (cast0 != NULL && cast1 != NULL) {
618  if (cast0->getOpcode() != cast1->getOpcode())
619  return false;
620  return lValuesAreEqual(cast0->getOperand(0), cast1->getOperand(0),
621  seenPhi0, seenPhi1);
622  }
623 
624  llvm::PHINode *phi0 = llvm::dyn_cast<llvm::PHINode>(v0);
625  llvm::PHINode *phi1 = llvm::dyn_cast<llvm::PHINode>(v1);
626  if (phi0 != NULL && phi1 != NULL) {
627  if (phi0->getNumIncomingValues() != phi1->getNumIncomingValues())
628  return false;
629 
630  seenPhi0.push_back(phi0);
631  seenPhi1.push_back(phi1);
632 
633  unsigned int numIncoming = phi0->getNumIncomingValues();
634  // Check all of the incoming values: if all of them are all equal,
635  // then we're good.
636  bool anyFailure = false;
637  for (unsigned int i = 0; i < numIncoming; ++i) {
638  // FIXME: should it be ok if the incoming blocks are different,
639  // where we just return faliure in this case?
640  Assert(phi0->getIncomingBlock(i) == phi1->getIncomingBlock(i));
641  if (!lValuesAreEqual(phi0->getIncomingValue(i),
642  phi1->getIncomingValue(i), seenPhi0, seenPhi1)) {
643  anyFailure = true;
644  break;
645  }
646  }
647 
648  seenPhi0.pop_back();
649  seenPhi1.pop_back();
650 
651  return !anyFailure;
652  }
653 
654  return false;
655 }
656 
657 
658 /** Given an llvm::Value known to be an integer, return its value as
659  an int64_t.
660 */
661 static int64_t
662 lGetIntValue(llvm::Value *offset) {
663  llvm::ConstantInt *intOffset = llvm::dyn_cast<llvm::ConstantInt>(offset);
664  Assert(intOffset && (intOffset->getBitWidth() == 32 ||
665  intOffset->getBitWidth() == 64));
666  return intOffset->getSExtValue();
667 }
668 
669 /** Recognizes constant vector with undef operands except the first one:
670  * <i64 4, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef>
671  */
672 static bool
673 lIsFirstElementConstVector(llvm::Value* v) {
674  llvm::ConstantVector *cv = llvm::dyn_cast<llvm::ConstantVector>(v);
675  if (cv != NULL) {
676  llvm::Constant *c =
677  llvm::dyn_cast<llvm::Constant>(cv->getOperand(0));
678  if (c == NULL) {
679  return false;
680  }
681 
682  for (int i = 1; i < (int)cv->getNumOperands(); ++i) {
683  if (!llvm::isa<llvm::UndefValue>(cv->getOperand(i))) {
684  return false;
685  }
686  }
687  return true;
688  }
689  return false;
690 }
691 
692 llvm::Value *
693 LLVMFlattenInsertChain(llvm::Value *inst, int vectorWidth,
694  bool compare, bool undef, bool searchFirstUndef) {
695  llvm::Value ** elements = new llvm::Value*[vectorWidth];
696  for (int i = 0; i < vectorWidth; ++i) {
697  elements[i] = NULL;
698  }
699 
700  // Catch a pattern of InsertElement chain.
701  if (llvm::InsertElementInst *ie =
702  llvm::dyn_cast<llvm::InsertElementInst>(inst)) {
703  //Gather elements of vector
704  while (ie != NULL) {
705  int64_t iOffset = lGetIntValue(ie->getOperand(2));
706  Assert(iOffset >= 0 && iOffset < vectorWidth);
707 
708  // Get the scalar value from this insert
709  if (elements[iOffset] == NULL) {
710  elements[iOffset] = ie->getOperand(1);
711  }
712 
713  // Do we have another insert?
714  llvm::Value *insertBase = ie->getOperand(0);
715  ie = llvm::dyn_cast<llvm::InsertElementInst>(insertBase);
716  if (ie != NULL) {
717  continue;
718  }
719 
720  if (llvm::isa<llvm::UndefValue>(insertBase)) {
721  break;
722  }
723 
724  if (llvm::isa<llvm::ConstantVector>(insertBase) ||
725  llvm::isa<llvm::ConstantAggregateZero>(insertBase)) {
726  llvm::Constant *cv = llvm::dyn_cast<llvm::Constant>(insertBase);
727  Assert(vectorWidth == (int)(cv->getNumOperands()));
728  for (int i=0; i<vectorWidth; i++) {
729  if (elements[i] == NULL) {
730  elements[i] = cv->getOperand(i);
731  }
732  }
733  break;
734  }
735  else {
736  // Here chain ends in llvm::LoadInst or some other.
737  // They are not equal to each other so we should return NULL if compare
738  // and first element if we have it.
739  Assert(compare == true || elements[0] != NULL);
740  if (compare) {
741  return NULL;
742  }
743  else {
744  return elements[0];
745  }
746  }
747  // TODO: Also, should we handle some other values like
748  // ConstantDataVectors.
749  }
750  if (compare == false) {
751  //We simply want first element
752  return elements[0];
753  }
754 
755  int null_number = 0;
756  int NonNull = 0;
757  for(int i = 0; i < vectorWidth; i++) {
758  if (elements[i] == NULL) {
759  null_number++;
760  }
761  else {
762  NonNull = i;
763  }
764  }
765  if (null_number == vectorWidth) {
766  //All of elements are NULLs
767  return NULL;
768  }
769  if ((undef == false) && (null_number != 0)) {
770  //We don't want NULLs in chain, but we have them
771  return NULL;
772  }
773 
774  // Compare elements of vector
775  for (int i = 0; i < vectorWidth; i++) {
776  if (elements[i] == NULL) {
777  continue;
778  }
779 
780  std::vector<llvm::PHINode *> seenPhi0;
781  std::vector<llvm::PHINode *> seenPhi1;
782  if (lValuesAreEqual(elements[NonNull], elements[i],
783  seenPhi0, seenPhi1) == false) {
784  return NULL;
785  }
786  }
787  return elements[NonNull];
788  }
789 
790  // Catch a pattern of broadcast implemented as InsertElement + Shuffle:
791  // %broadcast_init.0 = insertelement <4 x i32> undef, i32 %val, i32 0
792  // %broadcast.1 = shufflevector <4 x i32> %smear.0, <4 x i32> undef,
793  // <4 x i32> zeroinitializer
794  // Or:
795  // %gep_ptr2int_broadcast_init = insertelement <8 x i64> undef, i64 %gep_ptr2int, i32 0
796  // %0 = add <8 x i64> %gep_ptr2int_broadcast_init,
797  // <i64 4, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef>
798  // %gep_offset = shufflevector <8 x i64> %0, <8 x i64> undef, <8 x i32> zeroinitializer
799  else if (llvm::ShuffleVectorInst *shuf =
800  llvm::dyn_cast<llvm::ShuffleVectorInst>(inst)) {
801  llvm::Value *indices = shuf->getOperand(2);
802  if (llvm::isa<llvm::ConstantAggregateZero>(indices)) {
803  llvm::Value *op = shuf->getOperand(0);
804  llvm::InsertElementInst *ie = llvm::dyn_cast<llvm::InsertElementInst>(op);
805  if (ie == NULL && searchFirstUndef) {
806  // Trying to recognize 2nd pattern
807  llvm::BinaryOperator * bop = llvm::dyn_cast<llvm::BinaryOperator>(op);
808  if (bop != NULL && bop->getOpcode() == llvm::Instruction::Add) {
809  if (lIsFirstElementConstVector(bop->getOperand(1))) {
810  ie = llvm::dyn_cast<llvm::InsertElementInst>(bop->getOperand(0));
811  }
812  }
813  }
814  if (ie != NULL &&
815  llvm::isa<llvm::UndefValue>(ie->getOperand(0))) {
816  llvm::ConstantInt *ci =
817  llvm::dyn_cast<llvm::ConstantInt>(ie->getOperand(2));
818  if (ci->isZero()) {
819  return ie->getOperand(1);
820  }
821  }
822  }
823  }
824  return NULL;
825 }
826 
827 bool
828 LLVMExtractVectorInts(llvm::Value *v, int64_t ret[], int *nElts) {
829  // Make sure we do in fact have a vector of integer values here
830  llvm::VectorType *vt =
831  llvm::dyn_cast<llvm::VectorType>(v->getType());
832  Assert(vt != NULL);
833  Assert(llvm::isa<llvm::IntegerType>(vt->getElementType()));
834 
835  *nElts = (int)vt->getNumElements();
836 
837  if (llvm::isa<llvm::ConstantAggregateZero>(v)) {
838  for (int i = 0; i < (int)vt->getNumElements(); ++i)
839  ret[i] = 0;
840  return true;
841  }
842 
843  llvm::ConstantDataVector *cv = llvm::dyn_cast<llvm::ConstantDataVector>(v);
844  if (cv == NULL)
845  return false;
846 
847  for (int i = 0; i < (int)cv->getNumElements(); ++i)
848  ret[i] = cv->getElementAsInteger(i);
849  return true;
850 }
851 
852 
853 static bool
854 lVectorValuesAllEqual(llvm::Value *v, int vectorLength,
855  std::vector<llvm::PHINode *> &seenPhis,
856  llvm::Value **splatValue = NULL);
857 
858 
859 /** This function checks to see if the given (scalar or vector) value is an
860  exact multiple of baseValue. It returns true if so, and false if not
861  (or if it's not able to determine if it is). Any vector value passed
862  in is required to have the same value in all elements (so that we can
863  just check the first element to be a multiple of the given value.)
864  */
865 static bool
866 lIsExactMultiple(llvm::Value *val, int baseValue, int vectorLength,
867  std::vector<llvm::PHINode *> &seenPhis) {
868  if (llvm::isa<llvm::VectorType>(val->getType()) == false) {
869  // If we've worked down to a constant int, then the moment of truth
870  // has arrived...
871  llvm::ConstantInt *ci = llvm::dyn_cast<llvm::ConstantInt>(val);
872  if (ci != NULL)
873  return (ci->getZExtValue() % baseValue) == 0;
874  }
875  else
877 
878  if (llvm::isa<llvm::InsertElementInst>(val) ||
879  llvm::isa<llvm::ShuffleVectorInst>(val)) {
880  llvm::Value *element = LLVMFlattenInsertChain(val, g->target->getVectorWidth());
881  // We just need to check the scalar first value, since we know that
882  // all elements are equal
883  return lIsExactMultiple(element, baseValue, vectorLength, seenPhis);
884  }
885 
886  llvm::PHINode *phi = llvm::dyn_cast<llvm::PHINode>(val);
887  if (phi != NULL) {
888  for (unsigned int i = 0; i < seenPhis.size(); ++i)
889  if (phi == seenPhis[i])
890  return true;
891 
892  seenPhis.push_back(phi);
893  unsigned int numIncoming = phi->getNumIncomingValues();
894 
895  // Check all of the incoming values: if all of them pass, then
896  // we're good.
897  for (unsigned int i = 0; i < numIncoming; ++i) {
898  llvm::Value *incoming = phi->getIncomingValue(i);
899  bool mult = lIsExactMultiple(incoming, baseValue, vectorLength,
900  seenPhis);
901  if (mult == false) {
902  seenPhis.pop_back();
903  return false;
904  }
905  }
906  seenPhis.pop_back();
907  return true;
908  }
909 
910  llvm::BinaryOperator *bop = llvm::dyn_cast<llvm::BinaryOperator>(val);
911  if (bop != NULL && bop->getOpcode() == llvm::Instruction::Add) {
912  llvm::Value *op0 = bop->getOperand(0);
913  llvm::Value *op1 = bop->getOperand(1);
914 
915  bool be0 = lIsExactMultiple(op0, baseValue, vectorLength, seenPhis);
916  bool be1 = lIsExactMultiple(op1, baseValue, vectorLength, seenPhis);
917  return (be0 && be1);
918  }
919  // FIXME: mul? casts? ... ?
920 
921  return false;
922 }
923 
924 
925 /** Returns the next power of two greater than or equal to the given
926  value. */
927 static int
928 lRoundUpPow2(int v) {
929  v--;
930  v |= v >> 1;
931  v |= v >> 2;
932  v |= v >> 4;
933  v |= v >> 8;
934  v |= v >> 16;
935  return v+1;
936 }
937 
938 
939 /** Try to determine if all of the elements of the given vector value have
940  the same value when divided by the given baseValue. The function
941  returns true if this can be determined to be the case, and false
942  otherwise. (This function may fail to identify some cases where it
943  does in fact have this property, but should never report a given value
944  as being a multiple if it isn't!)
945  */
946 static bool
947 lAllDivBaseEqual(llvm::Value *val, int64_t baseValue, int vectorLength,
948  std::vector<llvm::PHINode *> &seenPhis,
949  bool &canAdd) {
950  Assert(llvm::isa<llvm::VectorType>(val->getType()));
951  // Make sure the base value is a positive power of 2
952  Assert(baseValue > 0 && (baseValue & (baseValue-1)) == 0);
953 
954  // The easy case
955  if (lVectorValuesAllEqual(val, vectorLength, seenPhis))
956  return true;
957 
958  int64_t vecVals[ISPC_MAX_NVEC];
959  int nElts;
960  if (llvm::isa<llvm::VectorType>(val->getType()) &&
961  LLVMExtractVectorInts(val, vecVals, &nElts)) {
962  // If we have a vector of compile-time constant integer values,
963  // then go ahead and check them directly..
964  int64_t firstDiv = vecVals[0] / baseValue;
965  for (int i = 1; i < nElts; ++i)
966  if ((vecVals[i] / baseValue) != firstDiv)
967  return false;
968 
969  return true;
970  }
971 
972  llvm::PHINode *phi = llvm::dyn_cast<llvm::PHINode>(val);
973  if (phi != NULL) {
974  for (unsigned int i = 0; i < seenPhis.size(); ++i)
975  if (phi == seenPhis[i])
976  return true;
977 
978  seenPhis.push_back(phi);
979  unsigned int numIncoming = phi->getNumIncomingValues();
980 
981  // Check all of the incoming values: if all of them pass, then
982  // we're good.
983  for (unsigned int i = 0; i < numIncoming; ++i) {
984  llvm::Value *incoming = phi->getIncomingValue(i);
985  bool ca = canAdd;
986  bool mult = lAllDivBaseEqual(incoming, baseValue, vectorLength,
987  seenPhis, ca);
988  if (mult == false) {
989  seenPhis.pop_back();
990  return false;
991  }
992  }
993  seenPhis.pop_back();
994  return true;
995  }
996 
997  llvm::BinaryOperator *bop = llvm::dyn_cast<llvm::BinaryOperator>(val);
998  if (bop != NULL && bop->getOpcode() == llvm::Instruction::Add &&
999  canAdd == true) {
1000  llvm::Value *op0 = bop->getOperand(0);
1001  llvm::Value *op1 = bop->getOperand(1);
1002 
1003  // Otherwise we're only going to worry about the following case,
1004  // which comes up often when looping over SOA data:
1005  // ashr %val, <constant shift>
1006  // where %val = add %smear, <0,1,2,3...>
1007  // and where the maximum of the <0,...> vector in the add is less than
1008  // 1<<(constant shift),
1009  // and where %smear is a smear of a value that is a multiple of
1010  // baseValue.
1011 
1012  int64_t addConstants[ISPC_MAX_NVEC];
1013  if (LLVMExtractVectorInts(op1, addConstants, &nElts) == false)
1014  return false;
1015  Assert(nElts == vectorLength);
1016 
1017  // Do all of them give the same value when divided by baseValue?
1018  int64_t firstConstDiv = addConstants[0] / baseValue;
1019  for (int i = 1; i < vectorLength; ++i)
1020  if ((addConstants[i] / baseValue) != firstConstDiv)
1021  return false;
1022 
1023  if (lVectorValuesAllEqual(op0, vectorLength, seenPhis) == false)
1024  return false;
1025 
1026  // Note that canAdd is a reference parameter; setting this ensures
1027  // that we don't allow multiple adds in other parts of the chain of
1028  // dependent values from here.
1029  canAdd = false;
1030 
1031  // Now we need to figure out the required alignment (in numbers of
1032  // elements of the underlying type being indexed) of the value to
1033  // which these integer addConstant[] values are being added to. We
1034  // know that we have addConstant[] values that all give the same
1035  // value when divided by baseValue, but we may need a less-strict
1036  // alignment than baseValue depending on the actual values.
1037  //
1038  // As an example, consider a case where the baseValue alignment is
1039  // 16, but the addConstants here are <0,1,2,3>. In that case, the
1040  // value to which addConstants is added to only needs to be a
1041  // multiple of 4. Conversely, if addConstants are <4,5,6,7>, then
1042  // we need a multiple of 8 to ensure that the final added result
1043  // will still have the same value for all vector elements when
1044  // divided by baseValue.
1045  //
1046  // All that said, here we need to find the maximum value of any of
1047  // the addConstants[], mod baseValue. If we round that up to the
1048  // next power of 2, we'll have a value that will be no greater than
1049  // baseValue and sometimes less.
1050  int maxMod = int(addConstants[0] % baseValue);
1051  for (int i = 1; i < vectorLength; ++i)
1052  maxMod = std::max(maxMod, int(addConstants[i] % baseValue));
1053  int requiredAlignment = lRoundUpPow2(maxMod);
1054 
1055  std::vector<llvm::PHINode *> seenPhisEEM;
1056  return lIsExactMultiple(op0, requiredAlignment, vectorLength,
1057  seenPhisEEM);
1058  }
1059  // TODO: could handle mul by a vector of equal constant integer values
1060  // and the like here and adjust the 'baseValue' value when it evenly
1061  // divides, but unclear if it's worthwhile...
1062 
1063  return false;
1064 }
1065 
1066 
1067 /** Given a vector shift right of some value by some amount, try to
1068  determine if all of the elements of the final result have the same
1069  value (i.e. whether the high bits are all equal, disregarding the low
1070  bits that are shifted out.) Returns true if so, and false otherwise.
1071  */
1072 static bool
1073 lVectorShiftRightAllEqual(llvm::Value *val, llvm::Value *shift,
1074  int vectorLength) {
1075  // Are we shifting all elements by a compile-time constant amount? If
1076  // not, give up.
1077  int64_t shiftAmount[ISPC_MAX_NVEC];
1078  int nElts;
1079  if (LLVMExtractVectorInts(shift, shiftAmount, &nElts) == false)
1080  return false;
1081  Assert(nElts == vectorLength);
1082 
1083  // Is it the same amount for all elements?
1084  for (int i = 0; i < vectorLength; ++i)
1085  if (shiftAmount[i] != shiftAmount[0])
1086  return false;
1087 
1088  // Now see if the value divided by (1 << shift) can be determined to
1089  // have the same value for all vector elements.
1090  int pow2 = 1 << shiftAmount[0];
1091  bool canAdd = true;
1092  std::vector<llvm::PHINode *> seenPhis;
1093  bool eq = lAllDivBaseEqual(val, pow2, vectorLength, seenPhis, canAdd);
1094 #if 0
1095  fprintf(stderr, "check all div base equal:\n");
1096  LLVMDumpValue(shift);
1097  LLVMDumpValue(val);
1098  fprintf(stderr, "----> %s\n\n", eq ? "true" : "false");
1099 #endif
1100  return eq;
1101 }
1102 
1103 
1104 static bool
1105 lVectorValuesAllEqual(llvm::Value *v, int vectorLength,
1106  std::vector<llvm::PHINode *> &seenPhis,
1107  llvm::Value **splatValue) {
1108  if (vectorLength == 1)
1109  return true;
1110 
1111  if (llvm::isa<llvm::ConstantAggregateZero>(v)) {
1112  if (splatValue) {
1113  llvm::ConstantAggregateZero *caz =
1114  llvm::dyn_cast<llvm::ConstantAggregateZero>(v);
1115  *splatValue = caz->getSequentialElement();
1116  }
1117  return true;
1118  }
1119 
1120  llvm::ConstantVector *cv = llvm::dyn_cast<llvm::ConstantVector>(v);
1121  if (cv != NULL) {
1122  llvm::Value* splat = cv->getSplatValue();
1123  if (splat != NULL && splatValue) {
1124  *splatValue = splat;
1125  }
1126  return (splat != NULL);
1127  }
1128 
1129  llvm::ConstantDataVector *cdv = llvm::dyn_cast<llvm::ConstantDataVector>(v);
1130  if (cdv != NULL) {
1131  llvm::Value* splat = cdv->getSplatValue();
1132  if (splat != NULL && splatValue) {
1133  *splatValue = splat;
1134  }
1135  return (splat != NULL);
1136  }
1137 
1138  llvm::BinaryOperator *bop = llvm::dyn_cast<llvm::BinaryOperator>(v);
1139  if (bop != NULL) {
1140  // Easy case: both operands are all equal -> return true
1141  if (lVectorValuesAllEqual(bop->getOperand(0), vectorLength,
1142  seenPhis) &&
1143  lVectorValuesAllEqual(bop->getOperand(1), vectorLength,
1144  seenPhis))
1145  return true;
1146 
1147  // If it's a shift, take a special path that tries to check if the
1148  // high (surviving) bits of the values are equal.
1149  if (bop->getOpcode() == llvm::Instruction::AShr ||
1150  bop->getOpcode() == llvm::Instruction::LShr)
1151  return lVectorShiftRightAllEqual(bop->getOperand(0),
1152  bop->getOperand(1), vectorLength);
1153 
1154  return false;
1155  }
1156 
1157  llvm::CastInst *cast = llvm::dyn_cast<llvm::CastInst>(v);
1158  if (cast != NULL)
1159  return lVectorValuesAllEqual(cast->getOperand(0), vectorLength,
1160  seenPhis);
1161 
1162  llvm::InsertElementInst *ie = llvm::dyn_cast<llvm::InsertElementInst>(v);
1163  if (ie != NULL) {
1164  return (LLVMFlattenInsertChain(ie, vectorLength) != NULL);
1165  }
1166 
1167  llvm::PHINode *phi = llvm::dyn_cast<llvm::PHINode>(v);
1168  if (phi) {
1169  for (unsigned int i = 0; i < seenPhis.size(); ++i)
1170  if (seenPhis[i] == phi)
1171  return true;
1172 
1173  seenPhis.push_back(phi);
1174 
1175  unsigned int numIncoming = phi->getNumIncomingValues();
1176  // Check all of the incoming values: if all of them are all equal,
1177  // then we're good.
1178  for (unsigned int i = 0; i < numIncoming; ++i) {
1179  if (!lVectorValuesAllEqual(phi->getIncomingValue(i), vectorLength,
1180  seenPhis)) {
1181  seenPhis.pop_back();
1182  return false;
1183  }
1184  }
1185 
1186  seenPhis.pop_back();
1187  return true;
1188  }
1189 
1190  if (llvm::isa<llvm::UndefValue>(v))
1191  // ?
1192  return false;
1193 
1194  Assert(!llvm::isa<llvm::Constant>(v));
1195 
1196  if (llvm::isa<llvm::CallInst>(v) || llvm::isa<llvm::LoadInst>(v) ||
1197  !llvm::isa<llvm::Instruction>(v))
1198  return false;
1199 
1200  llvm::ShuffleVectorInst *shuffle = llvm::dyn_cast<llvm::ShuffleVectorInst>(v);
1201  if (shuffle != NULL) {
1202  llvm::Value *indices = shuffle->getOperand(2);
1203  if (lVectorValuesAllEqual(indices, vectorLength, seenPhis))
1204  // The easy case--just a smear of the same element across the
1205  // whole vector.
1206  return true;
1207 
1208  // TODO: handle more general cases?
1209  return false;
1210  }
1211 
1212 #if 0
1213  fprintf(stderr, "all equal: ");
1214  v->dump();
1215  fprintf(stderr, "\n");
1216  llvm::Instruction *inst = llvm::dyn_cast<llvm::Instruction>(v);
1217  if (inst) {
1218  inst->getParent()->dump();
1219  fprintf(stderr, "\n");
1220  fprintf(stderr, "\n");
1221  }
1222 #endif
1223 
1224  return false;
1225 }
1226 
1227 
1228 /** Tests to see if all of the elements of the vector in the 'v' parameter
1229  are equal. This is a conservative test and may return false for arrays
1230  where the values are actually all equal.
1231 */
1232 bool
1233 LLVMVectorValuesAllEqual(llvm::Value *v, llvm::Value **splat) {
1234  llvm::VectorType *vt =
1235  llvm::dyn_cast<llvm::VectorType>(v->getType());
1236  Assert(vt != NULL);
1237  int vectorLength = vt->getNumElements();
1238 
1239  std::vector<llvm::PHINode *> seenPhis;
1240  bool equal = lVectorValuesAllEqual(v, vectorLength, seenPhis, splat);
1241 
1242  Debug(SourcePos(), "LLVMVectorValuesAllEqual(%s) -> %s.",
1243  v->getName().str().c_str(), equal ? "true" : "false");
1244  if (g->debugPrint)
1245  LLVMDumpValue(v);
1246 
1247  return equal;
1248 }
1249 
1250 
1251 static bool
1252 lVectorIsLinear(llvm::Value *v, int vectorLength, int stride,
1253  std::vector<llvm::PHINode *> &seenPhis);
1254 
1255 /** Given a vector of compile-time constant integer values, test to see if
1256  they are a linear sequence of constant integers starting from an
1257  arbirary value but then having a step of value "stride" between
1258  elements.
1259  */
1260 static bool
1261 lVectorIsLinearConstantInts(llvm::ConstantDataVector *cv,
1262  int vectorLength,
1263  int stride) {
1264  // Flatten the vector out into the elements array
1265  llvm::SmallVector<llvm::Constant *, ISPC_MAX_NVEC> elements;
1266  for (int i = 0; i < (int)cv->getNumElements(); ++i)
1267  elements.push_back(cv->getElementAsConstant(i));
1268  Assert((int)elements.size() == vectorLength);
1269 
1270  llvm::ConstantInt *ci = llvm::dyn_cast<llvm::ConstantInt>(elements[0]);
1271  if (ci == NULL)
1272  // Not a vector of integers
1273  return false;
1274 
1275  int64_t prevVal = ci->getSExtValue();
1276 
1277  // For each element in the array, see if it is both a ConstantInt and
1278  // if the difference between it and the value of the previous element
1279  // is stride. If not, fail.
1280  for (int i = 1; i < vectorLength; ++i) {
1281  ci = llvm::dyn_cast<llvm::ConstantInt>(elements[i]);
1282  if (ci == NULL)
1283  return false;
1284 
1285  int64_t nextVal = ci->getSExtValue();
1286  if (prevVal + stride != nextVal)
1287  return false;
1288 
1289  prevVal = nextVal;
1290  }
1291  return true;
1292 }
1293 
1294 
1295 /** Checks to see if (op0 * op1) is a linear vector where the result is a
1296  vector with values that increase by stride.
1297  */
1298 static bool
1299 lCheckMulForLinear(llvm::Value *op0, llvm::Value *op1, int vectorLength,
1300  int stride, std::vector<llvm::PHINode *> &seenPhis) {
1301  // Is the first operand a constant integer value splatted across all of
1302  // the lanes?
1303  llvm::ConstantDataVector *cv = llvm::dyn_cast<llvm::ConstantDataVector>(op0);
1304  if (cv == NULL)
1305  return false;
1306 
1307  llvm::Constant *csplat = cv->getSplatValue();
1308  if (csplat == NULL)
1309  return false;
1310 
1311  llvm::ConstantInt *splat = llvm::dyn_cast<llvm::ConstantInt>(csplat);
1312  if (splat == NULL)
1313  return false;
1314 
1315  // If the splat value doesn't evenly divide the stride we're looking
1316  // for, there's no way that we can get the linear sequence we're
1317  // looking or.
1318  int64_t splatVal = splat->getSExtValue();
1319  if (splatVal == 0 || splatVal > stride || (stride % splatVal) != 0)
1320  return false;
1321 
1322  // Check to see if the other operand is a linear vector with stride
1323  // given by stride/splatVal.
1324  return lVectorIsLinear(op1, vectorLength, (int)(stride / splatVal),
1325  seenPhis);
1326 }
1327 
1328 
1329 /** Checks to see if (op0 << op1) is a linear vector where the result is a
1330  vector with values that increase by stride.
1331  */
1332 static bool
1333 lCheckShlForLinear(llvm::Value *op0, llvm::Value *op1, int vectorLength,
1334  int stride, std::vector<llvm::PHINode *> &seenPhis) {
1335  // Is the second operand a constant integer value splatted across all of
1336  // the lanes?
1337  llvm::ConstantDataVector *cv = llvm::dyn_cast<llvm::ConstantDataVector>(op1);
1338  if (cv == NULL)
1339  return false;
1340 
1341  llvm::Constant *csplat = cv->getSplatValue();
1342  if (csplat == NULL)
1343  return false;
1344 
1345  llvm::ConstantInt *splat = llvm::dyn_cast<llvm::ConstantInt>(csplat);
1346  if (splat == NULL)
1347  return false;
1348 
1349  // If (1 << the splat value) doesn't evenly divide the stride we're
1350  // looking for, there's no way that we can get the linear sequence
1351  // we're looking or.
1352  int64_t equivalentMul = (1LL << splat->getSExtValue());
1353  if (equivalentMul > stride || (stride % equivalentMul) != 0)
1354  return false;
1355 
1356  // Check to see if the first operand is a linear vector with stride
1357  // given by stride/splatVal.
1358  return lVectorIsLinear(op0, vectorLength, (int)(stride / equivalentMul),
1359  seenPhis);
1360 }
1361 
1362 
1363 /** Given (op0 AND op1), try and see if we can determine if the result is a
1364  linear sequence with a step of "stride" between values. Returns true
1365  if so and false otherwise. This pattern comes up when accessing SOA
1366  data.
1367  */
1368 static bool
1369 lCheckAndForLinear(llvm::Value *op0, llvm::Value *op1, int vectorLength,
1370  int stride, std::vector<llvm::PHINode *> &seenPhis) {
1371  // Require op1 to be a compile-time constant
1372  int64_t maskValue[ISPC_MAX_NVEC];
1373  int nElts;
1374  if (LLVMExtractVectorInts(op1, maskValue, &nElts) == false)
1375  return false;
1376  Assert(nElts == vectorLength);
1377 
1378  // Is op1 a smear of the same value across all lanes? Give up if not.
1379  for (int i = 1; i < vectorLength; ++i)
1380  if (maskValue[i] != maskValue[0])
1381  return false;
1382 
1383  // If the op1 value isn't power of 2 minus one, then also give up.
1384  int64_t maskPlusOne = maskValue[0] + 1;
1385  bool isPowTwo = (maskPlusOne & (maskPlusOne - 1)) == 0;
1386  if (isPowTwo == false)
1387  return false;
1388 
1389  // The case we'll covert here is op0 being a linear vector with desired
1390  // stride, and where all of the values of op0, when divided by
1391  // maskPlusOne, have the same value.
1392  if (lVectorIsLinear(op0, vectorLength, stride, seenPhis) == false)
1393  return false;
1394 
1395  bool canAdd = true;
1396  bool isMult = lAllDivBaseEqual(op0, maskPlusOne, vectorLength, seenPhis,
1397  canAdd);
1398  return isMult;
1399 }
1400 
1401 
1402 static bool
1403 lVectorIsLinear(llvm::Value *v, int vectorLength, int stride,
1404  std::vector<llvm::PHINode *> &seenPhis) {
1405  // First try the easy case: if the values are all just constant
1406  // integers and have the expected stride between them, then we're done.
1407  llvm::ConstantDataVector *cv = llvm::dyn_cast<llvm::ConstantDataVector>(v);
1408  if (cv != NULL)
1409  return lVectorIsLinearConstantInts(cv, vectorLength, stride);
1410 
1411  llvm::BinaryOperator *bop = llvm::dyn_cast<llvm::BinaryOperator>(v);
1412  if (bop != NULL) {
1413  // FIXME: is it right to pass the seenPhis to the all equal check as well??
1414  llvm::Value *op0 = bop->getOperand(0), *op1 = bop->getOperand(1);
1415 
1416  if (bop->getOpcode() == llvm::Instruction::Add) {
1417  // There are two cases to check if we have an add:
1418  //
1419  // programIndex + unif -> ascending linear seqeuence
1420  // unif + programIndex -> ascending linear sequence
1421  bool l0 = lVectorIsLinear(op0, vectorLength, stride, seenPhis);
1422  bool e1 = lVectorValuesAllEqual(op1, vectorLength, seenPhis);
1423  if (l0 && e1)
1424  return true;
1425 
1426  bool e0 = lVectorValuesAllEqual(op0, vectorLength, seenPhis);
1427  bool l1 = lVectorIsLinear(op1, vectorLength, stride, seenPhis);
1428  return (e0 && l1);
1429  }
1430  else if (bop->getOpcode() == llvm::Instruction::Sub)
1431  // For subtraction, we only match:
1432  // programIndex - unif -> ascending linear seqeuence
1433  return (lVectorIsLinear(bop->getOperand(0), vectorLength,
1434  stride, seenPhis) &&
1435  lVectorValuesAllEqual(bop->getOperand(1), vectorLength,
1436  seenPhis));
1437  else if (bop->getOpcode() == llvm::Instruction::Mul) {
1438  // Multiplies are a bit trickier, so are handled in a separate
1439  // function.
1440  bool m0 = lCheckMulForLinear(op0, op1, vectorLength, stride, seenPhis);
1441  if (m0)
1442  return true;
1443  bool m1 = lCheckMulForLinear(op1, op0, vectorLength, stride, seenPhis);
1444  return m1;
1445  }
1446  else if (bop->getOpcode() == llvm::Instruction::Shl) {
1447  // Sometimes multiplies come in as shift lefts (especially in
1448  // LLVM 3.4+).
1449  bool linear = lCheckShlForLinear(op0, op1, vectorLength, stride, seenPhis);
1450  return linear;
1451  }
1452  else if (bop->getOpcode() == llvm::Instruction::And) {
1453  // Special case for some AND-related patterns that come up when
1454  // looping over SOA data
1455  bool linear = lCheckAndForLinear(op0, op1, vectorLength, stride, seenPhis);
1456  return linear;
1457  }
1458  else
1459  return false;
1460  }
1461 
1462  llvm::CastInst *ci = llvm::dyn_cast<llvm::CastInst>(v);
1463  if (ci != NULL)
1464  return lVectorIsLinear(ci->getOperand(0), vectorLength,
1465  stride, seenPhis);
1466 
1467  if (llvm::isa<llvm::CallInst>(v) || llvm::isa<llvm::LoadInst>(v))
1468  return false;
1469 
1470  llvm::PHINode *phi = llvm::dyn_cast<llvm::PHINode>(v);
1471  if (phi != NULL) {
1472  for (unsigned int i = 0; i < seenPhis.size(); ++i)
1473  if (seenPhis[i] == phi)
1474  return true;
1475 
1476  seenPhis.push_back(phi);
1477 
1478  unsigned int numIncoming = phi->getNumIncomingValues();
1479  // Check all of the incoming values: if all of them are all equal,
1480  // then we're good.
1481  for (unsigned int i = 0; i < numIncoming; ++i) {
1482  if (!lVectorIsLinear(phi->getIncomingValue(i), vectorLength, stride,
1483  seenPhis)) {
1484  seenPhis.pop_back();
1485  return false;
1486  }
1487  }
1488 
1489  seenPhis.pop_back();
1490  return true;
1491  }
1492 
1493  // TODO: is any reason to worry about these?
1494  if (llvm::isa<llvm::InsertElementInst>(v))
1495  return false;
1496 
1497  // TODO: we could also handle shuffles, but we haven't yet seen any
1498  // cases where doing so would detect cases where actually have a linear
1499  // vector.
1500  llvm::ShuffleVectorInst *shuffle = llvm::dyn_cast<llvm::ShuffleVectorInst>(v);
1501  if (shuffle != NULL)
1502  return false;
1503 
1504 #if 0
1505  fprintf(stderr, "linear check: ");
1506  v->dump();
1507  fprintf(stderr, "\n");
1508  llvm::Instruction *inst = llvm::dyn_cast<llvm::Instruction>(v);
1509  if (inst) {
1510  inst->getParent()->dump();
1511  fprintf(stderr, "\n");
1512  fprintf(stderr, "\n");
1513  }
1514 #endif
1515 
1516  return false;
1517 }
1518 
1519 
1520 /** Given vector of integer-typed values, see if the elements of the array
1521  have a step of 'stride' between their values. This function tries to
1522  handle as many possibilities as possible, including things like all
1523  elements equal to some non-constant value plus an integer offset, etc.
1524 */
1525 bool
1526 LLVMVectorIsLinear(llvm::Value *v, int stride) {
1527  llvm::VectorType *vt =
1528  llvm::dyn_cast<llvm::VectorType>(v->getType());
1529  Assert(vt != NULL);
1530  int vectorLength = vt->getNumElements();
1531 
1532  std::vector<llvm::PHINode *> seenPhis;
1533  bool linear = lVectorIsLinear(v, vectorLength, stride, seenPhis);
1534  Debug(SourcePos(), "LLVMVectorIsLinear(%s) -> %s.",
1535  v->getName().str().c_str(), linear ? "true" : "false");
1536  if (g->debugPrint)
1537  LLVMDumpValue(v);
1538 
1539  return linear;
1540 }
1541 
1542 
1543 static void
1544 lDumpValue(llvm::Value *v, std::set<llvm::Value *> &done) {
1545  if (done.find(v) != done.end())
1546  return;
1547 
1548  llvm::Instruction *inst = llvm::dyn_cast<llvm::Instruction>(v);
1549  if (done.size() > 0 && inst == NULL)
1550  return;
1551 
1552  fprintf(stderr, " ");
1553  v->dump();
1554  done.insert(v);
1555 
1556  if (inst == NULL)
1557  return;
1558 
1559  for (unsigned i = 0; i < inst->getNumOperands(); ++i)
1560  lDumpValue(inst->getOperand(i), done);
1561 }
1562 
1563 
1564 void
1565 LLVMDumpValue(llvm::Value *v) {
1566  std::set<llvm::Value *> done;
1567  lDumpValue(v, done);
1568  fprintf(stderr, "----\n");
1569 }
1570 
1571 
1572 static llvm::Value *
1574  std::map<llvm::PHINode *, llvm::PHINode *> &phiMap) {
1575  llvm::VectorType *vt =
1576  llvm::dyn_cast<llvm::VectorType>(v->getType());
1577  Assert(vt != NULL);
1578 
1579  // First, handle various constant types; do the extraction manually, as
1580  // appropriate.
1581  if (llvm::isa<llvm::ConstantAggregateZero>(v) == true) {
1582  return llvm::Constant::getNullValue(vt->getElementType());
1583  }
1584  if (llvm::ConstantVector *cv = llvm::dyn_cast<llvm::ConstantVector>(v)) {
1585  return cv->getOperand(0);
1586  }
1587  if (llvm::ConstantDataVector *cdv =
1588  llvm::dyn_cast<llvm::ConstantDataVector>(v))
1589  return cdv->getElementAsConstant(0);
1590 
1591  // Otherwise, all that we should have at this point is an instruction
1592  // of some sort
1593  Assert(llvm::isa<llvm::Constant>(v) == false);
1594  Assert(llvm::isa<llvm::Instruction>(v) == true);
1595 
1596  std::string newName = v->getName().str() + std::string(".elt0");
1597 
1598  // Rewrite regular binary operators and casts to the scalarized
1599  // equivalent.
1600  llvm::BinaryOperator *bop = llvm::dyn_cast<llvm::BinaryOperator>(v);
1601  if (bop != NULL) {
1602  llvm::Value *v0 = lExtractFirstVectorElement(bop->getOperand(0),
1603  phiMap);
1604  llvm::Value *v1 = lExtractFirstVectorElement(bop->getOperand(1),
1605  phiMap);
1606  Assert(v0 != NULL);
1607  Assert(v1 != NULL);
1608  // Note that the new binary operator is inserted immediately before
1609  // the previous vector one
1610  return llvm::BinaryOperator::Create(bop->getOpcode(), v0, v1,
1611  newName, bop);
1612  }
1613 
1614  llvm::CastInst *cast = llvm::dyn_cast<llvm::CastInst>(v);
1615  if (cast != NULL) {
1616  llvm::Value *v = lExtractFirstVectorElement(cast->getOperand(0),
1617  phiMap);
1618  // Similarly, the equivalent scalar cast instruction goes right
1619  // before the vector cast
1620  return llvm::CastInst::Create(cast->getOpcode(), v,
1621  vt->getElementType(), newName,
1622  cast);
1623  }
1624 
1625  llvm::PHINode *phi = llvm::dyn_cast<llvm::PHINode>(v);
1626  if (phi != NULL) {
1627  // For PHI notes, recursively scalarize them.
1628  if (phiMap.find(phi) != phiMap.end())
1629  return phiMap[phi];
1630 
1631  // We need to create the new scalar PHI node immediately, though,
1632  // and put it in the map<>, so that if we come back to this node
1633  // via a recursive lExtractFirstVectorElement() call, then we can
1634  // return the pointer and not get stuck in an infinite loop.
1635  //
1636  // The insertion point for the new phi node also has to be the
1637  // start of the bblock of the original phi node.
1638 
1639 #if ISPC_LLVM_VERSION <= ISPC_LLVM_3_7 /* 3.2, 3.3, 3.4, 3.5, 3.6, 3.7 */
1640  llvm::Instruction *phiInsertPos = phi->getParent()->begin();
1641 #else /* LLVM 3.8+ */
1642  llvm::Instruction *phiInsertPos = &*(phi->getParent()->begin());
1643 #endif
1644  llvm::PHINode *scalarPhi =
1645  llvm::PHINode::Create(vt->getElementType(),
1646  phi->getNumIncomingValues(),
1647  newName, phiInsertPos);
1648  phiMap[phi] = scalarPhi;
1649 
1650  for (unsigned i = 0; i < phi->getNumIncomingValues(); ++i) {
1651  llvm::Value *v = lExtractFirstVectorElement(phi->getIncomingValue(i),
1652  phiMap);
1653  scalarPhi->addIncoming(v, phi->getIncomingBlock(i));
1654  }
1655 
1656  return scalarPhi;
1657  }
1658 
1659  // We should consider "shuffle" case and "insertElement" case separately.
1660  // For example we can have shuffle(mul, undef, zero) but function
1661  // "LLVMFlattenInsertChain" can handle only case shuffle(insertElement, undef, zero).
1662  // Also if we have insertElement under shuffle we will handle it the next call of
1663  // "lExtractFirstVectorElement" function.
1664  if (llvm::isa<llvm::ShuffleVectorInst>(v)) {
1665  llvm::ShuffleVectorInst *shuf = llvm::dyn_cast<llvm::ShuffleVectorInst>(v);
1666  llvm::Value *indices = shuf->getOperand(2);
1667  if (llvm::isa<llvm::ConstantAggregateZero>(indices)) {
1668  return lExtractFirstVectorElement(shuf->getOperand(0), phiMap);
1669  }
1670  }
1671 
1672  // If we have a chain of insertelement instructions, then we can just
1673  // flatten them out and grab the value for the first one.
1674  if (llvm::isa<llvm::InsertElementInst>(v)) {
1675  return LLVMFlattenInsertChain(v, vt->getNumElements(), false);
1676  }
1677 
1678  // Worst case, for everything else, just do a regular extract element
1679  // instruction, which we insert immediately after the instruction we
1680  // have here.
1681  llvm::Instruction *insertAfter = llvm::dyn_cast<llvm::Instruction>(v);
1682  Assert(insertAfter != NULL);
1683  llvm::Instruction *ee =
1684  llvm::ExtractElementInst::Create(v, LLVMInt32(0), "first_elt",
1685  (llvm::Instruction *)NULL);
1686  ee->insertAfter(insertAfter);
1687  return ee;
1688 }
1689 
1690 
1691 llvm::Value *
1693  std::map<llvm::PHINode *, llvm::PHINode *> phiMap;
1694  llvm::Value *ret = lExtractFirstVectorElement(v, phiMap);
1695  return ret;
1696 }
1697 
1698 
1699 /** Given two vectors of the same type, concatenate them into a vector that
1700  has twice as many elements, where the first half has the elements from
1701  the first vector and the second half has the elements from the second
1702  vector.
1703  */
1704 llvm::Value *
1705 LLVMConcatVectors(llvm::Value *v1, llvm::Value *v2,
1706  llvm::Instruction *insertBefore) {
1707  Assert(v1->getType() == v2->getType());
1708 
1709  llvm::VectorType *vt =
1710  llvm::dyn_cast<llvm::VectorType>(v1->getType());
1711  Assert(vt != NULL);
1712 
1713  int32_t identity[ISPC_MAX_NVEC];
1714  int resultSize = 2*vt->getNumElements();
1715  Assert(resultSize <= ISPC_MAX_NVEC);
1716  for (int i = 0; i < resultSize; ++i)
1717  identity[i] = i;
1718 
1719  return LLVMShuffleVectors(v1, v2, identity, resultSize, insertBefore);
1720 }
1721 
1722 
1723 /** Shuffle two vectors together with a ShuffleVectorInst, returning a
1724  vector with shufSize elements, where the shuf[] array offsets are used
1725  to determine which element from the two given vectors is used for each
1726  result element. */
1727 llvm::Value *
1728 LLVMShuffleVectors(llvm::Value *v1, llvm::Value *v2, int32_t shuf[],
1729  int shufSize, llvm::Instruction *insertBefore) {
1730  std::vector<llvm::Constant *> shufVec;
1731  for (int i = 0; i < shufSize; ++i) {
1732  if (shuf[i] == -1)
1733  shufVec.push_back(llvm::UndefValue::get(LLVMTypes::Int32Type));
1734  else
1735  shufVec.push_back(LLVMInt32(shuf[i]));
1736  }
1737 
1738  llvm::ArrayRef<llvm::Constant *> aref(&shufVec[0], &shufVec[shufSize]);
1739  llvm::Value *vec = llvm::ConstantVector::get(aref);
1740 
1741  return new llvm::ShuffleVectorInst(v1, v2, vec, "shuffle", insertBefore);
1742 }
1743 
1744 
1745 const char *
1746 LLVMGetName(llvm::Value *v, const char *s) {
1747  if (v == NULL) return s;
1748  std::string ret = v->getName();
1749  ret += s;
1750  return strdup(ret.c_str());
1751 }
1752 
1753 
1754 const char *
1755 LLVMGetName(const char *op, llvm::Value *v1, llvm::Value *v2) {
1756  std::string r = op;
1757  r += "_";
1758  r += v1->getName().str();
1759  r += "_";
1760  r += v2->getName().str();
1761  return strdup(r.c_str());
1762 }
llvm::Constant * LLVMIntAsType(int64_t val, llvm::Type *type)
Definition: llvmutil.cpp:548
static llvm::Type * FloatType
Definition: llvmutil.h:80
static bool lVectorIsLinear(llvm::Value *v, int vectorLength, int stride, std::vector< llvm::PHINode * > &seenPhis)
Definition: llvmutil.cpp:1403
static llvm::Type * Int32VectorPointerType
Definition: llvmutil.h:103
llvm::Constant * LLVMUInt64Vector(uint64_t ival)
Definition: llvmutil.cpp:474
static int lRoundUpPow2(int v)
Definition: llvmutil.cpp:928
static bool lIsFirstElementConstVector(llvm::Value *v)
Definition: llvmutil.cpp:673
llvm::Constant * LLVMUInt32Vector(uint32_t ival)
Definition: llvmutil.cpp:398
llvm::ConstantInt * LLVMUInt32(uint32_t ival)
Definition: llvmutil.cpp:270
llvm::Constant * LLVMBoolVector(bool b)
Definition: llvmutil.cpp:493
static llvm::Type * DoubleType
Definition: llvmutil.h:81
static bool lVectorShiftRightAllEqual(llvm::Value *val, llvm::Value *shift, int vectorLength)
Definition: llvmutil.cpp:1073
llvm::ConstantInt * LLVMInt8(int8_t ival)
Definition: llvmutil.cpp:235
int getVectorWidth() const
Definition: ispc.h:286
llvm::Constant * LLVMInt64Vector(int64_t ival)
Definition: llvmutil.cpp:455
llvm::ConstantInt * LLVMUInt8(uint8_t ival)
Definition: llvmutil.cpp:242
Structure that defines a compilation target.
Definition: ispc.h:184
Target * target
Definition: ispc.h:550
static llvm::VectorType * VoidPointerVectorType
Definition: llvmutil.h:108
static llvm::VectorType * BoolVectorType
Definition: llvmutil.h:92
bool LLVMExtractVectorInts(llvm::Value *v, int64_t ret[], int *nElts)
Definition: llvmutil.cpp:828
const char * LLVMGetName(llvm::Value *v, const char *s)
Definition: llvmutil.cpp:1746
llvm::Constant * LLVMMaskAllOn
Definition: llvmutil.cpp:92
static void lDumpValue(llvm::Value *v, std::set< llvm::Value * > &done)
Definition: llvmutil.cpp:1544
static llvm::Type * BoolType
Definition: llvmutil.h:74
llvm::ConstantInt * LLVMInt16(int16_t ival)
Definition: llvmutil.cpp:249
#define Assert(expr)
Definition: ispc.h:173
static llvm::VectorType * Int32VectorType
Definition: llvmutil.h:96
static bool lCheckShlForLinear(llvm::Value *op0, llvm::Value *op1, int vectorLength, int stride, std::vector< llvm::PHINode * > &seenPhis)
Definition: llvmutil.cpp:1333
void InitLLVMUtil(llvm::LLVMContext *ctx, Target &target)
Definition: llvmutil.cpp:97
static llvm::Type * FloatVectorPointerType
Definition: llvmutil.h:105
static llvm::Type * Int8PointerType
Definition: llvmutil.h:83
static llvm::Type * Int32PointerType
Definition: llvmutil.h:85
llvm::Constant * LLVMUIntAsType(uint64_t val, llvm::Type *type)
Definition: llvmutil.cpp:566
static llvm::Value * lExtractFirstVectorElement(llvm::Value *v, std::map< llvm::PHINode *, llvm::PHINode * > &phiMap)
Definition: llvmutil.cpp:1573
static llvm::Type * Int16VectorPointerType
Definition: llvmutil.h:102
static llvm::Type * Int16Type
Definition: llvmutil.h:77
static llvm::Type * DoubleVectorPointerType
Definition: llvmutil.h:106
static llvm::VectorType * Int1VectorType
Definition: llvmutil.h:93
llvm::Constant * LLVMFloatVector(float fval)
Definition: llvmutil.cpp:417
static llvm::Type * VoidType
Definition: llvmutil.h:71
llvm::Constant * LLVMDoubleVector(double dval)
Definition: llvmutil.cpp:436
static llvm::Type * Int8VectorPointerType
Definition: llvmutil.h:101
Globals * g
Definition: ispc.cpp:100
bool debugPrint
Definition: ispc.h:567
int getMaskBitCount() const
Definition: ispc.h:292
static llvm::VectorType * Int8VectorType
Definition: llvmutil.h:94
llvm::Constant * LLVMInt32Vector(int32_t ival)
Definition: llvmutil.cpp:379
llvm::Constant * LLVMTrue
Definition: llvmutil.cpp:90
static llvm::VectorType * FloatVectorType
Definition: llvmutil.h:98
static llvm::Type * Int64Type
Definition: llvmutil.h:79
static llvm::Type * Int8Type
Definition: llvmutil.h:76
static llvm::VectorType * Int64VectorType
Definition: llvmutil.h:97
Header file with declarations for various LLVM utility stuff.
static llvm::Type * Int64PointerType
Definition: llvmutil.h:86
llvm::ConstantInt * LLVMUInt64(uint64_t ival)
Definition: llvmutil.cpp:284
static bool lCheckMulForLinear(llvm::Value *op0, llvm::Value *op1, int vectorLength, int stride, std::vector< llvm::PHINode * > &seenPhis)
Definition: llvmutil.cpp:1299
static llvm::Type * FloatPointerType
Definition: llvmutil.h:87
llvm::Constant * LLVMMaskAllOff
Definition: llvmutil.cpp:93
Representation of a range of positions in a source file.
Definition: ispc.h:137
static llvm::Type * Int16PointerType
Definition: llvmutil.h:84
llvm::Value * LLVMConcatVectors(llvm::Value *v1, llvm::Value *v2, llvm::Instruction *insertBefore)
Definition: llvmutil.cpp:1705
static bool lAllDivBaseEqual(llvm::Value *val, int64_t baseValue, int vectorLength, std::vector< llvm::PHINode * > &seenPhis, bool &canAdd)
Definition: llvmutil.cpp:947
llvm::ConstantInt * LLVMInt32(int32_t ival)
Definition: llvmutil.cpp:263
bool LLVMVectorIsLinear(llvm::Value *v, int stride)
Definition: llvmutil.cpp:1526
llvm::Constant * LLVMUInt8Vector(uint8_t ival)
Definition: llvmutil.cpp:322
static llvm::Type * PointerIntType
Definition: llvmutil.h:73
static llvm::PointerType * VoidPointerType
Definition: llvmutil.h:72
llvm::Constant * LLVMFloat(float fval)
Definition: llvmutil.cpp:291
#define FATAL(message)
Definition: util.h:113
llvm::Constant * LLVMDouble(double dval)
Definition: llvmutil.cpp:297
void LLVMDumpValue(llvm::Value *v)
Definition: llvmutil.cpp:1565
static llvm::Type * Int64VectorPointerType
Definition: llvmutil.h:104
static llvm::Type * Int32Type
Definition: llvmutil.h:78
static llvm::Type * DoublePointerType
Definition: llvmutil.h:88
#define ISPC_MAX_NVEC
Definition: ispc.h:69
static bool lIsExactMultiple(llvm::Value *val, int baseValue, int vectorLength, std::vector< llvm::PHINode * > &seenPhis)
Definition: llvmutil.cpp:866
llvm::Constant * LLVMFalse
Definition: llvmutil.cpp:91
llvm::Value * LLVMExtractFirstVectorElement(llvm::Value *v)
Definition: llvmutil.cpp:1692
llvm::Constant * LLVMInt16Vector(int16_t ival)
Definition: llvmutil.cpp:341
static llvm::VectorType * MaskType
Definition: llvmutil.h:90
static bool lVectorValuesAllEqual(llvm::Value *v, int vectorLength, std::vector< llvm::PHINode * > &seenPhis, llvm::Value **splatValue=NULL)
Definition: llvmutil.cpp:1105
llvm::ConstantInt * LLVMUInt16(uint16_t ival)
Definition: llvmutil.cpp:256
bool LLVMVectorValuesAllEqual(llvm::Value *v, llvm::Value **splat)
Definition: llvmutil.cpp:1233
static llvm::VectorType * DoubleVectorType
Definition: llvmutil.h:99
static bool lCheckAndForLinear(llvm::Value *op0, llvm::Value *op1, int vectorLength, int stride, std::vector< llvm::PHINode * > &seenPhis)
Definition: llvmutil.cpp:1369
llvm::Constant * LLVMInt8Vector(int8_t ival)
Definition: llvmutil.cpp:303
llvm::Constant * LLVMUInt16Vector(uint16_t ival)
Definition: llvmutil.cpp:360
static int64_t lGetIntValue(llvm::Value *offset)
Definition: llvmutil.cpp:662
llvm::ConstantInt * LLVMInt64(int64_t ival)
Definition: llvmutil.cpp:277
static llvm::VectorType * Int16VectorType
Definition: llvmutil.h:95
static bool lValuesAreEqual(llvm::Value *v0, llvm::Value *v1, std::vector< llvm::PHINode * > &seenPhi0, std::vector< llvm::PHINode * > &seenPhi1)
Definition: llvmutil.cpp:589
llvm::LLVMContext * ctx
Definition: ispc.h:639
Main ispc.header file. Defines Target, Globals and Opt classes.
llvm::Value * LLVMShuffleVectors(llvm::Value *v1, llvm::Value *v2, int32_t shuf[], int shufSize, llvm::Instruction *insertBefore)
Definition: llvmutil.cpp:1728
void Debug(SourcePos p, const char *format,...) PRINTF_FUNC
Definition: util.cpp:398
bool is32Bit() const
Definition: ispc.h:276
static bool lVectorIsLinearConstantInts(llvm::ConstantDataVector *cv, int vectorLength, int stride)
Definition: llvmutil.cpp:1261
File with declarations for classes related to type representation.
llvm::Value * LLVMFlattenInsertChain(llvm::Value *inst, int vectorWidth, bool compare, bool undef, bool searchFirstUndef)
Definition: llvmutil.cpp:693