]>
Commit | Line | Data |
---|---|---|
223e47cc LB |
1 | //===-- LegalizeDAG.cpp - Implement SelectionDAG::Legalize ----------------===// |
2 | // | |
3 | // The LLVM Compiler Infrastructure | |
4 | // | |
5 | // This file is distributed under the University of Illinois Open Source | |
6 | // License. See LICENSE.TXT for details. | |
7 | // | |
8 | //===----------------------------------------------------------------------===// | |
9 | // | |
10 | // This file implements the SelectionDAG::Legalize method. | |
11 | // | |
12 | //===----------------------------------------------------------------------===// | |
13 | ||
970d7e83 | 14 | #include "llvm/CodeGen/SelectionDAG.h" |
1a4d82fc | 15 | #include "llvm/ADT/SetVector.h" |
970d7e83 | 16 | #include "llvm/ADT/SmallPtrSet.h" |
1a4d82fc | 17 | #include "llvm/ADT/SmallSet.h" |
970d7e83 LB |
18 | #include "llvm/ADT/SmallVector.h" |
19 | #include "llvm/ADT/Triple.h" | |
223e47cc LB |
20 | #include "llvm/CodeGen/Analysis.h" |
21 | #include "llvm/CodeGen/MachineFunction.h" | |
22 | #include "llvm/CodeGen/MachineJumpTableInfo.h" | |
970d7e83 LB |
23 | #include "llvm/IR/CallingConv.h" |
24 | #include "llvm/IR/Constants.h" | |
25 | #include "llvm/IR/DataLayout.h" | |
1a4d82fc | 26 | #include "llvm/IR/DebugInfo.h" |
970d7e83 LB |
27 | #include "llvm/IR/DerivedTypes.h" |
28 | #include "llvm/IR/Function.h" | |
29 | #include "llvm/IR/LLVMContext.h" | |
223e47cc LB |
30 | #include "llvm/Support/Debug.h" |
31 | #include "llvm/Support/ErrorHandling.h" | |
32 | #include "llvm/Support/MathExtras.h" | |
33 | #include "llvm/Support/raw_ostream.h" | |
970d7e83 LB |
34 | #include "llvm/Target/TargetFrameLowering.h" |
35 | #include "llvm/Target/TargetLowering.h" | |
36 | #include "llvm/Target/TargetMachine.h" | |
1a4d82fc | 37 | #include "llvm/Target/TargetSubtargetInfo.h" |
223e47cc LB |
38 | using namespace llvm; |
39 | ||
1a4d82fc JJ |
40 | #define DEBUG_TYPE "legalizedag" |
41 | ||
223e47cc | 42 | //===----------------------------------------------------------------------===// |
85aaf69f | 43 | /// This takes an arbitrary SelectionDAG as input and |
223e47cc LB |
44 | /// hacks on it until the target machine can handle it. This involves |
45 | /// eliminating value sizes the machine cannot handle (promoting small sizes to | |
46 | /// large sizes or splitting up large values into small values) as well as | |
47 | /// eliminating operations the machine cannot handle. | |
48 | /// | |
49 | /// This code also does a small amount of optimization and recognition of idioms | |
50 | /// as part of its processing. For example, if a target does not support a | |
51 | /// 'setcc' instruction efficiently, but does support 'brcc' instruction, this | |
52 | /// will attempt merge setcc and brc instructions into brcc's. | |
53 | /// | |
54 | namespace { | |
1a4d82fc | 55 | class SelectionDAGLegalize { |
223e47cc LB |
56 | const TargetMachine &TM; |
57 | const TargetLowering &TLI; | |
58 | SelectionDAG &DAG; | |
59 | ||
1a4d82fc JJ |
60 | /// \brief The set of nodes which have already been legalized. We hold a |
61 | /// reference to it in order to update as necessary on node deletion. | |
62 | SmallPtrSetImpl<SDNode *> &LegalizedNodes; | |
223e47cc | 63 | |
1a4d82fc JJ |
64 | /// \brief A set of all the nodes updated during legalization. |
65 | SmallSetVector<SDNode *, 16> *UpdatedNodes; | |
66 | ||
67 | EVT getSetCCResultType(EVT VT) const { | |
68 | return TLI.getSetCCResultType(*DAG.getContext(), VT); | |
69 | } | |
223e47cc LB |
70 | |
71 | // Libcall insertion helpers. | |
72 | ||
73 | public: | |
1a4d82fc JJ |
74 | SelectionDAGLegalize(SelectionDAG &DAG, |
75 | SmallPtrSetImpl<SDNode *> &LegalizedNodes, | |
76 | SmallSetVector<SDNode *, 16> *UpdatedNodes = nullptr) | |
77 | : TM(DAG.getTarget()), TLI(DAG.getTargetLoweringInfo()), DAG(DAG), | |
78 | LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {} | |
223e47cc | 79 | |
1a4d82fc | 80 | /// \brief Legalizes the given operation. |
223e47cc LB |
81 | void LegalizeOp(SDNode *Node); |
82 | ||
1a4d82fc | 83 | private: |
223e47cc LB |
84 | SDValue OptimizeFloatStore(StoreSDNode *ST); |
85 | ||
86 | void LegalizeLoadOps(SDNode *Node); | |
87 | void LegalizeStoreOps(SDNode *Node); | |
88 | ||
85aaf69f | 89 | /// Some targets cannot handle a variable |
223e47cc LB |
90 | /// insertion index for the INSERT_VECTOR_ELT instruction. In this case, it |
91 | /// is necessary to spill the vector being inserted into to memory, perform | |
92 | /// the insert there, and then read the result back. | |
93 | SDValue PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, | |
1a4d82fc | 94 | SDValue Idx, SDLoc dl); |
223e47cc | 95 | SDValue ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, |
1a4d82fc | 96 | SDValue Idx, SDLoc dl); |
223e47cc | 97 | |
85aaf69f | 98 | /// Return a vector shuffle operation which |
223e47cc LB |
99 | /// performs the same shuffe in terms of order or result bytes, but on a type |
100 | /// whose vector element type is narrower than the original shuffle type. | |
101 | /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3> | |
1a4d82fc | 102 | SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, SDLoc dl, |
223e47cc LB |
103 | SDValue N1, SDValue N2, |
104 | ArrayRef<int> Mask) const; | |
105 | ||
1a4d82fc JJ |
106 | bool LegalizeSetCCCondCode(EVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC, |
107 | bool &NeedInvert, SDLoc dl); | |
223e47cc LB |
108 | |
109 | SDValue ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned); | |
110 | SDValue ExpandLibCall(RTLIB::Libcall LC, EVT RetVT, const SDValue *Ops, | |
1a4d82fc | 111 | unsigned NumOps, bool isSigned, SDLoc dl); |
223e47cc LB |
112 | |
113 | std::pair<SDValue, SDValue> ExpandChainLibCall(RTLIB::Libcall LC, | |
114 | SDNode *Node, bool isSigned); | |
115 | SDValue ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32, | |
116 | RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80, | |
970d7e83 | 117 | RTLIB::Libcall Call_F128, |
223e47cc LB |
118 | RTLIB::Libcall Call_PPCF128); |
119 | SDValue ExpandIntLibCall(SDNode *Node, bool isSigned, | |
120 | RTLIB::Libcall Call_I8, | |
121 | RTLIB::Libcall Call_I16, | |
122 | RTLIB::Libcall Call_I32, | |
123 | RTLIB::Libcall Call_I64, | |
124 | RTLIB::Libcall Call_I128); | |
125 | void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results); | |
970d7e83 | 126 | void ExpandSinCosLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results); |
223e47cc | 127 | |
1a4d82fc | 128 | SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT, SDLoc dl); |
223e47cc LB |
129 | SDValue ExpandBUILD_VECTOR(SDNode *Node); |
130 | SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node); | |
131 | void ExpandDYNAMIC_STACKALLOC(SDNode *Node, | |
132 | SmallVectorImpl<SDValue> &Results); | |
133 | SDValue ExpandFCOPYSIGN(SDNode *Node); | |
134 | SDValue ExpandLegalINT_TO_FP(bool isSigned, SDValue LegalOp, EVT DestVT, | |
1a4d82fc | 135 | SDLoc dl); |
223e47cc | 136 | SDValue PromoteLegalINT_TO_FP(SDValue LegalOp, EVT DestVT, bool isSigned, |
1a4d82fc | 137 | SDLoc dl); |
223e47cc | 138 | SDValue PromoteLegalFP_TO_INT(SDValue LegalOp, EVT DestVT, bool isSigned, |
1a4d82fc | 139 | SDLoc dl); |
223e47cc | 140 | |
1a4d82fc JJ |
141 | SDValue ExpandBSWAP(SDValue Op, SDLoc dl); |
142 | SDValue ExpandBitCount(unsigned Opc, SDValue Op, SDLoc dl); | |
223e47cc LB |
143 | |
144 | SDValue ExpandExtractFromVectorThroughStack(SDValue Op); | |
145 | SDValue ExpandInsertToVectorThroughStack(SDValue Op); | |
146 | SDValue ExpandVectorBuildThroughStack(SDNode* Node); | |
147 | ||
148 | SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP); | |
149 | ||
150 | std::pair<SDValue, SDValue> ExpandAtomic(SDNode *Node); | |
151 | ||
152 | void ExpandNode(SDNode *Node); | |
153 | void PromoteNode(SDNode *Node); | |
154 | ||
223e47cc | 155 | public: |
223e47cc LB |
156 | // Node replacement helpers |
157 | void ReplacedNode(SDNode *N) { | |
1a4d82fc JJ |
158 | LegalizedNodes.erase(N); |
159 | if (UpdatedNodes) | |
160 | UpdatedNodes->insert(N); | |
223e47cc LB |
161 | } |
162 | void ReplaceNode(SDNode *Old, SDNode *New) { | |
1a4d82fc JJ |
163 | DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG); |
164 | dbgs() << " with: "; New->dump(&DAG)); | |
165 | ||
166 | assert(Old->getNumValues() == New->getNumValues() && | |
167 | "Replacing one node with another that produces a different number " | |
168 | "of values!"); | |
223e47cc | 169 | DAG.ReplaceAllUsesWith(Old, New); |
1a4d82fc JJ |
170 | for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) |
171 | DAG.TransferDbgValues(SDValue(Old, i), SDValue(New, i)); | |
172 | if (UpdatedNodes) | |
173 | UpdatedNodes->insert(New); | |
223e47cc LB |
174 | ReplacedNode(Old); |
175 | } | |
176 | void ReplaceNode(SDValue Old, SDValue New) { | |
1a4d82fc JJ |
177 | DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG); |
178 | dbgs() << " with: "; New->dump(&DAG)); | |
179 | ||
223e47cc | 180 | DAG.ReplaceAllUsesWith(Old, New); |
1a4d82fc JJ |
181 | DAG.TransferDbgValues(Old, New); |
182 | if (UpdatedNodes) | |
183 | UpdatedNodes->insert(New.getNode()); | |
223e47cc LB |
184 | ReplacedNode(Old.getNode()); |
185 | } | |
186 | void ReplaceNode(SDNode *Old, const SDValue *New) { | |
1a4d82fc JJ |
187 | DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG)); |
188 | ||
223e47cc | 189 | DAG.ReplaceAllUsesWith(Old, New); |
1a4d82fc JJ |
190 | for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) { |
191 | DEBUG(dbgs() << (i == 0 ? " with: " | |
192 | : " and: "); | |
193 | New[i]->dump(&DAG)); | |
194 | DAG.TransferDbgValues(SDValue(Old, i), New[i]); | |
195 | if (UpdatedNodes) | |
196 | UpdatedNodes->insert(New[i].getNode()); | |
197 | } | |
223e47cc LB |
198 | ReplacedNode(Old); |
199 | } | |
200 | }; | |
201 | } | |
202 | ||
85aaf69f | 203 | /// Return a vector shuffle operation which |
223e47cc LB |
204 | /// performs the same shuffe in terms of order or result bytes, but on a type |
205 | /// whose vector element type is narrower than the original shuffle type. | |
206 | /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3> | |
207 | SDValue | |
1a4d82fc | 208 | SelectionDAGLegalize::ShuffleWithNarrowerEltType(EVT NVT, EVT VT, SDLoc dl, |
223e47cc LB |
209 | SDValue N1, SDValue N2, |
210 | ArrayRef<int> Mask) const { | |
211 | unsigned NumMaskElts = VT.getVectorNumElements(); | |
212 | unsigned NumDestElts = NVT.getVectorNumElements(); | |
213 | unsigned NumEltsGrowth = NumDestElts / NumMaskElts; | |
214 | ||
215 | assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!"); | |
216 | ||
217 | if (NumEltsGrowth == 1) | |
218 | return DAG.getVectorShuffle(NVT, dl, N1, N2, &Mask[0]); | |
219 | ||
220 | SmallVector<int, 8> NewMask; | |
221 | for (unsigned i = 0; i != NumMaskElts; ++i) { | |
222 | int Idx = Mask[i]; | |
223 | for (unsigned j = 0; j != NumEltsGrowth; ++j) { | |
224 | if (Idx < 0) | |
225 | NewMask.push_back(-1); | |
226 | else | |
227 | NewMask.push_back(Idx * NumEltsGrowth + j); | |
228 | } | |
229 | } | |
230 | assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?"); | |
231 | assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?"); | |
232 | return DAG.getVectorShuffle(NVT, dl, N1, N2, &NewMask[0]); | |
233 | } | |
234 | ||
85aaf69f | 235 | /// Expands the ConstantFP node to an integer constant or |
223e47cc LB |
236 | /// a load from the constant pool. |
237 | SDValue | |
238 | SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) { | |
239 | bool Extend = false; | |
1a4d82fc | 240 | SDLoc dl(CFP); |
223e47cc LB |
241 | |
242 | // If a FP immediate is precise when represented as a float and if the | |
243 | // target can do an extending load from float to double, we put it into | |
244 | // the constant pool as a float, even if it's is statically typed as a | |
245 | // double. This shrinks FP constants and canonicalizes them for targets where | |
246 | // an FP extending load is the same cost as a normal load (such as on the x87 | |
247 | // fp stack or PPC FP unit). | |
248 | EVT VT = CFP->getValueType(0); | |
249 | ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue()); | |
250 | if (!UseCP) { | |
251 | assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion"); | |
252 | return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(), | |
253 | (VT == MVT::f64) ? MVT::i64 : MVT::i32); | |
254 | } | |
255 | ||
256 | EVT OrigVT = VT; | |
257 | EVT SVT = VT; | |
1a4d82fc | 258 | while (SVT != MVT::f32 && SVT != MVT::f16) { |
223e47cc LB |
259 | SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1); |
260 | if (ConstantFPSDNode::isValueValidForType(SVT, CFP->getValueAPF()) && | |
261 | // Only do this if the target has a native EXTLOAD instruction from | |
262 | // smaller type. | |
85aaf69f | 263 | TLI.isLoadExtLegal(ISD::EXTLOAD, OrigVT, SVT) && |
223e47cc LB |
264 | TLI.ShouldShrinkFPConstant(OrigVT)) { |
265 | Type *SType = SVT.getTypeForEVT(*DAG.getContext()); | |
266 | LLVMC = cast<ConstantFP>(ConstantExpr::getFPTrunc(LLVMC, SType)); | |
267 | VT = SVT; | |
268 | Extend = true; | |
269 | } | |
270 | } | |
271 | ||
272 | SDValue CPIdx = DAG.getConstantPool(LLVMC, TLI.getPointerTy()); | |
273 | unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment(); | |
274 | if (Extend) { | |
275 | SDValue Result = | |
276 | DAG.getExtLoad(ISD::EXTLOAD, dl, OrigVT, | |
277 | DAG.getEntryNode(), | |
278 | CPIdx, MachinePointerInfo::getConstantPool(), | |
1a4d82fc | 279 | VT, false, false, false, Alignment); |
223e47cc LB |
280 | return Result; |
281 | } | |
282 | SDValue Result = | |
283 | DAG.getLoad(OrigVT, dl, DAG.getEntryNode(), CPIdx, | |
284 | MachinePointerInfo::getConstantPool(), false, false, false, | |
285 | Alignment); | |
286 | return Result; | |
287 | } | |
288 | ||
85aaf69f | 289 | /// Expands an unaligned store to 2 half-size stores. |
223e47cc LB |
290 | static void ExpandUnalignedStore(StoreSDNode *ST, SelectionDAG &DAG, |
291 | const TargetLowering &TLI, | |
292 | SelectionDAGLegalize *DAGLegalize) { | |
293 | assert(ST->getAddressingMode() == ISD::UNINDEXED && | |
294 | "unaligned indexed stores not implemented!"); | |
295 | SDValue Chain = ST->getChain(); | |
296 | SDValue Ptr = ST->getBasePtr(); | |
297 | SDValue Val = ST->getValue(); | |
298 | EVT VT = Val.getValueType(); | |
299 | int Alignment = ST->getAlignment(); | |
1a4d82fc JJ |
300 | unsigned AS = ST->getAddressSpace(); |
301 | ||
302 | SDLoc dl(ST); | |
223e47cc LB |
303 | if (ST->getMemoryVT().isFloatingPoint() || |
304 | ST->getMemoryVT().isVector()) { | |
305 | EVT intVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); | |
306 | if (TLI.isTypeLegal(intVT)) { | |
307 | // Expand to a bitconvert of the value to the integer type of the | |
308 | // same size, then a (misaligned) int store. | |
309 | // FIXME: Does not handle truncating floating point stores! | |
310 | SDValue Result = DAG.getNode(ISD::BITCAST, dl, intVT, Val); | |
311 | Result = DAG.getStore(Chain, dl, Result, Ptr, ST->getPointerInfo(), | |
312 | ST->isVolatile(), ST->isNonTemporal(), Alignment); | |
313 | DAGLegalize->ReplaceNode(SDValue(ST, 0), Result); | |
314 | return; | |
315 | } | |
316 | // Do a (aligned) store to a stack slot, then copy from the stack slot | |
317 | // to the final destination using (unaligned) integer loads and stores. | |
318 | EVT StoredVT = ST->getMemoryVT(); | |
970d7e83 | 319 | MVT RegVT = |
223e47cc LB |
320 | TLI.getRegisterType(*DAG.getContext(), |
321 | EVT::getIntegerVT(*DAG.getContext(), | |
322 | StoredVT.getSizeInBits())); | |
323 | unsigned StoredBytes = StoredVT.getSizeInBits() / 8; | |
324 | unsigned RegBytes = RegVT.getSizeInBits() / 8; | |
325 | unsigned NumRegs = (StoredBytes + RegBytes - 1) / RegBytes; | |
326 | ||
327 | // Make sure the stack slot is also aligned for the register type. | |
328 | SDValue StackPtr = DAG.CreateStackTemporary(StoredVT, RegVT); | |
329 | ||
330 | // Perform the original store, only redirected to the stack slot. | |
331 | SDValue Store = DAG.getTruncStore(Chain, dl, | |
332 | Val, StackPtr, MachinePointerInfo(), | |
333 | StoredVT, false, false, 0); | |
1a4d82fc | 334 | SDValue Increment = DAG.getConstant(RegBytes, TLI.getPointerTy(AS)); |
223e47cc LB |
335 | SmallVector<SDValue, 8> Stores; |
336 | unsigned Offset = 0; | |
337 | ||
338 | // Do all but one copies using the full register width. | |
339 | for (unsigned i = 1; i < NumRegs; i++) { | |
340 | // Load one integer register's worth from the stack slot. | |
341 | SDValue Load = DAG.getLoad(RegVT, dl, Store, StackPtr, | |
342 | MachinePointerInfo(), | |
343 | false, false, false, 0); | |
344 | // Store it to the final location. Remember the store. | |
345 | Stores.push_back(DAG.getStore(Load.getValue(1), dl, Load, Ptr, | |
346 | ST->getPointerInfo().getWithOffset(Offset), | |
347 | ST->isVolatile(), ST->isNonTemporal(), | |
348 | MinAlign(ST->getAlignment(), Offset))); | |
349 | // Increment the pointers. | |
350 | Offset += RegBytes; | |
351 | StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr, | |
352 | Increment); | |
353 | Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, Increment); | |
354 | } | |
355 | ||
356 | // The last store may be partial. Do a truncating store. On big-endian | |
357 | // machines this requires an extending load from the stack slot to ensure | |
358 | // that the bits are in the right place. | |
359 | EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), | |
360 | 8 * (StoredBytes - Offset)); | |
361 | ||
362 | // Load from the stack slot. | |
363 | SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, RegVT, Store, StackPtr, | |
364 | MachinePointerInfo(), | |
1a4d82fc | 365 | MemVT, false, false, false, 0); |
223e47cc LB |
366 | |
367 | Stores.push_back(DAG.getTruncStore(Load.getValue(1), dl, Load, Ptr, | |
368 | ST->getPointerInfo() | |
369 | .getWithOffset(Offset), | |
370 | MemVT, ST->isVolatile(), | |
371 | ST->isNonTemporal(), | |
1a4d82fc JJ |
372 | MinAlign(ST->getAlignment(), Offset), |
373 | ST->getAAInfo())); | |
223e47cc | 374 | // The order of the stores doesn't matter - say it with a TokenFactor. |
1a4d82fc | 375 | SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores); |
223e47cc LB |
376 | DAGLegalize->ReplaceNode(SDValue(ST, 0), Result); |
377 | return; | |
378 | } | |
379 | assert(ST->getMemoryVT().isInteger() && | |
380 | !ST->getMemoryVT().isVector() && | |
381 | "Unaligned store of unknown type."); | |
382 | // Get the half-size VT | |
383 | EVT NewStoredVT = ST->getMemoryVT().getHalfSizedIntegerVT(*DAG.getContext()); | |
384 | int NumBits = NewStoredVT.getSizeInBits(); | |
385 | int IncrementSize = NumBits / 8; | |
386 | ||
387 | // Divide the stored value in two parts. | |
388 | SDValue ShiftAmount = DAG.getConstant(NumBits, | |
389 | TLI.getShiftAmountTy(Val.getValueType())); | |
390 | SDValue Lo = Val; | |
391 | SDValue Hi = DAG.getNode(ISD::SRL, dl, VT, Val, ShiftAmount); | |
392 | ||
393 | // Store the two parts | |
394 | SDValue Store1, Store2; | |
395 | Store1 = DAG.getTruncStore(Chain, dl, TLI.isLittleEndian()?Lo:Hi, Ptr, | |
396 | ST->getPointerInfo(), NewStoredVT, | |
397 | ST->isVolatile(), ST->isNonTemporal(), Alignment); | |
1a4d82fc | 398 | |
223e47cc | 399 | Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, |
1a4d82fc | 400 | DAG.getConstant(IncrementSize, TLI.getPointerTy(AS))); |
223e47cc LB |
401 | Alignment = MinAlign(Alignment, IncrementSize); |
402 | Store2 = DAG.getTruncStore(Chain, dl, TLI.isLittleEndian()?Hi:Lo, Ptr, | |
403 | ST->getPointerInfo().getWithOffset(IncrementSize), | |
404 | NewStoredVT, ST->isVolatile(), ST->isNonTemporal(), | |
1a4d82fc | 405 | Alignment, ST->getAAInfo()); |
223e47cc LB |
406 | |
407 | SDValue Result = | |
408 | DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2); | |
409 | DAGLegalize->ReplaceNode(SDValue(ST, 0), Result); | |
410 | } | |
411 | ||
85aaf69f | 412 | /// Expands an unaligned load to 2 half-size loads. |
223e47cc LB |
413 | static void |
414 | ExpandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG, | |
415 | const TargetLowering &TLI, | |
416 | SDValue &ValResult, SDValue &ChainResult) { | |
417 | assert(LD->getAddressingMode() == ISD::UNINDEXED && | |
418 | "unaligned indexed loads not implemented!"); | |
419 | SDValue Chain = LD->getChain(); | |
420 | SDValue Ptr = LD->getBasePtr(); | |
421 | EVT VT = LD->getValueType(0); | |
422 | EVT LoadedVT = LD->getMemoryVT(); | |
1a4d82fc | 423 | SDLoc dl(LD); |
223e47cc LB |
424 | if (VT.isFloatingPoint() || VT.isVector()) { |
425 | EVT intVT = EVT::getIntegerVT(*DAG.getContext(), LoadedVT.getSizeInBits()); | |
426 | if (TLI.isTypeLegal(intVT) && TLI.isTypeLegal(LoadedVT)) { | |
427 | // Expand to a (misaligned) integer load of the same size, | |
428 | // then bitconvert to floating point or vector. | |
1a4d82fc JJ |
429 | SDValue newLoad = DAG.getLoad(intVT, dl, Chain, Ptr, |
430 | LD->getMemOperand()); | |
223e47cc LB |
431 | SDValue Result = DAG.getNode(ISD::BITCAST, dl, LoadedVT, newLoad); |
432 | if (LoadedVT != VT) | |
433 | Result = DAG.getNode(VT.isFloatingPoint() ? ISD::FP_EXTEND : | |
434 | ISD::ANY_EXTEND, dl, VT, Result); | |
435 | ||
436 | ValResult = Result; | |
437 | ChainResult = Chain; | |
438 | return; | |
439 | } | |
440 | ||
441 | // Copy the value to a (aligned) stack slot using (unaligned) integer | |
442 | // loads and stores, then do a (aligned) load from the stack slot. | |
970d7e83 | 443 | MVT RegVT = TLI.getRegisterType(*DAG.getContext(), intVT); |
223e47cc LB |
444 | unsigned LoadedBytes = LoadedVT.getSizeInBits() / 8; |
445 | unsigned RegBytes = RegVT.getSizeInBits() / 8; | |
446 | unsigned NumRegs = (LoadedBytes + RegBytes - 1) / RegBytes; | |
447 | ||
448 | // Make sure the stack slot is also aligned for the register type. | |
449 | SDValue StackBase = DAG.CreateStackTemporary(LoadedVT, RegVT); | |
450 | ||
451 | SDValue Increment = DAG.getConstant(RegBytes, TLI.getPointerTy()); | |
452 | SmallVector<SDValue, 8> Stores; | |
453 | SDValue StackPtr = StackBase; | |
454 | unsigned Offset = 0; | |
455 | ||
456 | // Do all but one copies using the full register width. | |
457 | for (unsigned i = 1; i < NumRegs; i++) { | |
458 | // Load one integer register's worth from the original location. | |
459 | SDValue Load = DAG.getLoad(RegVT, dl, Chain, Ptr, | |
460 | LD->getPointerInfo().getWithOffset(Offset), | |
461 | LD->isVolatile(), LD->isNonTemporal(), | |
462 | LD->isInvariant(), | |
1a4d82fc JJ |
463 | MinAlign(LD->getAlignment(), Offset), |
464 | LD->getAAInfo()); | |
223e47cc LB |
465 | // Follow the load with a store to the stack slot. Remember the store. |
466 | Stores.push_back(DAG.getStore(Load.getValue(1), dl, Load, StackPtr, | |
467 | MachinePointerInfo(), false, false, 0)); | |
468 | // Increment the pointers. | |
469 | Offset += RegBytes; | |
470 | Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, Increment); | |
471 | StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr, | |
472 | Increment); | |
473 | } | |
474 | ||
475 | // The last copy may be partial. Do an extending load. | |
476 | EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), | |
477 | 8 * (LoadedBytes - Offset)); | |
478 | SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, RegVT, Chain, Ptr, | |
479 | LD->getPointerInfo().getWithOffset(Offset), | |
480 | MemVT, LD->isVolatile(), | |
481 | LD->isNonTemporal(), | |
1a4d82fc JJ |
482 | LD->isInvariant(), |
483 | MinAlign(LD->getAlignment(), Offset), | |
484 | LD->getAAInfo()); | |
223e47cc LB |
485 | // Follow the load with a store to the stack slot. Remember the store. |
486 | // On big-endian machines this requires a truncating store to ensure | |
487 | // that the bits end up in the right place. | |
488 | Stores.push_back(DAG.getTruncStore(Load.getValue(1), dl, Load, StackPtr, | |
489 | MachinePointerInfo(), MemVT, | |
490 | false, false, 0)); | |
491 | ||
492 | // The order of the stores doesn't matter - say it with a TokenFactor. | |
1a4d82fc | 493 | SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores); |
223e47cc LB |
494 | |
495 | // Finally, perform the original load only redirected to the stack slot. | |
496 | Load = DAG.getExtLoad(LD->getExtensionType(), dl, VT, TF, StackBase, | |
1a4d82fc JJ |
497 | MachinePointerInfo(), LoadedVT, false,false, false, |
498 | 0); | |
223e47cc LB |
499 | |
500 | // Callers expect a MERGE_VALUES node. | |
501 | ValResult = Load; | |
502 | ChainResult = TF; | |
503 | return; | |
504 | } | |
505 | assert(LoadedVT.isInteger() && !LoadedVT.isVector() && | |
506 | "Unaligned load of unsupported type."); | |
507 | ||
508 | // Compute the new VT that is half the size of the old one. This is an | |
509 | // integer MVT. | |
510 | unsigned NumBits = LoadedVT.getSizeInBits(); | |
511 | EVT NewLoadedVT; | |
512 | NewLoadedVT = EVT::getIntegerVT(*DAG.getContext(), NumBits/2); | |
513 | NumBits >>= 1; | |
514 | ||
515 | unsigned Alignment = LD->getAlignment(); | |
516 | unsigned IncrementSize = NumBits / 8; | |
517 | ISD::LoadExtType HiExtType = LD->getExtensionType(); | |
518 | ||
519 | // If the original load is NON_EXTLOAD, the hi part load must be ZEXTLOAD. | |
520 | if (HiExtType == ISD::NON_EXTLOAD) | |
521 | HiExtType = ISD::ZEXTLOAD; | |
522 | ||
523 | // Load the value in two parts | |
524 | SDValue Lo, Hi; | |
525 | if (TLI.isLittleEndian()) { | |
526 | Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr, LD->getPointerInfo(), | |
527 | NewLoadedVT, LD->isVolatile(), | |
1a4d82fc JJ |
528 | LD->isNonTemporal(), LD->isInvariant(), Alignment, |
529 | LD->getAAInfo()); | |
223e47cc | 530 | Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, |
1a4d82fc | 531 | DAG.getConstant(IncrementSize, Ptr.getValueType())); |
223e47cc LB |
532 | Hi = DAG.getExtLoad(HiExtType, dl, VT, Chain, Ptr, |
533 | LD->getPointerInfo().getWithOffset(IncrementSize), | |
534 | NewLoadedVT, LD->isVolatile(), | |
1a4d82fc JJ |
535 | LD->isNonTemporal(),LD->isInvariant(), |
536 | MinAlign(Alignment, IncrementSize), LD->getAAInfo()); | |
223e47cc LB |
537 | } else { |
538 | Hi = DAG.getExtLoad(HiExtType, dl, VT, Chain, Ptr, LD->getPointerInfo(), | |
539 | NewLoadedVT, LD->isVolatile(), | |
1a4d82fc JJ |
540 | LD->isNonTemporal(), LD->isInvariant(), Alignment, |
541 | LD->getAAInfo()); | |
223e47cc | 542 | Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, |
1a4d82fc | 543 | DAG.getConstant(IncrementSize, Ptr.getValueType())); |
223e47cc LB |
544 | Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr, |
545 | LD->getPointerInfo().getWithOffset(IncrementSize), | |
546 | NewLoadedVT, LD->isVolatile(), | |
1a4d82fc JJ |
547 | LD->isNonTemporal(), LD->isInvariant(), |
548 | MinAlign(Alignment, IncrementSize), LD->getAAInfo()); | |
223e47cc LB |
549 | } |
550 | ||
551 | // aggregate the two parts | |
552 | SDValue ShiftAmount = DAG.getConstant(NumBits, | |
553 | TLI.getShiftAmountTy(Hi.getValueType())); | |
554 | SDValue Result = DAG.getNode(ISD::SHL, dl, VT, Hi, ShiftAmount); | |
555 | Result = DAG.getNode(ISD::OR, dl, VT, Result, Lo); | |
556 | ||
557 | SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), | |
558 | Hi.getValue(1)); | |
559 | ||
560 | ValResult = Result; | |
561 | ChainResult = TF; | |
562 | } | |
563 | ||
85aaf69f SL |
564 | /// Some target cannot handle a variable insertion index for the |
565 | /// INSERT_VECTOR_ELT instruction. In this case, it | |
223e47cc LB |
566 | /// is necessary to spill the vector being inserted into to memory, perform |
567 | /// the insert there, and then read the result back. | |
568 | SDValue SelectionDAGLegalize:: | |
569 | PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx, | |
1a4d82fc | 570 | SDLoc dl) { |
223e47cc LB |
571 | SDValue Tmp1 = Vec; |
572 | SDValue Tmp2 = Val; | |
573 | SDValue Tmp3 = Idx; | |
574 | ||
575 | // If the target doesn't support this, we have to spill the input vector | |
576 | // to a temporary stack slot, update the element, then reload it. This is | |
577 | // badness. We could also load the value into a vector register (either | |
578 | // with a "move to register" or "extload into register" instruction, then | |
579 | // permute it into place, if the idx is a constant and if the idx is | |
580 | // supported by the target. | |
581 | EVT VT = Tmp1.getValueType(); | |
582 | EVT EltVT = VT.getVectorElementType(); | |
583 | EVT IdxVT = Tmp3.getValueType(); | |
584 | EVT PtrVT = TLI.getPointerTy(); | |
585 | SDValue StackPtr = DAG.CreateStackTemporary(VT); | |
586 | ||
587 | int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); | |
588 | ||
589 | // Store the vector. | |
590 | SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Tmp1, StackPtr, | |
591 | MachinePointerInfo::getFixedStack(SPFI), | |
592 | false, false, 0); | |
593 | ||
594 | // Truncate or zero extend offset to target pointer type. | |
595 | unsigned CastOpc = IdxVT.bitsGT(PtrVT) ? ISD::TRUNCATE : ISD::ZERO_EXTEND; | |
596 | Tmp3 = DAG.getNode(CastOpc, dl, PtrVT, Tmp3); | |
597 | // Add the offset to the index. | |
598 | unsigned EltSize = EltVT.getSizeInBits()/8; | |
599 | Tmp3 = DAG.getNode(ISD::MUL, dl, IdxVT, Tmp3,DAG.getConstant(EltSize, IdxVT)); | |
600 | SDValue StackPtr2 = DAG.getNode(ISD::ADD, dl, IdxVT, Tmp3, StackPtr); | |
601 | // Store the scalar value. | |
602 | Ch = DAG.getTruncStore(Ch, dl, Tmp2, StackPtr2, MachinePointerInfo(), EltVT, | |
603 | false, false, 0); | |
604 | // Load the updated vector. | |
605 | return DAG.getLoad(VT, dl, Ch, StackPtr, | |
1a4d82fc | 606 | MachinePointerInfo::getFixedStack(SPFI), false, false, |
223e47cc LB |
607 | false, 0); |
608 | } | |
609 | ||
610 | ||
611 | SDValue SelectionDAGLegalize:: | |
1a4d82fc | 612 | ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, SDValue Idx, SDLoc dl) { |
223e47cc LB |
613 | if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Idx)) { |
614 | // SCALAR_TO_VECTOR requires that the type of the value being inserted | |
615 | // match the element type of the vector being created, except for | |
616 | // integers in which case the inserted value can be over width. | |
617 | EVT EltVT = Vec.getValueType().getVectorElementType(); | |
618 | if (Val.getValueType() == EltVT || | |
619 | (EltVT.isInteger() && Val.getValueType().bitsGE(EltVT))) { | |
620 | SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, | |
621 | Vec.getValueType(), Val); | |
622 | ||
623 | unsigned NumElts = Vec.getValueType().getVectorNumElements(); | |
624 | // We generate a shuffle of InVec and ScVec, so the shuffle mask | |
625 | // should be 0,1,2,3,4,5... with the appropriate element replaced with | |
626 | // elt 0 of the RHS. | |
627 | SmallVector<int, 8> ShufOps; | |
628 | for (unsigned i = 0; i != NumElts; ++i) | |
629 | ShufOps.push_back(i != InsertPos->getZExtValue() ? i : NumElts); | |
630 | ||
631 | return DAG.getVectorShuffle(Vec.getValueType(), dl, Vec, ScVec, | |
632 | &ShufOps[0]); | |
633 | } | |
634 | } | |
635 | return PerformInsertVectorEltInMemory(Vec, Val, Idx, dl); | |
636 | } | |
637 | ||
638 | SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) { | |
639 | // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr' | |
640 | // FIXME: We shouldn't do this for TargetConstantFP's. | |
641 | // FIXME: move this to the DAG Combiner! Note that we can't regress due | |
642 | // to phase ordering between legalized code and the dag combiner. This | |
643 | // probably means that we need to integrate dag combiner and legalizer | |
644 | // together. | |
645 | // We generally can't do this one for long doubles. | |
646 | SDValue Chain = ST->getChain(); | |
647 | SDValue Ptr = ST->getBasePtr(); | |
648 | unsigned Alignment = ST->getAlignment(); | |
649 | bool isVolatile = ST->isVolatile(); | |
650 | bool isNonTemporal = ST->isNonTemporal(); | |
1a4d82fc JJ |
651 | AAMDNodes AAInfo = ST->getAAInfo(); |
652 | SDLoc dl(ST); | |
223e47cc LB |
653 | if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(ST->getValue())) { |
654 | if (CFP->getValueType(0) == MVT::f32 && | |
655 | TLI.isTypeLegal(MVT::i32)) { | |
656 | SDValue Con = DAG.getConstant(CFP->getValueAPF(). | |
657 | bitcastToAPInt().zextOrTrunc(32), | |
658 | MVT::i32); | |
659 | return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(), | |
1a4d82fc | 660 | isVolatile, isNonTemporal, Alignment, AAInfo); |
223e47cc LB |
661 | } |
662 | ||
663 | if (CFP->getValueType(0) == MVT::f64) { | |
664 | // If this target supports 64-bit registers, do a single 64-bit store. | |
665 | if (TLI.isTypeLegal(MVT::i64)) { | |
666 | SDValue Con = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt(). | |
667 | zextOrTrunc(64), MVT::i64); | |
668 | return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(), | |
1a4d82fc | 669 | isVolatile, isNonTemporal, Alignment, AAInfo); |
223e47cc LB |
670 | } |
671 | ||
672 | if (TLI.isTypeLegal(MVT::i32) && !ST->isVolatile()) { | |
673 | // Otherwise, if the target supports 32-bit registers, use 2 32-bit | |
674 | // stores. If the target supports neither 32- nor 64-bits, this | |
675 | // xform is certainly not worth it. | |
676 | const APInt &IntVal =CFP->getValueAPF().bitcastToAPInt(); | |
677 | SDValue Lo = DAG.getConstant(IntVal.trunc(32), MVT::i32); | |
678 | SDValue Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), MVT::i32); | |
679 | if (TLI.isBigEndian()) std::swap(Lo, Hi); | |
680 | ||
681 | Lo = DAG.getStore(Chain, dl, Lo, Ptr, ST->getPointerInfo(), isVolatile, | |
1a4d82fc | 682 | isNonTemporal, Alignment, AAInfo); |
223e47cc | 683 | Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, |
1a4d82fc | 684 | DAG.getConstant(4, Ptr.getValueType())); |
223e47cc LB |
685 | Hi = DAG.getStore(Chain, dl, Hi, Ptr, |
686 | ST->getPointerInfo().getWithOffset(4), | |
1a4d82fc JJ |
687 | isVolatile, isNonTemporal, MinAlign(Alignment, 4U), |
688 | AAInfo); | |
223e47cc LB |
689 | |
690 | return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); | |
691 | } | |
692 | } | |
693 | } | |
1a4d82fc | 694 | return SDValue(nullptr, 0); |
223e47cc LB |
695 | } |
696 | ||
697 | void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) { | |
698 | StoreSDNode *ST = cast<StoreSDNode>(Node); | |
699 | SDValue Chain = ST->getChain(); | |
700 | SDValue Ptr = ST->getBasePtr(); | |
1a4d82fc | 701 | SDLoc dl(Node); |
223e47cc LB |
702 | |
703 | unsigned Alignment = ST->getAlignment(); | |
704 | bool isVolatile = ST->isVolatile(); | |
705 | bool isNonTemporal = ST->isNonTemporal(); | |
1a4d82fc | 706 | AAMDNodes AAInfo = ST->getAAInfo(); |
223e47cc LB |
707 | |
708 | if (!ST->isTruncatingStore()) { | |
709 | if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) { | |
710 | ReplaceNode(ST, OptStore); | |
711 | return; | |
712 | } | |
713 | ||
714 | { | |
715 | SDValue Value = ST->getValue(); | |
970d7e83 | 716 | MVT VT = Value.getSimpleValueType(); |
223e47cc LB |
717 | switch (TLI.getOperationAction(ISD::STORE, VT)) { |
718 | default: llvm_unreachable("This action is not supported yet!"); | |
1a4d82fc | 719 | case TargetLowering::Legal: { |
223e47cc LB |
720 | // If this is an unaligned store and the target doesn't support it, |
721 | // expand it. | |
1a4d82fc JJ |
722 | unsigned AS = ST->getAddressSpace(); |
723 | unsigned Align = ST->getAlignment(); | |
724 | if (!TLI.allowsMisalignedMemoryAccesses(ST->getMemoryVT(), AS, Align)) { | |
223e47cc | 725 | Type *Ty = ST->getMemoryVT().getTypeForEVT(*DAG.getContext()); |
970d7e83 | 726 | unsigned ABIAlignment= TLI.getDataLayout()->getABITypeAlignment(Ty); |
1a4d82fc | 727 | if (Align < ABIAlignment) |
85aaf69f | 728 | ExpandUnalignedStore(cast<StoreSDNode>(Node), DAG, TLI, this); |
223e47cc LB |
729 | } |
730 | break; | |
1a4d82fc | 731 | } |
223e47cc LB |
732 | case TargetLowering::Custom: { |
733 | SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG); | |
734 | if (Res.getNode()) | |
735 | ReplaceNode(SDValue(Node, 0), Res); | |
736 | return; | |
737 | } | |
738 | case TargetLowering::Promote: { | |
970d7e83 LB |
739 | MVT NVT = TLI.getTypeToPromoteTo(ISD::STORE, VT); |
740 | assert(NVT.getSizeInBits() == VT.getSizeInBits() && | |
741 | "Can only promote stores to same size type"); | |
742 | Value = DAG.getNode(ISD::BITCAST, dl, NVT, Value); | |
223e47cc LB |
743 | SDValue Result = |
744 | DAG.getStore(Chain, dl, Value, Ptr, | |
745 | ST->getPointerInfo(), isVolatile, | |
1a4d82fc | 746 | isNonTemporal, Alignment, AAInfo); |
223e47cc LB |
747 | ReplaceNode(SDValue(Node, 0), Result); |
748 | break; | |
749 | } | |
750 | } | |
751 | return; | |
752 | } | |
753 | } else { | |
754 | SDValue Value = ST->getValue(); | |
755 | ||
756 | EVT StVT = ST->getMemoryVT(); | |
757 | unsigned StWidth = StVT.getSizeInBits(); | |
758 | ||
759 | if (StWidth != StVT.getStoreSizeInBits()) { | |
760 | // Promote to a byte-sized store with upper bits zero if not | |
761 | // storing an integral number of bytes. For example, promote | |
762 | // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1) | |
763 | EVT NVT = EVT::getIntegerVT(*DAG.getContext(), | |
764 | StVT.getStoreSizeInBits()); | |
765 | Value = DAG.getZeroExtendInReg(Value, dl, StVT); | |
766 | SDValue Result = | |
767 | DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), | |
85aaf69f | 768 | NVT, isVolatile, isNonTemporal, Alignment, AAInfo); |
223e47cc LB |
769 | ReplaceNode(SDValue(Node, 0), Result); |
770 | } else if (StWidth & (StWidth - 1)) { | |
771 | // If not storing a power-of-2 number of bits, expand as two stores. | |
772 | assert(!StVT.isVector() && "Unsupported truncstore!"); | |
773 | unsigned RoundWidth = 1 << Log2_32(StWidth); | |
774 | assert(RoundWidth < StWidth); | |
775 | unsigned ExtraWidth = StWidth - RoundWidth; | |
776 | assert(ExtraWidth < RoundWidth); | |
777 | assert(!(RoundWidth % 8) && !(ExtraWidth % 8) && | |
778 | "Store size not an integral number of bytes!"); | |
779 | EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth); | |
780 | EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth); | |
781 | SDValue Lo, Hi; | |
782 | unsigned IncrementSize; | |
783 | ||
784 | if (TLI.isLittleEndian()) { | |
785 | // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16) | |
786 | // Store the bottom RoundWidth bits. | |
787 | Lo = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), | |
788 | RoundVT, | |
1a4d82fc JJ |
789 | isVolatile, isNonTemporal, Alignment, |
790 | AAInfo); | |
223e47cc LB |
791 | |
792 | // Store the remaining ExtraWidth bits. | |
793 | IncrementSize = RoundWidth / 8; | |
794 | Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, | |
1a4d82fc | 795 | DAG.getConstant(IncrementSize, Ptr.getValueType())); |
223e47cc LB |
796 | Hi = DAG.getNode(ISD::SRL, dl, Value.getValueType(), Value, |
797 | DAG.getConstant(RoundWidth, | |
1a4d82fc | 798 | TLI.getShiftAmountTy(Value.getValueType()))); |
223e47cc LB |
799 | Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, |
800 | ST->getPointerInfo().getWithOffset(IncrementSize), | |
801 | ExtraVT, isVolatile, isNonTemporal, | |
1a4d82fc | 802 | MinAlign(Alignment, IncrementSize), AAInfo); |
223e47cc LB |
803 | } else { |
804 | // Big endian - avoid unaligned stores. | |
805 | // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X | |
806 | // Store the top RoundWidth bits. | |
807 | Hi = DAG.getNode(ISD::SRL, dl, Value.getValueType(), Value, | |
808 | DAG.getConstant(ExtraWidth, | |
1a4d82fc | 809 | TLI.getShiftAmountTy(Value.getValueType()))); |
223e47cc | 810 | Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, ST->getPointerInfo(), |
1a4d82fc JJ |
811 | RoundVT, isVolatile, isNonTemporal, Alignment, |
812 | AAInfo); | |
223e47cc LB |
813 | |
814 | // Store the remaining ExtraWidth bits. | |
815 | IncrementSize = RoundWidth / 8; | |
816 | Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, | |
1a4d82fc | 817 | DAG.getConstant(IncrementSize, Ptr.getValueType())); |
223e47cc LB |
818 | Lo = DAG.getTruncStore(Chain, dl, Value, Ptr, |
819 | ST->getPointerInfo().getWithOffset(IncrementSize), | |
820 | ExtraVT, isVolatile, isNonTemporal, | |
1a4d82fc | 821 | MinAlign(Alignment, IncrementSize), AAInfo); |
223e47cc LB |
822 | } |
823 | ||
824 | // The order of the stores doesn't matter. | |
825 | SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); | |
826 | ReplaceNode(SDValue(Node, 0), Result); | |
827 | } else { | |
970d7e83 LB |
828 | switch (TLI.getTruncStoreAction(ST->getValue().getSimpleValueType(), |
829 | StVT.getSimpleVT())) { | |
223e47cc | 830 | default: llvm_unreachable("This action is not supported yet!"); |
1a4d82fc JJ |
831 | case TargetLowering::Legal: { |
832 | unsigned AS = ST->getAddressSpace(); | |
833 | unsigned Align = ST->getAlignment(); | |
223e47cc LB |
834 | // If this is an unaligned store and the target doesn't support it, |
835 | // expand it. | |
1a4d82fc | 836 | if (!TLI.allowsMisalignedMemoryAccesses(ST->getMemoryVT(), AS, Align)) { |
223e47cc | 837 | Type *Ty = ST->getMemoryVT().getTypeForEVT(*DAG.getContext()); |
970d7e83 | 838 | unsigned ABIAlignment= TLI.getDataLayout()->getABITypeAlignment(Ty); |
1a4d82fc | 839 | if (Align < ABIAlignment) |
223e47cc LB |
840 | ExpandUnalignedStore(cast<StoreSDNode>(Node), DAG, TLI, this); |
841 | } | |
842 | break; | |
1a4d82fc | 843 | } |
223e47cc LB |
844 | case TargetLowering::Custom: { |
845 | SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG); | |
846 | if (Res.getNode()) | |
847 | ReplaceNode(SDValue(Node, 0), Res); | |
848 | return; | |
849 | } | |
850 | case TargetLowering::Expand: | |
851 | assert(!StVT.isVector() && | |
852 | "Vector Stores are handled in LegalizeVectorOps"); | |
853 | ||
854 | // TRUNCSTORE:i16 i32 -> STORE i16 | |
855 | assert(TLI.isTypeLegal(StVT) && | |
856 | "Do not know how to expand this store!"); | |
857 | Value = DAG.getNode(ISD::TRUNCATE, dl, StVT, Value); | |
858 | SDValue Result = | |
859 | DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), | |
1a4d82fc | 860 | isVolatile, isNonTemporal, Alignment, AAInfo); |
223e47cc LB |
861 | ReplaceNode(SDValue(Node, 0), Result); |
862 | break; | |
863 | } | |
864 | } | |
865 | } | |
866 | } | |
867 | ||
868 | void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) { | |
869 | LoadSDNode *LD = cast<LoadSDNode>(Node); | |
870 | SDValue Chain = LD->getChain(); // The chain. | |
871 | SDValue Ptr = LD->getBasePtr(); // The base pointer. | |
872 | SDValue Value; // The value returned by the load op. | |
1a4d82fc | 873 | SDLoc dl(Node); |
223e47cc LB |
874 | |
875 | ISD::LoadExtType ExtType = LD->getExtensionType(); | |
876 | if (ExtType == ISD::NON_EXTLOAD) { | |
970d7e83 | 877 | MVT VT = Node->getSimpleValueType(0); |
223e47cc LB |
878 | SDValue RVal = SDValue(Node, 0); |
879 | SDValue RChain = SDValue(Node, 1); | |
880 | ||
881 | switch (TLI.getOperationAction(Node->getOpcode(), VT)) { | |
882 | default: llvm_unreachable("This action is not supported yet!"); | |
1a4d82fc JJ |
883 | case TargetLowering::Legal: { |
884 | unsigned AS = LD->getAddressSpace(); | |
885 | unsigned Align = LD->getAlignment(); | |
223e47cc LB |
886 | // If this is an unaligned load and the target doesn't support it, |
887 | // expand it. | |
1a4d82fc | 888 | if (!TLI.allowsMisalignedMemoryAccesses(LD->getMemoryVT(), AS, Align)) { |
223e47cc LB |
889 | Type *Ty = LD->getMemoryVT().getTypeForEVT(*DAG.getContext()); |
890 | unsigned ABIAlignment = | |
970d7e83 | 891 | TLI.getDataLayout()->getABITypeAlignment(Ty); |
1a4d82fc | 892 | if (Align < ABIAlignment){ |
223e47cc LB |
893 | ExpandUnalignedLoad(cast<LoadSDNode>(Node), DAG, TLI, RVal, RChain); |
894 | } | |
895 | } | |
896 | break; | |
1a4d82fc | 897 | } |
223e47cc LB |
898 | case TargetLowering::Custom: { |
899 | SDValue Res = TLI.LowerOperation(RVal, DAG); | |
900 | if (Res.getNode()) { | |
901 | RVal = Res; | |
902 | RChain = Res.getValue(1); | |
903 | } | |
904 | break; | |
905 | } | |
906 | case TargetLowering::Promote: { | |
970d7e83 LB |
907 | MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT); |
908 | assert(NVT.getSizeInBits() == VT.getSizeInBits() && | |
909 | "Can only promote loads to same size type"); | |
223e47cc | 910 | |
1a4d82fc | 911 | SDValue Res = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getMemOperand()); |
223e47cc LB |
912 | RVal = DAG.getNode(ISD::BITCAST, dl, VT, Res); |
913 | RChain = Res.getValue(1); | |
914 | break; | |
915 | } | |
916 | } | |
917 | if (RChain.getNode() != Node) { | |
918 | assert(RVal.getNode() != Node && "Load must be completely replaced"); | |
919 | DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), RVal); | |
920 | DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), RChain); | |
1a4d82fc JJ |
921 | if (UpdatedNodes) { |
922 | UpdatedNodes->insert(RVal.getNode()); | |
923 | UpdatedNodes->insert(RChain.getNode()); | |
924 | } | |
223e47cc LB |
925 | ReplacedNode(Node); |
926 | } | |
927 | return; | |
928 | } | |
929 | ||
930 | EVT SrcVT = LD->getMemoryVT(); | |
931 | unsigned SrcWidth = SrcVT.getSizeInBits(); | |
932 | unsigned Alignment = LD->getAlignment(); | |
933 | bool isVolatile = LD->isVolatile(); | |
934 | bool isNonTemporal = LD->isNonTemporal(); | |
1a4d82fc JJ |
935 | bool isInvariant = LD->isInvariant(); |
936 | AAMDNodes AAInfo = LD->getAAInfo(); | |
223e47cc LB |
937 | |
938 | if (SrcWidth != SrcVT.getStoreSizeInBits() && | |
939 | // Some targets pretend to have an i1 loading operation, and actually | |
940 | // load an i8. This trick is correct for ZEXTLOAD because the top 7 | |
941 | // bits are guaranteed to be zero; it helps the optimizers understand | |
942 | // that these bits are zero. It is also useful for EXTLOAD, since it | |
943 | // tells the optimizers that those bits are undefined. It would be | |
944 | // nice to have an effective generic way of getting these benefits... | |
945 | // Until such a way is found, don't insist on promoting i1 here. | |
946 | (SrcVT != MVT::i1 || | |
85aaf69f SL |
947 | TLI.getLoadExtAction(ExtType, Node->getValueType(0), MVT::i1) == |
948 | TargetLowering::Promote)) { | |
223e47cc LB |
949 | // Promote to a byte-sized load if not loading an integral number of |
950 | // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24. | |
951 | unsigned NewWidth = SrcVT.getStoreSizeInBits(); | |
952 | EVT NVT = EVT::getIntegerVT(*DAG.getContext(), NewWidth); | |
953 | SDValue Ch; | |
954 | ||
955 | // The extra bits are guaranteed to be zero, since we stored them that | |
956 | // way. A zext load from NVT thus automatically gives zext from SrcVT. | |
957 | ||
958 | ISD::LoadExtType NewExtType = | |
959 | ExtType == ISD::ZEXTLOAD ? ISD::ZEXTLOAD : ISD::EXTLOAD; | |
960 | ||
961 | SDValue Result = | |
962 | DAG.getExtLoad(NewExtType, dl, Node->getValueType(0), | |
963 | Chain, Ptr, LD->getPointerInfo(), | |
1a4d82fc JJ |
964 | NVT, isVolatile, isNonTemporal, isInvariant, Alignment, |
965 | AAInfo); | |
223e47cc LB |
966 | |
967 | Ch = Result.getValue(1); // The chain. | |
968 | ||
969 | if (ExtType == ISD::SEXTLOAD) | |
970 | // Having the top bits zero doesn't help when sign extending. | |
971 | Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, | |
972 | Result.getValueType(), | |
973 | Result, DAG.getValueType(SrcVT)); | |
974 | else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType()) | |
975 | // All the top bits are guaranteed to be zero - inform the optimizers. | |
976 | Result = DAG.getNode(ISD::AssertZext, dl, | |
977 | Result.getValueType(), Result, | |
978 | DAG.getValueType(SrcVT)); | |
979 | ||
980 | Value = Result; | |
981 | Chain = Ch; | |
982 | } else if (SrcWidth & (SrcWidth - 1)) { | |
983 | // If not loading a power-of-2 number of bits, expand as two loads. | |
984 | assert(!SrcVT.isVector() && "Unsupported extload!"); | |
985 | unsigned RoundWidth = 1 << Log2_32(SrcWidth); | |
986 | assert(RoundWidth < SrcWidth); | |
987 | unsigned ExtraWidth = SrcWidth - RoundWidth; | |
988 | assert(ExtraWidth < RoundWidth); | |
989 | assert(!(RoundWidth % 8) && !(ExtraWidth % 8) && | |
990 | "Load size not an integral number of bytes!"); | |
991 | EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth); | |
992 | EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth); | |
993 | SDValue Lo, Hi, Ch; | |
994 | unsigned IncrementSize; | |
995 | ||
996 | if (TLI.isLittleEndian()) { | |
997 | // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16) | |
998 | // Load the bottom RoundWidth bits. | |
999 | Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), | |
1000 | Chain, Ptr, | |
1001 | LD->getPointerInfo(), RoundVT, isVolatile, | |
1a4d82fc | 1002 | isNonTemporal, isInvariant, Alignment, AAInfo); |
223e47cc LB |
1003 | |
1004 | // Load the remaining ExtraWidth bits. | |
1005 | IncrementSize = RoundWidth / 8; | |
1006 | Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, | |
1a4d82fc | 1007 | DAG.getConstant(IncrementSize, Ptr.getValueType())); |
223e47cc LB |
1008 | Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr, |
1009 | LD->getPointerInfo().getWithOffset(IncrementSize), | |
1a4d82fc JJ |
1010 | ExtraVT, isVolatile, isNonTemporal, isInvariant, |
1011 | MinAlign(Alignment, IncrementSize), AAInfo); | |
223e47cc LB |
1012 | |
1013 | // Build a factor node to remember that this load is independent of | |
1014 | // the other one. | |
1015 | Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), | |
1016 | Hi.getValue(1)); | |
1017 | ||
1018 | // Move the top bits to the right place. | |
1019 | Hi = DAG.getNode(ISD::SHL, dl, Hi.getValueType(), Hi, | |
1020 | DAG.getConstant(RoundWidth, | |
1a4d82fc | 1021 | TLI.getShiftAmountTy(Hi.getValueType()))); |
223e47cc LB |
1022 | |
1023 | // Join the hi and lo parts. | |
1024 | Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi); | |
1025 | } else { | |
1026 | // Big endian - avoid unaligned loads. | |
1027 | // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8 | |
1028 | // Load the top RoundWidth bits. | |
1029 | Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr, | |
1030 | LD->getPointerInfo(), RoundVT, isVolatile, | |
1a4d82fc | 1031 | isNonTemporal, isInvariant, Alignment, AAInfo); |
223e47cc LB |
1032 | |
1033 | // Load the remaining ExtraWidth bits. | |
1034 | IncrementSize = RoundWidth / 8; | |
1035 | Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, | |
1a4d82fc | 1036 | DAG.getConstant(IncrementSize, Ptr.getValueType())); |
223e47cc LB |
1037 | Lo = DAG.getExtLoad(ISD::ZEXTLOAD, |
1038 | dl, Node->getValueType(0), Chain, Ptr, | |
1039 | LD->getPointerInfo().getWithOffset(IncrementSize), | |
1a4d82fc JJ |
1040 | ExtraVT, isVolatile, isNonTemporal, isInvariant, |
1041 | MinAlign(Alignment, IncrementSize), AAInfo); | |
223e47cc LB |
1042 | |
1043 | // Build a factor node to remember that this load is independent of | |
1044 | // the other one. | |
1045 | Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), | |
1046 | Hi.getValue(1)); | |
1047 | ||
1048 | // Move the top bits to the right place. | |
1049 | Hi = DAG.getNode(ISD::SHL, dl, Hi.getValueType(), Hi, | |
1050 | DAG.getConstant(ExtraWidth, | |
1a4d82fc | 1051 | TLI.getShiftAmountTy(Hi.getValueType()))); |
223e47cc LB |
1052 | |
1053 | // Join the hi and lo parts. | |
1054 | Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi); | |
1055 | } | |
1056 | ||
1057 | Chain = Ch; | |
1058 | } else { | |
1059 | bool isCustom = false; | |
85aaf69f SL |
1060 | switch (TLI.getLoadExtAction(ExtType, Node->getValueType(0), |
1061 | SrcVT.getSimpleVT())) { | |
223e47cc LB |
1062 | default: llvm_unreachable("This action is not supported yet!"); |
1063 | case TargetLowering::Custom: | |
1a4d82fc JJ |
1064 | isCustom = true; |
1065 | // FALLTHROUGH | |
223e47cc | 1066 | case TargetLowering::Legal: { |
1a4d82fc JJ |
1067 | Value = SDValue(Node, 0); |
1068 | Chain = SDValue(Node, 1); | |
1069 | ||
1070 | if (isCustom) { | |
1071 | SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG); | |
1072 | if (Res.getNode()) { | |
1073 | Value = Res; | |
1074 | Chain = Res.getValue(1); | |
1075 | } | |
1076 | } else { | |
1077 | // If this is an unaligned load and the target doesn't support | |
1078 | // it, expand it. | |
1079 | EVT MemVT = LD->getMemoryVT(); | |
1080 | unsigned AS = LD->getAddressSpace(); | |
1081 | unsigned Align = LD->getAlignment(); | |
1082 | if (!TLI.allowsMisalignedMemoryAccesses(MemVT, AS, Align)) { | |
85aaf69f SL |
1083 | Type *Ty = LD->getMemoryVT().getTypeForEVT(*DAG.getContext()); |
1084 | unsigned ABIAlignment = TLI.getDataLayout()->getABITypeAlignment(Ty); | |
1a4d82fc | 1085 | if (Align < ABIAlignment){ |
85aaf69f | 1086 | ExpandUnalignedLoad(cast<LoadSDNode>(Node), DAG, TLI, Value, Chain); |
1a4d82fc JJ |
1087 | } |
1088 | } | |
1089 | } | |
1090 | break; | |
223e47cc LB |
1091 | } |
1092 | case TargetLowering::Expand: | |
85aaf69f SL |
1093 | if (!TLI.isLoadExtLegal(ISD::EXTLOAD, Node->getValueType(0), SrcVT)) { |
1094 | // If the source type is not legal, see if there is a legal extload to | |
1095 | // an intermediate type that we can then extend further. | |
1096 | EVT LoadVT = TLI.getRegisterType(SrcVT.getSimpleVT()); | |
1097 | if (TLI.isTypeLegal(SrcVT) || // Same as SrcVT == LoadVT? | |
1098 | TLI.isLoadExtLegal(ExtType, LoadVT, SrcVT)) { | |
1099 | // If we are loading a legal type, this is a non-extload followed by a | |
1100 | // full extend. | |
1101 | ISD::LoadExtType MidExtType = | |
1102 | (LoadVT == SrcVT) ? ISD::NON_EXTLOAD : ExtType; | |
1103 | ||
1104 | SDValue Load = DAG.getExtLoad(MidExtType, dl, LoadVT, Chain, Ptr, | |
1105 | SrcVT, LD->getMemOperand()); | |
1106 | unsigned ExtendOp = | |
1107 | ISD::getExtForLoadExtType(SrcVT.isFloatingPoint(), ExtType); | |
1108 | Value = DAG.getNode(ExtendOp, dl, Node->getValueType(0), Load); | |
1109 | Chain = Load.getValue(1); | |
1a4d82fc | 1110 | break; |
1a4d82fc | 1111 | } |
1a4d82fc JJ |
1112 | } |
1113 | ||
1114 | assert(!SrcVT.isVector() && | |
1115 | "Vector Loads are handled in LegalizeVectorOps"); | |
1116 | ||
1117 | // FIXME: This does not work for vectors on most targets. Sign- | |
1118 | // and zero-extend operations are currently folded into extending | |
1119 | // loads, whether they are legal or not, and then we end up here | |
1120 | // without any support for legalizing them. | |
1121 | assert(ExtType != ISD::EXTLOAD && | |
1122 | "EXTLOAD should always be supported!"); | |
1123 | // Turn the unsupported load into an EXTLOAD followed by an | |
1124 | // explicit zero/sign extend inreg. | |
1125 | SDValue Result = DAG.getExtLoad(ISD::EXTLOAD, dl, | |
1126 | Node->getValueType(0), | |
1127 | Chain, Ptr, SrcVT, | |
1128 | LD->getMemOperand()); | |
1129 | SDValue ValRes; | |
1130 | if (ExtType == ISD::SEXTLOAD) | |
1131 | ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, | |
1132 | Result.getValueType(), | |
1133 | Result, DAG.getValueType(SrcVT)); | |
1134 | else | |
85aaf69f | 1135 | ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT.getScalarType()); |
1a4d82fc JJ |
1136 | Value = ValRes; |
1137 | Chain = Result.getValue(1); | |
1138 | break; | |
223e47cc LB |
1139 | } |
1140 | } | |
1141 | ||
1142 | // Since loads produce two values, make sure to remember that we legalized | |
1143 | // both of them. | |
1144 | if (Chain.getNode() != Node) { | |
1145 | assert(Value.getNode() != Node && "Load must be completely replaced"); | |
1146 | DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Value); | |
1147 | DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain); | |
1a4d82fc JJ |
1148 | if (UpdatedNodes) { |
1149 | UpdatedNodes->insert(Value.getNode()); | |
1150 | UpdatedNodes->insert(Chain.getNode()); | |
1151 | } | |
223e47cc LB |
1152 | ReplacedNode(Node); |
1153 | } | |
1154 | } | |
1155 | ||
85aaf69f | 1156 | /// Return a legal replacement for the given operation, with all legal operands. |
223e47cc | 1157 | void SelectionDAGLegalize::LegalizeOp(SDNode *Node) { |
1a4d82fc JJ |
1158 | DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG)); |
1159 | ||
223e47cc LB |
1160 | if (Node->getOpcode() == ISD::TargetConstant) // Allow illegal target nodes. |
1161 | return; | |
1162 | ||
1163 | for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) | |
1164 | assert(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) == | |
1165 | TargetLowering::TypeLegal && | |
1166 | "Unexpected illegal type!"); | |
1167 | ||
1168 | for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) | |
1169 | assert((TLI.getTypeAction(*DAG.getContext(), | |
1170 | Node->getOperand(i).getValueType()) == | |
1171 | TargetLowering::TypeLegal || | |
1172 | Node->getOperand(i).getOpcode() == ISD::TargetConstant) && | |
1173 | "Unexpected illegal type!"); | |
1174 | ||
1175 | // Figure out the correct action; the way to query this varies by opcode | |
1176 | TargetLowering::LegalizeAction Action = TargetLowering::Legal; | |
1177 | bool SimpleFinishLegalizing = true; | |
1178 | switch (Node->getOpcode()) { | |
1179 | case ISD::INTRINSIC_W_CHAIN: | |
1180 | case ISD::INTRINSIC_WO_CHAIN: | |
1181 | case ISD::INTRINSIC_VOID: | |
1182 | case ISD::STACKSAVE: | |
1183 | Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other); | |
1184 | break; | |
1185 | case ISD::VAARG: | |
1186 | Action = TLI.getOperationAction(Node->getOpcode(), | |
1187 | Node->getValueType(0)); | |
1188 | if (Action != TargetLowering::Promote) | |
1189 | Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other); | |
1190 | break; | |
1a4d82fc | 1191 | case ISD::FP_TO_FP16: |
223e47cc LB |
1192 | case ISD::SINT_TO_FP: |
1193 | case ISD::UINT_TO_FP: | |
1194 | case ISD::EXTRACT_VECTOR_ELT: | |
1195 | Action = TLI.getOperationAction(Node->getOpcode(), | |
1196 | Node->getOperand(0).getValueType()); | |
1197 | break; | |
1198 | case ISD::FP_ROUND_INREG: | |
1199 | case ISD::SIGN_EXTEND_INREG: { | |
1200 | EVT InnerType = cast<VTSDNode>(Node->getOperand(1))->getVT(); | |
1201 | Action = TLI.getOperationAction(Node->getOpcode(), InnerType); | |
1202 | break; | |
1203 | } | |
1204 | case ISD::ATOMIC_STORE: { | |
1205 | Action = TLI.getOperationAction(Node->getOpcode(), | |
1206 | Node->getOperand(2).getValueType()); | |
1207 | break; | |
1208 | } | |
1209 | case ISD::SELECT_CC: | |
1210 | case ISD::SETCC: | |
1211 | case ISD::BR_CC: { | |
1212 | unsigned CCOperand = Node->getOpcode() == ISD::SELECT_CC ? 4 : | |
1213 | Node->getOpcode() == ISD::SETCC ? 2 : 1; | |
1214 | unsigned CompareOperand = Node->getOpcode() == ISD::BR_CC ? 2 : 0; | |
970d7e83 | 1215 | MVT OpVT = Node->getOperand(CompareOperand).getSimpleValueType(); |
223e47cc LB |
1216 | ISD::CondCode CCCode = |
1217 | cast<CondCodeSDNode>(Node->getOperand(CCOperand))->get(); | |
1218 | Action = TLI.getCondCodeAction(CCCode, OpVT); | |
1219 | if (Action == TargetLowering::Legal) { | |
1220 | if (Node->getOpcode() == ISD::SELECT_CC) | |
1221 | Action = TLI.getOperationAction(Node->getOpcode(), | |
1222 | Node->getValueType(0)); | |
1223 | else | |
1224 | Action = TLI.getOperationAction(Node->getOpcode(), OpVT); | |
1225 | } | |
1226 | break; | |
1227 | } | |
1228 | case ISD::LOAD: | |
1229 | case ISD::STORE: | |
1230 | // FIXME: Model these properly. LOAD and STORE are complicated, and | |
1231 | // STORE expects the unlegalized operand in some cases. | |
1232 | SimpleFinishLegalizing = false; | |
1233 | break; | |
1234 | case ISD::CALLSEQ_START: | |
1235 | case ISD::CALLSEQ_END: | |
1236 | // FIXME: This shouldn't be necessary. These nodes have special properties | |
1237 | // dealing with the recursive nature of legalization. Removing this | |
1238 | // special case should be done as part of making LegalizeDAG non-recursive. | |
1239 | SimpleFinishLegalizing = false; | |
1240 | break; | |
1241 | case ISD::EXTRACT_ELEMENT: | |
1242 | case ISD::FLT_ROUNDS_: | |
1243 | case ISD::SADDO: | |
1244 | case ISD::SSUBO: | |
1245 | case ISD::UADDO: | |
1246 | case ISD::USUBO: | |
1247 | case ISD::SMULO: | |
1248 | case ISD::UMULO: | |
1249 | case ISD::FPOWI: | |
1250 | case ISD::MERGE_VALUES: | |
1251 | case ISD::EH_RETURN: | |
1252 | case ISD::FRAME_TO_ARGS_OFFSET: | |
1253 | case ISD::EH_SJLJ_SETJMP: | |
1254 | case ISD::EH_SJLJ_LONGJMP: | |
1255 | // These operations lie about being legal: when they claim to be legal, | |
1256 | // they should actually be expanded. | |
1257 | Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); | |
1258 | if (Action == TargetLowering::Legal) | |
1259 | Action = TargetLowering::Expand; | |
1260 | break; | |
1261 | case ISD::INIT_TRAMPOLINE: | |
1262 | case ISD::ADJUST_TRAMPOLINE: | |
1263 | case ISD::FRAMEADDR: | |
1264 | case ISD::RETURNADDR: | |
1265 | // These operations lie about being legal: when they claim to be legal, | |
1266 | // they should actually be custom-lowered. | |
1267 | Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); | |
1268 | if (Action == TargetLowering::Legal) | |
1269 | Action = TargetLowering::Custom; | |
1270 | break; | |
1a4d82fc JJ |
1271 | case ISD::READ_REGISTER: |
1272 | case ISD::WRITE_REGISTER: | |
1273 | // Named register is legal in the DAG, but blocked by register name | |
1274 | // selection if not implemented by target (to chose the correct register) | |
1275 | // They'll be converted to Copy(To/From)Reg. | |
1276 | Action = TargetLowering::Legal; | |
1277 | break; | |
970d7e83 LB |
1278 | case ISD::DEBUGTRAP: |
1279 | Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); | |
1280 | if (Action == TargetLowering::Expand) { | |
1281 | // replace ISD::DEBUGTRAP with ISD::TRAP | |
1282 | SDValue NewVal; | |
1a4d82fc | 1283 | NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(), |
970d7e83 LB |
1284 | Node->getOperand(0)); |
1285 | ReplaceNode(Node, NewVal.getNode()); | |
1286 | LegalizeOp(NewVal.getNode()); | |
1287 | return; | |
1288 | } | |
1289 | break; | |
1290 | ||
223e47cc LB |
1291 | default: |
1292 | if (Node->getOpcode() >= ISD::BUILTIN_OP_END) { | |
1293 | Action = TargetLowering::Legal; | |
1294 | } else { | |
1295 | Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); | |
1296 | } | |
1297 | break; | |
1298 | } | |
1299 | ||
1300 | if (SimpleFinishLegalizing) { | |
1301 | SDNode *NewNode = Node; | |
1302 | switch (Node->getOpcode()) { | |
1303 | default: break; | |
1304 | case ISD::SHL: | |
1305 | case ISD::SRL: | |
1306 | case ISD::SRA: | |
1307 | case ISD::ROTL: | |
1308 | case ISD::ROTR: | |
1309 | // Legalizing shifts/rotates requires adjusting the shift amount | |
1310 | // to the appropriate width. | |
1311 | if (!Node->getOperand(1).getValueType().isVector()) { | |
1312 | SDValue SAO = | |
1313 | DAG.getShiftAmountOperand(Node->getOperand(0).getValueType(), | |
1314 | Node->getOperand(1)); | |
1315 | HandleSDNode Handle(SAO); | |
1316 | LegalizeOp(SAO.getNode()); | |
1317 | NewNode = DAG.UpdateNodeOperands(Node, Node->getOperand(0), | |
1318 | Handle.getValue()); | |
1319 | } | |
1320 | break; | |
1321 | case ISD::SRL_PARTS: | |
1322 | case ISD::SRA_PARTS: | |
1323 | case ISD::SHL_PARTS: | |
1324 | // Legalizing shifts/rotates requires adjusting the shift amount | |
1325 | // to the appropriate width. | |
1326 | if (!Node->getOperand(2).getValueType().isVector()) { | |
1327 | SDValue SAO = | |
1328 | DAG.getShiftAmountOperand(Node->getOperand(0).getValueType(), | |
1329 | Node->getOperand(2)); | |
1330 | HandleSDNode Handle(SAO); | |
1331 | LegalizeOp(SAO.getNode()); | |
1332 | NewNode = DAG.UpdateNodeOperands(Node, Node->getOperand(0), | |
1333 | Node->getOperand(1), | |
1334 | Handle.getValue()); | |
1335 | } | |
1336 | break; | |
1337 | } | |
1338 | ||
1339 | if (NewNode != Node) { | |
1a4d82fc | 1340 | ReplaceNode(Node, NewNode); |
223e47cc LB |
1341 | Node = NewNode; |
1342 | } | |
1343 | switch (Action) { | |
1344 | case TargetLowering::Legal: | |
1345 | return; | |
1346 | case TargetLowering::Custom: { | |
1347 | // FIXME: The handling for custom lowering with multiple results is | |
1348 | // a complete mess. | |
1349 | SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG); | |
1350 | if (Res.getNode()) { | |
1a4d82fc JJ |
1351 | if (!(Res.getNode() != Node || Res.getResNo() != 0)) |
1352 | return; | |
1353 | ||
1354 | if (Node->getNumValues() == 1) { | |
1355 | // We can just directly replace this node with the lowered value. | |
1356 | ReplaceNode(SDValue(Node, 0), Res); | |
1357 | return; | |
223e47cc | 1358 | } |
1a4d82fc JJ |
1359 | |
1360 | SmallVector<SDValue, 8> ResultVals; | |
1361 | for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) | |
1362 | ResultVals.push_back(Res.getValue(i)); | |
1363 | ReplaceNode(Node, ResultVals.data()); | |
223e47cc LB |
1364 | return; |
1365 | } | |
1366 | } | |
1367 | // FALL THROUGH | |
1368 | case TargetLowering::Expand: | |
1369 | ExpandNode(Node); | |
1370 | return; | |
1371 | case TargetLowering::Promote: | |
1372 | PromoteNode(Node); | |
1373 | return; | |
1374 | } | |
1375 | } | |
1376 | ||
1377 | switch (Node->getOpcode()) { | |
1378 | default: | |
1379 | #ifndef NDEBUG | |
1380 | dbgs() << "NODE: "; | |
1381 | Node->dump( &DAG); | |
1382 | dbgs() << "\n"; | |
1383 | #endif | |
1384 | llvm_unreachable("Do not know how to legalize this operator!"); | |
1385 | ||
1386 | case ISD::CALLSEQ_START: | |
1387 | case ISD::CALLSEQ_END: | |
1388 | break; | |
1389 | case ISD::LOAD: { | |
1390 | return LegalizeLoadOps(Node); | |
1391 | } | |
1392 | case ISD::STORE: { | |
1393 | return LegalizeStoreOps(Node); | |
1394 | } | |
1395 | } | |
1396 | } | |
1397 | ||
1398 | SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) { | |
1399 | SDValue Vec = Op.getOperand(0); | |
1400 | SDValue Idx = Op.getOperand(1); | |
1a4d82fc JJ |
1401 | SDLoc dl(Op); |
1402 | ||
1403 | // Before we generate a new store to a temporary stack slot, see if there is | |
1404 | // already one that we can use. There often is because when we scalarize | |
1405 | // vector operations (using SelectionDAG::UnrollVectorOp for example) a whole | |
1406 | // series of EXTRACT_VECTOR_ELT nodes are generated, one for each element in | |
1407 | // the vector. If all are expanded here, we don't want one store per vector | |
1408 | // element. | |
1409 | SDValue StackPtr, Ch; | |
1410 | for (SDNode::use_iterator UI = Vec.getNode()->use_begin(), | |
1411 | UE = Vec.getNode()->use_end(); UI != UE; ++UI) { | |
1412 | SDNode *User = *UI; | |
1413 | if (StoreSDNode *ST = dyn_cast<StoreSDNode>(User)) { | |
1414 | if (ST->isIndexed() || ST->isTruncatingStore() || | |
1415 | ST->getValue() != Vec) | |
1416 | continue; | |
1417 | ||
1418 | // Make sure that nothing else could have stored into the destination of | |
1419 | // this store. | |
1420 | if (!ST->getChain().reachesChainWithoutSideEffects(DAG.getEntryNode())) | |
1421 | continue; | |
1422 | ||
1423 | StackPtr = ST->getBasePtr(); | |
1424 | Ch = SDValue(ST, 0); | |
1425 | break; | |
1426 | } | |
1427 | } | |
1428 | ||
1429 | if (!Ch.getNode()) { | |
1430 | // Store the value to a temporary stack slot, then LOAD the returned part. | |
1431 | StackPtr = DAG.CreateStackTemporary(Vec.getValueType()); | |
1432 | Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, | |
1433 | MachinePointerInfo(), false, false, 0); | |
1434 | } | |
223e47cc LB |
1435 | |
1436 | // Add the offset to the index. | |
1437 | unsigned EltSize = | |
1438 | Vec.getValueType().getVectorElementType().getSizeInBits()/8; | |
1439 | Idx = DAG.getNode(ISD::MUL, dl, Idx.getValueType(), Idx, | |
1440 | DAG.getConstant(EltSize, Idx.getValueType())); | |
1441 | ||
1a4d82fc | 1442 | Idx = DAG.getZExtOrTrunc(Idx, dl, TLI.getPointerTy()); |
223e47cc LB |
1443 | StackPtr = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, StackPtr); |
1444 | ||
1445 | if (Op.getValueType().isVector()) | |
1446 | return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr,MachinePointerInfo(), | |
1447 | false, false, false, 0); | |
1448 | return DAG.getExtLoad(ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr, | |
1449 | MachinePointerInfo(), | |
1450 | Vec.getValueType().getVectorElementType(), | |
1a4d82fc | 1451 | false, false, false, 0); |
223e47cc LB |
1452 | } |
1453 | ||
1454 | SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) { | |
1455 | assert(Op.getValueType().isVector() && "Non-vector insert subvector!"); | |
1456 | ||
1457 | SDValue Vec = Op.getOperand(0); | |
1458 | SDValue Part = Op.getOperand(1); | |
1459 | SDValue Idx = Op.getOperand(2); | |
1a4d82fc | 1460 | SDLoc dl(Op); |
223e47cc LB |
1461 | |
1462 | // Store the value to a temporary stack slot, then LOAD the returned part. | |
1463 | ||
1464 | SDValue StackPtr = DAG.CreateStackTemporary(Vec.getValueType()); | |
1465 | int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); | |
1466 | MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(FI); | |
1467 | ||
1468 | // First store the whole vector. | |
1469 | SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo, | |
1470 | false, false, 0); | |
1471 | ||
1472 | // Then store the inserted part. | |
1473 | ||
1474 | // Add the offset to the index. | |
1475 | unsigned EltSize = | |
1476 | Vec.getValueType().getVectorElementType().getSizeInBits()/8; | |
1477 | ||
1478 | Idx = DAG.getNode(ISD::MUL, dl, Idx.getValueType(), Idx, | |
1479 | DAG.getConstant(EltSize, Idx.getValueType())); | |
1a4d82fc | 1480 | Idx = DAG.getZExtOrTrunc(Idx, dl, TLI.getPointerTy()); |
223e47cc LB |
1481 | |
1482 | SDValue SubStackPtr = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, | |
1483 | StackPtr); | |
1484 | ||
1485 | // Store the subvector. | |
85aaf69f | 1486 | Ch = DAG.getStore(Ch, dl, Part, SubStackPtr, |
223e47cc LB |
1487 | MachinePointerInfo(), false, false, 0); |
1488 | ||
1489 | // Finally, load the updated vector. | |
1490 | return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, PtrInfo, | |
1491 | false, false, false, 0); | |
1492 | } | |
1493 | ||
1494 | SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) { | |
1495 | // We can't handle this case efficiently. Allocate a sufficiently | |
1496 | // aligned object on the stack, store each element into it, then load | |
1497 | // the result as a vector. | |
1498 | // Create the stack frame object. | |
1499 | EVT VT = Node->getValueType(0); | |
1500 | EVT EltVT = VT.getVectorElementType(); | |
1a4d82fc | 1501 | SDLoc dl(Node); |
223e47cc LB |
1502 | SDValue FIPtr = DAG.CreateStackTemporary(VT); |
1503 | int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex(); | |
1504 | MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(FI); | |
1505 | ||
1506 | // Emit a store of each element to the stack slot. | |
1507 | SmallVector<SDValue, 8> Stores; | |
1508 | unsigned TypeByteSize = EltVT.getSizeInBits() / 8; | |
1509 | // Store (in the right endianness) the elements to memory. | |
1510 | for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) { | |
1511 | // Ignore undef elements. | |
1512 | if (Node->getOperand(i).getOpcode() == ISD::UNDEF) continue; | |
1513 | ||
1514 | unsigned Offset = TypeByteSize*i; | |
1515 | ||
1516 | SDValue Idx = DAG.getConstant(Offset, FIPtr.getValueType()); | |
1517 | Idx = DAG.getNode(ISD::ADD, dl, FIPtr.getValueType(), FIPtr, Idx); | |
1518 | ||
1519 | // If the destination vector element type is narrower than the source | |
1520 | // element type, only store the bits necessary. | |
1521 | if (EltVT.bitsLT(Node->getOperand(i).getValueType().getScalarType())) { | |
1522 | Stores.push_back(DAG.getTruncStore(DAG.getEntryNode(), dl, | |
1523 | Node->getOperand(i), Idx, | |
1524 | PtrInfo.getWithOffset(Offset), | |
1525 | EltVT, false, false, 0)); | |
1526 | } else | |
1527 | Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, | |
1528 | Node->getOperand(i), Idx, | |
1529 | PtrInfo.getWithOffset(Offset), | |
1530 | false, false, 0)); | |
1531 | } | |
1532 | ||
1533 | SDValue StoreChain; | |
1534 | if (!Stores.empty()) // Not all undef elements? | |
1a4d82fc | 1535 | StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores); |
223e47cc LB |
1536 | else |
1537 | StoreChain = DAG.getEntryNode(); | |
1538 | ||
1539 | // Result is a load from the stack slot. | |
1a4d82fc | 1540 | return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo, |
223e47cc LB |
1541 | false, false, false, 0); |
1542 | } | |
1543 | ||
1544 | SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode* Node) { | |
1a4d82fc | 1545 | SDLoc dl(Node); |
223e47cc LB |
1546 | SDValue Tmp1 = Node->getOperand(0); |
1547 | SDValue Tmp2 = Node->getOperand(1); | |
1548 | ||
1549 | // Get the sign bit of the RHS. First obtain a value that has the same | |
1550 | // sign as the sign bit, i.e. negative if and only if the sign bit is 1. | |
1551 | SDValue SignBit; | |
1552 | EVT FloatVT = Tmp2.getValueType(); | |
1553 | EVT IVT = EVT::getIntegerVT(*DAG.getContext(), FloatVT.getSizeInBits()); | |
1554 | if (TLI.isTypeLegal(IVT)) { | |
1555 | // Convert to an integer with the same sign bit. | |
1556 | SignBit = DAG.getNode(ISD::BITCAST, dl, IVT, Tmp2); | |
1557 | } else { | |
1558 | // Store the float to memory, then load the sign part out as an integer. | |
1559 | MVT LoadTy = TLI.getPointerTy(); | |
1560 | // First create a temporary that is aligned for both the load and store. | |
1561 | SDValue StackPtr = DAG.CreateStackTemporary(FloatVT, LoadTy); | |
1562 | // Then store the float to it. | |
1563 | SDValue Ch = | |
1564 | DAG.getStore(DAG.getEntryNode(), dl, Tmp2, StackPtr, MachinePointerInfo(), | |
1565 | false, false, 0); | |
1566 | if (TLI.isBigEndian()) { | |
1567 | assert(FloatVT.isByteSized() && "Unsupported floating point type!"); | |
1568 | // Load out a legal integer with the same sign bit as the float. | |
1569 | SignBit = DAG.getLoad(LoadTy, dl, Ch, StackPtr, MachinePointerInfo(), | |
1570 | false, false, false, 0); | |
1571 | } else { // Little endian | |
1572 | SDValue LoadPtr = StackPtr; | |
1573 | // The float may be wider than the integer we are going to load. Advance | |
1574 | // the pointer so that the loaded integer will contain the sign bit. | |
1575 | unsigned Strides = (FloatVT.getSizeInBits()-1)/LoadTy.getSizeInBits(); | |
1576 | unsigned ByteOffset = (Strides * LoadTy.getSizeInBits()) / 8; | |
1a4d82fc JJ |
1577 | LoadPtr = DAG.getNode(ISD::ADD, dl, LoadPtr.getValueType(), LoadPtr, |
1578 | DAG.getConstant(ByteOffset, LoadPtr.getValueType())); | |
223e47cc LB |
1579 | // Load a legal integer containing the sign bit. |
1580 | SignBit = DAG.getLoad(LoadTy, dl, Ch, LoadPtr, MachinePointerInfo(), | |
1581 | false, false, false, 0); | |
1582 | // Move the sign bit to the top bit of the loaded integer. | |
1583 | unsigned BitShift = LoadTy.getSizeInBits() - | |
1584 | (FloatVT.getSizeInBits() - 8 * ByteOffset); | |
1585 | assert(BitShift < LoadTy.getSizeInBits() && "Pointer advanced wrong?"); | |
1586 | if (BitShift) | |
1587 | SignBit = DAG.getNode(ISD::SHL, dl, LoadTy, SignBit, | |
1588 | DAG.getConstant(BitShift, | |
1589 | TLI.getShiftAmountTy(SignBit.getValueType()))); | |
1590 | } | |
1591 | } | |
1592 | // Now get the sign bit proper, by seeing whether the value is negative. | |
1a4d82fc | 1593 | SignBit = DAG.getSetCC(dl, getSetCCResultType(SignBit.getValueType()), |
223e47cc LB |
1594 | SignBit, DAG.getConstant(0, SignBit.getValueType()), |
1595 | ISD::SETLT); | |
1596 | // Get the absolute value of the result. | |
1597 | SDValue AbsVal = DAG.getNode(ISD::FABS, dl, Tmp1.getValueType(), Tmp1); | |
1598 | // Select between the nabs and abs value based on the sign bit of | |
1599 | // the input. | |
1a4d82fc JJ |
1600 | return DAG.getSelect(dl, AbsVal.getValueType(), SignBit, |
1601 | DAG.getNode(ISD::FNEG, dl, AbsVal.getValueType(), AbsVal), | |
1602 | AbsVal); | |
223e47cc LB |
1603 | } |
1604 | ||
1605 | void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node, | |
1606 | SmallVectorImpl<SDValue> &Results) { | |
1607 | unsigned SPReg = TLI.getStackPointerRegisterToSaveRestore(); | |
1608 | assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and" | |
1609 | " not tell us which reg is the stack pointer!"); | |
1a4d82fc | 1610 | SDLoc dl(Node); |
223e47cc LB |
1611 | EVT VT = Node->getValueType(0); |
1612 | SDValue Tmp1 = SDValue(Node, 0); | |
1613 | SDValue Tmp2 = SDValue(Node, 1); | |
1614 | SDValue Tmp3 = Node->getOperand(2); | |
1615 | SDValue Chain = Tmp1.getOperand(0); | |
1616 | ||
1617 | // Chain the dynamic stack allocation so that it doesn't modify the stack | |
1618 | // pointer when other instructions are using the stack. | |
1a4d82fc JJ |
1619 | Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(0, true), |
1620 | SDLoc(Node)); | |
223e47cc LB |
1621 | |
1622 | SDValue Size = Tmp2.getOperand(1); | |
1623 | SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT); | |
1624 | Chain = SP.getValue(1); | |
1625 | unsigned Align = cast<ConstantSDNode>(Tmp3)->getZExtValue(); | |
1a4d82fc | 1626 | unsigned StackAlign = |
85aaf69f | 1627 | DAG.getSubtarget().getFrameLowering()->getStackAlignment(); |
223e47cc | 1628 | Tmp1 = DAG.getNode(ISD::SUB, dl, VT, SP, Size); // Value |
1a4d82fc JJ |
1629 | if (Align > StackAlign) |
1630 | Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1, | |
1631 | DAG.getConstant(-(uint64_t)Align, VT)); | |
223e47cc LB |
1632 | Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1); // Output chain |
1633 | ||
1634 | Tmp2 = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, true), | |
1a4d82fc JJ |
1635 | DAG.getIntPtrConstant(0, true), SDValue(), |
1636 | SDLoc(Node)); | |
223e47cc LB |
1637 | |
1638 | Results.push_back(Tmp1); | |
1639 | Results.push_back(Tmp2); | |
1640 | } | |
1641 | ||
85aaf69f SL |
1642 | /// Legalize a SETCC with given LHS and RHS and condition code CC on the current |
1643 | /// target. | |
1a4d82fc JJ |
1644 | /// |
1645 | /// If the SETCC has been legalized using AND / OR, then the legalized node | |
1646 | /// will be stored in LHS. RHS and CC will be set to SDValue(). NeedInvert | |
1647 | /// will be set to false. | |
1648 | /// | |
1649 | /// If the SETCC has been legalized by using getSetCCSwappedOperands(), | |
1650 | /// then the values of LHS and RHS will be swapped, CC will be set to the | |
1651 | /// new condition, and NeedInvert will be set to false. | |
1652 | /// | |
1653 | /// If the SETCC has been legalized using the inverse condcode, then LHS and | |
1654 | /// RHS will be unchanged, CC will set to the inverted condcode, and NeedInvert | |
1655 | /// will be set to true. The caller must invert the result of the SETCC with | |
1656 | /// SelectionDAG::getLogicalNOT() or take equivalent action to swap the effect | |
1657 | /// of a true/false result. | |
1658 | /// | |
1659 | /// \returns true if the SetCC has been legalized, false if it hasn't. | |
1660 | bool SelectionDAGLegalize::LegalizeSetCCCondCode(EVT VT, | |
223e47cc LB |
1661 | SDValue &LHS, SDValue &RHS, |
1662 | SDValue &CC, | |
1a4d82fc JJ |
1663 | bool &NeedInvert, |
1664 | SDLoc dl) { | |
970d7e83 | 1665 | MVT OpVT = LHS.getSimpleValueType(); |
223e47cc | 1666 | ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get(); |
1a4d82fc | 1667 | NeedInvert = false; |
223e47cc LB |
1668 | switch (TLI.getCondCodeAction(CCCode, OpVT)) { |
1669 | default: llvm_unreachable("Unknown condition code action!"); | |
1670 | case TargetLowering::Legal: | |
1671 | // Nothing to do. | |
1672 | break; | |
1673 | case TargetLowering::Expand: { | |
1a4d82fc JJ |
1674 | ISD::CondCode InvCC = ISD::getSetCCSwappedOperands(CCCode); |
1675 | if (TLI.isCondCodeLegal(InvCC, OpVT)) { | |
1676 | std::swap(LHS, RHS); | |
1677 | CC = DAG.getCondCode(InvCC); | |
1678 | return true; | |
1679 | } | |
223e47cc LB |
1680 | ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID; |
1681 | unsigned Opc = 0; | |
1682 | switch (CCCode) { | |
1683 | default: llvm_unreachable("Don't know how to expand this condition!"); | |
1a4d82fc | 1684 | case ISD::SETO: |
970d7e83 LB |
1685 | assert(TLI.getCondCodeAction(ISD::SETOEQ, OpVT) |
1686 | == TargetLowering::Legal | |
1687 | && "If SETO is expanded, SETOEQ must be legal!"); | |
1688 | CC1 = ISD::SETOEQ; CC2 = ISD::SETOEQ; Opc = ISD::AND; break; | |
1a4d82fc | 1689 | case ISD::SETUO: |
970d7e83 LB |
1690 | assert(TLI.getCondCodeAction(ISD::SETUNE, OpVT) |
1691 | == TargetLowering::Legal | |
1692 | && "If SETUO is expanded, SETUNE must be legal!"); | |
1693 | CC1 = ISD::SETUNE; CC2 = ISD::SETUNE; Opc = ISD::OR; break; | |
1694 | case ISD::SETOEQ: | |
1695 | case ISD::SETOGT: | |
1696 | case ISD::SETOGE: | |
1697 | case ISD::SETOLT: | |
1698 | case ISD::SETOLE: | |
1a4d82fc JJ |
1699 | case ISD::SETONE: |
1700 | case ISD::SETUEQ: | |
1701 | case ISD::SETUNE: | |
1702 | case ISD::SETUGT: | |
1703 | case ISD::SETUGE: | |
1704 | case ISD::SETULT: | |
970d7e83 LB |
1705 | case ISD::SETULE: |
1706 | // If we are floating point, assign and break, otherwise fall through. | |
1707 | if (!OpVT.isInteger()) { | |
1708 | // We can use the 4th bit to tell if we are the unordered | |
1709 | // or ordered version of the opcode. | |
1710 | CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO; | |
1711 | Opc = ((unsigned)CCCode & 0x8U) ? ISD::OR : ISD::AND; | |
1712 | CC1 = (ISD::CondCode)(((int)CCCode & 0x7) | 0x10); | |
1713 | break; | |
1714 | } | |
1715 | // Fallthrough if we are unsigned integer. | |
1716 | case ISD::SETLE: | |
1717 | case ISD::SETGT: | |
1718 | case ISD::SETGE: | |
1719 | case ISD::SETLT: | |
1a4d82fc JJ |
1720 | // We only support using the inverted operation, which is computed above |
1721 | // and not a different manner of supporting expanding these cases. | |
1722 | llvm_unreachable("Don't know how to expand this condition!"); | |
970d7e83 LB |
1723 | case ISD::SETNE: |
1724 | case ISD::SETEQ: | |
1a4d82fc JJ |
1725 | // Try inverting the result of the inverse condition. |
1726 | InvCC = CCCode == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ; | |
1727 | if (TLI.isCondCodeLegal(InvCC, OpVT)) { | |
1728 | CC = DAG.getCondCode(InvCC); | |
1729 | NeedInvert = true; | |
1730 | return true; | |
970d7e83 | 1731 | } |
1a4d82fc JJ |
1732 | // If inverting the condition didn't work then we have no means to expand |
1733 | // the condition. | |
1734 | llvm_unreachable("Don't know how to expand this condition!"); | |
970d7e83 | 1735 | } |
1a4d82fc | 1736 | |
970d7e83 LB |
1737 | SDValue SetCC1, SetCC2; |
1738 | if (CCCode != ISD::SETO && CCCode != ISD::SETUO) { | |
1739 | // If we aren't the ordered or unorder operation, | |
1740 | // then the pattern is (LHS CC1 RHS) Opc (LHS CC2 RHS). | |
1741 | SetCC1 = DAG.getSetCC(dl, VT, LHS, RHS, CC1); | |
1742 | SetCC2 = DAG.getSetCC(dl, VT, LHS, RHS, CC2); | |
1743 | } else { | |
1744 | // Otherwise, the pattern is (LHS CC1 LHS) Opc (RHS CC2 RHS) | |
1745 | SetCC1 = DAG.getSetCC(dl, VT, LHS, LHS, CC1); | |
1746 | SetCC2 = DAG.getSetCC(dl, VT, RHS, RHS, CC2); | |
1747 | } | |
223e47cc LB |
1748 | LHS = DAG.getNode(Opc, dl, VT, SetCC1, SetCC2); |
1749 | RHS = SDValue(); | |
1750 | CC = SDValue(); | |
1a4d82fc | 1751 | return true; |
223e47cc LB |
1752 | } |
1753 | } | |
1a4d82fc | 1754 | return false; |
223e47cc LB |
1755 | } |
1756 | ||
85aaf69f | 1757 | /// Emit a store/load combination to the stack. This stores |
223e47cc LB |
1758 | /// SrcOp to a stack slot of type SlotVT, truncating it if needed. It then does |
1759 | /// a load from the stack slot to DestVT, extending it if needed. | |
1760 | /// The resultant code need not be legal. | |
1761 | SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, | |
1762 | EVT SlotVT, | |
1763 | EVT DestVT, | |
1a4d82fc | 1764 | SDLoc dl) { |
223e47cc LB |
1765 | // Create the stack frame object. |
1766 | unsigned SrcAlign = | |
970d7e83 | 1767 | TLI.getDataLayout()->getPrefTypeAlignment(SrcOp.getValueType(). |
223e47cc LB |
1768 | getTypeForEVT(*DAG.getContext())); |
1769 | SDValue FIPtr = DAG.CreateStackTemporary(SlotVT, SrcAlign); | |
1770 | ||
1771 | FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(FIPtr); | |
1772 | int SPFI = StackPtrFI->getIndex(); | |
1773 | MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(SPFI); | |
1774 | ||
1775 | unsigned SrcSize = SrcOp.getValueType().getSizeInBits(); | |
1776 | unsigned SlotSize = SlotVT.getSizeInBits(); | |
1777 | unsigned DestSize = DestVT.getSizeInBits(); | |
1778 | Type *DestType = DestVT.getTypeForEVT(*DAG.getContext()); | |
970d7e83 | 1779 | unsigned DestAlign = TLI.getDataLayout()->getPrefTypeAlignment(DestType); |
223e47cc LB |
1780 | |
1781 | // Emit a store to the stack slot. Use a truncstore if the input value is | |
1782 | // later than DestVT. | |
1783 | SDValue Store; | |
1784 | ||
1785 | if (SrcSize > SlotSize) | |
1786 | Store = DAG.getTruncStore(DAG.getEntryNode(), dl, SrcOp, FIPtr, | |
1787 | PtrInfo, SlotVT, false, false, SrcAlign); | |
1788 | else { | |
1789 | assert(SrcSize == SlotSize && "Invalid store"); | |
1790 | Store = DAG.getStore(DAG.getEntryNode(), dl, SrcOp, FIPtr, | |
1791 | PtrInfo, false, false, SrcAlign); | |
1792 | } | |
1793 | ||
1794 | // Result is a load from the stack slot. | |
1795 | if (SlotSize == DestSize) | |
1796 | return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, | |
1797 | false, false, false, DestAlign); | |
1798 | ||
1799 | assert(SlotSize < DestSize && "Unknown extension!"); | |
1800 | return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr, | |
1a4d82fc | 1801 | PtrInfo, SlotVT, false, false, false, DestAlign); |
223e47cc LB |
1802 | } |
1803 | ||
1804 | SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) { | |
1a4d82fc | 1805 | SDLoc dl(Node); |
223e47cc LB |
1806 | // Create a vector sized/aligned stack slot, store the value to element #0, |
1807 | // then load the whole vector back out. | |
1808 | SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0)); | |
1809 | ||
1810 | FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(StackPtr); | |
1811 | int SPFI = StackPtrFI->getIndex(); | |
1812 | ||
1813 | SDValue Ch = DAG.getTruncStore(DAG.getEntryNode(), dl, Node->getOperand(0), | |
1814 | StackPtr, | |
1815 | MachinePointerInfo::getFixedStack(SPFI), | |
1816 | Node->getValueType(0).getVectorElementType(), | |
1817 | false, false, 0); | |
1818 | return DAG.getLoad(Node->getValueType(0), dl, Ch, StackPtr, | |
1819 | MachinePointerInfo::getFixedStack(SPFI), | |
1820 | false, false, false, 0); | |
1821 | } | |
1822 | ||
1a4d82fc JJ |
1823 | static bool |
1824 | ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG, | |
1825 | const TargetLowering &TLI, SDValue &Res) { | |
1826 | unsigned NumElems = Node->getNumOperands(); | |
1827 | SDLoc dl(Node); | |
1828 | EVT VT = Node->getValueType(0); | |
1829 | ||
1830 | // Try to group the scalars into pairs, shuffle the pairs together, then | |
1831 | // shuffle the pairs of pairs together, etc. until the vector has | |
1832 | // been built. This will work only if all of the necessary shuffle masks | |
1833 | // are legal. | |
1834 | ||
1835 | // We do this in two phases; first to check the legality of the shuffles, | |
1836 | // and next, assuming that all shuffles are legal, to create the new nodes. | |
1837 | for (int Phase = 0; Phase < 2; ++Phase) { | |
1838 | SmallVector<std::pair<SDValue, SmallVector<int, 16> >, 16> IntermedVals, | |
1839 | NewIntermedVals; | |
1840 | for (unsigned i = 0; i < NumElems; ++i) { | |
1841 | SDValue V = Node->getOperand(i); | |
1842 | if (V.getOpcode() == ISD::UNDEF) | |
1843 | continue; | |
1844 | ||
1845 | SDValue Vec; | |
1846 | if (Phase) | |
1847 | Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, V); | |
1848 | IntermedVals.push_back(std::make_pair(Vec, SmallVector<int, 16>(1, i))); | |
1849 | } | |
1850 | ||
1851 | while (IntermedVals.size() > 2) { | |
1852 | NewIntermedVals.clear(); | |
1853 | for (unsigned i = 0, e = (IntermedVals.size() & ~1u); i < e; i += 2) { | |
1854 | // This vector and the next vector are shuffled together (simply to | |
1855 | // append the one to the other). | |
1856 | SmallVector<int, 16> ShuffleVec(NumElems, -1); | |
1857 | ||
1858 | SmallVector<int, 16> FinalIndices; | |
1859 | FinalIndices.reserve(IntermedVals[i].second.size() + | |
1860 | IntermedVals[i+1].second.size()); | |
1861 | ||
1862 | int k = 0; | |
1863 | for (unsigned j = 0, f = IntermedVals[i].second.size(); j != f; | |
1864 | ++j, ++k) { | |
1865 | ShuffleVec[k] = j; | |
1866 | FinalIndices.push_back(IntermedVals[i].second[j]); | |
1867 | } | |
1868 | for (unsigned j = 0, f = IntermedVals[i+1].second.size(); j != f; | |
1869 | ++j, ++k) { | |
1870 | ShuffleVec[k] = NumElems + j; | |
1871 | FinalIndices.push_back(IntermedVals[i+1].second[j]); | |
1872 | } | |
1873 | ||
1874 | SDValue Shuffle; | |
1875 | if (Phase) | |
1876 | Shuffle = DAG.getVectorShuffle(VT, dl, IntermedVals[i].first, | |
1877 | IntermedVals[i+1].first, | |
1878 | ShuffleVec.data()); | |
1879 | else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT)) | |
1880 | return false; | |
85aaf69f SL |
1881 | NewIntermedVals.push_back( |
1882 | std::make_pair(Shuffle, std::move(FinalIndices))); | |
1a4d82fc JJ |
1883 | } |
1884 | ||
1885 | // If we had an odd number of defined values, then append the last | |
1886 | // element to the array of new vectors. | |
1887 | if ((IntermedVals.size() & 1) != 0) | |
1888 | NewIntermedVals.push_back(IntermedVals.back()); | |
1889 | ||
1890 | IntermedVals.swap(NewIntermedVals); | |
1891 | } | |
1892 | ||
1893 | assert(IntermedVals.size() <= 2 && IntermedVals.size() > 0 && | |
1894 | "Invalid number of intermediate vectors"); | |
1895 | SDValue Vec1 = IntermedVals[0].first; | |
1896 | SDValue Vec2; | |
1897 | if (IntermedVals.size() > 1) | |
1898 | Vec2 = IntermedVals[1].first; | |
1899 | else if (Phase) | |
1900 | Vec2 = DAG.getUNDEF(VT); | |
1901 | ||
1902 | SmallVector<int, 16> ShuffleVec(NumElems, -1); | |
1903 | for (unsigned i = 0, e = IntermedVals[0].second.size(); i != e; ++i) | |
1904 | ShuffleVec[IntermedVals[0].second[i]] = i; | |
1905 | for (unsigned i = 0, e = IntermedVals[1].second.size(); i != e; ++i) | |
1906 | ShuffleVec[IntermedVals[1].second[i]] = NumElems + i; | |
1907 | ||
1908 | if (Phase) | |
1909 | Res = DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec.data()); | |
1910 | else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT)) | |
1911 | return false; | |
1912 | } | |
1913 | ||
1914 | return true; | |
1915 | } | |
223e47cc | 1916 | |
85aaf69f | 1917 | /// Expand a BUILD_VECTOR node on targets that don't |
223e47cc LB |
1918 | /// support the operation, but do support the resultant vector type. |
1919 | SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) { | |
1920 | unsigned NumElems = Node->getNumOperands(); | |
1921 | SDValue Value1, Value2; | |
1a4d82fc | 1922 | SDLoc dl(Node); |
223e47cc LB |
1923 | EVT VT = Node->getValueType(0); |
1924 | EVT OpVT = Node->getOperand(0).getValueType(); | |
1925 | EVT EltVT = VT.getVectorElementType(); | |
1926 | ||
1927 | // If the only non-undef value is the low element, turn this into a | |
1928 | // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X. | |
1929 | bool isOnlyLowElement = true; | |
1930 | bool MoreThanTwoValues = false; | |
1931 | bool isConstant = true; | |
1932 | for (unsigned i = 0; i < NumElems; ++i) { | |
1933 | SDValue V = Node->getOperand(i); | |
1934 | if (V.getOpcode() == ISD::UNDEF) | |
1935 | continue; | |
1936 | if (i > 0) | |
1937 | isOnlyLowElement = false; | |
1938 | if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V)) | |
1939 | isConstant = false; | |
1940 | ||
1941 | if (!Value1.getNode()) { | |
1942 | Value1 = V; | |
1943 | } else if (!Value2.getNode()) { | |
1944 | if (V != Value1) | |
1945 | Value2 = V; | |
1946 | } else if (V != Value1 && V != Value2) { | |
1947 | MoreThanTwoValues = true; | |
1948 | } | |
1949 | } | |
1950 | ||
1951 | if (!Value1.getNode()) | |
1952 | return DAG.getUNDEF(VT); | |
1953 | ||
1954 | if (isOnlyLowElement) | |
1955 | return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0)); | |
1956 | ||
1957 | // If all elements are constants, create a load from the constant pool. | |
1958 | if (isConstant) { | |
1959 | SmallVector<Constant*, 16> CV; | |
1960 | for (unsigned i = 0, e = NumElems; i != e; ++i) { | |
1961 | if (ConstantFPSDNode *V = | |
1962 | dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) { | |
1963 | CV.push_back(const_cast<ConstantFP *>(V->getConstantFPValue())); | |
1964 | } else if (ConstantSDNode *V = | |
1965 | dyn_cast<ConstantSDNode>(Node->getOperand(i))) { | |
1966 | if (OpVT==EltVT) | |
1967 | CV.push_back(const_cast<ConstantInt *>(V->getConstantIntValue())); | |
1968 | else { | |
1969 | // If OpVT and EltVT don't match, EltVT is not legal and the | |
1970 | // element values have been promoted/truncated earlier. Undo this; | |
1971 | // we don't want a v16i8 to become a v16i32 for example. | |
1972 | const ConstantInt *CI = V->getConstantIntValue(); | |
1973 | CV.push_back(ConstantInt::get(EltVT.getTypeForEVT(*DAG.getContext()), | |
1974 | CI->getZExtValue())); | |
1975 | } | |
1976 | } else { | |
1977 | assert(Node->getOperand(i).getOpcode() == ISD::UNDEF); | |
1978 | Type *OpNTy = EltVT.getTypeForEVT(*DAG.getContext()); | |
1979 | CV.push_back(UndefValue::get(OpNTy)); | |
1980 | } | |
1981 | } | |
1982 | Constant *CP = ConstantVector::get(CV); | |
1983 | SDValue CPIdx = DAG.getConstantPool(CP, TLI.getPointerTy()); | |
1984 | unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment(); | |
1985 | return DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx, | |
1986 | MachinePointerInfo::getConstantPool(), | |
1987 | false, false, false, Alignment); | |
1988 | } | |
1989 | ||
1a4d82fc JJ |
1990 | SmallSet<SDValue, 16> DefinedValues; |
1991 | for (unsigned i = 0; i < NumElems; ++i) { | |
1992 | if (Node->getOperand(i).getOpcode() == ISD::UNDEF) | |
1993 | continue; | |
1994 | DefinedValues.insert(Node->getOperand(i)); | |
1995 | } | |
223e47cc | 1996 | |
1a4d82fc JJ |
1997 | if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues.size())) { |
1998 | if (!MoreThanTwoValues) { | |
1999 | SmallVector<int, 8> ShuffleVec(NumElems, -1); | |
2000 | for (unsigned i = 0; i < NumElems; ++i) { | |
2001 | SDValue V = Node->getOperand(i); | |
2002 | if (V.getOpcode() == ISD::UNDEF) | |
2003 | continue; | |
2004 | ShuffleVec[i] = V == Value1 ? 0 : NumElems; | |
2005 | } | |
2006 | if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(0))) { | |
2007 | // Get the splatted value into the low element of a vector register. | |
2008 | SDValue Vec1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value1); | |
2009 | SDValue Vec2; | |
2010 | if (Value2.getNode()) | |
2011 | Vec2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value2); | |
2012 | else | |
2013 | Vec2 = DAG.getUNDEF(VT); | |
2014 | ||
2015 | // Return shuffle(LowValVec, undef, <0,0,0,0>) | |
2016 | return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec.data()); | |
2017 | } | |
2018 | } else { | |
2019 | SDValue Res; | |
2020 | if (ExpandBVWithShuffles(Node, DAG, TLI, Res)) | |
2021 | return Res; | |
223e47cc LB |
2022 | } |
2023 | } | |
2024 | ||
2025 | // Otherwise, we can't handle this case efficiently. | |
2026 | return ExpandVectorBuildThroughStack(Node); | |
2027 | } | |
2028 | ||
85aaf69f | 2029 | // Expand a node into a call to a libcall. If the result value |
223e47cc LB |
2030 | // does not fit into a register, return the lo part and set the hi part to the |
2031 | // by-reg argument. If it does fit into a single register, return the result | |
2032 | // and leave the Hi part unset. | |
2033 | SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, | |
2034 | bool isSigned) { | |
2035 | TargetLowering::ArgListTy Args; | |
2036 | TargetLowering::ArgListEntry Entry; | |
2037 | for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) { | |
2038 | EVT ArgVT = Node->getOperand(i).getValueType(); | |
2039 | Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); | |
2040 | Entry.Node = Node->getOperand(i); Entry.Ty = ArgTy; | |
2041 | Entry.isSExt = isSigned; | |
2042 | Entry.isZExt = !isSigned; | |
2043 | Args.push_back(Entry); | |
2044 | } | |
2045 | SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), | |
2046 | TLI.getPointerTy()); | |
2047 | ||
2048 | Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext()); | |
2049 | ||
2050 | // By default, the input chain to this libcall is the entry node of the | |
2051 | // function. If the libcall is going to be emitted as a tail call then | |
2052 | // TLI.isUsedByReturnOnly will change it to the right chain if the return | |
2053 | // node which is being folded has a non-entry input chain. | |
2054 | SDValue InChain = DAG.getEntryNode(); | |
2055 | ||
2056 | // isTailCall may be true since the callee does not reference caller stack | |
2057 | // frame. Check if it's in the right position. | |
2058 | SDValue TCChain = InChain; | |
970d7e83 | 2059 | bool isTailCall = TLI.isInTailCallPosition(DAG, Node, TCChain); |
223e47cc LB |
2060 | if (isTailCall) |
2061 | InChain = TCChain; | |
2062 | ||
1a4d82fc JJ |
2063 | TargetLowering::CallLoweringInfo CLI(DAG); |
2064 | CLI.setDebugLoc(SDLoc(Node)).setChain(InChain) | |
2065 | .setCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee, std::move(Args), 0) | |
2066 | .setTailCall(isTailCall).setSExtResult(isSigned).setZExtResult(!isSigned); | |
223e47cc | 2067 | |
1a4d82fc | 2068 | std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); |
223e47cc LB |
2069 | |
2070 | if (!CallInfo.second.getNode()) | |
2071 | // It's a tailcall, return the chain (which is the DAG root). | |
2072 | return DAG.getRoot(); | |
2073 | ||
2074 | return CallInfo.first; | |
2075 | } | |
2076 | ||
85aaf69f | 2077 | /// Generate a libcall taking the given operands as arguments |
223e47cc LB |
2078 | /// and returning a result of type RetVT. |
2079 | SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, EVT RetVT, | |
2080 | const SDValue *Ops, unsigned NumOps, | |
1a4d82fc | 2081 | bool isSigned, SDLoc dl) { |
223e47cc LB |
2082 | TargetLowering::ArgListTy Args; |
2083 | Args.reserve(NumOps); | |
2084 | ||
2085 | TargetLowering::ArgListEntry Entry; | |
2086 | for (unsigned i = 0; i != NumOps; ++i) { | |
2087 | Entry.Node = Ops[i]; | |
2088 | Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext()); | |
2089 | Entry.isSExt = isSigned; | |
2090 | Entry.isZExt = !isSigned; | |
2091 | Args.push_back(Entry); | |
2092 | } | |
2093 | SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), | |
2094 | TLI.getPointerTy()); | |
2095 | ||
2096 | Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext()); | |
1a4d82fc JJ |
2097 | |
2098 | TargetLowering::CallLoweringInfo CLI(DAG); | |
2099 | CLI.setDebugLoc(dl).setChain(DAG.getEntryNode()) | |
2100 | .setCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee, std::move(Args), 0) | |
2101 | .setSExtResult(isSigned).setZExtResult(!isSigned); | |
2102 | ||
223e47cc LB |
2103 | std::pair<SDValue,SDValue> CallInfo = TLI.LowerCallTo(CLI); |
2104 | ||
2105 | return CallInfo.first; | |
2106 | } | |
2107 | ||
85aaf69f | 2108 | // Expand a node into a call to a libcall. Similar to |
223e47cc LB |
2109 | // ExpandLibCall except that the first operand is the in-chain. |
2110 | std::pair<SDValue, SDValue> | |
2111 | SelectionDAGLegalize::ExpandChainLibCall(RTLIB::Libcall LC, | |
2112 | SDNode *Node, | |
2113 | bool isSigned) { | |
2114 | SDValue InChain = Node->getOperand(0); | |
2115 | ||
2116 | TargetLowering::ArgListTy Args; | |
2117 | TargetLowering::ArgListEntry Entry; | |
2118 | for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i) { | |
2119 | EVT ArgVT = Node->getOperand(i).getValueType(); | |
2120 | Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); | |
2121 | Entry.Node = Node->getOperand(i); | |
2122 | Entry.Ty = ArgTy; | |
2123 | Entry.isSExt = isSigned; | |
2124 | Entry.isZExt = !isSigned; | |
2125 | Args.push_back(Entry); | |
2126 | } | |
2127 | SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), | |
2128 | TLI.getPointerTy()); | |
2129 | ||
2130 | Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext()); | |
1a4d82fc JJ |
2131 | |
2132 | TargetLowering::CallLoweringInfo CLI(DAG); | |
2133 | CLI.setDebugLoc(SDLoc(Node)).setChain(InChain) | |
2134 | .setCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee, std::move(Args), 0) | |
2135 | .setSExtResult(isSigned).setZExtResult(!isSigned); | |
2136 | ||
223e47cc LB |
2137 | std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); |
2138 | ||
2139 | return CallInfo; | |
2140 | } | |
2141 | ||
2142 | SDValue SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node, | |
2143 | RTLIB::Libcall Call_F32, | |
2144 | RTLIB::Libcall Call_F64, | |
2145 | RTLIB::Libcall Call_F80, | |
970d7e83 | 2146 | RTLIB::Libcall Call_F128, |
223e47cc LB |
2147 | RTLIB::Libcall Call_PPCF128) { |
2148 | RTLIB::Libcall LC; | |
1a4d82fc | 2149 | switch (Node->getSimpleValueType(0).SimpleTy) { |
223e47cc LB |
2150 | default: llvm_unreachable("Unexpected request for libcall!"); |
2151 | case MVT::f32: LC = Call_F32; break; | |
2152 | case MVT::f64: LC = Call_F64; break; | |
2153 | case MVT::f80: LC = Call_F80; break; | |
970d7e83 | 2154 | case MVT::f128: LC = Call_F128; break; |
223e47cc LB |
2155 | case MVT::ppcf128: LC = Call_PPCF128; break; |
2156 | } | |
2157 | return ExpandLibCall(LC, Node, false); | |
2158 | } | |
2159 | ||
2160 | SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned, | |
2161 | RTLIB::Libcall Call_I8, | |
2162 | RTLIB::Libcall Call_I16, | |
2163 | RTLIB::Libcall Call_I32, | |
2164 | RTLIB::Libcall Call_I64, | |
2165 | RTLIB::Libcall Call_I128) { | |
2166 | RTLIB::Libcall LC; | |
1a4d82fc | 2167 | switch (Node->getSimpleValueType(0).SimpleTy) { |
223e47cc LB |
2168 | default: llvm_unreachable("Unexpected request for libcall!"); |
2169 | case MVT::i8: LC = Call_I8; break; | |
2170 | case MVT::i16: LC = Call_I16; break; | |
2171 | case MVT::i32: LC = Call_I32; break; | |
2172 | case MVT::i64: LC = Call_I64; break; | |
2173 | case MVT::i128: LC = Call_I128; break; | |
2174 | } | |
2175 | return ExpandLibCall(LC, Node, isSigned); | |
2176 | } | |
2177 | ||
85aaf69f | 2178 | /// Return true if divmod libcall is available. |
223e47cc LB |
2179 | static bool isDivRemLibcallAvailable(SDNode *Node, bool isSigned, |
2180 | const TargetLowering &TLI) { | |
2181 | RTLIB::Libcall LC; | |
1a4d82fc | 2182 | switch (Node->getSimpleValueType(0).SimpleTy) { |
223e47cc LB |
2183 | default: llvm_unreachable("Unexpected request for libcall!"); |
2184 | case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break; | |
2185 | case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break; | |
2186 | case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break; | |
2187 | case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break; | |
2188 | case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break; | |
2189 | } | |
2190 | ||
1a4d82fc | 2191 | return TLI.getLibcallName(LC) != nullptr; |
223e47cc LB |
2192 | } |
2193 | ||
85aaf69f | 2194 | /// Only issue divrem libcall if both quotient and remainder are needed. |
223e47cc LB |
2195 | static bool useDivRem(SDNode *Node, bool isSigned, bool isDIV) { |
2196 | // The other use might have been replaced with a divrem already. | |
2197 | unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM; | |
2198 | unsigned OtherOpcode = 0; | |
2199 | if (isSigned) | |
2200 | OtherOpcode = isDIV ? ISD::SREM : ISD::SDIV; | |
2201 | else | |
2202 | OtherOpcode = isDIV ? ISD::UREM : ISD::UDIV; | |
2203 | ||
2204 | SDValue Op0 = Node->getOperand(0); | |
2205 | SDValue Op1 = Node->getOperand(1); | |
2206 | for (SDNode::use_iterator UI = Op0.getNode()->use_begin(), | |
2207 | UE = Op0.getNode()->use_end(); UI != UE; ++UI) { | |
2208 | SDNode *User = *UI; | |
2209 | if (User == Node) | |
2210 | continue; | |
2211 | if ((User->getOpcode() == OtherOpcode || User->getOpcode() == DivRemOpc) && | |
2212 | User->getOperand(0) == Op0 && | |
2213 | User->getOperand(1) == Op1) | |
2214 | return true; | |
2215 | } | |
2216 | return false; | |
2217 | } | |
2218 | ||
85aaf69f | 2219 | /// Issue libcalls to __{u}divmod to compute div / rem pairs. |
223e47cc LB |
2220 | void |
2221 | SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node, | |
2222 | SmallVectorImpl<SDValue> &Results) { | |
2223 | unsigned Opcode = Node->getOpcode(); | |
2224 | bool isSigned = Opcode == ISD::SDIVREM; | |
2225 | ||
2226 | RTLIB::Libcall LC; | |
1a4d82fc | 2227 | switch (Node->getSimpleValueType(0).SimpleTy) { |
223e47cc LB |
2228 | default: llvm_unreachable("Unexpected request for libcall!"); |
2229 | case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break; | |
2230 | case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break; | |
2231 | case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break; | |
2232 | case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break; | |
2233 | case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break; | |
2234 | } | |
2235 | ||
2236 | // The input chain to this libcall is the entry node of the function. | |
2237 | // Legalizing the call will automatically add the previous call to the | |
2238 | // dependence. | |
2239 | SDValue InChain = DAG.getEntryNode(); | |
2240 | ||
2241 | EVT RetVT = Node->getValueType(0); | |
2242 | Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext()); | |
2243 | ||
2244 | TargetLowering::ArgListTy Args; | |
2245 | TargetLowering::ArgListEntry Entry; | |
2246 | for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) { | |
2247 | EVT ArgVT = Node->getOperand(i).getValueType(); | |
2248 | Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); | |
2249 | Entry.Node = Node->getOperand(i); Entry.Ty = ArgTy; | |
2250 | Entry.isSExt = isSigned; | |
2251 | Entry.isZExt = !isSigned; | |
2252 | Args.push_back(Entry); | |
2253 | } | |
2254 | ||
2255 | // Also pass the return address of the remainder. | |
2256 | SDValue FIPtr = DAG.CreateStackTemporary(RetVT); | |
2257 | Entry.Node = FIPtr; | |
2258 | Entry.Ty = RetTy->getPointerTo(); | |
2259 | Entry.isSExt = isSigned; | |
2260 | Entry.isZExt = !isSigned; | |
2261 | Args.push_back(Entry); | |
2262 | ||
2263 | SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), | |
2264 | TLI.getPointerTy()); | |
2265 | ||
1a4d82fc JJ |
2266 | SDLoc dl(Node); |
2267 | TargetLowering::CallLoweringInfo CLI(DAG); | |
2268 | CLI.setDebugLoc(dl).setChain(InChain) | |
2269 | .setCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee, std::move(Args), 0) | |
2270 | .setSExtResult(isSigned).setZExtResult(!isSigned); | |
2271 | ||
223e47cc LB |
2272 | std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); |
2273 | ||
2274 | // Remainder is loaded back from the stack frame. | |
2275 | SDValue Rem = DAG.getLoad(RetVT, dl, CallInfo.second, FIPtr, | |
2276 | MachinePointerInfo(), false, false, false, 0); | |
2277 | Results.push_back(CallInfo.first); | |
2278 | Results.push_back(Rem); | |
2279 | } | |
2280 | ||
85aaf69f | 2281 | /// Return true if sincos libcall is available. |
970d7e83 LB |
2282 | static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI) { |
2283 | RTLIB::Libcall LC; | |
1a4d82fc | 2284 | switch (Node->getSimpleValueType(0).SimpleTy) { |
970d7e83 LB |
2285 | default: llvm_unreachable("Unexpected request for libcall!"); |
2286 | case MVT::f32: LC = RTLIB::SINCOS_F32; break; | |
2287 | case MVT::f64: LC = RTLIB::SINCOS_F64; break; | |
2288 | case MVT::f80: LC = RTLIB::SINCOS_F80; break; | |
2289 | case MVT::f128: LC = RTLIB::SINCOS_F128; break; | |
2290 | case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break; | |
2291 | } | |
1a4d82fc | 2292 | return TLI.getLibcallName(LC) != nullptr; |
970d7e83 LB |
2293 | } |
2294 | ||
85aaf69f SL |
2295 | /// Return true if sincos libcall is available and can be used to combine sin |
2296 | /// and cos. | |
970d7e83 LB |
2297 | static bool canCombineSinCosLibcall(SDNode *Node, const TargetLowering &TLI, |
2298 | const TargetMachine &TM) { | |
2299 | if (!isSinCosLibcallAvailable(Node, TLI)) | |
2300 | return false; | |
2301 | // GNU sin/cos functions set errno while sincos does not. Therefore | |
2302 | // combining sin and cos is only safe if unsafe-fpmath is enabled. | |
2303 | bool isGNU = Triple(TM.getTargetTriple()).getEnvironment() == Triple::GNU; | |
2304 | if (isGNU && !TM.Options.UnsafeFPMath) | |
2305 | return false; | |
2306 | return true; | |
2307 | } | |
2308 | ||
85aaf69f | 2309 | /// Only issue sincos libcall if both sin and cos are needed. |
970d7e83 LB |
2310 | static bool useSinCos(SDNode *Node) { |
2311 | unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN | |
2312 | ? ISD::FCOS : ISD::FSIN; | |
1a4d82fc | 2313 | |
970d7e83 LB |
2314 | SDValue Op0 = Node->getOperand(0); |
2315 | for (SDNode::use_iterator UI = Op0.getNode()->use_begin(), | |
2316 | UE = Op0.getNode()->use_end(); UI != UE; ++UI) { | |
2317 | SDNode *User = *UI; | |
2318 | if (User == Node) | |
2319 | continue; | |
2320 | // The other user might have been turned into sincos already. | |
2321 | if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS) | |
2322 | return true; | |
2323 | } | |
2324 | return false; | |
2325 | } | |
2326 | ||
85aaf69f | 2327 | /// Issue libcalls to sincos to compute sin / cos pairs. |
970d7e83 LB |
2328 | void |
2329 | SelectionDAGLegalize::ExpandSinCosLibCall(SDNode *Node, | |
2330 | SmallVectorImpl<SDValue> &Results) { | |
2331 | RTLIB::Libcall LC; | |
1a4d82fc | 2332 | switch (Node->getSimpleValueType(0).SimpleTy) { |
970d7e83 LB |
2333 | default: llvm_unreachable("Unexpected request for libcall!"); |
2334 | case MVT::f32: LC = RTLIB::SINCOS_F32; break; | |
2335 | case MVT::f64: LC = RTLIB::SINCOS_F64; break; | |
2336 | case MVT::f80: LC = RTLIB::SINCOS_F80; break; | |
2337 | case MVT::f128: LC = RTLIB::SINCOS_F128; break; | |
2338 | case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break; | |
2339 | } | |
1a4d82fc | 2340 | |
970d7e83 LB |
2341 | // The input chain to this libcall is the entry node of the function. |
2342 | // Legalizing the call will automatically add the previous call to the | |
2343 | // dependence. | |
2344 | SDValue InChain = DAG.getEntryNode(); | |
1a4d82fc | 2345 | |
970d7e83 LB |
2346 | EVT RetVT = Node->getValueType(0); |
2347 | Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext()); | |
1a4d82fc | 2348 | |
970d7e83 LB |
2349 | TargetLowering::ArgListTy Args; |
2350 | TargetLowering::ArgListEntry Entry; | |
1a4d82fc | 2351 | |
970d7e83 LB |
2352 | // Pass the argument. |
2353 | Entry.Node = Node->getOperand(0); | |
2354 | Entry.Ty = RetTy; | |
2355 | Entry.isSExt = false; | |
2356 | Entry.isZExt = false; | |
2357 | Args.push_back(Entry); | |
1a4d82fc | 2358 | |
970d7e83 LB |
2359 | // Pass the return address of sin. |
2360 | SDValue SinPtr = DAG.CreateStackTemporary(RetVT); | |
2361 | Entry.Node = SinPtr; | |
2362 | Entry.Ty = RetTy->getPointerTo(); | |
2363 | Entry.isSExt = false; | |
2364 | Entry.isZExt = false; | |
2365 | Args.push_back(Entry); | |
1a4d82fc | 2366 | |
970d7e83 LB |
2367 | // Also pass the return address of the cos. |
2368 | SDValue CosPtr = DAG.CreateStackTemporary(RetVT); | |
2369 | Entry.Node = CosPtr; | |
2370 | Entry.Ty = RetTy->getPointerTo(); | |
2371 | Entry.isSExt = false; | |
2372 | Entry.isZExt = false; | |
2373 | Args.push_back(Entry); | |
1a4d82fc | 2374 | |
970d7e83 LB |
2375 | SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), |
2376 | TLI.getPointerTy()); | |
1a4d82fc JJ |
2377 | |
2378 | SDLoc dl(Node); | |
2379 | TargetLowering::CallLoweringInfo CLI(DAG); | |
2380 | CLI.setDebugLoc(dl).setChain(InChain) | |
2381 | .setCallee(TLI.getLibcallCallingConv(LC), | |
2382 | Type::getVoidTy(*DAG.getContext()), Callee, std::move(Args), 0); | |
2383 | ||
970d7e83 LB |
2384 | std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); |
2385 | ||
2386 | Results.push_back(DAG.getLoad(RetVT, dl, CallInfo.second, SinPtr, | |
2387 | MachinePointerInfo(), false, false, false, 0)); | |
2388 | Results.push_back(DAG.getLoad(RetVT, dl, CallInfo.second, CosPtr, | |
2389 | MachinePointerInfo(), false, false, false, 0)); | |
2390 | } | |
2391 | ||
85aaf69f | 2392 | /// This function is responsible for legalizing a |
223e47cc LB |
2393 | /// INT_TO_FP operation of the specified operand when the target requests that |
2394 | /// we expand it. At this point, we know that the result and operand types are | |
2395 | /// legal for the target. | |
2396 | SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned, | |
2397 | SDValue Op0, | |
2398 | EVT DestVT, | |
1a4d82fc | 2399 | SDLoc dl) { |
223e47cc LB |
2400 | if (Op0.getValueType() == MVT::i32 && TLI.isTypeLegal(MVT::f64)) { |
2401 | // simple 32-bit [signed|unsigned] integer to float/double expansion | |
2402 | ||
2403 | // Get the stack frame index of a 8 byte buffer. | |
2404 | SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64); | |
2405 | ||
2406 | // word offset constant for Hi/Lo address computation | |
1a4d82fc | 2407 | SDValue WordOff = DAG.getConstant(sizeof(int), StackSlot.getValueType()); |
223e47cc LB |
2408 | // set up Hi and Lo (into buffer) address based on endian |
2409 | SDValue Hi = StackSlot; | |
1a4d82fc JJ |
2410 | SDValue Lo = DAG.getNode(ISD::ADD, dl, StackSlot.getValueType(), |
2411 | StackSlot, WordOff); | |
223e47cc LB |
2412 | if (TLI.isLittleEndian()) |
2413 | std::swap(Hi, Lo); | |
2414 | ||
2415 | // if signed map to unsigned space | |
2416 | SDValue Op0Mapped; | |
2417 | if (isSigned) { | |
2418 | // constant used to invert sign bit (signed to unsigned mapping) | |
2419 | SDValue SignBit = DAG.getConstant(0x80000000u, MVT::i32); | |
2420 | Op0Mapped = DAG.getNode(ISD::XOR, dl, MVT::i32, Op0, SignBit); | |
2421 | } else { | |
2422 | Op0Mapped = Op0; | |
2423 | } | |
2424 | // store the lo of the constructed double - based on integer input | |
2425 | SDValue Store1 = DAG.getStore(DAG.getEntryNode(), dl, | |
2426 | Op0Mapped, Lo, MachinePointerInfo(), | |
2427 | false, false, 0); | |
2428 | // initial hi portion of constructed double | |
2429 | SDValue InitialHi = DAG.getConstant(0x43300000u, MVT::i32); | |
2430 | // store the hi of the constructed double - biased exponent | |
2431 | SDValue Store2 = DAG.getStore(Store1, dl, InitialHi, Hi, | |
2432 | MachinePointerInfo(), | |
2433 | false, false, 0); | |
2434 | // load the constructed double | |
2435 | SDValue Load = DAG.getLoad(MVT::f64, dl, Store2, StackSlot, | |
2436 | MachinePointerInfo(), false, false, false, 0); | |
2437 | // FP constant to bias correct the final result | |
2438 | SDValue Bias = DAG.getConstantFP(isSigned ? | |
2439 | BitsToDouble(0x4330000080000000ULL) : | |
2440 | BitsToDouble(0x4330000000000000ULL), | |
2441 | MVT::f64); | |
2442 | // subtract the bias | |
2443 | SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias); | |
2444 | // final result | |
2445 | SDValue Result; | |
2446 | // handle final rounding | |
2447 | if (DestVT == MVT::f64) { | |
2448 | // do nothing | |
2449 | Result = Sub; | |
2450 | } else if (DestVT.bitsLT(MVT::f64)) { | |
2451 | Result = DAG.getNode(ISD::FP_ROUND, dl, DestVT, Sub, | |
2452 | DAG.getIntPtrConstant(0)); | |
2453 | } else if (DestVT.bitsGT(MVT::f64)) { | |
2454 | Result = DAG.getNode(ISD::FP_EXTEND, dl, DestVT, Sub); | |
2455 | } | |
2456 | return Result; | |
2457 | } | |
2458 | assert(!isSigned && "Legalize cannot Expand SINT_TO_FP for i64 yet"); | |
2459 | // Code below here assumes !isSigned without checking again. | |
2460 | ||
2461 | // Implementation of unsigned i64 to f64 following the algorithm in | |
2462 | // __floatundidf in compiler_rt. This implementation has the advantage | |
2463 | // of performing rounding correctly, both in the default rounding mode | |
2464 | // and in all alternate rounding modes. | |
2465 | // TODO: Generalize this for use with other types. | |
2466 | if (Op0.getValueType() == MVT::i64 && DestVT == MVT::f64) { | |
2467 | SDValue TwoP52 = | |
2468 | DAG.getConstant(UINT64_C(0x4330000000000000), MVT::i64); | |
2469 | SDValue TwoP84PlusTwoP52 = | |
2470 | DAG.getConstantFP(BitsToDouble(UINT64_C(0x4530000000100000)), MVT::f64); | |
2471 | SDValue TwoP84 = | |
2472 | DAG.getConstant(UINT64_C(0x4530000000000000), MVT::i64); | |
2473 | ||
2474 | SDValue Lo = DAG.getZeroExtendInReg(Op0, dl, MVT::i32); | |
2475 | SDValue Hi = DAG.getNode(ISD::SRL, dl, MVT::i64, Op0, | |
2476 | DAG.getConstant(32, MVT::i64)); | |
2477 | SDValue LoOr = DAG.getNode(ISD::OR, dl, MVT::i64, Lo, TwoP52); | |
2478 | SDValue HiOr = DAG.getNode(ISD::OR, dl, MVT::i64, Hi, TwoP84); | |
2479 | SDValue LoFlt = DAG.getNode(ISD::BITCAST, dl, MVT::f64, LoOr); | |
2480 | SDValue HiFlt = DAG.getNode(ISD::BITCAST, dl, MVT::f64, HiOr); | |
2481 | SDValue HiSub = DAG.getNode(ISD::FSUB, dl, MVT::f64, HiFlt, | |
2482 | TwoP84PlusTwoP52); | |
2483 | return DAG.getNode(ISD::FADD, dl, MVT::f64, LoFlt, HiSub); | |
2484 | } | |
2485 | ||
2486 | // Implementation of unsigned i64 to f32. | |
2487 | // TODO: Generalize this for use with other types. | |
2488 | if (Op0.getValueType() == MVT::i64 && DestVT == MVT::f32) { | |
2489 | // For unsigned conversions, convert them to signed conversions using the | |
2490 | // algorithm from the x86_64 __floatundidf in compiler_rt. | |
2491 | if (!isSigned) { | |
2492 | SDValue Fast = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, Op0); | |
2493 | ||
2494 | SDValue ShiftConst = | |
2495 | DAG.getConstant(1, TLI.getShiftAmountTy(Op0.getValueType())); | |
2496 | SDValue Shr = DAG.getNode(ISD::SRL, dl, MVT::i64, Op0, ShiftConst); | |
2497 | SDValue AndConst = DAG.getConstant(1, MVT::i64); | |
2498 | SDValue And = DAG.getNode(ISD::AND, dl, MVT::i64, Op0, AndConst); | |
2499 | SDValue Or = DAG.getNode(ISD::OR, dl, MVT::i64, And, Shr); | |
2500 | ||
2501 | SDValue SignCvt = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, Or); | |
2502 | SDValue Slow = DAG.getNode(ISD::FADD, dl, MVT::f32, SignCvt, SignCvt); | |
2503 | ||
2504 | // TODO: This really should be implemented using a branch rather than a | |
2505 | // select. We happen to get lucky and machinesink does the right | |
2506 | // thing most of the time. This would be a good candidate for a | |
2507 | //pseudo-op, or, even better, for whole-function isel. | |
1a4d82fc | 2508 | SDValue SignBitTest = DAG.getSetCC(dl, getSetCCResultType(MVT::i64), |
223e47cc | 2509 | Op0, DAG.getConstant(0, MVT::i64), ISD::SETLT); |
1a4d82fc | 2510 | return DAG.getSelect(dl, MVT::f32, SignBitTest, Slow, Fast); |
223e47cc LB |
2511 | } |
2512 | ||
2513 | // Otherwise, implement the fully general conversion. | |
2514 | ||
2515 | SDValue And = DAG.getNode(ISD::AND, dl, MVT::i64, Op0, | |
2516 | DAG.getConstant(UINT64_C(0xfffffffffffff800), MVT::i64)); | |
2517 | SDValue Or = DAG.getNode(ISD::OR, dl, MVT::i64, And, | |
2518 | DAG.getConstant(UINT64_C(0x800), MVT::i64)); | |
2519 | SDValue And2 = DAG.getNode(ISD::AND, dl, MVT::i64, Op0, | |
2520 | DAG.getConstant(UINT64_C(0x7ff), MVT::i64)); | |
1a4d82fc | 2521 | SDValue Ne = DAG.getSetCC(dl, getSetCCResultType(MVT::i64), |
223e47cc | 2522 | And2, DAG.getConstant(UINT64_C(0), MVT::i64), ISD::SETNE); |
1a4d82fc JJ |
2523 | SDValue Sel = DAG.getSelect(dl, MVT::i64, Ne, Or, Op0); |
2524 | SDValue Ge = DAG.getSetCC(dl, getSetCCResultType(MVT::i64), | |
223e47cc LB |
2525 | Op0, DAG.getConstant(UINT64_C(0x0020000000000000), MVT::i64), |
2526 | ISD::SETUGE); | |
1a4d82fc | 2527 | SDValue Sel2 = DAG.getSelect(dl, MVT::i64, Ge, Sel, Op0); |
223e47cc LB |
2528 | EVT SHVT = TLI.getShiftAmountTy(Sel2.getValueType()); |
2529 | ||
2530 | SDValue Sh = DAG.getNode(ISD::SRL, dl, MVT::i64, Sel2, | |
2531 | DAG.getConstant(32, SHVT)); | |
2532 | SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Sh); | |
2533 | SDValue Fcvt = DAG.getNode(ISD::UINT_TO_FP, dl, MVT::f64, Trunc); | |
2534 | SDValue TwoP32 = | |
2535 | DAG.getConstantFP(BitsToDouble(UINT64_C(0x41f0000000000000)), MVT::f64); | |
2536 | SDValue Fmul = DAG.getNode(ISD::FMUL, dl, MVT::f64, TwoP32, Fcvt); | |
2537 | SDValue Lo = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Sel2); | |
2538 | SDValue Fcvt2 = DAG.getNode(ISD::UINT_TO_FP, dl, MVT::f64, Lo); | |
2539 | SDValue Fadd = DAG.getNode(ISD::FADD, dl, MVT::f64, Fmul, Fcvt2); | |
2540 | return DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Fadd, | |
2541 | DAG.getIntPtrConstant(0)); | |
2542 | } | |
2543 | ||
2544 | SDValue Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0); | |
2545 | ||
1a4d82fc | 2546 | SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(Op0.getValueType()), |
223e47cc LB |
2547 | Op0, DAG.getConstant(0, Op0.getValueType()), |
2548 | ISD::SETLT); | |
2549 | SDValue Zero = DAG.getIntPtrConstant(0), Four = DAG.getIntPtrConstant(4); | |
1a4d82fc | 2550 | SDValue CstOffset = DAG.getSelect(dl, Zero.getValueType(), |
223e47cc LB |
2551 | SignSet, Four, Zero); |
2552 | ||
2553 | // If the sign bit of the integer is set, the large number will be treated | |
2554 | // as a negative number. To counteract this, the dynamic code adds an | |
2555 | // offset depending on the data type. | |
2556 | uint64_t FF; | |
1a4d82fc | 2557 | switch (Op0.getSimpleValueType().SimpleTy) { |
223e47cc LB |
2558 | default: llvm_unreachable("Unsupported integer type!"); |
2559 | case MVT::i8 : FF = 0x43800000ULL; break; // 2^8 (as a float) | |
2560 | case MVT::i16: FF = 0x47800000ULL; break; // 2^16 (as a float) | |
2561 | case MVT::i32: FF = 0x4F800000ULL; break; // 2^32 (as a float) | |
2562 | case MVT::i64: FF = 0x5F800000ULL; break; // 2^64 (as a float) | |
2563 | } | |
2564 | if (TLI.isLittleEndian()) FF <<= 32; | |
2565 | Constant *FudgeFactor = ConstantInt::get( | |
2566 | Type::getInt64Ty(*DAG.getContext()), FF); | |
2567 | ||
2568 | SDValue CPIdx = DAG.getConstantPool(FudgeFactor, TLI.getPointerTy()); | |
2569 | unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment(); | |
1a4d82fc | 2570 | CPIdx = DAG.getNode(ISD::ADD, dl, CPIdx.getValueType(), CPIdx, CstOffset); |
223e47cc LB |
2571 | Alignment = std::min(Alignment, 4u); |
2572 | SDValue FudgeInReg; | |
2573 | if (DestVT == MVT::f32) | |
2574 | FudgeInReg = DAG.getLoad(MVT::f32, dl, DAG.getEntryNode(), CPIdx, | |
2575 | MachinePointerInfo::getConstantPool(), | |
2576 | false, false, false, Alignment); | |
2577 | else { | |
2578 | SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, | |
2579 | DAG.getEntryNode(), CPIdx, | |
2580 | MachinePointerInfo::getConstantPool(), | |
1a4d82fc | 2581 | MVT::f32, false, false, false, Alignment); |
223e47cc LB |
2582 | HandleSDNode Handle(Load); |
2583 | LegalizeOp(Load.getNode()); | |
2584 | FudgeInReg = Handle.getValue(); | |
2585 | } | |
2586 | ||
2587 | return DAG.getNode(ISD::FADD, dl, DestVT, Tmp1, FudgeInReg); | |
2588 | } | |
2589 | ||
85aaf69f | 2590 | /// This function is responsible for legalizing a |
223e47cc LB |
2591 | /// *INT_TO_FP operation of the specified operand when the target requests that |
2592 | /// we promote it. At this point, we know that the result and operand types are | |
2593 | /// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP | |
2594 | /// operation that takes a larger input. | |
2595 | SDValue SelectionDAGLegalize::PromoteLegalINT_TO_FP(SDValue LegalOp, | |
2596 | EVT DestVT, | |
2597 | bool isSigned, | |
1a4d82fc | 2598 | SDLoc dl) { |
223e47cc LB |
2599 | // First step, figure out the appropriate *INT_TO_FP operation to use. |
2600 | EVT NewInTy = LegalOp.getValueType(); | |
2601 | ||
2602 | unsigned OpToUse = 0; | |
2603 | ||
2604 | // Scan for the appropriate larger type to use. | |
2605 | while (1) { | |
2606 | NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1); | |
2607 | assert(NewInTy.isInteger() && "Ran out of possibilities!"); | |
2608 | ||
2609 | // If the target supports SINT_TO_FP of this type, use it. | |
2610 | if (TLI.isOperationLegalOrCustom(ISD::SINT_TO_FP, NewInTy)) { | |
2611 | OpToUse = ISD::SINT_TO_FP; | |
2612 | break; | |
2613 | } | |
2614 | if (isSigned) continue; | |
2615 | ||
2616 | // If the target supports UINT_TO_FP of this type, use it. | |
2617 | if (TLI.isOperationLegalOrCustom(ISD::UINT_TO_FP, NewInTy)) { | |
2618 | OpToUse = ISD::UINT_TO_FP; | |
2619 | break; | |
2620 | } | |
2621 | ||
2622 | // Otherwise, try a larger type. | |
2623 | } | |
2624 | ||
2625 | // Okay, we found the operation and type to use. Zero extend our input to the | |
2626 | // desired type then run the operation on it. | |
2627 | return DAG.getNode(OpToUse, dl, DestVT, | |
2628 | DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, | |
2629 | dl, NewInTy, LegalOp)); | |
2630 | } | |
2631 | ||
85aaf69f | 2632 | /// This function is responsible for legalizing a |
223e47cc LB |
2633 | /// FP_TO_*INT operation of the specified operand when the target requests that |
2634 | /// we promote it. At this point, we know that the result and operand types are | |
2635 | /// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT | |
2636 | /// operation that returns a larger result. | |
2637 | SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDValue LegalOp, | |
2638 | EVT DestVT, | |
2639 | bool isSigned, | |
1a4d82fc | 2640 | SDLoc dl) { |
223e47cc LB |
2641 | // First step, figure out the appropriate FP_TO*INT operation to use. |
2642 | EVT NewOutTy = DestVT; | |
2643 | ||
2644 | unsigned OpToUse = 0; | |
2645 | ||
2646 | // Scan for the appropriate larger type to use. | |
2647 | while (1) { | |
2648 | NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1); | |
2649 | assert(NewOutTy.isInteger() && "Ran out of possibilities!"); | |
2650 | ||
1a4d82fc JJ |
2651 | // A larger signed type can hold all unsigned values of the requested type, |
2652 | // so using FP_TO_SINT is valid | |
223e47cc LB |
2653 | if (TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NewOutTy)) { |
2654 | OpToUse = ISD::FP_TO_SINT; | |
2655 | break; | |
2656 | } | |
2657 | ||
1a4d82fc JJ |
2658 | // However, if the value may be < 0.0, we *must* use some FP_TO_SINT. |
2659 | if (!isSigned && TLI.isOperationLegalOrCustom(ISD::FP_TO_UINT, NewOutTy)) { | |
223e47cc LB |
2660 | OpToUse = ISD::FP_TO_UINT; |
2661 | break; | |
2662 | } | |
2663 | ||
2664 | // Otherwise, try a larger type. | |
2665 | } | |
2666 | ||
2667 | ||
2668 | // Okay, we found the operation and type to use. | |
2669 | SDValue Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp); | |
2670 | ||
2671 | // Truncate the result of the extended FP_TO_*INT operation to the desired | |
2672 | // size. | |
2673 | return DAG.getNode(ISD::TRUNCATE, dl, DestVT, Operation); | |
2674 | } | |
2675 | ||
85aaf69f | 2676 | /// Open code the operations for BSWAP of the specified operation. |
1a4d82fc | 2677 | SDValue SelectionDAGLegalize::ExpandBSWAP(SDValue Op, SDLoc dl) { |
223e47cc LB |
2678 | EVT VT = Op.getValueType(); |
2679 | EVT SHVT = TLI.getShiftAmountTy(VT); | |
2680 | SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8; | |
2681 | switch (VT.getSimpleVT().SimpleTy) { | |
2682 | default: llvm_unreachable("Unhandled Expand type in BSWAP!"); | |
2683 | case MVT::i16: | |
2684 | Tmp2 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, SHVT)); | |
2685 | Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, SHVT)); | |
2686 | return DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2); | |
2687 | case MVT::i32: | |
2688 | Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, SHVT)); | |
2689 | Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, SHVT)); | |
2690 | Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, SHVT)); | |
2691 | Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, SHVT)); | |
2692 | Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3, DAG.getConstant(0xFF0000, VT)); | |
2693 | Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, VT)); | |
2694 | Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3); | |
2695 | Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1); | |
2696 | return DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2); | |
2697 | case MVT::i64: | |
2698 | Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, SHVT)); | |
2699 | Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(40, SHVT)); | |
2700 | Tmp6 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, SHVT)); | |
2701 | Tmp5 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, SHVT)); | |
2702 | Tmp4 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, SHVT)); | |
2703 | Tmp3 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, SHVT)); | |
2704 | Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(40, SHVT)); | |
2705 | Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(56, SHVT)); | |
2706 | Tmp7 = DAG.getNode(ISD::AND, dl, VT, Tmp7, DAG.getConstant(255ULL<<48, VT)); | |
2707 | Tmp6 = DAG.getNode(ISD::AND, dl, VT, Tmp6, DAG.getConstant(255ULL<<40, VT)); | |
2708 | Tmp5 = DAG.getNode(ISD::AND, dl, VT, Tmp5, DAG.getConstant(255ULL<<32, VT)); | |
2709 | Tmp4 = DAG.getNode(ISD::AND, dl, VT, Tmp4, DAG.getConstant(255ULL<<24, VT)); | |
2710 | Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3, DAG.getConstant(255ULL<<16, VT)); | |
2711 | Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(255ULL<<8 , VT)); | |
2712 | Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp7); | |
2713 | Tmp6 = DAG.getNode(ISD::OR, dl, VT, Tmp6, Tmp5); | |
2714 | Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3); | |
2715 | Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1); | |
2716 | Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp6); | |
2717 | Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2); | |
2718 | return DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp4); | |
2719 | } | |
2720 | } | |
2721 | ||
85aaf69f | 2722 | /// Expand the specified bitcount instruction into operations. |
223e47cc | 2723 | SDValue SelectionDAGLegalize::ExpandBitCount(unsigned Opc, SDValue Op, |
1a4d82fc | 2724 | SDLoc dl) { |
223e47cc LB |
2725 | switch (Opc) { |
2726 | default: llvm_unreachable("Cannot expand this yet!"); | |
2727 | case ISD::CTPOP: { | |
2728 | EVT VT = Op.getValueType(); | |
2729 | EVT ShVT = TLI.getShiftAmountTy(VT); | |
2730 | unsigned Len = VT.getSizeInBits(); | |
2731 | ||
2732 | assert(VT.isInteger() && Len <= 128 && Len % 8 == 0 && | |
2733 | "CTPOP not implemented for this type."); | |
2734 | ||
2735 | // This is the "best" algorithm from | |
2736 | // http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel | |
2737 | ||
970d7e83 LB |
2738 | SDValue Mask55 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x55)), VT); |
2739 | SDValue Mask33 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x33)), VT); | |
2740 | SDValue Mask0F = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x0F)), VT); | |
2741 | SDValue Mask01 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x01)), VT); | |
223e47cc LB |
2742 | |
2743 | // v = v - ((v >> 1) & 0x55555555...) | |
2744 | Op = DAG.getNode(ISD::SUB, dl, VT, Op, | |
2745 | DAG.getNode(ISD::AND, dl, VT, | |
2746 | DAG.getNode(ISD::SRL, dl, VT, Op, | |
2747 | DAG.getConstant(1, ShVT)), | |
2748 | Mask55)); | |
2749 | // v = (v & 0x33333333...) + ((v >> 2) & 0x33333333...) | |
2750 | Op = DAG.getNode(ISD::ADD, dl, VT, | |
2751 | DAG.getNode(ISD::AND, dl, VT, Op, Mask33), | |
2752 | DAG.getNode(ISD::AND, dl, VT, | |
2753 | DAG.getNode(ISD::SRL, dl, VT, Op, | |
2754 | DAG.getConstant(2, ShVT)), | |
2755 | Mask33)); | |
2756 | // v = (v + (v >> 4)) & 0x0F0F0F0F... | |
2757 | Op = DAG.getNode(ISD::AND, dl, VT, | |
2758 | DAG.getNode(ISD::ADD, dl, VT, Op, | |
2759 | DAG.getNode(ISD::SRL, dl, VT, Op, | |
2760 | DAG.getConstant(4, ShVT))), | |
2761 | Mask0F); | |
2762 | // v = (v * 0x01010101...) >> (Len - 8) | |
2763 | Op = DAG.getNode(ISD::SRL, dl, VT, | |
2764 | DAG.getNode(ISD::MUL, dl, VT, Op, Mask01), | |
2765 | DAG.getConstant(Len - 8, ShVT)); | |
2766 | ||
2767 | return Op; | |
2768 | } | |
2769 | case ISD::CTLZ_ZERO_UNDEF: | |
2770 | // This trivially expands to CTLZ. | |
2771 | return DAG.getNode(ISD::CTLZ, dl, Op.getValueType(), Op); | |
2772 | case ISD::CTLZ: { | |
2773 | // for now, we do this: | |
2774 | // x = x | (x >> 1); | |
2775 | // x = x | (x >> 2); | |
2776 | // ... | |
2777 | // x = x | (x >>16); | |
2778 | // x = x | (x >>32); // for 64-bit input | |
2779 | // return popcount(~x); | |
2780 | // | |
1a4d82fc | 2781 | // Ref: "Hacker's Delight" by Henry Warren |
223e47cc LB |
2782 | EVT VT = Op.getValueType(); |
2783 | EVT ShVT = TLI.getShiftAmountTy(VT); | |
2784 | unsigned len = VT.getSizeInBits(); | |
2785 | for (unsigned i = 0; (1U << i) <= (len / 2); ++i) { | |
2786 | SDValue Tmp3 = DAG.getConstant(1ULL << i, ShVT); | |
2787 | Op = DAG.getNode(ISD::OR, dl, VT, Op, | |
2788 | DAG.getNode(ISD::SRL, dl, VT, Op, Tmp3)); | |
2789 | } | |
2790 | Op = DAG.getNOT(dl, Op, VT); | |
2791 | return DAG.getNode(ISD::CTPOP, dl, VT, Op); | |
2792 | } | |
2793 | case ISD::CTTZ_ZERO_UNDEF: | |
2794 | // This trivially expands to CTTZ. | |
2795 | return DAG.getNode(ISD::CTTZ, dl, Op.getValueType(), Op); | |
2796 | case ISD::CTTZ: { | |
2797 | // for now, we use: { return popcount(~x & (x - 1)); } | |
2798 | // unless the target has ctlz but not ctpop, in which case we use: | |
2799 | // { return 32 - nlz(~x & (x-1)); } | |
1a4d82fc | 2800 | // Ref: "Hacker's Delight" by Henry Warren |
223e47cc LB |
2801 | EVT VT = Op.getValueType(); |
2802 | SDValue Tmp3 = DAG.getNode(ISD::AND, dl, VT, | |
2803 | DAG.getNOT(dl, Op, VT), | |
2804 | DAG.getNode(ISD::SUB, dl, VT, Op, | |
2805 | DAG.getConstant(1, VT))); | |
2806 | // If ISD::CTLZ is legal and CTPOP isn't, then do that instead. | |
2807 | if (!TLI.isOperationLegalOrCustom(ISD::CTPOP, VT) && | |
2808 | TLI.isOperationLegalOrCustom(ISD::CTLZ, VT)) | |
2809 | return DAG.getNode(ISD::SUB, dl, VT, | |
2810 | DAG.getConstant(VT.getSizeInBits(), VT), | |
2811 | DAG.getNode(ISD::CTLZ, dl, VT, Tmp3)); | |
2812 | return DAG.getNode(ISD::CTPOP, dl, VT, Tmp3); | |
2813 | } | |
2814 | } | |
2815 | } | |
2816 | ||
2817 | std::pair <SDValue, SDValue> SelectionDAGLegalize::ExpandAtomic(SDNode *Node) { | |
2818 | unsigned Opc = Node->getOpcode(); | |
2819 | MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT(); | |
2820 | RTLIB::Libcall LC; | |
2821 | ||
2822 | switch (Opc) { | |
2823 | default: | |
2824 | llvm_unreachable("Unhandled atomic intrinsic Expand!"); | |
2825 | case ISD::ATOMIC_SWAP: | |
2826 | switch (VT.SimpleTy) { | |
2827 | default: llvm_unreachable("Unexpected value type for atomic!"); | |
2828 | case MVT::i8: LC = RTLIB::SYNC_LOCK_TEST_AND_SET_1; break; | |
2829 | case MVT::i16: LC = RTLIB::SYNC_LOCK_TEST_AND_SET_2; break; | |
2830 | case MVT::i32: LC = RTLIB::SYNC_LOCK_TEST_AND_SET_4; break; | |
2831 | case MVT::i64: LC = RTLIB::SYNC_LOCK_TEST_AND_SET_8; break; | |
1a4d82fc | 2832 | case MVT::i128:LC = RTLIB::SYNC_LOCK_TEST_AND_SET_16;break; |
223e47cc LB |
2833 | } |
2834 | break; | |
2835 | case ISD::ATOMIC_CMP_SWAP: | |
2836 | switch (VT.SimpleTy) { | |
2837 | default: llvm_unreachable("Unexpected value type for atomic!"); | |
2838 | case MVT::i8: LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_1; break; | |
2839 | case MVT::i16: LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_2; break; | |
2840 | case MVT::i32: LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_4; break; | |
2841 | case MVT::i64: LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_8; break; | |
1a4d82fc | 2842 | case MVT::i128:LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_16;break; |
223e47cc LB |
2843 | } |
2844 | break; | |
2845 | case ISD::ATOMIC_LOAD_ADD: | |
2846 | switch (VT.SimpleTy) { | |
2847 | default: llvm_unreachable("Unexpected value type for atomic!"); | |
2848 | case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_ADD_1; break; | |
2849 | case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_ADD_2; break; | |
2850 | case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_ADD_4; break; | |
2851 | case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_ADD_8; break; | |
1a4d82fc | 2852 | case MVT::i128:LC = RTLIB::SYNC_FETCH_AND_ADD_16;break; |
223e47cc LB |
2853 | } |
2854 | break; | |
2855 | case ISD::ATOMIC_LOAD_SUB: | |
2856 | switch (VT.SimpleTy) { | |
2857 | default: llvm_unreachable("Unexpected value type for atomic!"); | |
2858 | case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_SUB_1; break; | |
2859 | case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_SUB_2; break; | |
2860 | case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_SUB_4; break; | |
2861 | case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_SUB_8; break; | |
1a4d82fc | 2862 | case MVT::i128:LC = RTLIB::SYNC_FETCH_AND_SUB_16;break; |
223e47cc LB |
2863 | } |
2864 | break; | |
2865 | case ISD::ATOMIC_LOAD_AND: | |
2866 | switch (VT.SimpleTy) { | |
2867 | default: llvm_unreachable("Unexpected value type for atomic!"); | |
2868 | case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_AND_1; break; | |
2869 | case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_AND_2; break; | |
2870 | case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_AND_4; break; | |
2871 | case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_AND_8; break; | |
1a4d82fc | 2872 | case MVT::i128:LC = RTLIB::SYNC_FETCH_AND_AND_16;break; |
223e47cc LB |
2873 | } |
2874 | break; | |
2875 | case ISD::ATOMIC_LOAD_OR: | |
2876 | switch (VT.SimpleTy) { | |
2877 | default: llvm_unreachable("Unexpected value type for atomic!"); | |
2878 | case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_OR_1; break; | |
2879 | case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_OR_2; break; | |
2880 | case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_OR_4; break; | |
2881 | case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_OR_8; break; | |
1a4d82fc | 2882 | case MVT::i128:LC = RTLIB::SYNC_FETCH_AND_OR_16;break; |
223e47cc LB |
2883 | } |
2884 | break; | |
2885 | case ISD::ATOMIC_LOAD_XOR: | |
2886 | switch (VT.SimpleTy) { | |
2887 | default: llvm_unreachable("Unexpected value type for atomic!"); | |
2888 | case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_XOR_1; break; | |
2889 | case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_XOR_2; break; | |
2890 | case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_XOR_4; break; | |
2891 | case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_XOR_8; break; | |
1a4d82fc | 2892 | case MVT::i128:LC = RTLIB::SYNC_FETCH_AND_XOR_16;break; |
223e47cc LB |
2893 | } |
2894 | break; | |
2895 | case ISD::ATOMIC_LOAD_NAND: | |
2896 | switch (VT.SimpleTy) { | |
2897 | default: llvm_unreachable("Unexpected value type for atomic!"); | |
2898 | case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_NAND_1; break; | |
2899 | case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_NAND_2; break; | |
2900 | case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_NAND_4; break; | |
2901 | case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_NAND_8; break; | |
1a4d82fc JJ |
2902 | case MVT::i128:LC = RTLIB::SYNC_FETCH_AND_NAND_16;break; |
2903 | } | |
2904 | break; | |
2905 | case ISD::ATOMIC_LOAD_MAX: | |
2906 | switch (VT.SimpleTy) { | |
2907 | default: llvm_unreachable("Unexpected value type for atomic!"); | |
2908 | case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_MAX_1; break; | |
2909 | case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_MAX_2; break; | |
2910 | case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_MAX_4; break; | |
2911 | case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_MAX_8; break; | |
2912 | case MVT::i128:LC = RTLIB::SYNC_FETCH_AND_MAX_16;break; | |
2913 | } | |
2914 | break; | |
2915 | case ISD::ATOMIC_LOAD_UMAX: | |
2916 | switch (VT.SimpleTy) { | |
2917 | default: llvm_unreachable("Unexpected value type for atomic!"); | |
2918 | case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_UMAX_1; break; | |
2919 | case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_UMAX_2; break; | |
2920 | case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_UMAX_4; break; | |
2921 | case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_UMAX_8; break; | |
2922 | case MVT::i128:LC = RTLIB::SYNC_FETCH_AND_UMAX_16;break; | |
2923 | } | |
2924 | break; | |
2925 | case ISD::ATOMIC_LOAD_MIN: | |
2926 | switch (VT.SimpleTy) { | |
2927 | default: llvm_unreachable("Unexpected value type for atomic!"); | |
2928 | case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_MIN_1; break; | |
2929 | case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_MIN_2; break; | |
2930 | case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_MIN_4; break; | |
2931 | case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_MIN_8; break; | |
2932 | case MVT::i128:LC = RTLIB::SYNC_FETCH_AND_MIN_16;break; | |
2933 | } | |
2934 | break; | |
2935 | case ISD::ATOMIC_LOAD_UMIN: | |
2936 | switch (VT.SimpleTy) { | |
2937 | default: llvm_unreachable("Unexpected value type for atomic!"); | |
2938 | case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_UMIN_1; break; | |
2939 | case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_UMIN_2; break; | |
2940 | case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_UMIN_4; break; | |
2941 | case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_UMIN_8; break; | |
2942 | case MVT::i128:LC = RTLIB::SYNC_FETCH_AND_UMIN_16;break; | |
223e47cc LB |
2943 | } |
2944 | break; | |
2945 | } | |
2946 | ||
2947 | return ExpandChainLibCall(LC, Node, false); | |
2948 | } | |
2949 | ||
2950 | void SelectionDAGLegalize::ExpandNode(SDNode *Node) { | |
2951 | SmallVector<SDValue, 8> Results; | |
1a4d82fc | 2952 | SDLoc dl(Node); |
223e47cc | 2953 | SDValue Tmp1, Tmp2, Tmp3, Tmp4; |
1a4d82fc | 2954 | bool NeedInvert; |
223e47cc LB |
2955 | switch (Node->getOpcode()) { |
2956 | case ISD::CTPOP: | |
2957 | case ISD::CTLZ: | |
2958 | case ISD::CTLZ_ZERO_UNDEF: | |
2959 | case ISD::CTTZ: | |
2960 | case ISD::CTTZ_ZERO_UNDEF: | |
2961 | Tmp1 = ExpandBitCount(Node->getOpcode(), Node->getOperand(0), dl); | |
2962 | Results.push_back(Tmp1); | |
2963 | break; | |
2964 | case ISD::BSWAP: | |
2965 | Results.push_back(ExpandBSWAP(Node->getOperand(0), dl)); | |
2966 | break; | |
2967 | case ISD::FRAMEADDR: | |
2968 | case ISD::RETURNADDR: | |
2969 | case ISD::FRAME_TO_ARGS_OFFSET: | |
2970 | Results.push_back(DAG.getConstant(0, Node->getValueType(0))); | |
2971 | break; | |
2972 | case ISD::FLT_ROUNDS_: | |
2973 | Results.push_back(DAG.getConstant(1, Node->getValueType(0))); | |
2974 | break; | |
2975 | case ISD::EH_RETURN: | |
2976 | case ISD::EH_LABEL: | |
2977 | case ISD::PREFETCH: | |
2978 | case ISD::VAEND: | |
2979 | case ISD::EH_SJLJ_LONGJMP: | |
2980 | // If the target didn't expand these, there's nothing to do, so just | |
2981 | // preserve the chain and be done. | |
2982 | Results.push_back(Node->getOperand(0)); | |
2983 | break; | |
2984 | case ISD::EH_SJLJ_SETJMP: | |
2985 | // If the target didn't expand this, just return 'zero' and preserve the | |
2986 | // chain. | |
2987 | Results.push_back(DAG.getConstant(0, MVT::i32)); | |
2988 | Results.push_back(Node->getOperand(0)); | |
2989 | break; | |
1a4d82fc | 2990 | case ISD::ATOMIC_FENCE: { |
223e47cc LB |
2991 | // If the target didn't lower this, lower it to '__sync_synchronize()' call |
2992 | // FIXME: handle "fence singlethread" more efficiently. | |
2993 | TargetLowering::ArgListTy Args; | |
1a4d82fc JJ |
2994 | |
2995 | TargetLowering::CallLoweringInfo CLI(DAG); | |
2996 | CLI.setDebugLoc(dl).setChain(Node->getOperand(0)) | |
2997 | .setCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()), | |
2998 | DAG.getExternalSymbol("__sync_synchronize", | |
2999 | TLI.getPointerTy()), std::move(Args), 0); | |
3000 | ||
223e47cc LB |
3001 | std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI); |
3002 | ||
3003 | Results.push_back(CallResult.second); | |
3004 | break; | |
3005 | } | |
3006 | case ISD::ATOMIC_LOAD: { | |
3007 | // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP. | |
3008 | SDValue Zero = DAG.getConstant(0, Node->getValueType(0)); | |
1a4d82fc JJ |
3009 | SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other); |
3010 | SDValue Swap = DAG.getAtomicCmpSwap( | |
3011 | ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs, | |
3012 | Node->getOperand(0), Node->getOperand(1), Zero, Zero, | |
3013 | cast<AtomicSDNode>(Node)->getMemOperand(), | |
3014 | cast<AtomicSDNode>(Node)->getOrdering(), | |
3015 | cast<AtomicSDNode>(Node)->getOrdering(), | |
3016 | cast<AtomicSDNode>(Node)->getSynchScope()); | |
223e47cc LB |
3017 | Results.push_back(Swap.getValue(0)); |
3018 | Results.push_back(Swap.getValue(1)); | |
3019 | break; | |
3020 | } | |
3021 | case ISD::ATOMIC_STORE: { | |
3022 | // There is no libcall for atomic store; fake it with ATOMIC_SWAP. | |
3023 | SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, | |
3024 | cast<AtomicSDNode>(Node)->getMemoryVT(), | |
3025 | Node->getOperand(0), | |
3026 | Node->getOperand(1), Node->getOperand(2), | |
3027 | cast<AtomicSDNode>(Node)->getMemOperand(), | |
3028 | cast<AtomicSDNode>(Node)->getOrdering(), | |
3029 | cast<AtomicSDNode>(Node)->getSynchScope()); | |
3030 | Results.push_back(Swap.getValue(1)); | |
3031 | break; | |
3032 | } | |
3033 | // By default, atomic intrinsics are marked Legal and lowered. Targets | |
3034 | // which don't support them directly, however, may want libcalls, in which | |
3035 | // case they mark them Expand, and we get here. | |
3036 | case ISD::ATOMIC_SWAP: | |
3037 | case ISD::ATOMIC_LOAD_ADD: | |
3038 | case ISD::ATOMIC_LOAD_SUB: | |
3039 | case ISD::ATOMIC_LOAD_AND: | |
3040 | case ISD::ATOMIC_LOAD_OR: | |
3041 | case ISD::ATOMIC_LOAD_XOR: | |
3042 | case ISD::ATOMIC_LOAD_NAND: | |
3043 | case ISD::ATOMIC_LOAD_MIN: | |
3044 | case ISD::ATOMIC_LOAD_MAX: | |
3045 | case ISD::ATOMIC_LOAD_UMIN: | |
3046 | case ISD::ATOMIC_LOAD_UMAX: | |
3047 | case ISD::ATOMIC_CMP_SWAP: { | |
3048 | std::pair<SDValue, SDValue> Tmp = ExpandAtomic(Node); | |
3049 | Results.push_back(Tmp.first); | |
3050 | Results.push_back(Tmp.second); | |
3051 | break; | |
3052 | } | |
1a4d82fc JJ |
3053 | case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: { |
3054 | // Expanding an ATOMIC_CMP_SWAP_WITH_SUCCESS produces an ATOMIC_CMP_SWAP and | |
3055 | // splits out the success value as a comparison. Expanding the resulting | |
3056 | // ATOMIC_CMP_SWAP will produce a libcall. | |
3057 | SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other); | |
3058 | SDValue Res = DAG.getAtomicCmpSwap( | |
3059 | ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs, | |
3060 | Node->getOperand(0), Node->getOperand(1), Node->getOperand(2), | |
3061 | Node->getOperand(3), cast<MemSDNode>(Node)->getMemOperand(), | |
3062 | cast<AtomicSDNode>(Node)->getSuccessOrdering(), | |
3063 | cast<AtomicSDNode>(Node)->getFailureOrdering(), | |
3064 | cast<AtomicSDNode>(Node)->getSynchScope()); | |
3065 | ||
3066 | SDValue Success = DAG.getSetCC(SDLoc(Node), Node->getValueType(1), | |
3067 | Res, Node->getOperand(2), ISD::SETEQ); | |
3068 | ||
3069 | Results.push_back(Res.getValue(0)); | |
3070 | Results.push_back(Success); | |
3071 | Results.push_back(Res.getValue(1)); | |
3072 | break; | |
3073 | } | |
223e47cc LB |
3074 | case ISD::DYNAMIC_STACKALLOC: |
3075 | ExpandDYNAMIC_STACKALLOC(Node, Results); | |
3076 | break; | |
3077 | case ISD::MERGE_VALUES: | |
3078 | for (unsigned i = 0; i < Node->getNumValues(); i++) | |
3079 | Results.push_back(Node->getOperand(i)); | |
3080 | break; | |
3081 | case ISD::UNDEF: { | |
3082 | EVT VT = Node->getValueType(0); | |
3083 | if (VT.isInteger()) | |
3084 | Results.push_back(DAG.getConstant(0, VT)); | |
3085 | else { | |
3086 | assert(VT.isFloatingPoint() && "Unknown value type!"); | |
3087 | Results.push_back(DAG.getConstantFP(0, VT)); | |
3088 | } | |
3089 | break; | |
3090 | } | |
3091 | case ISD::TRAP: { | |
3092 | // If this operation is not supported, lower it to 'abort()' call | |
3093 | TargetLowering::ArgListTy Args; | |
1a4d82fc JJ |
3094 | TargetLowering::CallLoweringInfo CLI(DAG); |
3095 | CLI.setDebugLoc(dl).setChain(Node->getOperand(0)) | |
3096 | .setCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()), | |
3097 | DAG.getExternalSymbol("abort", TLI.getPointerTy()), | |
3098 | std::move(Args), 0); | |
223e47cc LB |
3099 | std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI); |
3100 | ||
3101 | Results.push_back(CallResult.second); | |
3102 | break; | |
3103 | } | |
3104 | case ISD::FP_ROUND: | |
3105 | case ISD::BITCAST: | |
3106 | Tmp1 = EmitStackConvert(Node->getOperand(0), Node->getValueType(0), | |
3107 | Node->getValueType(0), dl); | |
3108 | Results.push_back(Tmp1); | |
3109 | break; | |
3110 | case ISD::FP_EXTEND: | |
3111 | Tmp1 = EmitStackConvert(Node->getOperand(0), | |
3112 | Node->getOperand(0).getValueType(), | |
3113 | Node->getValueType(0), dl); | |
3114 | Results.push_back(Tmp1); | |
3115 | break; | |
3116 | case ISD::SIGN_EXTEND_INREG: { | |
3117 | // NOTE: we could fall back on load/store here too for targets without | |
3118 | // SAR. However, it is doubtful that any exist. | |
3119 | EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT(); | |
3120 | EVT VT = Node->getValueType(0); | |
3121 | EVT ShiftAmountTy = TLI.getShiftAmountTy(VT); | |
3122 | if (VT.isVector()) | |
3123 | ShiftAmountTy = VT; | |
3124 | unsigned BitsDiff = VT.getScalarType().getSizeInBits() - | |
3125 | ExtraVT.getScalarType().getSizeInBits(); | |
3126 | SDValue ShiftCst = DAG.getConstant(BitsDiff, ShiftAmountTy); | |
3127 | Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0), | |
3128 | Node->getOperand(0), ShiftCst); | |
3129 | Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst); | |
3130 | Results.push_back(Tmp1); | |
3131 | break; | |
3132 | } | |
3133 | case ISD::FP_ROUND_INREG: { | |
3134 | // The only way we can lower this is to turn it into a TRUNCSTORE, | |
3135 | // EXTLOAD pair, targeting a temporary location (a stack slot). | |
3136 | ||
3137 | // NOTE: there is a choice here between constantly creating new stack | |
3138 | // slots and always reusing the same one. We currently always create | |
3139 | // new ones, as reuse may inhibit scheduling. | |
3140 | EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT(); | |
3141 | Tmp1 = EmitStackConvert(Node->getOperand(0), ExtraVT, | |
3142 | Node->getValueType(0), dl); | |
3143 | Results.push_back(Tmp1); | |
3144 | break; | |
3145 | } | |
3146 | case ISD::SINT_TO_FP: | |
3147 | case ISD::UINT_TO_FP: | |
3148 | Tmp1 = ExpandLegalINT_TO_FP(Node->getOpcode() == ISD::SINT_TO_FP, | |
3149 | Node->getOperand(0), Node->getValueType(0), dl); | |
3150 | Results.push_back(Tmp1); | |
3151 | break; | |
1a4d82fc JJ |
3152 | case ISD::FP_TO_SINT: |
3153 | if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG)) | |
3154 | Results.push_back(Tmp1); | |
3155 | break; | |
223e47cc LB |
3156 | case ISD::FP_TO_UINT: { |
3157 | SDValue True, False; | |
3158 | EVT VT = Node->getOperand(0).getValueType(); | |
3159 | EVT NVT = Node->getValueType(0); | |
970d7e83 LB |
3160 | APFloat apf(DAG.EVTToAPFloatSemantics(VT), |
3161 | APInt::getNullValue(VT.getSizeInBits())); | |
223e47cc LB |
3162 | APInt x = APInt::getSignBit(NVT.getSizeInBits()); |
3163 | (void)apf.convertFromAPInt(x, false, APFloat::rmNearestTiesToEven); | |
3164 | Tmp1 = DAG.getConstantFP(apf, VT); | |
1a4d82fc | 3165 | Tmp2 = DAG.getSetCC(dl, getSetCCResultType(VT), |
223e47cc LB |
3166 | Node->getOperand(0), |
3167 | Tmp1, ISD::SETLT); | |
3168 | True = DAG.getNode(ISD::FP_TO_SINT, dl, NVT, Node->getOperand(0)); | |
3169 | False = DAG.getNode(ISD::FP_TO_SINT, dl, NVT, | |
3170 | DAG.getNode(ISD::FSUB, dl, VT, | |
3171 | Node->getOperand(0), Tmp1)); | |
3172 | False = DAG.getNode(ISD::XOR, dl, NVT, False, | |
3173 | DAG.getConstant(x, NVT)); | |
1a4d82fc | 3174 | Tmp1 = DAG.getSelect(dl, NVT, Tmp2, True, False); |
223e47cc LB |
3175 | Results.push_back(Tmp1); |
3176 | break; | |
3177 | } | |
3178 | case ISD::VAARG: { | |
3179 | const Value *V = cast<SrcValueSDNode>(Node->getOperand(2))->getValue(); | |
3180 | EVT VT = Node->getValueType(0); | |
3181 | Tmp1 = Node->getOperand(0); | |
3182 | Tmp2 = Node->getOperand(1); | |
3183 | unsigned Align = Node->getConstantOperandVal(3); | |
3184 | ||
3185 | SDValue VAListLoad = DAG.getLoad(TLI.getPointerTy(), dl, Tmp1, Tmp2, | |
1a4d82fc | 3186 | MachinePointerInfo(V), |
223e47cc LB |
3187 | false, false, false, 0); |
3188 | SDValue VAList = VAListLoad; | |
3189 | ||
3190 | if (Align > TLI.getMinStackArgumentAlignment()) { | |
3191 | assert(((Align & (Align-1)) == 0) && "Expected Align to be a power of 2"); | |
3192 | ||
1a4d82fc | 3193 | VAList = DAG.getNode(ISD::ADD, dl, VAList.getValueType(), VAList, |
223e47cc | 3194 | DAG.getConstant(Align - 1, |
1a4d82fc | 3195 | VAList.getValueType())); |
223e47cc | 3196 | |
1a4d82fc | 3197 | VAList = DAG.getNode(ISD::AND, dl, VAList.getValueType(), VAList, |
223e47cc | 3198 | DAG.getConstant(-(int64_t)Align, |
1a4d82fc | 3199 | VAList.getValueType())); |
223e47cc LB |
3200 | } |
3201 | ||
3202 | // Increment the pointer, VAList, to the next vaarg | |
1a4d82fc | 3203 | Tmp3 = DAG.getNode(ISD::ADD, dl, VAList.getValueType(), VAList, |
970d7e83 | 3204 | DAG.getConstant(TLI.getDataLayout()-> |
223e47cc | 3205 | getTypeAllocSize(VT.getTypeForEVT(*DAG.getContext())), |
1a4d82fc | 3206 | VAList.getValueType())); |
223e47cc LB |
3207 | // Store the incremented VAList to the legalized pointer |
3208 | Tmp3 = DAG.getStore(VAListLoad.getValue(1), dl, Tmp3, Tmp2, | |
3209 | MachinePointerInfo(V), false, false, 0); | |
3210 | // Load the actual argument out of the pointer VAList | |
3211 | Results.push_back(DAG.getLoad(VT, dl, Tmp3, VAList, MachinePointerInfo(), | |
3212 | false, false, false, 0)); | |
3213 | Results.push_back(Results[0].getValue(1)); | |
3214 | break; | |
3215 | } | |
3216 | case ISD::VACOPY: { | |
3217 | // This defaults to loading a pointer from the input and storing it to the | |
3218 | // output, returning the chain. | |
3219 | const Value *VD = cast<SrcValueSDNode>(Node->getOperand(3))->getValue(); | |
3220 | const Value *VS = cast<SrcValueSDNode>(Node->getOperand(4))->getValue(); | |
3221 | Tmp1 = DAG.getLoad(TLI.getPointerTy(), dl, Node->getOperand(0), | |
3222 | Node->getOperand(2), MachinePointerInfo(VS), | |
3223 | false, false, false, 0); | |
3224 | Tmp1 = DAG.getStore(Tmp1.getValue(1), dl, Tmp1, Node->getOperand(1), | |
3225 | MachinePointerInfo(VD), false, false, 0); | |
3226 | Results.push_back(Tmp1); | |
3227 | break; | |
3228 | } | |
3229 | case ISD::EXTRACT_VECTOR_ELT: | |
3230 | if (Node->getOperand(0).getValueType().getVectorNumElements() == 1) | |
3231 | // This must be an access of the only element. Return it. | |
3232 | Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), | |
3233 | Node->getOperand(0)); | |
3234 | else | |
3235 | Tmp1 = ExpandExtractFromVectorThroughStack(SDValue(Node, 0)); | |
3236 | Results.push_back(Tmp1); | |
3237 | break; | |
3238 | case ISD::EXTRACT_SUBVECTOR: | |
3239 | Results.push_back(ExpandExtractFromVectorThroughStack(SDValue(Node, 0))); | |
3240 | break; | |
3241 | case ISD::INSERT_SUBVECTOR: | |
3242 | Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0))); | |
3243 | break; | |
3244 | case ISD::CONCAT_VECTORS: { | |
3245 | Results.push_back(ExpandVectorBuildThroughStack(Node)); | |
3246 | break; | |
3247 | } | |
3248 | case ISD::SCALAR_TO_VECTOR: | |
3249 | Results.push_back(ExpandSCALAR_TO_VECTOR(Node)); | |
3250 | break; | |
3251 | case ISD::INSERT_VECTOR_ELT: | |
3252 | Results.push_back(ExpandINSERT_VECTOR_ELT(Node->getOperand(0), | |
3253 | Node->getOperand(1), | |
3254 | Node->getOperand(2), dl)); | |
3255 | break; | |
3256 | case ISD::VECTOR_SHUFFLE: { | |
3257 | SmallVector<int, 32> NewMask; | |
3258 | ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask(); | |
3259 | ||
3260 | EVT VT = Node->getValueType(0); | |
3261 | EVT EltVT = VT.getVectorElementType(); | |
3262 | SDValue Op0 = Node->getOperand(0); | |
3263 | SDValue Op1 = Node->getOperand(1); | |
3264 | if (!TLI.isTypeLegal(EltVT)) { | |
3265 | ||
3266 | EVT NewEltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT); | |
3267 | ||
3268 | // BUILD_VECTOR operands are allowed to be wider than the element type. | |
1a4d82fc JJ |
3269 | // But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept |
3270 | // it. | |
223e47cc LB |
3271 | if (NewEltVT.bitsLT(EltVT)) { |
3272 | ||
3273 | // Convert shuffle node. | |
3274 | // If original node was v4i64 and the new EltVT is i32, | |
3275 | // cast operands to v8i32 and re-build the mask. | |
3276 | ||
3277 | // Calculate new VT, the size of the new VT should be equal to original. | |
1a4d82fc JJ |
3278 | EVT NewVT = |
3279 | EVT::getVectorVT(*DAG.getContext(), NewEltVT, | |
3280 | VT.getSizeInBits() / NewEltVT.getSizeInBits()); | |
223e47cc LB |
3281 | assert(NewVT.bitsEq(VT)); |
3282 | ||
3283 | // cast operands to new VT | |
3284 | Op0 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op0); | |
3285 | Op1 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op1); | |
3286 | ||
3287 | // Convert the shuffle mask | |
1a4d82fc JJ |
3288 | unsigned int factor = |
3289 | NewVT.getVectorNumElements()/VT.getVectorNumElements(); | |
223e47cc LB |
3290 | |
3291 | // EltVT gets smaller | |
3292 | assert(factor > 0); | |
3293 | ||
3294 | for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) { | |
3295 | if (Mask[i] < 0) { | |
3296 | for (unsigned fi = 0; fi < factor; ++fi) | |
3297 | NewMask.push_back(Mask[i]); | |
3298 | } | |
3299 | else { | |
3300 | for (unsigned fi = 0; fi < factor; ++fi) | |
3301 | NewMask.push_back(Mask[i]*factor+fi); | |
3302 | } | |
3303 | } | |
3304 | Mask = NewMask; | |
3305 | VT = NewVT; | |
3306 | } | |
3307 | EltVT = NewEltVT; | |
3308 | } | |
3309 | unsigned NumElems = VT.getVectorNumElements(); | |
3310 | SmallVector<SDValue, 16> Ops; | |
3311 | for (unsigned i = 0; i != NumElems; ++i) { | |
3312 | if (Mask[i] < 0) { | |
3313 | Ops.push_back(DAG.getUNDEF(EltVT)); | |
3314 | continue; | |
3315 | } | |
3316 | unsigned Idx = Mask[i]; | |
3317 | if (Idx < NumElems) | |
3318 | Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, | |
3319 | Op0, | |
1a4d82fc | 3320 | DAG.getConstant(Idx, TLI.getVectorIdxTy()))); |
223e47cc LB |
3321 | else |
3322 | Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, | |
3323 | Op1, | |
1a4d82fc JJ |
3324 | DAG.getConstant(Idx - NumElems, |
3325 | TLI.getVectorIdxTy()))); | |
223e47cc LB |
3326 | } |
3327 | ||
1a4d82fc | 3328 | Tmp1 = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops); |
223e47cc LB |
3329 | // We may have changed the BUILD_VECTOR type. Cast it back to the Node type. |
3330 | Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1); | |
3331 | Results.push_back(Tmp1); | |
3332 | break; | |
3333 | } | |
3334 | case ISD::EXTRACT_ELEMENT: { | |
3335 | EVT OpTy = Node->getOperand(0).getValueType(); | |
3336 | if (cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue()) { | |
3337 | // 1 -> Hi | |
3338 | Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0), | |
3339 | DAG.getConstant(OpTy.getSizeInBits()/2, | |
3340 | TLI.getShiftAmountTy(Node->getOperand(0).getValueType()))); | |
3341 | Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1); | |
3342 | } else { | |
3343 | // 0 -> Lo | |
3344 | Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), | |
3345 | Node->getOperand(0)); | |
3346 | } | |
3347 | Results.push_back(Tmp1); | |
3348 | break; | |
3349 | } | |
3350 | case ISD::STACKSAVE: | |
3351 | // Expand to CopyFromReg if the target set | |
3352 | // StackPointerRegisterToSaveRestore. | |
3353 | if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) { | |
3354 | Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, SP, | |
3355 | Node->getValueType(0))); | |
3356 | Results.push_back(Results[0].getValue(1)); | |
3357 | } else { | |
3358 | Results.push_back(DAG.getUNDEF(Node->getValueType(0))); | |
3359 | Results.push_back(Node->getOperand(0)); | |
3360 | } | |
3361 | break; | |
3362 | case ISD::STACKRESTORE: | |
3363 | // Expand to CopyToReg if the target set | |
3364 | // StackPointerRegisterToSaveRestore. | |
3365 | if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) { | |
3366 | Results.push_back(DAG.getCopyToReg(Node->getOperand(0), dl, SP, | |
3367 | Node->getOperand(1))); | |
3368 | } else { | |
3369 | Results.push_back(Node->getOperand(0)); | |
3370 | } | |
3371 | break; | |
3372 | case ISD::FCOPYSIGN: | |
3373 | Results.push_back(ExpandFCOPYSIGN(Node)); | |
3374 | break; | |
3375 | case ISD::FNEG: | |
3376 | // Expand Y = FNEG(X) -> Y = SUB -0.0, X | |
3377 | Tmp1 = DAG.getConstantFP(-0.0, Node->getValueType(0)); | |
3378 | Tmp1 = DAG.getNode(ISD::FSUB, dl, Node->getValueType(0), Tmp1, | |
3379 | Node->getOperand(0)); | |
3380 | Results.push_back(Tmp1); | |
3381 | break; | |
3382 | case ISD::FABS: { | |
3383 | // Expand Y = FABS(X) -> Y = (X >u 0.0) ? X : fneg(X). | |
3384 | EVT VT = Node->getValueType(0); | |
3385 | Tmp1 = Node->getOperand(0); | |
3386 | Tmp2 = DAG.getConstantFP(0.0, VT); | |
1a4d82fc | 3387 | Tmp2 = DAG.getSetCC(dl, getSetCCResultType(Tmp1.getValueType()), |
223e47cc LB |
3388 | Tmp1, Tmp2, ISD::SETUGT); |
3389 | Tmp3 = DAG.getNode(ISD::FNEG, dl, VT, Tmp1); | |
1a4d82fc | 3390 | Tmp1 = DAG.getSelect(dl, VT, Tmp2, Tmp1, Tmp3); |
223e47cc LB |
3391 | Results.push_back(Tmp1); |
3392 | break; | |
3393 | } | |
85aaf69f SL |
3394 | case ISD::FMINNUM: |
3395 | Results.push_back(ExpandFPLibCall(Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64, | |
3396 | RTLIB::FMIN_F80, RTLIB::FMIN_F128, | |
3397 | RTLIB::FMIN_PPCF128)); | |
3398 | break; | |
3399 | case ISD::FMAXNUM: | |
3400 | Results.push_back(ExpandFPLibCall(Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64, | |
3401 | RTLIB::FMAX_F80, RTLIB::FMAX_F128, | |
3402 | RTLIB::FMAX_PPCF128)); | |
3403 | break; | |
223e47cc LB |
3404 | case ISD::FSQRT: |
3405 | Results.push_back(ExpandFPLibCall(Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64, | |
970d7e83 LB |
3406 | RTLIB::SQRT_F80, RTLIB::SQRT_F128, |
3407 | RTLIB::SQRT_PPCF128)); | |
223e47cc LB |
3408 | break; |
3409 | case ISD::FSIN: | |
970d7e83 LB |
3410 | case ISD::FCOS: { |
3411 | EVT VT = Node->getValueType(0); | |
3412 | bool isSIN = Node->getOpcode() == ISD::FSIN; | |
3413 | // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin / | |
3414 | // fcos which share the same operand and both are used. | |
3415 | if ((TLI.isOperationLegalOrCustom(ISD::FSINCOS, VT) || | |
3416 | canCombineSinCosLibcall(Node, TLI, TM)) | |
3417 | && useSinCos(Node)) { | |
3418 | SDVTList VTs = DAG.getVTList(VT, VT); | |
3419 | Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0)); | |
3420 | if (!isSIN) | |
3421 | Tmp1 = Tmp1.getValue(1); | |
3422 | Results.push_back(Tmp1); | |
3423 | } else if (isSIN) { | |
3424 | Results.push_back(ExpandFPLibCall(Node, RTLIB::SIN_F32, RTLIB::SIN_F64, | |
3425 | RTLIB::SIN_F80, RTLIB::SIN_F128, | |
3426 | RTLIB::SIN_PPCF128)); | |
3427 | } else { | |
3428 | Results.push_back(ExpandFPLibCall(Node, RTLIB::COS_F32, RTLIB::COS_F64, | |
3429 | RTLIB::COS_F80, RTLIB::COS_F128, | |
3430 | RTLIB::COS_PPCF128)); | |
3431 | } | |
223e47cc | 3432 | break; |
970d7e83 LB |
3433 | } |
3434 | case ISD::FSINCOS: | |
3435 | // Expand into sincos libcall. | |
3436 | ExpandSinCosLibCall(Node, Results); | |
223e47cc LB |
3437 | break; |
3438 | case ISD::FLOG: | |
3439 | Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG_F32, RTLIB::LOG_F64, | |
970d7e83 LB |
3440 | RTLIB::LOG_F80, RTLIB::LOG_F128, |
3441 | RTLIB::LOG_PPCF128)); | |
223e47cc LB |
3442 | break; |
3443 | case ISD::FLOG2: | |
3444 | Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64, | |
970d7e83 LB |
3445 | RTLIB::LOG2_F80, RTLIB::LOG2_F128, |
3446 | RTLIB::LOG2_PPCF128)); | |
223e47cc LB |
3447 | break; |
3448 | case ISD::FLOG10: | |
3449 | Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64, | |
970d7e83 LB |
3450 | RTLIB::LOG10_F80, RTLIB::LOG10_F128, |
3451 | RTLIB::LOG10_PPCF128)); | |
223e47cc LB |
3452 | break; |
3453 | case ISD::FEXP: | |
3454 | Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP_F32, RTLIB::EXP_F64, | |
970d7e83 LB |
3455 | RTLIB::EXP_F80, RTLIB::EXP_F128, |
3456 | RTLIB::EXP_PPCF128)); | |
223e47cc LB |
3457 | break; |
3458 | case ISD::FEXP2: | |
3459 | Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64, | |
970d7e83 LB |
3460 | RTLIB::EXP2_F80, RTLIB::EXP2_F128, |
3461 | RTLIB::EXP2_PPCF128)); | |
223e47cc LB |
3462 | break; |
3463 | case ISD::FTRUNC: | |
3464 | Results.push_back(ExpandFPLibCall(Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64, | |
970d7e83 LB |
3465 | RTLIB::TRUNC_F80, RTLIB::TRUNC_F128, |
3466 | RTLIB::TRUNC_PPCF128)); | |
223e47cc LB |
3467 | break; |
3468 | case ISD::FFLOOR: | |
3469 | Results.push_back(ExpandFPLibCall(Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64, | |
970d7e83 LB |
3470 | RTLIB::FLOOR_F80, RTLIB::FLOOR_F128, |
3471 | RTLIB::FLOOR_PPCF128)); | |
223e47cc LB |
3472 | break; |
3473 | case ISD::FCEIL: | |
3474 | Results.push_back(ExpandFPLibCall(Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64, | |
970d7e83 LB |
3475 | RTLIB::CEIL_F80, RTLIB::CEIL_F128, |
3476 | RTLIB::CEIL_PPCF128)); | |
223e47cc LB |
3477 | break; |
3478 | case ISD::FRINT: | |
3479 | Results.push_back(ExpandFPLibCall(Node, RTLIB::RINT_F32, RTLIB::RINT_F64, | |
970d7e83 LB |
3480 | RTLIB::RINT_F80, RTLIB::RINT_F128, |
3481 | RTLIB::RINT_PPCF128)); | |
223e47cc LB |
3482 | break; |
3483 | case ISD::FNEARBYINT: | |
3484 | Results.push_back(ExpandFPLibCall(Node, RTLIB::NEARBYINT_F32, | |
3485 | RTLIB::NEARBYINT_F64, | |
3486 | RTLIB::NEARBYINT_F80, | |
970d7e83 | 3487 | RTLIB::NEARBYINT_F128, |
223e47cc LB |
3488 | RTLIB::NEARBYINT_PPCF128)); |
3489 | break; | |
1a4d82fc JJ |
3490 | case ISD::FROUND: |
3491 | Results.push_back(ExpandFPLibCall(Node, RTLIB::ROUND_F32, | |
3492 | RTLIB::ROUND_F64, | |
3493 | RTLIB::ROUND_F80, | |
3494 | RTLIB::ROUND_F128, | |
3495 | RTLIB::ROUND_PPCF128)); | |
3496 | break; | |
223e47cc LB |
3497 | case ISD::FPOWI: |
3498 | Results.push_back(ExpandFPLibCall(Node, RTLIB::POWI_F32, RTLIB::POWI_F64, | |
970d7e83 LB |
3499 | RTLIB::POWI_F80, RTLIB::POWI_F128, |
3500 | RTLIB::POWI_PPCF128)); | |
223e47cc LB |
3501 | break; |
3502 | case ISD::FPOW: | |
3503 | Results.push_back(ExpandFPLibCall(Node, RTLIB::POW_F32, RTLIB::POW_F64, | |
970d7e83 LB |
3504 | RTLIB::POW_F80, RTLIB::POW_F128, |
3505 | RTLIB::POW_PPCF128)); | |
223e47cc LB |
3506 | break; |
3507 | case ISD::FDIV: | |
3508 | Results.push_back(ExpandFPLibCall(Node, RTLIB::DIV_F32, RTLIB::DIV_F64, | |
970d7e83 LB |
3509 | RTLIB::DIV_F80, RTLIB::DIV_F128, |
3510 | RTLIB::DIV_PPCF128)); | |
223e47cc LB |
3511 | break; |
3512 | case ISD::FREM: | |
3513 | Results.push_back(ExpandFPLibCall(Node, RTLIB::REM_F32, RTLIB::REM_F64, | |
970d7e83 LB |
3514 | RTLIB::REM_F80, RTLIB::REM_F128, |
3515 | RTLIB::REM_PPCF128)); | |
223e47cc LB |
3516 | break; |
3517 | case ISD::FMA: | |
3518 | Results.push_back(ExpandFPLibCall(Node, RTLIB::FMA_F32, RTLIB::FMA_F64, | |
970d7e83 LB |
3519 | RTLIB::FMA_F80, RTLIB::FMA_F128, |
3520 | RTLIB::FMA_PPCF128)); | |
223e47cc | 3521 | break; |
1a4d82fc JJ |
3522 | case ISD::FADD: |
3523 | Results.push_back(ExpandFPLibCall(Node, RTLIB::ADD_F32, RTLIB::ADD_F64, | |
3524 | RTLIB::ADD_F80, RTLIB::ADD_F128, | |
3525 | RTLIB::ADD_PPCF128)); | |
3526 | break; | |
3527 | case ISD::FMUL: | |
3528 | Results.push_back(ExpandFPLibCall(Node, RTLIB::MUL_F32, RTLIB::MUL_F64, | |
3529 | RTLIB::MUL_F80, RTLIB::MUL_F128, | |
3530 | RTLIB::MUL_PPCF128)); | |
3531 | break; | |
3532 | case ISD::FP16_TO_FP: { | |
3533 | if (Node->getValueType(0) == MVT::f32) { | |
3534 | Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32, Node, false)); | |
3535 | break; | |
3536 | } | |
3537 | ||
3538 | // We can extend to types bigger than f32 in two steps without changing the | |
3539 | // result. Since "f16 -> f32" is much more commonly available, give CodeGen | |
3540 | // the option of emitting that before resorting to a libcall. | |
3541 | SDValue Res = | |
3542 | DAG.getNode(ISD::FP16_TO_FP, dl, MVT::f32, Node->getOperand(0)); | |
3543 | Results.push_back( | |
3544 | DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Res)); | |
223e47cc | 3545 | break; |
1a4d82fc JJ |
3546 | } |
3547 | case ISD::FP_TO_FP16: { | |
3548 | RTLIB::Libcall LC = | |
3549 | RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::f16); | |
3550 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16"); | |
3551 | Results.push_back(ExpandLibCall(LC, Node, false)); | |
223e47cc | 3552 | break; |
1a4d82fc | 3553 | } |
223e47cc LB |
3554 | case ISD::ConstantFP: { |
3555 | ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node); | |
3556 | // Check to see if this FP immediate is already legal. | |
3557 | // If this is a legal constant, turn it into a TargetConstantFP node. | |
3558 | if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(0))) | |
3559 | Results.push_back(ExpandConstantFP(CFP, true)); | |
3560 | break; | |
3561 | } | |
223e47cc LB |
3562 | case ISD::FSUB: { |
3563 | EVT VT = Node->getValueType(0); | |
1a4d82fc JJ |
3564 | if (TLI.isOperationLegalOrCustom(ISD::FADD, VT) && |
3565 | TLI.isOperationLegalOrCustom(ISD::FNEG, VT)) { | |
3566 | Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1)); | |
3567 | Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1); | |
3568 | Results.push_back(Tmp1); | |
3569 | } else { | |
3570 | Results.push_back(ExpandFPLibCall(Node, RTLIB::SUB_F32, RTLIB::SUB_F64, | |
3571 | RTLIB::SUB_F80, RTLIB::SUB_F128, | |
3572 | RTLIB::SUB_PPCF128)); | |
3573 | } | |
223e47cc LB |
3574 | break; |
3575 | } | |
3576 | case ISD::SUB: { | |
3577 | EVT VT = Node->getValueType(0); | |
3578 | assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) && | |
3579 | TLI.isOperationLegalOrCustom(ISD::XOR, VT) && | |
3580 | "Don't know how to expand this subtraction!"); | |
3581 | Tmp1 = DAG.getNode(ISD::XOR, dl, VT, Node->getOperand(1), | |
3582 | DAG.getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), VT)); | |
3583 | Tmp1 = DAG.getNode(ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, VT)); | |
3584 | Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1)); | |
3585 | break; | |
3586 | } | |
3587 | case ISD::UREM: | |
3588 | case ISD::SREM: { | |
3589 | EVT VT = Node->getValueType(0); | |
223e47cc LB |
3590 | bool isSigned = Node->getOpcode() == ISD::SREM; |
3591 | unsigned DivOpc = isSigned ? ISD::SDIV : ISD::UDIV; | |
3592 | unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM; | |
3593 | Tmp2 = Node->getOperand(0); | |
3594 | Tmp3 = Node->getOperand(1); | |
3595 | if (TLI.isOperationLegalOrCustom(DivRemOpc, VT) || | |
3596 | (isDivRemLibcallAvailable(Node, isSigned, TLI) && | |
970d7e83 LB |
3597 | // If div is legal, it's better to do the normal expansion |
3598 | !TLI.isOperationLegalOrCustom(DivOpc, Node->getValueType(0)) && | |
223e47cc | 3599 | useDivRem(Node, isSigned, false))) { |
970d7e83 | 3600 | SDVTList VTs = DAG.getVTList(VT, VT); |
223e47cc LB |
3601 | Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Tmp2, Tmp3).getValue(1); |
3602 | } else if (TLI.isOperationLegalOrCustom(DivOpc, VT)) { | |
3603 | // X % Y -> X-X/Y*Y | |
3604 | Tmp1 = DAG.getNode(DivOpc, dl, VT, Tmp2, Tmp3); | |
3605 | Tmp1 = DAG.getNode(ISD::MUL, dl, VT, Tmp1, Tmp3); | |
3606 | Tmp1 = DAG.getNode(ISD::SUB, dl, VT, Tmp2, Tmp1); | |
3607 | } else if (isSigned) | |
3608 | Tmp1 = ExpandIntLibCall(Node, true, | |
3609 | RTLIB::SREM_I8, | |
3610 | RTLIB::SREM_I16, RTLIB::SREM_I32, | |
3611 | RTLIB::SREM_I64, RTLIB::SREM_I128); | |
3612 | else | |
3613 | Tmp1 = ExpandIntLibCall(Node, false, | |
3614 | RTLIB::UREM_I8, | |
3615 | RTLIB::UREM_I16, RTLIB::UREM_I32, | |
3616 | RTLIB::UREM_I64, RTLIB::UREM_I128); | |
3617 | Results.push_back(Tmp1); | |
3618 | break; | |
3619 | } | |
3620 | case ISD::UDIV: | |
3621 | case ISD::SDIV: { | |
3622 | bool isSigned = Node->getOpcode() == ISD::SDIV; | |
3623 | unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM; | |
3624 | EVT VT = Node->getValueType(0); | |
3625 | SDVTList VTs = DAG.getVTList(VT, VT); | |
3626 | if (TLI.isOperationLegalOrCustom(DivRemOpc, VT) || | |
3627 | (isDivRemLibcallAvailable(Node, isSigned, TLI) && | |
3628 | useDivRem(Node, isSigned, true))) | |
3629 | Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Node->getOperand(0), | |
3630 | Node->getOperand(1)); | |
3631 | else if (isSigned) | |
3632 | Tmp1 = ExpandIntLibCall(Node, true, | |
3633 | RTLIB::SDIV_I8, | |
3634 | RTLIB::SDIV_I16, RTLIB::SDIV_I32, | |
3635 | RTLIB::SDIV_I64, RTLIB::SDIV_I128); | |
3636 | else | |
3637 | Tmp1 = ExpandIntLibCall(Node, false, | |
3638 | RTLIB::UDIV_I8, | |
3639 | RTLIB::UDIV_I16, RTLIB::UDIV_I32, | |
3640 | RTLIB::UDIV_I64, RTLIB::UDIV_I128); | |
3641 | Results.push_back(Tmp1); | |
3642 | break; | |
3643 | } | |
3644 | case ISD::MULHU: | |
3645 | case ISD::MULHS: { | |
3646 | unsigned ExpandOpcode = Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : | |
3647 | ISD::SMUL_LOHI; | |
3648 | EVT VT = Node->getValueType(0); | |
3649 | SDVTList VTs = DAG.getVTList(VT, VT); | |
3650 | assert(TLI.isOperationLegalOrCustom(ExpandOpcode, VT) && | |
3651 | "If this wasn't legal, it shouldn't have been created!"); | |
3652 | Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0), | |
3653 | Node->getOperand(1)); | |
3654 | Results.push_back(Tmp1.getValue(1)); | |
3655 | break; | |
3656 | } | |
3657 | case ISD::SDIVREM: | |
3658 | case ISD::UDIVREM: | |
3659 | // Expand into divrem libcall | |
3660 | ExpandDivRemLibCall(Node, Results); | |
3661 | break; | |
3662 | case ISD::MUL: { | |
3663 | EVT VT = Node->getValueType(0); | |
3664 | SDVTList VTs = DAG.getVTList(VT, VT); | |
3665 | // See if multiply or divide can be lowered using two-result operations. | |
3666 | // We just need the low half of the multiply; try both the signed | |
3667 | // and unsigned forms. If the target supports both SMUL_LOHI and | |
3668 | // UMUL_LOHI, form a preference by checking which forms of plain | |
3669 | // MULH it supports. | |
3670 | bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT); | |
3671 | bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT); | |
3672 | bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, VT); | |
3673 | bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, VT); | |
3674 | unsigned OpToUse = 0; | |
3675 | if (HasSMUL_LOHI && !HasMULHS) { | |
3676 | OpToUse = ISD::SMUL_LOHI; | |
3677 | } else if (HasUMUL_LOHI && !HasMULHU) { | |
3678 | OpToUse = ISD::UMUL_LOHI; | |
3679 | } else if (HasSMUL_LOHI) { | |
3680 | OpToUse = ISD::SMUL_LOHI; | |
3681 | } else if (HasUMUL_LOHI) { | |
3682 | OpToUse = ISD::UMUL_LOHI; | |
3683 | } | |
3684 | if (OpToUse) { | |
3685 | Results.push_back(DAG.getNode(OpToUse, dl, VTs, Node->getOperand(0), | |
3686 | Node->getOperand(1))); | |
3687 | break; | |
3688 | } | |
1a4d82fc JJ |
3689 | |
3690 | SDValue Lo, Hi; | |
3691 | EVT HalfType = VT.getHalfSizedIntegerVT(*DAG.getContext()); | |
3692 | if (TLI.isOperationLegalOrCustom(ISD::ZERO_EXTEND, VT) && | |
3693 | TLI.isOperationLegalOrCustom(ISD::ANY_EXTEND, VT) && | |
3694 | TLI.isOperationLegalOrCustom(ISD::SHL, VT) && | |
3695 | TLI.isOperationLegalOrCustom(ISD::OR, VT) && | |
3696 | TLI.expandMUL(Node, Lo, Hi, HalfType, DAG)) { | |
3697 | Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo); | |
3698 | Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Hi); | |
3699 | SDValue Shift = DAG.getConstant(HalfType.getSizeInBits(), | |
3700 | TLI.getShiftAmountTy(HalfType)); | |
3701 | Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift); | |
3702 | Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi)); | |
3703 | break; | |
3704 | } | |
3705 | ||
223e47cc LB |
3706 | Tmp1 = ExpandIntLibCall(Node, false, |
3707 | RTLIB::MUL_I8, | |
3708 | RTLIB::MUL_I16, RTLIB::MUL_I32, | |
3709 | RTLIB::MUL_I64, RTLIB::MUL_I128); | |
3710 | Results.push_back(Tmp1); | |
3711 | break; | |
3712 | } | |
3713 | case ISD::SADDO: | |
3714 | case ISD::SSUBO: { | |
3715 | SDValue LHS = Node->getOperand(0); | |
3716 | SDValue RHS = Node->getOperand(1); | |
3717 | SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ? | |
3718 | ISD::ADD : ISD::SUB, dl, LHS.getValueType(), | |
3719 | LHS, RHS); | |
3720 | Results.push_back(Sum); | |
1a4d82fc JJ |
3721 | EVT ResultType = Node->getValueType(1); |
3722 | EVT OType = getSetCCResultType(Node->getValueType(0)); | |
223e47cc LB |
3723 | |
3724 | SDValue Zero = DAG.getConstant(0, LHS.getValueType()); | |
3725 | ||
3726 | // LHSSign -> LHS >= 0 | |
3727 | // RHSSign -> RHS >= 0 | |
3728 | // SumSign -> Sum >= 0 | |
3729 | // | |
3730 | // Add: | |
3731 | // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign) | |
3732 | // Sub: | |
3733 | // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign) | |
3734 | // | |
3735 | SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE); | |
3736 | SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE); | |
3737 | SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign, | |
3738 | Node->getOpcode() == ISD::SADDO ? | |
3739 | ISD::SETEQ : ISD::SETNE); | |
3740 | ||
3741 | SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE); | |
3742 | SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE); | |
3743 | ||
3744 | SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE); | |
1a4d82fc | 3745 | Results.push_back(DAG.getBoolExtOrTrunc(Cmp, dl, ResultType, ResultType)); |
223e47cc LB |
3746 | break; |
3747 | } | |
3748 | case ISD::UADDO: | |
3749 | case ISD::USUBO: { | |
3750 | SDValue LHS = Node->getOperand(0); | |
3751 | SDValue RHS = Node->getOperand(1); | |
3752 | SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::UADDO ? | |
3753 | ISD::ADD : ISD::SUB, dl, LHS.getValueType(), | |
3754 | LHS, RHS); | |
3755 | Results.push_back(Sum); | |
1a4d82fc JJ |
3756 | |
3757 | EVT ResultType = Node->getValueType(1); | |
3758 | EVT SetCCType = getSetCCResultType(Node->getValueType(0)); | |
3759 | ISD::CondCode CC | |
3760 | = Node->getOpcode() == ISD::UADDO ? ISD::SETULT : ISD::SETUGT; | |
3761 | SDValue SetCC = DAG.getSetCC(dl, SetCCType, Sum, LHS, CC); | |
3762 | ||
3763 | Results.push_back(DAG.getBoolExtOrTrunc(SetCC, dl, ResultType, ResultType)); | |
223e47cc LB |
3764 | break; |
3765 | } | |
3766 | case ISD::UMULO: | |
3767 | case ISD::SMULO: { | |
3768 | EVT VT = Node->getValueType(0); | |
3769 | EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits() * 2); | |
3770 | SDValue LHS = Node->getOperand(0); | |
3771 | SDValue RHS = Node->getOperand(1); | |
3772 | SDValue BottomHalf; | |
3773 | SDValue TopHalf; | |
3774 | static const unsigned Ops[2][3] = | |
3775 | { { ISD::MULHU, ISD::UMUL_LOHI, ISD::ZERO_EXTEND }, | |
3776 | { ISD::MULHS, ISD::SMUL_LOHI, ISD::SIGN_EXTEND }}; | |
3777 | bool isSigned = Node->getOpcode() == ISD::SMULO; | |
3778 | if (TLI.isOperationLegalOrCustom(Ops[isSigned][0], VT)) { | |
3779 | BottomHalf = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS); | |
3780 | TopHalf = DAG.getNode(Ops[isSigned][0], dl, VT, LHS, RHS); | |
3781 | } else if (TLI.isOperationLegalOrCustom(Ops[isSigned][1], VT)) { | |
3782 | BottomHalf = DAG.getNode(Ops[isSigned][1], dl, DAG.getVTList(VT, VT), LHS, | |
3783 | RHS); | |
3784 | TopHalf = BottomHalf.getValue(1); | |
1a4d82fc | 3785 | } else if (TLI.isTypeLegal(WideVT)) { |
223e47cc LB |
3786 | LHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, LHS); |
3787 | RHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, RHS); | |
3788 | Tmp1 = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS); | |
3789 | BottomHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1, | |
3790 | DAG.getIntPtrConstant(0)); | |
3791 | TopHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1, | |
3792 | DAG.getIntPtrConstant(1)); | |
3793 | } else { | |
3794 | // We can fall back to a libcall with an illegal type for the MUL if we | |
3795 | // have a libcall big enough. | |
3796 | // Also, we can fall back to a division in some cases, but that's a big | |
3797 | // performance hit in the general case. | |
3798 | RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; | |
3799 | if (WideVT == MVT::i16) | |
3800 | LC = RTLIB::MUL_I16; | |
3801 | else if (WideVT == MVT::i32) | |
3802 | LC = RTLIB::MUL_I32; | |
3803 | else if (WideVT == MVT::i64) | |
3804 | LC = RTLIB::MUL_I64; | |
3805 | else if (WideVT == MVT::i128) | |
3806 | LC = RTLIB::MUL_I128; | |
3807 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Cannot expand this operation!"); | |
3808 | ||
3809 | // The high part is obtained by SRA'ing all but one of the bits of low | |
3810 | // part. | |
3811 | unsigned LoSize = VT.getSizeInBits(); | |
3812 | SDValue HiLHS = DAG.getNode(ISD::SRA, dl, VT, RHS, | |
3813 | DAG.getConstant(LoSize-1, TLI.getPointerTy())); | |
3814 | SDValue HiRHS = DAG.getNode(ISD::SRA, dl, VT, LHS, | |
3815 | DAG.getConstant(LoSize-1, TLI.getPointerTy())); | |
3816 | ||
3817 | // Here we're passing the 2 arguments explicitly as 4 arguments that are | |
3818 | // pre-lowered to the correct types. This all depends upon WideVT not | |
3819 | // being a legal type for the architecture and thus has to be split to | |
3820 | // two arguments. | |
3821 | SDValue Args[] = { LHS, HiLHS, RHS, HiRHS }; | |
3822 | SDValue Ret = ExpandLibCall(LC, WideVT, Args, 4, isSigned, dl); | |
3823 | BottomHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Ret, | |
3824 | DAG.getIntPtrConstant(0)); | |
3825 | TopHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Ret, | |
3826 | DAG.getIntPtrConstant(1)); | |
3827 | // Ret is a node with an illegal type. Because such things are not | |
1a4d82fc JJ |
3828 | // generally permitted during this phase of legalization, make sure the |
3829 | // node has no more uses. The above EXTRACT_ELEMENT nodes should have been | |
3830 | // folded. | |
3831 | assert(Ret->use_empty() && | |
3832 | "Unexpected uses of illegally type from expanded lib call."); | |
223e47cc LB |
3833 | } |
3834 | ||
3835 | if (isSigned) { | |
3836 | Tmp1 = DAG.getConstant(VT.getSizeInBits() - 1, | |
3837 | TLI.getShiftAmountTy(BottomHalf.getValueType())); | |
3838 | Tmp1 = DAG.getNode(ISD::SRA, dl, VT, BottomHalf, Tmp1); | |
1a4d82fc | 3839 | TopHalf = DAG.getSetCC(dl, getSetCCResultType(VT), TopHalf, Tmp1, |
223e47cc LB |
3840 | ISD::SETNE); |
3841 | } else { | |
1a4d82fc | 3842 | TopHalf = DAG.getSetCC(dl, getSetCCResultType(VT), TopHalf, |
223e47cc LB |
3843 | DAG.getConstant(0, VT), ISD::SETNE); |
3844 | } | |
3845 | Results.push_back(BottomHalf); | |
3846 | Results.push_back(TopHalf); | |
3847 | break; | |
3848 | } | |
3849 | case ISD::BUILD_PAIR: { | |
3850 | EVT PairTy = Node->getValueType(0); | |
3851 | Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0)); | |
3852 | Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1)); | |
3853 | Tmp2 = DAG.getNode(ISD::SHL, dl, PairTy, Tmp2, | |
3854 | DAG.getConstant(PairTy.getSizeInBits()/2, | |
3855 | TLI.getShiftAmountTy(PairTy))); | |
3856 | Results.push_back(DAG.getNode(ISD::OR, dl, PairTy, Tmp1, Tmp2)); | |
3857 | break; | |
3858 | } | |
3859 | case ISD::SELECT: | |
3860 | Tmp1 = Node->getOperand(0); | |
3861 | Tmp2 = Node->getOperand(1); | |
3862 | Tmp3 = Node->getOperand(2); | |
3863 | if (Tmp1.getOpcode() == ISD::SETCC) { | |
3864 | Tmp1 = DAG.getSelectCC(dl, Tmp1.getOperand(0), Tmp1.getOperand(1), | |
3865 | Tmp2, Tmp3, | |
3866 | cast<CondCodeSDNode>(Tmp1.getOperand(2))->get()); | |
3867 | } else { | |
3868 | Tmp1 = DAG.getSelectCC(dl, Tmp1, | |
3869 | DAG.getConstant(0, Tmp1.getValueType()), | |
3870 | Tmp2, Tmp3, ISD::SETNE); | |
3871 | } | |
3872 | Results.push_back(Tmp1); | |
3873 | break; | |
3874 | case ISD::BR_JT: { | |
3875 | SDValue Chain = Node->getOperand(0); | |
3876 | SDValue Table = Node->getOperand(1); | |
3877 | SDValue Index = Node->getOperand(2); | |
3878 | ||
3879 | EVT PTy = TLI.getPointerTy(); | |
3880 | ||
970d7e83 | 3881 | const DataLayout &TD = *TLI.getDataLayout(); |
223e47cc LB |
3882 | unsigned EntrySize = |
3883 | DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD); | |
3884 | ||
1a4d82fc JJ |
3885 | Index = DAG.getNode(ISD::MUL, dl, Index.getValueType(), |
3886 | Index, DAG.getConstant(EntrySize, Index.getValueType())); | |
3887 | SDValue Addr = DAG.getNode(ISD::ADD, dl, Index.getValueType(), | |
3888 | Index, Table); | |
223e47cc LB |
3889 | |
3890 | EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), EntrySize * 8); | |
3891 | SDValue LD = DAG.getExtLoad(ISD::SEXTLOAD, dl, PTy, Chain, Addr, | |
3892 | MachinePointerInfo::getJumpTable(), MemVT, | |
1a4d82fc | 3893 | false, false, false, 0); |
223e47cc LB |
3894 | Addr = LD; |
3895 | if (TM.getRelocationModel() == Reloc::PIC_) { | |
3896 | // For PIC, the sequence is: | |
3897 | // BRIND(load(Jumptable + index) + RelocBase) | |
3898 | // RelocBase can be JumpTable, GOT or some sort of global base. | |
3899 | Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr, | |
3900 | TLI.getPICJumpTableRelocBase(Table, DAG)); | |
3901 | } | |
3902 | Tmp1 = DAG.getNode(ISD::BRIND, dl, MVT::Other, LD.getValue(1), Addr); | |
3903 | Results.push_back(Tmp1); | |
3904 | break; | |
3905 | } | |
3906 | case ISD::BRCOND: | |
3907 | // Expand brcond's setcc into its constituent parts and create a BR_CC | |
3908 | // Node. | |
3909 | Tmp1 = Node->getOperand(0); | |
3910 | Tmp2 = Node->getOperand(1); | |
3911 | if (Tmp2.getOpcode() == ISD::SETCC) { | |
3912 | Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, | |
3913 | Tmp1, Tmp2.getOperand(2), | |
3914 | Tmp2.getOperand(0), Tmp2.getOperand(1), | |
3915 | Node->getOperand(2)); | |
3916 | } else { | |
3917 | // We test only the i1 bit. Skip the AND if UNDEF. | |
3918 | Tmp3 = (Tmp2.getOpcode() == ISD::UNDEF) ? Tmp2 : | |
3919 | DAG.getNode(ISD::AND, dl, Tmp2.getValueType(), Tmp2, | |
3920 | DAG.getConstant(1, Tmp2.getValueType())); | |
3921 | Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1, | |
3922 | DAG.getCondCode(ISD::SETNE), Tmp3, | |
3923 | DAG.getConstant(0, Tmp3.getValueType()), | |
3924 | Node->getOperand(2)); | |
3925 | } | |
3926 | Results.push_back(Tmp1); | |
3927 | break; | |
3928 | case ISD::SETCC: { | |
3929 | Tmp1 = Node->getOperand(0); | |
3930 | Tmp2 = Node->getOperand(1); | |
3931 | Tmp3 = Node->getOperand(2); | |
1a4d82fc JJ |
3932 | bool Legalized = LegalizeSetCCCondCode(Node->getValueType(0), Tmp1, Tmp2, |
3933 | Tmp3, NeedInvert, dl); | |
3934 | ||
3935 | if (Legalized) { | |
3936 | // If we expanded the SETCC by swapping LHS and RHS, or by inverting the | |
3937 | // condition code, create a new SETCC node. | |
3938 | if (Tmp3.getNode()) | |
3939 | Tmp1 = DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), | |
3940 | Tmp1, Tmp2, Tmp3); | |
3941 | ||
3942 | // If we expanded the SETCC by inverting the condition code, then wrap | |
3943 | // the existing SETCC in a NOT to restore the intended condition. | |
3944 | if (NeedInvert) | |
3945 | Tmp1 = DAG.getLogicalNOT(dl, Tmp1, Tmp1->getValueType(0)); | |
223e47cc | 3946 | |
223e47cc LB |
3947 | Results.push_back(Tmp1); |
3948 | break; | |
3949 | } | |
3950 | ||
3951 | // Otherwise, SETCC for the given comparison type must be completely | |
3952 | // illegal; expand it into a SELECT_CC. | |
3953 | EVT VT = Node->getValueType(0); | |
970d7e83 | 3954 | int TrueValue; |
1a4d82fc | 3955 | switch (TLI.getBooleanContents(Tmp1->getValueType(0))) { |
970d7e83 LB |
3956 | case TargetLowering::ZeroOrOneBooleanContent: |
3957 | case TargetLowering::UndefinedBooleanContent: | |
3958 | TrueValue = 1; | |
3959 | break; | |
3960 | case TargetLowering::ZeroOrNegativeOneBooleanContent: | |
3961 | TrueValue = -1; | |
3962 | break; | |
3963 | } | |
223e47cc | 3964 | Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2, |
970d7e83 LB |
3965 | DAG.getConstant(TrueValue, VT), DAG.getConstant(0, VT), |
3966 | Tmp3); | |
223e47cc LB |
3967 | Results.push_back(Tmp1); |
3968 | break; | |
3969 | } | |
3970 | case ISD::SELECT_CC: { | |
3971 | Tmp1 = Node->getOperand(0); // LHS | |
3972 | Tmp2 = Node->getOperand(1); // RHS | |
3973 | Tmp3 = Node->getOperand(2); // True | |
3974 | Tmp4 = Node->getOperand(3); // False | |
1a4d82fc | 3975 | EVT VT = Node->getValueType(0); |
223e47cc | 3976 | SDValue CC = Node->getOperand(4); |
1a4d82fc JJ |
3977 | ISD::CondCode CCOp = cast<CondCodeSDNode>(CC)->get(); |
3978 | ||
3979 | if (TLI.isCondCodeLegal(CCOp, Tmp1.getSimpleValueType())) { | |
3980 | // If the condition code is legal, then we need to expand this | |
3981 | // node using SETCC and SELECT. | |
3982 | EVT CmpVT = Tmp1.getValueType(); | |
3983 | assert(!TLI.isOperationExpand(ISD::SELECT, VT) && | |
3984 | "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be " | |
3985 | "expanded."); | |
3986 | EVT CCVT = TLI.getSetCCResultType(*DAG.getContext(), CmpVT); | |
3987 | SDValue Cond = DAG.getNode(ISD::SETCC, dl, CCVT, Tmp1, Tmp2, CC); | |
3988 | Results.push_back(DAG.getSelect(dl, VT, Cond, Tmp3, Tmp4)); | |
3989 | break; | |
3990 | } | |
3991 | ||
3992 | // SELECT_CC is legal, so the condition code must not be. | |
3993 | bool Legalized = false; | |
3994 | // Try to legalize by inverting the condition. This is for targets that | |
3995 | // might support an ordered version of a condition, but not the unordered | |
3996 | // version (or vice versa). | |
3997 | ISD::CondCode InvCC = ISD::getSetCCInverse(CCOp, | |
3998 | Tmp1.getValueType().isInteger()); | |
3999 | if (TLI.isCondCodeLegal(InvCC, Tmp1.getSimpleValueType())) { | |
4000 | // Use the new condition code and swap true and false | |
4001 | Legalized = true; | |
4002 | Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC); | |
4003 | } else { | |
4004 | // If The inverse is not legal, then try to swap the arguments using | |
4005 | // the inverse condition code. | |
4006 | ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(InvCC); | |
4007 | if (TLI.isCondCodeLegal(SwapInvCC, Tmp1.getSimpleValueType())) { | |
4008 | // The swapped inverse condition is legal, so swap true and false, | |
4009 | // lhs and rhs. | |
4010 | Legalized = true; | |
4011 | Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC); | |
4012 | } | |
4013 | } | |
4014 | ||
4015 | if (!Legalized) { | |
4016 | Legalized = LegalizeSetCCCondCode( | |
4017 | getSetCCResultType(Tmp1.getValueType()), Tmp1, Tmp2, CC, NeedInvert, | |
4018 | dl); | |
223e47cc | 4019 | |
1a4d82fc | 4020 | assert(Legalized && "Can't legalize SELECT_CC with legal condition!"); |
223e47cc | 4021 | |
1a4d82fc JJ |
4022 | // If we expanded the SETCC by inverting the condition code, then swap |
4023 | // the True/False operands to match. | |
4024 | if (NeedInvert) | |
4025 | std::swap(Tmp3, Tmp4); | |
4026 | ||
4027 | // If we expanded the SETCC by swapping LHS and RHS, or by inverting the | |
4028 | // condition code, create a new SELECT_CC node. | |
4029 | if (CC.getNode()) { | |
4030 | Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), | |
4031 | Tmp1, Tmp2, Tmp3, Tmp4, CC); | |
4032 | } else { | |
4033 | Tmp2 = DAG.getConstant(0, Tmp1.getValueType()); | |
4034 | CC = DAG.getCondCode(ISD::SETNE); | |
4035 | Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1, | |
4036 | Tmp2, Tmp3, Tmp4, CC); | |
4037 | } | |
4038 | } | |
223e47cc LB |
4039 | Results.push_back(Tmp1); |
4040 | break; | |
4041 | } | |
4042 | case ISD::BR_CC: { | |
4043 | Tmp1 = Node->getOperand(0); // Chain | |
4044 | Tmp2 = Node->getOperand(2); // LHS | |
4045 | Tmp3 = Node->getOperand(3); // RHS | |
4046 | Tmp4 = Node->getOperand(1); // CC | |
4047 | ||
1a4d82fc JJ |
4048 | bool Legalized = LegalizeSetCCCondCode(getSetCCResultType( |
4049 | Tmp2.getValueType()), Tmp2, Tmp3, Tmp4, NeedInvert, dl); | |
4050 | (void)Legalized; | |
4051 | assert(Legalized && "Can't legalize BR_CC with legal condition!"); | |
4052 | ||
4053 | // If we expanded the SETCC by inverting the condition code, then wrap | |
4054 | // the existing SETCC in a NOT to restore the intended condition. | |
4055 | if (NeedInvert) | |
4056 | Tmp4 = DAG.getNOT(dl, Tmp4, Tmp4->getValueType(0)); | |
4057 | ||
4058 | // If we expanded the SETCC by swapping LHS and RHS, create a new BR_CC | |
4059 | // node. | |
4060 | if (Tmp4.getNode()) { | |
4061 | Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, | |
4062 | Tmp4, Tmp2, Tmp3, Node->getOperand(4)); | |
4063 | } else { | |
4064 | Tmp3 = DAG.getConstant(0, Tmp2.getValueType()); | |
4065 | Tmp4 = DAG.getCondCode(ISD::SETNE); | |
4066 | Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4, | |
4067 | Tmp2, Tmp3, Node->getOperand(4)); | |
4068 | } | |
223e47cc LB |
4069 | Results.push_back(Tmp1); |
4070 | break; | |
4071 | } | |
4072 | case ISD::BUILD_VECTOR: | |
4073 | Results.push_back(ExpandBUILD_VECTOR(Node)); | |
4074 | break; | |
4075 | case ISD::SRA: | |
4076 | case ISD::SRL: | |
4077 | case ISD::SHL: { | |
4078 | // Scalarize vector SRA/SRL/SHL. | |
4079 | EVT VT = Node->getValueType(0); | |
4080 | assert(VT.isVector() && "Unable to legalize non-vector shift"); | |
4081 | assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal"); | |
4082 | unsigned NumElem = VT.getVectorNumElements(); | |
4083 | ||
4084 | SmallVector<SDValue, 8> Scalars; | |
4085 | for (unsigned Idx = 0; Idx < NumElem; Idx++) { | |
4086 | SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, | |
4087 | VT.getScalarType(), | |
1a4d82fc JJ |
4088 | Node->getOperand(0), DAG.getConstant(Idx, |
4089 | TLI.getVectorIdxTy())); | |
223e47cc LB |
4090 | SDValue Sh = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, |
4091 | VT.getScalarType(), | |
1a4d82fc JJ |
4092 | Node->getOperand(1), DAG.getConstant(Idx, |
4093 | TLI.getVectorIdxTy())); | |
223e47cc LB |
4094 | Scalars.push_back(DAG.getNode(Node->getOpcode(), dl, |
4095 | VT.getScalarType(), Ex, Sh)); | |
4096 | } | |
4097 | SDValue Result = | |
1a4d82fc | 4098 | DAG.getNode(ISD::BUILD_VECTOR, dl, Node->getValueType(0), Scalars); |
223e47cc LB |
4099 | ReplaceNode(SDValue(Node, 0), Result); |
4100 | break; | |
4101 | } | |
4102 | case ISD::GLOBAL_OFFSET_TABLE: | |
4103 | case ISD::GlobalAddress: | |
4104 | case ISD::GlobalTLSAddress: | |
4105 | case ISD::ExternalSymbol: | |
4106 | case ISD::ConstantPool: | |
4107 | case ISD::JumpTable: | |
4108 | case ISD::INTRINSIC_W_CHAIN: | |
4109 | case ISD::INTRINSIC_WO_CHAIN: | |
4110 | case ISD::INTRINSIC_VOID: | |
4111 | // FIXME: Custom lowering for these operations shouldn't return null! | |
4112 | break; | |
4113 | } | |
4114 | ||
4115 | // Replace the original node with the legalized result. | |
4116 | if (!Results.empty()) | |
4117 | ReplaceNode(Node, Results.data()); | |
4118 | } | |
4119 | ||
4120 | void SelectionDAGLegalize::PromoteNode(SDNode *Node) { | |
4121 | SmallVector<SDValue, 8> Results; | |
970d7e83 | 4122 | MVT OVT = Node->getSimpleValueType(0); |
223e47cc LB |
4123 | if (Node->getOpcode() == ISD::UINT_TO_FP || |
4124 | Node->getOpcode() == ISD::SINT_TO_FP || | |
4125 | Node->getOpcode() == ISD::SETCC) { | |
970d7e83 | 4126 | OVT = Node->getOperand(0).getSimpleValueType(); |
223e47cc | 4127 | } |
970d7e83 | 4128 | MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT); |
1a4d82fc | 4129 | SDLoc dl(Node); |
223e47cc LB |
4130 | SDValue Tmp1, Tmp2, Tmp3; |
4131 | switch (Node->getOpcode()) { | |
4132 | case ISD::CTTZ: | |
4133 | case ISD::CTTZ_ZERO_UNDEF: | |
4134 | case ISD::CTLZ: | |
4135 | case ISD::CTLZ_ZERO_UNDEF: | |
4136 | case ISD::CTPOP: | |
4137 | // Zero extend the argument. | |
4138 | Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0)); | |
4139 | // Perform the larger operation. For CTPOP and CTTZ_ZERO_UNDEF, this is | |
4140 | // already the correct result. | |
4141 | Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); | |
4142 | if (Node->getOpcode() == ISD::CTTZ) { | |
4143 | // FIXME: This should set a bit in the zero extended value instead. | |
1a4d82fc | 4144 | Tmp2 = DAG.getSetCC(dl, getSetCCResultType(NVT), |
223e47cc LB |
4145 | Tmp1, DAG.getConstant(NVT.getSizeInBits(), NVT), |
4146 | ISD::SETEQ); | |
1a4d82fc JJ |
4147 | Tmp1 = DAG.getSelect(dl, NVT, Tmp2, |
4148 | DAG.getConstant(OVT.getSizeInBits(), NVT), Tmp1); | |
223e47cc LB |
4149 | } else if (Node->getOpcode() == ISD::CTLZ || |
4150 | Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF) { | |
4151 | // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT)) | |
4152 | Tmp1 = DAG.getNode(ISD::SUB, dl, NVT, Tmp1, | |
4153 | DAG.getConstant(NVT.getSizeInBits() - | |
4154 | OVT.getSizeInBits(), NVT)); | |
4155 | } | |
4156 | Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1)); | |
4157 | break; | |
4158 | case ISD::BSWAP: { | |
4159 | unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits(); | |
4160 | Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0)); | |
4161 | Tmp1 = DAG.getNode(ISD::BSWAP, dl, NVT, Tmp1); | |
4162 | Tmp1 = DAG.getNode(ISD::SRL, dl, NVT, Tmp1, | |
4163 | DAG.getConstant(DiffBits, TLI.getShiftAmountTy(NVT))); | |
4164 | Results.push_back(Tmp1); | |
4165 | break; | |
4166 | } | |
4167 | case ISD::FP_TO_UINT: | |
4168 | case ISD::FP_TO_SINT: | |
4169 | Tmp1 = PromoteLegalFP_TO_INT(Node->getOperand(0), Node->getValueType(0), | |
4170 | Node->getOpcode() == ISD::FP_TO_SINT, dl); | |
4171 | Results.push_back(Tmp1); | |
4172 | break; | |
4173 | case ISD::UINT_TO_FP: | |
4174 | case ISD::SINT_TO_FP: | |
4175 | Tmp1 = PromoteLegalINT_TO_FP(Node->getOperand(0), Node->getValueType(0), | |
4176 | Node->getOpcode() == ISD::SINT_TO_FP, dl); | |
4177 | Results.push_back(Tmp1); | |
4178 | break; | |
4179 | case ISD::VAARG: { | |
4180 | SDValue Chain = Node->getOperand(0); // Get the chain. | |
4181 | SDValue Ptr = Node->getOperand(1); // Get the pointer. | |
4182 | ||
4183 | unsigned TruncOp; | |
4184 | if (OVT.isVector()) { | |
4185 | TruncOp = ISD::BITCAST; | |
4186 | } else { | |
4187 | assert(OVT.isInteger() | |
4188 | && "VAARG promotion is supported only for vectors or integer types"); | |
4189 | TruncOp = ISD::TRUNCATE; | |
4190 | } | |
4191 | ||
4192 | // Perform the larger operation, then convert back | |
4193 | Tmp1 = DAG.getVAArg(NVT, dl, Chain, Ptr, Node->getOperand(2), | |
4194 | Node->getConstantOperandVal(3)); | |
4195 | Chain = Tmp1.getValue(1); | |
4196 | ||
4197 | Tmp2 = DAG.getNode(TruncOp, dl, OVT, Tmp1); | |
4198 | ||
4199 | // Modified the chain result - switch anything that used the old chain to | |
4200 | // use the new one. | |
4201 | DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Tmp2); | |
4202 | DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain); | |
1a4d82fc JJ |
4203 | if (UpdatedNodes) { |
4204 | UpdatedNodes->insert(Tmp2.getNode()); | |
4205 | UpdatedNodes->insert(Chain.getNode()); | |
4206 | } | |
223e47cc LB |
4207 | ReplacedNode(Node); |
4208 | break; | |
4209 | } | |
4210 | case ISD::AND: | |
4211 | case ISD::OR: | |
4212 | case ISD::XOR: { | |
4213 | unsigned ExtOp, TruncOp; | |
4214 | if (OVT.isVector()) { | |
4215 | ExtOp = ISD::BITCAST; | |
4216 | TruncOp = ISD::BITCAST; | |
4217 | } else { | |
4218 | assert(OVT.isInteger() && "Cannot promote logic operation"); | |
4219 | ExtOp = ISD::ANY_EXTEND; | |
4220 | TruncOp = ISD::TRUNCATE; | |
4221 | } | |
4222 | // Promote each of the values to the new type. | |
4223 | Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); | |
4224 | Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); | |
4225 | // Perform the larger operation, then convert back | |
4226 | Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); | |
4227 | Results.push_back(DAG.getNode(TruncOp, dl, OVT, Tmp1)); | |
4228 | break; | |
4229 | } | |
4230 | case ISD::SELECT: { | |
4231 | unsigned ExtOp, TruncOp; | |
1a4d82fc JJ |
4232 | if (Node->getValueType(0).isVector() || |
4233 | Node->getValueType(0).getSizeInBits() == NVT.getSizeInBits()) { | |
223e47cc LB |
4234 | ExtOp = ISD::BITCAST; |
4235 | TruncOp = ISD::BITCAST; | |
4236 | } else if (Node->getValueType(0).isInteger()) { | |
4237 | ExtOp = ISD::ANY_EXTEND; | |
4238 | TruncOp = ISD::TRUNCATE; | |
4239 | } else { | |
4240 | ExtOp = ISD::FP_EXTEND; | |
4241 | TruncOp = ISD::FP_ROUND; | |
4242 | } | |
4243 | Tmp1 = Node->getOperand(0); | |
4244 | // Promote each of the values to the new type. | |
4245 | Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); | |
4246 | Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2)); | |
4247 | // Perform the larger operation, then round down. | |
1a4d82fc | 4248 | Tmp1 = DAG.getSelect(dl, NVT, Tmp1, Tmp2, Tmp3); |
223e47cc LB |
4249 | if (TruncOp != ISD::FP_ROUND) |
4250 | Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1); | |
4251 | else | |
4252 | Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1, | |
4253 | DAG.getIntPtrConstant(0)); | |
4254 | Results.push_back(Tmp1); | |
4255 | break; | |
4256 | } | |
4257 | case ISD::VECTOR_SHUFFLE: { | |
4258 | ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask(); | |
4259 | ||
4260 | // Cast the two input vectors. | |
4261 | Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0)); | |
4262 | Tmp2 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(1)); | |
4263 | ||
4264 | // Convert the shuffle mask to the right # elements. | |
4265 | Tmp1 = ShuffleWithNarrowerEltType(NVT, OVT, dl, Tmp1, Tmp2, Mask); | |
4266 | Tmp1 = DAG.getNode(ISD::BITCAST, dl, OVT, Tmp1); | |
4267 | Results.push_back(Tmp1); | |
4268 | break; | |
4269 | } | |
4270 | case ISD::SETCC: { | |
4271 | unsigned ExtOp = ISD::FP_EXTEND; | |
4272 | if (NVT.isInteger()) { | |
4273 | ISD::CondCode CCCode = | |
4274 | cast<CondCodeSDNode>(Node->getOperand(2))->get(); | |
4275 | ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; | |
4276 | } | |
4277 | Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); | |
4278 | Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); | |
4279 | Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), | |
4280 | Tmp1, Tmp2, Node->getOperand(2))); | |
4281 | break; | |
4282 | } | |
1a4d82fc JJ |
4283 | case ISD::FADD: |
4284 | case ISD::FSUB: | |
4285 | case ISD::FMUL: | |
223e47cc LB |
4286 | case ISD::FDIV: |
4287 | case ISD::FREM: | |
4288 | case ISD::FPOW: { | |
4289 | Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); | |
4290 | Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1)); | |
4291 | Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); | |
4292 | Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT, | |
4293 | Tmp3, DAG.getIntPtrConstant(0))); | |
4294 | break; | |
4295 | } | |
4296 | case ISD::FLOG2: | |
4297 | case ISD::FEXP2: | |
4298 | case ISD::FLOG: | |
4299 | case ISD::FEXP: { | |
4300 | Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); | |
4301 | Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); | |
4302 | Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT, | |
4303 | Tmp2, DAG.getIntPtrConstant(0))); | |
4304 | break; | |
4305 | } | |
4306 | } | |
4307 | ||
4308 | // Replace the original node with the legalized result. | |
4309 | if (!Results.empty()) | |
4310 | ReplaceNode(Node, Results.data()); | |
4311 | } | |
4312 | ||
85aaf69f | 4313 | /// This is the entry point for the file. |
223e47cc | 4314 | void SelectionDAG::Legalize() { |
1a4d82fc JJ |
4315 | AssignTopologicalOrder(); |
4316 | ||
4317 | SmallPtrSet<SDNode *, 16> LegalizedNodes; | |
4318 | SelectionDAGLegalize Legalizer(*this, LegalizedNodes); | |
4319 | ||
4320 | // Visit all the nodes. We start in topological order, so that we see | |
4321 | // nodes with their original operands intact. Legalization can produce | |
4322 | // new nodes which may themselves need to be legalized. Iterate until all | |
4323 | // nodes have been legalized. | |
4324 | for (;;) { | |
4325 | bool AnyLegalized = false; | |
4326 | for (auto NI = allnodes_end(); NI != allnodes_begin();) { | |
4327 | --NI; | |
4328 | ||
4329 | SDNode *N = NI; | |
4330 | if (N->use_empty() && N != getRoot().getNode()) { | |
4331 | ++NI; | |
4332 | DeleteNode(N); | |
4333 | continue; | |
4334 | } | |
4335 | ||
85aaf69f | 4336 | if (LegalizedNodes.insert(N).second) { |
1a4d82fc JJ |
4337 | AnyLegalized = true; |
4338 | Legalizer.LegalizeOp(N); | |
4339 | ||
4340 | if (N->use_empty() && N != getRoot().getNode()) { | |
4341 | ++NI; | |
4342 | DeleteNode(N); | |
4343 | } | |
4344 | } | |
4345 | } | |
4346 | if (!AnyLegalized) | |
4347 | break; | |
4348 | ||
4349 | } | |
4350 | ||
4351 | // Remove dead nodes now. | |
4352 | RemoveDeadNodes(); | |
4353 | } | |
4354 | ||
4355 | bool SelectionDAG::LegalizeOp(SDNode *N, | |
4356 | SmallSetVector<SDNode *, 16> &UpdatedNodes) { | |
4357 | SmallPtrSet<SDNode *, 16> LegalizedNodes; | |
4358 | SelectionDAGLegalize Legalizer(*this, LegalizedNodes, &UpdatedNodes); | |
4359 | ||
4360 | // Directly insert the node in question, and legalize it. This will recurse | |
4361 | // as needed through operands. | |
4362 | LegalizedNodes.insert(N); | |
4363 | Legalizer.LegalizeOp(N); | |
4364 | ||
4365 | return LegalizedNodes.count(N); | |
223e47cc | 4366 | } |