]>
Commit | Line | Data |
---|---|---|
223e47cc LB |
1 | //===-- ARMISelDAGToDAG.cpp - A dag to dag inst selector for ARM ----------===// |
2 | // | |
3 | // The LLVM Compiler Infrastructure | |
4 | // | |
5 | // This file is distributed under the University of Illinois Open Source | |
6 | // License. See LICENSE.TXT for details. | |
7 | // | |
8 | //===----------------------------------------------------------------------===// | |
9 | // | |
10 | // This file defines an instruction selector for the ARM target. | |
11 | // | |
12 | //===----------------------------------------------------------------------===// | |
13 | ||
223e47cc LB |
14 | #include "ARM.h" |
15 | #include "ARMBaseInstrInfo.h" | |
16 | #include "ARMTargetMachine.h" | |
17 | #include "MCTargetDesc/ARMAddressingModes.h" | |
223e47cc LB |
18 | #include "llvm/CodeGen/MachineFrameInfo.h" |
19 | #include "llvm/CodeGen/MachineFunction.h" | |
20 | #include "llvm/CodeGen/MachineInstrBuilder.h" | |
970d7e83 | 21 | #include "llvm/CodeGen/MachineRegisterInfo.h" |
223e47cc LB |
22 | #include "llvm/CodeGen/SelectionDAG.h" |
23 | #include "llvm/CodeGen/SelectionDAGISel.h" | |
970d7e83 LB |
24 | #include "llvm/IR/CallingConv.h" |
25 | #include "llvm/IR/Constants.h" | |
26 | #include "llvm/IR/DerivedTypes.h" | |
27 | #include "llvm/IR/Function.h" | |
28 | #include "llvm/IR/Intrinsics.h" | |
29 | #include "llvm/IR/LLVMContext.h" | |
223e47cc LB |
30 | #include "llvm/Support/CommandLine.h" |
31 | #include "llvm/Support/Compiler.h" | |
32 | #include "llvm/Support/Debug.h" | |
33 | #include "llvm/Support/ErrorHandling.h" | |
970d7e83 LB |
34 | #include "llvm/Target/TargetLowering.h" |
35 | #include "llvm/Target/TargetOptions.h" | |
223e47cc LB |
36 | |
37 | using namespace llvm; | |
38 | ||
1a4d82fc JJ |
39 | #define DEBUG_TYPE "arm-isel" |
40 | ||
223e47cc LB |
41 | static cl::opt<bool> |
42 | DisableShifterOp("disable-shifter-op", cl::Hidden, | |
43 | cl::desc("Disable isel of shifter-op"), | |
44 | cl::init(false)); | |
45 | ||
46 | static cl::opt<bool> | |
47 | CheckVMLxHazard("check-vmlx-hazard", cl::Hidden, | |
48 | cl::desc("Check fp vmla / vmls hazard at isel time"), | |
49 | cl::init(true)); | |
50 | ||
51 | //===--------------------------------------------------------------------===// | |
52 | /// ARMDAGToDAGISel - ARM specific code to select ARM machine | |
53 | /// instructions for SelectionDAG operations. | |
54 | /// | |
55 | namespace { | |
56 | ||
57 | enum AddrMode2Type { | |
58 | AM2_BASE, // Simple AM2 (+-imm12) | |
59 | AM2_SHOP // Shifter-op AM2 | |
60 | }; | |
61 | ||
62 | class ARMDAGToDAGISel : public SelectionDAGISel { | |
223e47cc LB |
63 | /// Subtarget - Keep a pointer to the ARMSubtarget around so that we can |
64 | /// make the right decision when generating code for different targets. | |
65 | const ARMSubtarget *Subtarget; | |
66 | ||
67 | public: | |
1a4d82fc JJ |
68 | explicit ARMDAGToDAGISel(ARMBaseTargetMachine &tm, CodeGenOpt::Level OptLevel) |
69 | : SelectionDAGISel(tm, OptLevel) {} | |
70 | ||
71 | bool runOnMachineFunction(MachineFunction &MF) override { | |
72 | // Reset the subtarget each time through. | |
73 | Subtarget = &MF.getTarget().getSubtarget<ARMSubtarget>(); | |
74 | SelectionDAGISel::runOnMachineFunction(MF); | |
75 | return true; | |
223e47cc LB |
76 | } |
77 | ||
1a4d82fc | 78 | const char *getPassName() const override { |
223e47cc LB |
79 | return "ARM Instruction Selection"; |
80 | } | |
81 | ||
1a4d82fc | 82 | void PreprocessISelDAG() override; |
970d7e83 | 83 | |
223e47cc LB |
84 | /// getI32Imm - Return a target constant of type i32 with the specified |
85 | /// value. | |
86 | inline SDValue getI32Imm(unsigned Imm) { | |
87 | return CurDAG->getTargetConstant(Imm, MVT::i32); | |
88 | } | |
89 | ||
1a4d82fc | 90 | SDNode *Select(SDNode *N) override; |
223e47cc LB |
91 | |
92 | ||
93 | bool hasNoVMLxHazardUse(SDNode *N) const; | |
94 | bool isShifterOpProfitable(const SDValue &Shift, | |
95 | ARM_AM::ShiftOpc ShOpcVal, unsigned ShAmt); | |
96 | bool SelectRegShifterOperand(SDValue N, SDValue &A, | |
97 | SDValue &B, SDValue &C, | |
98 | bool CheckProfitability = true); | |
99 | bool SelectImmShifterOperand(SDValue N, SDValue &A, | |
100 | SDValue &B, bool CheckProfitability = true); | |
101 | bool SelectShiftRegShifterOperand(SDValue N, SDValue &A, | |
102 | SDValue &B, SDValue &C) { | |
103 | // Don't apply the profitability check | |
104 | return SelectRegShifterOperand(N, A, B, C, false); | |
105 | } | |
106 | bool SelectShiftImmShifterOperand(SDValue N, SDValue &A, | |
107 | SDValue &B) { | |
108 | // Don't apply the profitability check | |
109 | return SelectImmShifterOperand(N, A, B, false); | |
110 | } | |
111 | ||
112 | bool SelectAddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm); | |
113 | bool SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset, SDValue &Opc); | |
114 | ||
115 | AddrMode2Type SelectAddrMode2Worker(SDValue N, SDValue &Base, | |
116 | SDValue &Offset, SDValue &Opc); | |
117 | bool SelectAddrMode2Base(SDValue N, SDValue &Base, SDValue &Offset, | |
118 | SDValue &Opc) { | |
119 | return SelectAddrMode2Worker(N, Base, Offset, Opc) == AM2_BASE; | |
120 | } | |
121 | ||
122 | bool SelectAddrMode2ShOp(SDValue N, SDValue &Base, SDValue &Offset, | |
123 | SDValue &Opc) { | |
124 | return SelectAddrMode2Worker(N, Base, Offset, Opc) == AM2_SHOP; | |
125 | } | |
126 | ||
127 | bool SelectAddrMode2(SDValue N, SDValue &Base, SDValue &Offset, | |
128 | SDValue &Opc) { | |
129 | SelectAddrMode2Worker(N, Base, Offset, Opc); | |
130 | // return SelectAddrMode2ShOp(N, Base, Offset, Opc); | |
131 | // This always matches one way or another. | |
132 | return true; | |
133 | } | |
134 | ||
1a4d82fc JJ |
135 | bool SelectCMOVPred(SDValue N, SDValue &Pred, SDValue &Reg) { |
136 | const ConstantSDNode *CN = cast<ConstantSDNode>(N); | |
137 | Pred = CurDAG->getTargetConstant(CN->getZExtValue(), MVT::i32); | |
138 | Reg = CurDAG->getRegister(ARM::CPSR, MVT::i32); | |
139 | return true; | |
140 | } | |
141 | ||
223e47cc LB |
142 | bool SelectAddrMode2OffsetReg(SDNode *Op, SDValue N, |
143 | SDValue &Offset, SDValue &Opc); | |
144 | bool SelectAddrMode2OffsetImm(SDNode *Op, SDValue N, | |
145 | SDValue &Offset, SDValue &Opc); | |
146 | bool SelectAddrMode2OffsetImmPre(SDNode *Op, SDValue N, | |
147 | SDValue &Offset, SDValue &Opc); | |
148 | bool SelectAddrOffsetNone(SDValue N, SDValue &Base); | |
149 | bool SelectAddrMode3(SDValue N, SDValue &Base, | |
150 | SDValue &Offset, SDValue &Opc); | |
151 | bool SelectAddrMode3Offset(SDNode *Op, SDValue N, | |
152 | SDValue &Offset, SDValue &Opc); | |
153 | bool SelectAddrMode5(SDValue N, SDValue &Base, | |
154 | SDValue &Offset); | |
155 | bool SelectAddrMode6(SDNode *Parent, SDValue N, SDValue &Addr,SDValue &Align); | |
156 | bool SelectAddrMode6Offset(SDNode *Op, SDValue N, SDValue &Offset); | |
157 | ||
158 | bool SelectAddrModePC(SDValue N, SDValue &Offset, SDValue &Label); | |
159 | ||
160 | // Thumb Addressing Modes: | |
161 | bool SelectThumbAddrModeRR(SDValue N, SDValue &Base, SDValue &Offset); | |
162 | bool SelectThumbAddrModeRI(SDValue N, SDValue &Base, SDValue &Offset, | |
163 | unsigned Scale); | |
164 | bool SelectThumbAddrModeRI5S1(SDValue N, SDValue &Base, SDValue &Offset); | |
165 | bool SelectThumbAddrModeRI5S2(SDValue N, SDValue &Base, SDValue &Offset); | |
166 | bool SelectThumbAddrModeRI5S4(SDValue N, SDValue &Base, SDValue &Offset); | |
167 | bool SelectThumbAddrModeImm5S(SDValue N, unsigned Scale, SDValue &Base, | |
168 | SDValue &OffImm); | |
169 | bool SelectThumbAddrModeImm5S1(SDValue N, SDValue &Base, | |
170 | SDValue &OffImm); | |
171 | bool SelectThumbAddrModeImm5S2(SDValue N, SDValue &Base, | |
172 | SDValue &OffImm); | |
173 | bool SelectThumbAddrModeImm5S4(SDValue N, SDValue &Base, | |
174 | SDValue &OffImm); | |
175 | bool SelectThumbAddrModeSP(SDValue N, SDValue &Base, SDValue &OffImm); | |
176 | ||
177 | // Thumb 2 Addressing Modes: | |
178 | bool SelectT2ShifterOperandReg(SDValue N, | |
179 | SDValue &BaseReg, SDValue &Opc); | |
180 | bool SelectT2AddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm); | |
181 | bool SelectT2AddrModeImm8(SDValue N, SDValue &Base, | |
182 | SDValue &OffImm); | |
183 | bool SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N, | |
184 | SDValue &OffImm); | |
185 | bool SelectT2AddrModeSoReg(SDValue N, SDValue &Base, | |
186 | SDValue &OffReg, SDValue &ShImm); | |
1a4d82fc | 187 | bool SelectT2AddrModeExclusive(SDValue N, SDValue &Base, SDValue &OffImm); |
223e47cc LB |
188 | |
189 | inline bool is_so_imm(unsigned Imm) const { | |
190 | return ARM_AM::getSOImmVal(Imm) != -1; | |
191 | } | |
192 | ||
193 | inline bool is_so_imm_not(unsigned Imm) const { | |
194 | return ARM_AM::getSOImmVal(~Imm) != -1; | |
195 | } | |
196 | ||
197 | inline bool is_t2_so_imm(unsigned Imm) const { | |
198 | return ARM_AM::getT2SOImmVal(Imm) != -1; | |
199 | } | |
200 | ||
201 | inline bool is_t2_so_imm_not(unsigned Imm) const { | |
202 | return ARM_AM::getT2SOImmVal(~Imm) != -1; | |
203 | } | |
204 | ||
205 | // Include the pieces autogenerated from the target description. | |
206 | #include "ARMGenDAGISel.inc" | |
207 | ||
208 | private: | |
209 | /// SelectARMIndexedLoad - Indexed (pre/post inc/dec) load matching code for | |
210 | /// ARM. | |
211 | SDNode *SelectARMIndexedLoad(SDNode *N); | |
212 | SDNode *SelectT2IndexedLoad(SDNode *N); | |
213 | ||
214 | /// SelectVLD - Select NEON load intrinsics. NumVecs should be | |
215 | /// 1, 2, 3 or 4. The opcode arrays specify the instructions used for | |
216 | /// loads of D registers and even subregs and odd subregs of Q registers. | |
217 | /// For NumVecs <= 2, QOpcodes1 is not used. | |
218 | SDNode *SelectVLD(SDNode *N, bool isUpdating, unsigned NumVecs, | |
219 | const uint16_t *DOpcodes, | |
220 | const uint16_t *QOpcodes0, const uint16_t *QOpcodes1); | |
221 | ||
222 | /// SelectVST - Select NEON store intrinsics. NumVecs should | |
223 | /// be 1, 2, 3 or 4. The opcode arrays specify the instructions used for | |
224 | /// stores of D registers and even subregs and odd subregs of Q registers. | |
225 | /// For NumVecs <= 2, QOpcodes1 is not used. | |
226 | SDNode *SelectVST(SDNode *N, bool isUpdating, unsigned NumVecs, | |
227 | const uint16_t *DOpcodes, | |
228 | const uint16_t *QOpcodes0, const uint16_t *QOpcodes1); | |
229 | ||
230 | /// SelectVLDSTLane - Select NEON load/store lane intrinsics. NumVecs should | |
231 | /// be 2, 3 or 4. The opcode arrays specify the instructions used for | |
232 | /// load/store of D registers and Q registers. | |
233 | SDNode *SelectVLDSTLane(SDNode *N, bool IsLoad, | |
234 | bool isUpdating, unsigned NumVecs, | |
235 | const uint16_t *DOpcodes, const uint16_t *QOpcodes); | |
236 | ||
237 | /// SelectVLDDup - Select NEON load-duplicate intrinsics. NumVecs | |
238 | /// should be 2, 3 or 4. The opcode array specifies the instructions used | |
239 | /// for loading D registers. (Q registers are not supported.) | |
240 | SDNode *SelectVLDDup(SDNode *N, bool isUpdating, unsigned NumVecs, | |
241 | const uint16_t *Opcodes); | |
242 | ||
243 | /// SelectVTBL - Select NEON VTBL and VTBX intrinsics. NumVecs should be 2, | |
244 | /// 3 or 4. These are custom-selected so that a REG_SEQUENCE can be | |
245 | /// generated to force the table registers to be consecutive. | |
246 | SDNode *SelectVTBL(SDNode *N, bool IsExt, unsigned NumVecs, unsigned Opc); | |
247 | ||
248 | /// SelectV6T2BitfieldExtractOp - Select SBFX/UBFX instructions for ARM. | |
249 | SDNode *SelectV6T2BitfieldExtractOp(SDNode *N, bool isSigned); | |
250 | ||
223e47cc LB |
251 | // Select special operations if node forms integer ABS pattern |
252 | SDNode *SelectABSOp(SDNode *N); | |
253 | ||
970d7e83 LB |
254 | SDNode *SelectInlineAsm(SDNode *N); |
255 | ||
223e47cc LB |
256 | SDNode *SelectConcatVector(SDNode *N); |
257 | ||
223e47cc LB |
258 | /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for |
259 | /// inline asm expressions. | |
1a4d82fc JJ |
260 | bool SelectInlineAsmMemoryOperand(const SDValue &Op, char ConstraintCode, |
261 | std::vector<SDValue> &OutOps) override; | |
223e47cc | 262 | |
970d7e83 LB |
263 | // Form pairs of consecutive R, S, D, or Q registers. |
264 | SDNode *createGPRPairNode(EVT VT, SDValue V0, SDValue V1); | |
265 | SDNode *createSRegPairNode(EVT VT, SDValue V0, SDValue V1); | |
266 | SDNode *createDRegPairNode(EVT VT, SDValue V0, SDValue V1); | |
267 | SDNode *createQRegPairNode(EVT VT, SDValue V0, SDValue V1); | |
223e47cc LB |
268 | |
269 | // Form sequences of 4 consecutive S, D, or Q registers. | |
970d7e83 LB |
270 | SDNode *createQuadSRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3); |
271 | SDNode *createQuadDRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3); | |
272 | SDNode *createQuadQRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3); | |
223e47cc LB |
273 | |
274 | // Get the alignment operand for a NEON VLD or VST instruction. | |
275 | SDValue GetVLDSTAlign(SDValue Align, unsigned NumVecs, bool is64BitVector); | |
276 | }; | |
277 | } | |
278 | ||
279 | /// isInt32Immediate - This method tests to see if the node is a 32-bit constant | |
280 | /// operand. If so Imm will receive the 32-bit value. | |
281 | static bool isInt32Immediate(SDNode *N, unsigned &Imm) { | |
282 | if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i32) { | |
283 | Imm = cast<ConstantSDNode>(N)->getZExtValue(); | |
284 | return true; | |
285 | } | |
286 | return false; | |
287 | } | |
288 | ||
289 | // isInt32Immediate - This method tests to see if a constant operand. | |
290 | // If so Imm will receive the 32 bit value. | |
291 | static bool isInt32Immediate(SDValue N, unsigned &Imm) { | |
292 | return isInt32Immediate(N.getNode(), Imm); | |
293 | } | |
294 | ||
295 | // isOpcWithIntImmediate - This method tests to see if the node is a specific | |
296 | // opcode and that it has a immediate integer right operand. | |
297 | // If so Imm will receive the 32 bit value. | |
298 | static bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned& Imm) { | |
299 | return N->getOpcode() == Opc && | |
300 | isInt32Immediate(N->getOperand(1).getNode(), Imm); | |
301 | } | |
302 | ||
303 | /// \brief Check whether a particular node is a constant value representable as | |
304 | /// (N * Scale) where (N in [\p RangeMin, \p RangeMax). | |
305 | /// | |
306 | /// \param ScaledConstant [out] - On success, the pre-scaled constant value. | |
307 | static bool isScaledConstantInRange(SDValue Node, int Scale, | |
308 | int RangeMin, int RangeMax, | |
309 | int &ScaledConstant) { | |
310 | assert(Scale > 0 && "Invalid scale!"); | |
311 | ||
312 | // Check that this is a constant. | |
313 | const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Node); | |
314 | if (!C) | |
315 | return false; | |
316 | ||
317 | ScaledConstant = (int) C->getZExtValue(); | |
318 | if ((ScaledConstant % Scale) != 0) | |
319 | return false; | |
320 | ||
321 | ScaledConstant /= Scale; | |
322 | return ScaledConstant >= RangeMin && ScaledConstant < RangeMax; | |
323 | } | |
324 | ||
970d7e83 LB |
325 | void ARMDAGToDAGISel::PreprocessISelDAG() { |
326 | if (!Subtarget->hasV6T2Ops()) | |
327 | return; | |
328 | ||
329 | bool isThumb2 = Subtarget->isThumb(); | |
330 | for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(), | |
331 | E = CurDAG->allnodes_end(); I != E; ) { | |
332 | SDNode *N = I++; // Preincrement iterator to avoid invalidation issues. | |
333 | ||
334 | if (N->getOpcode() != ISD::ADD) | |
335 | continue; | |
336 | ||
337 | // Look for (add X1, (and (srl X2, c1), c2)) where c2 is constant with | |
338 | // leading zeros, followed by consecutive set bits, followed by 1 or 2 | |
339 | // trailing zeros, e.g. 1020. | |
340 | // Transform the expression to | |
341 | // (add X1, (shl (and (srl X2, c1), (c2>>tz)), tz)) where tz is the number | |
342 | // of trailing zeros of c2. The left shift would be folded as an shifter | |
343 | // operand of 'add' and the 'and' and 'srl' would become a bits extraction | |
344 | // node (UBFX). | |
345 | ||
346 | SDValue N0 = N->getOperand(0); | |
347 | SDValue N1 = N->getOperand(1); | |
348 | unsigned And_imm = 0; | |
349 | if (!isOpcWithIntImmediate(N1.getNode(), ISD::AND, And_imm)) { | |
350 | if (isOpcWithIntImmediate(N0.getNode(), ISD::AND, And_imm)) | |
351 | std::swap(N0, N1); | |
352 | } | |
353 | if (!And_imm) | |
354 | continue; | |
355 | ||
356 | // Check if the AND mask is an immediate of the form: 000.....1111111100 | |
1a4d82fc | 357 | unsigned TZ = countTrailingZeros(And_imm); |
970d7e83 LB |
358 | if (TZ != 1 && TZ != 2) |
359 | // Be conservative here. Shifter operands aren't always free. e.g. On | |
360 | // Swift, left shifter operand of 1 / 2 for free but others are not. | |
361 | // e.g. | |
362 | // ubfx r3, r1, #16, #8 | |
363 | // ldr.w r3, [r0, r3, lsl #2] | |
364 | // vs. | |
365 | // mov.w r9, #1020 | |
366 | // and.w r2, r9, r1, lsr #14 | |
367 | // ldr r2, [r0, r2] | |
368 | continue; | |
369 | And_imm >>= TZ; | |
370 | if (And_imm & (And_imm + 1)) | |
371 | continue; | |
372 | ||
373 | // Look for (and (srl X, c1), c2). | |
374 | SDValue Srl = N1.getOperand(0); | |
375 | unsigned Srl_imm = 0; | |
376 | if (!isOpcWithIntImmediate(Srl.getNode(), ISD::SRL, Srl_imm) || | |
377 | (Srl_imm <= 2)) | |
378 | continue; | |
379 | ||
380 | // Make sure first operand is not a shifter operand which would prevent | |
381 | // folding of the left shift. | |
382 | SDValue CPTmp0; | |
383 | SDValue CPTmp1; | |
384 | SDValue CPTmp2; | |
385 | if (isThumb2) { | |
386 | if (SelectT2ShifterOperandReg(N0, CPTmp0, CPTmp1)) | |
387 | continue; | |
388 | } else { | |
389 | if (SelectImmShifterOperand(N0, CPTmp0, CPTmp1) || | |
390 | SelectRegShifterOperand(N0, CPTmp0, CPTmp1, CPTmp2)) | |
391 | continue; | |
392 | } | |
393 | ||
394 | // Now make the transformation. | |
1a4d82fc | 395 | Srl = CurDAG->getNode(ISD::SRL, SDLoc(Srl), MVT::i32, |
970d7e83 LB |
396 | Srl.getOperand(0), |
397 | CurDAG->getConstant(Srl_imm+TZ, MVT::i32)); | |
1a4d82fc | 398 | N1 = CurDAG->getNode(ISD::AND, SDLoc(N1), MVT::i32, |
970d7e83 | 399 | Srl, CurDAG->getConstant(And_imm, MVT::i32)); |
1a4d82fc | 400 | N1 = CurDAG->getNode(ISD::SHL, SDLoc(N1), MVT::i32, |
970d7e83 LB |
401 | N1, CurDAG->getConstant(TZ, MVT::i32)); |
402 | CurDAG->UpdateNodeOperands(N, N0, N1); | |
1a4d82fc | 403 | } |
970d7e83 LB |
404 | } |
405 | ||
223e47cc LB |
406 | /// hasNoVMLxHazardUse - Return true if it's desirable to select a FP MLA / MLS |
407 | /// node. VFP / NEON fp VMLA / VMLS instructions have special RAW hazards (at | |
408 | /// least on current ARM implementations) which should be avoidded. | |
409 | bool ARMDAGToDAGISel::hasNoVMLxHazardUse(SDNode *N) const { | |
410 | if (OptLevel == CodeGenOpt::None) | |
411 | return true; | |
412 | ||
413 | if (!CheckVMLxHazard) | |
414 | return true; | |
415 | ||
1a4d82fc JJ |
416 | if (!Subtarget->isCortexA7() && !Subtarget->isCortexA8() && |
417 | !Subtarget->isCortexA9() && !Subtarget->isSwift()) | |
223e47cc LB |
418 | return true; |
419 | ||
420 | if (!N->hasOneUse()) | |
421 | return false; | |
422 | ||
423 | SDNode *Use = *N->use_begin(); | |
424 | if (Use->getOpcode() == ISD::CopyToReg) | |
425 | return true; | |
426 | if (Use->isMachineOpcode()) { | |
1a4d82fc JJ |
427 | const ARMBaseInstrInfo *TII = static_cast<const ARMBaseInstrInfo *>( |
428 | CurDAG->getSubtarget().getInstrInfo()); | |
429 | ||
223e47cc LB |
430 | const MCInstrDesc &MCID = TII->get(Use->getMachineOpcode()); |
431 | if (MCID.mayStore()) | |
432 | return true; | |
433 | unsigned Opcode = MCID.getOpcode(); | |
434 | if (Opcode == ARM::VMOVRS || Opcode == ARM::VMOVRRD) | |
435 | return true; | |
436 | // vmlx feeding into another vmlx. We actually want to unfold | |
437 | // the use later in the MLxExpansion pass. e.g. | |
438 | // vmla | |
439 | // vmla (stall 8 cycles) | |
440 | // | |
441 | // vmul (5 cycles) | |
442 | // vadd (5 cycles) | |
443 | // vmla | |
444 | // This adds up to about 18 - 19 cycles. | |
445 | // | |
446 | // vmla | |
447 | // vmul (stall 4 cycles) | |
448 | // vadd adds up to about 14 cycles. | |
449 | return TII->isFpMLxInstruction(Opcode); | |
450 | } | |
451 | ||
452 | return false; | |
453 | } | |
454 | ||
455 | bool ARMDAGToDAGISel::isShifterOpProfitable(const SDValue &Shift, | |
456 | ARM_AM::ShiftOpc ShOpcVal, | |
457 | unsigned ShAmt) { | |
458 | if (!Subtarget->isLikeA9() && !Subtarget->isSwift()) | |
459 | return true; | |
460 | if (Shift.hasOneUse()) | |
461 | return true; | |
462 | // R << 2 is free. | |
463 | return ShOpcVal == ARM_AM::lsl && | |
464 | (ShAmt == 2 || (Subtarget->isSwift() && ShAmt == 1)); | |
465 | } | |
466 | ||
467 | bool ARMDAGToDAGISel::SelectImmShifterOperand(SDValue N, | |
468 | SDValue &BaseReg, | |
469 | SDValue &Opc, | |
470 | bool CheckProfitability) { | |
471 | if (DisableShifterOp) | |
472 | return false; | |
473 | ||
474 | ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode()); | |
475 | ||
476 | // Don't match base register only case. That is matched to a separate | |
477 | // lower complexity pattern with explicit register operand. | |
478 | if (ShOpcVal == ARM_AM::no_shift) return false; | |
479 | ||
480 | BaseReg = N.getOperand(0); | |
481 | unsigned ShImmVal = 0; | |
482 | ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1)); | |
483 | if (!RHS) return false; | |
484 | ShImmVal = RHS->getZExtValue() & 31; | |
485 | Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal), | |
486 | MVT::i32); | |
487 | return true; | |
488 | } | |
489 | ||
490 | bool ARMDAGToDAGISel::SelectRegShifterOperand(SDValue N, | |
491 | SDValue &BaseReg, | |
492 | SDValue &ShReg, | |
493 | SDValue &Opc, | |
494 | bool CheckProfitability) { | |
495 | if (DisableShifterOp) | |
496 | return false; | |
497 | ||
498 | ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode()); | |
499 | ||
500 | // Don't match base register only case. That is matched to a separate | |
501 | // lower complexity pattern with explicit register operand. | |
502 | if (ShOpcVal == ARM_AM::no_shift) return false; | |
503 | ||
504 | BaseReg = N.getOperand(0); | |
505 | unsigned ShImmVal = 0; | |
506 | ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1)); | |
507 | if (RHS) return false; | |
508 | ||
509 | ShReg = N.getOperand(1); | |
510 | if (CheckProfitability && !isShifterOpProfitable(N, ShOpcVal, ShImmVal)) | |
511 | return false; | |
512 | Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal), | |
513 | MVT::i32); | |
514 | return true; | |
515 | } | |
516 | ||
517 | ||
518 | bool ARMDAGToDAGISel::SelectAddrModeImm12(SDValue N, | |
519 | SDValue &Base, | |
520 | SDValue &OffImm) { | |
521 | // Match simple R + imm12 operands. | |
522 | ||
523 | // Base only. | |
524 | if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB && | |
525 | !CurDAG->isBaseWithConstantOffset(N)) { | |
526 | if (N.getOpcode() == ISD::FrameIndex) { | |
527 | // Match frame index. | |
528 | int FI = cast<FrameIndexSDNode>(N)->getIndex(); | |
85aaf69f | 529 | Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy()); |
223e47cc LB |
530 | OffImm = CurDAG->getTargetConstant(0, MVT::i32); |
531 | return true; | |
532 | } | |
533 | ||
534 | if (N.getOpcode() == ARMISD::Wrapper && | |
1a4d82fc | 535 | N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress) { |
223e47cc LB |
536 | Base = N.getOperand(0); |
537 | } else | |
538 | Base = N; | |
539 | OffImm = CurDAG->getTargetConstant(0, MVT::i32); | |
540 | return true; | |
541 | } | |
542 | ||
543 | if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) { | |
1a4d82fc | 544 | int RHSC = (int)RHS->getSExtValue(); |
223e47cc LB |
545 | if (N.getOpcode() == ISD::SUB) |
546 | RHSC = -RHSC; | |
547 | ||
1a4d82fc | 548 | if (RHSC > -0x1000 && RHSC < 0x1000) { // 12 bits |
223e47cc LB |
549 | Base = N.getOperand(0); |
550 | if (Base.getOpcode() == ISD::FrameIndex) { | |
551 | int FI = cast<FrameIndexSDNode>(Base)->getIndex(); | |
85aaf69f | 552 | Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy()); |
223e47cc LB |
553 | } |
554 | OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32); | |
555 | return true; | |
556 | } | |
557 | } | |
558 | ||
559 | // Base only. | |
560 | Base = N; | |
561 | OffImm = CurDAG->getTargetConstant(0, MVT::i32); | |
562 | return true; | |
563 | } | |
564 | ||
565 | ||
566 | ||
567 | bool ARMDAGToDAGISel::SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset, | |
568 | SDValue &Opc) { | |
569 | if (N.getOpcode() == ISD::MUL && | |
570 | ((!Subtarget->isLikeA9() && !Subtarget->isSwift()) || N.hasOneUse())) { | |
571 | if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) { | |
572 | // X * [3,5,9] -> X + X * [2,4,8] etc. | |
573 | int RHSC = (int)RHS->getZExtValue(); | |
574 | if (RHSC & 1) { | |
575 | RHSC = RHSC & ~1; | |
576 | ARM_AM::AddrOpc AddSub = ARM_AM::add; | |
577 | if (RHSC < 0) { | |
578 | AddSub = ARM_AM::sub; | |
579 | RHSC = - RHSC; | |
580 | } | |
581 | if (isPowerOf2_32(RHSC)) { | |
582 | unsigned ShAmt = Log2_32(RHSC); | |
583 | Base = Offset = N.getOperand(0); | |
584 | Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, | |
585 | ARM_AM::lsl), | |
586 | MVT::i32); | |
587 | return true; | |
588 | } | |
589 | } | |
590 | } | |
591 | } | |
592 | ||
593 | if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB && | |
594 | // ISD::OR that is equivalent to an ISD::ADD. | |
595 | !CurDAG->isBaseWithConstantOffset(N)) | |
596 | return false; | |
597 | ||
598 | // Leave simple R +/- imm12 operands for LDRi12 | |
599 | if (N.getOpcode() == ISD::ADD || N.getOpcode() == ISD::OR) { | |
600 | int RHSC; | |
601 | if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1, | |
602 | -0x1000+1, 0x1000, RHSC)) // 12 bits. | |
603 | return false; | |
604 | } | |
605 | ||
606 | // Otherwise this is R +/- [possibly shifted] R. | |
607 | ARM_AM::AddrOpc AddSub = N.getOpcode() == ISD::SUB ? ARM_AM::sub:ARM_AM::add; | |
608 | ARM_AM::ShiftOpc ShOpcVal = | |
609 | ARM_AM::getShiftOpcForNode(N.getOperand(1).getOpcode()); | |
610 | unsigned ShAmt = 0; | |
611 | ||
612 | Base = N.getOperand(0); | |
613 | Offset = N.getOperand(1); | |
614 | ||
615 | if (ShOpcVal != ARM_AM::no_shift) { | |
616 | // Check to see if the RHS of the shift is a constant, if not, we can't fold | |
617 | // it. | |
618 | if (ConstantSDNode *Sh = | |
619 | dyn_cast<ConstantSDNode>(N.getOperand(1).getOperand(1))) { | |
620 | ShAmt = Sh->getZExtValue(); | |
621 | if (isShifterOpProfitable(Offset, ShOpcVal, ShAmt)) | |
622 | Offset = N.getOperand(1).getOperand(0); | |
623 | else { | |
624 | ShAmt = 0; | |
625 | ShOpcVal = ARM_AM::no_shift; | |
626 | } | |
627 | } else { | |
628 | ShOpcVal = ARM_AM::no_shift; | |
629 | } | |
630 | } | |
631 | ||
632 | // Try matching (R shl C) + (R). | |
633 | if (N.getOpcode() != ISD::SUB && ShOpcVal == ARM_AM::no_shift && | |
634 | !(Subtarget->isLikeA9() || Subtarget->isSwift() || | |
635 | N.getOperand(0).hasOneUse())) { | |
636 | ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(0).getOpcode()); | |
637 | if (ShOpcVal != ARM_AM::no_shift) { | |
638 | // Check to see if the RHS of the shift is a constant, if not, we can't | |
639 | // fold it. | |
640 | if (ConstantSDNode *Sh = | |
641 | dyn_cast<ConstantSDNode>(N.getOperand(0).getOperand(1))) { | |
642 | ShAmt = Sh->getZExtValue(); | |
643 | if (isShifterOpProfitable(N.getOperand(0), ShOpcVal, ShAmt)) { | |
644 | Offset = N.getOperand(0).getOperand(0); | |
645 | Base = N.getOperand(1); | |
646 | } else { | |
647 | ShAmt = 0; | |
648 | ShOpcVal = ARM_AM::no_shift; | |
649 | } | |
650 | } else { | |
651 | ShOpcVal = ARM_AM::no_shift; | |
652 | } | |
653 | } | |
654 | } | |
655 | ||
656 | Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal), | |
657 | MVT::i32); | |
658 | return true; | |
659 | } | |
660 | ||
661 | ||
662 | //----- | |
663 | ||
664 | AddrMode2Type ARMDAGToDAGISel::SelectAddrMode2Worker(SDValue N, | |
665 | SDValue &Base, | |
666 | SDValue &Offset, | |
667 | SDValue &Opc) { | |
668 | if (N.getOpcode() == ISD::MUL && | |
669 | (!(Subtarget->isLikeA9() || Subtarget->isSwift()) || N.hasOneUse())) { | |
670 | if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) { | |
671 | // X * [3,5,9] -> X + X * [2,4,8] etc. | |
672 | int RHSC = (int)RHS->getZExtValue(); | |
673 | if (RHSC & 1) { | |
674 | RHSC = RHSC & ~1; | |
675 | ARM_AM::AddrOpc AddSub = ARM_AM::add; | |
676 | if (RHSC < 0) { | |
677 | AddSub = ARM_AM::sub; | |
678 | RHSC = - RHSC; | |
679 | } | |
680 | if (isPowerOf2_32(RHSC)) { | |
681 | unsigned ShAmt = Log2_32(RHSC); | |
682 | Base = Offset = N.getOperand(0); | |
683 | Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, | |
684 | ARM_AM::lsl), | |
685 | MVT::i32); | |
686 | return AM2_SHOP; | |
687 | } | |
688 | } | |
689 | } | |
690 | } | |
691 | ||
692 | if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB && | |
693 | // ISD::OR that is equivalent to an ADD. | |
694 | !CurDAG->isBaseWithConstantOffset(N)) { | |
695 | Base = N; | |
696 | if (N.getOpcode() == ISD::FrameIndex) { | |
697 | int FI = cast<FrameIndexSDNode>(N)->getIndex(); | |
85aaf69f | 698 | Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy()); |
223e47cc | 699 | } else if (N.getOpcode() == ARMISD::Wrapper && |
1a4d82fc | 700 | N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress) { |
223e47cc LB |
701 | Base = N.getOperand(0); |
702 | } | |
703 | Offset = CurDAG->getRegister(0, MVT::i32); | |
704 | Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(ARM_AM::add, 0, | |
705 | ARM_AM::no_shift), | |
706 | MVT::i32); | |
707 | return AM2_BASE; | |
708 | } | |
709 | ||
710 | // Match simple R +/- imm12 operands. | |
711 | if (N.getOpcode() != ISD::SUB) { | |
712 | int RHSC; | |
713 | if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1, | |
714 | -0x1000+1, 0x1000, RHSC)) { // 12 bits. | |
715 | Base = N.getOperand(0); | |
716 | if (Base.getOpcode() == ISD::FrameIndex) { | |
717 | int FI = cast<FrameIndexSDNode>(Base)->getIndex(); | |
85aaf69f | 718 | Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy()); |
223e47cc LB |
719 | } |
720 | Offset = CurDAG->getRegister(0, MVT::i32); | |
721 | ||
722 | ARM_AM::AddrOpc AddSub = ARM_AM::add; | |
723 | if (RHSC < 0) { | |
724 | AddSub = ARM_AM::sub; | |
725 | RHSC = - RHSC; | |
726 | } | |
727 | Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, RHSC, | |
728 | ARM_AM::no_shift), | |
729 | MVT::i32); | |
730 | return AM2_BASE; | |
731 | } | |
732 | } | |
733 | ||
734 | if ((Subtarget->isLikeA9() || Subtarget->isSwift()) && !N.hasOneUse()) { | |
735 | // Compute R +/- (R << N) and reuse it. | |
736 | Base = N; | |
737 | Offset = CurDAG->getRegister(0, MVT::i32); | |
738 | Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(ARM_AM::add, 0, | |
739 | ARM_AM::no_shift), | |
740 | MVT::i32); | |
741 | return AM2_BASE; | |
742 | } | |
743 | ||
744 | // Otherwise this is R +/- [possibly shifted] R. | |
745 | ARM_AM::AddrOpc AddSub = N.getOpcode() != ISD::SUB ? ARM_AM::add:ARM_AM::sub; | |
746 | ARM_AM::ShiftOpc ShOpcVal = | |
747 | ARM_AM::getShiftOpcForNode(N.getOperand(1).getOpcode()); | |
748 | unsigned ShAmt = 0; | |
749 | ||
750 | Base = N.getOperand(0); | |
751 | Offset = N.getOperand(1); | |
752 | ||
753 | if (ShOpcVal != ARM_AM::no_shift) { | |
754 | // Check to see if the RHS of the shift is a constant, if not, we can't fold | |
755 | // it. | |
756 | if (ConstantSDNode *Sh = | |
757 | dyn_cast<ConstantSDNode>(N.getOperand(1).getOperand(1))) { | |
758 | ShAmt = Sh->getZExtValue(); | |
759 | if (isShifterOpProfitable(Offset, ShOpcVal, ShAmt)) | |
760 | Offset = N.getOperand(1).getOperand(0); | |
761 | else { | |
762 | ShAmt = 0; | |
763 | ShOpcVal = ARM_AM::no_shift; | |
764 | } | |
765 | } else { | |
766 | ShOpcVal = ARM_AM::no_shift; | |
767 | } | |
768 | } | |
769 | ||
770 | // Try matching (R shl C) + (R). | |
771 | if (N.getOpcode() != ISD::SUB && ShOpcVal == ARM_AM::no_shift && | |
772 | !(Subtarget->isLikeA9() || Subtarget->isSwift() || | |
773 | N.getOperand(0).hasOneUse())) { | |
774 | ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(0).getOpcode()); | |
775 | if (ShOpcVal != ARM_AM::no_shift) { | |
776 | // Check to see if the RHS of the shift is a constant, if not, we can't | |
777 | // fold it. | |
778 | if (ConstantSDNode *Sh = | |
779 | dyn_cast<ConstantSDNode>(N.getOperand(0).getOperand(1))) { | |
780 | ShAmt = Sh->getZExtValue(); | |
781 | if (isShifterOpProfitable(N.getOperand(0), ShOpcVal, ShAmt)) { | |
782 | Offset = N.getOperand(0).getOperand(0); | |
783 | Base = N.getOperand(1); | |
784 | } else { | |
785 | ShAmt = 0; | |
786 | ShOpcVal = ARM_AM::no_shift; | |
787 | } | |
788 | } else { | |
789 | ShOpcVal = ARM_AM::no_shift; | |
790 | } | |
791 | } | |
792 | } | |
793 | ||
794 | Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal), | |
795 | MVT::i32); | |
796 | return AM2_SHOP; | |
797 | } | |
798 | ||
799 | bool ARMDAGToDAGISel::SelectAddrMode2OffsetReg(SDNode *Op, SDValue N, | |
800 | SDValue &Offset, SDValue &Opc) { | |
801 | unsigned Opcode = Op->getOpcode(); | |
802 | ISD::MemIndexedMode AM = (Opcode == ISD::LOAD) | |
803 | ? cast<LoadSDNode>(Op)->getAddressingMode() | |
804 | : cast<StoreSDNode>(Op)->getAddressingMode(); | |
805 | ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC) | |
806 | ? ARM_AM::add : ARM_AM::sub; | |
807 | int Val; | |
808 | if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) | |
809 | return false; | |
810 | ||
811 | Offset = N; | |
812 | ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode()); | |
813 | unsigned ShAmt = 0; | |
814 | if (ShOpcVal != ARM_AM::no_shift) { | |
815 | // Check to see if the RHS of the shift is a constant, if not, we can't fold | |
816 | // it. | |
817 | if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(N.getOperand(1))) { | |
818 | ShAmt = Sh->getZExtValue(); | |
819 | if (isShifterOpProfitable(N, ShOpcVal, ShAmt)) | |
820 | Offset = N.getOperand(0); | |
821 | else { | |
822 | ShAmt = 0; | |
823 | ShOpcVal = ARM_AM::no_shift; | |
824 | } | |
825 | } else { | |
826 | ShOpcVal = ARM_AM::no_shift; | |
827 | } | |
828 | } | |
829 | ||
830 | Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal), | |
831 | MVT::i32); | |
832 | return true; | |
833 | } | |
834 | ||
835 | bool ARMDAGToDAGISel::SelectAddrMode2OffsetImmPre(SDNode *Op, SDValue N, | |
836 | SDValue &Offset, SDValue &Opc) { | |
837 | unsigned Opcode = Op->getOpcode(); | |
838 | ISD::MemIndexedMode AM = (Opcode == ISD::LOAD) | |
839 | ? cast<LoadSDNode>(Op)->getAddressingMode() | |
840 | : cast<StoreSDNode>(Op)->getAddressingMode(); | |
841 | ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC) | |
842 | ? ARM_AM::add : ARM_AM::sub; | |
843 | int Val; | |
844 | if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) { // 12 bits. | |
845 | if (AddSub == ARM_AM::sub) Val *= -1; | |
846 | Offset = CurDAG->getRegister(0, MVT::i32); | |
847 | Opc = CurDAG->getTargetConstant(Val, MVT::i32); | |
848 | return true; | |
849 | } | |
850 | ||
851 | return false; | |
852 | } | |
853 | ||
854 | ||
855 | bool ARMDAGToDAGISel::SelectAddrMode2OffsetImm(SDNode *Op, SDValue N, | |
856 | SDValue &Offset, SDValue &Opc) { | |
857 | unsigned Opcode = Op->getOpcode(); | |
858 | ISD::MemIndexedMode AM = (Opcode == ISD::LOAD) | |
859 | ? cast<LoadSDNode>(Op)->getAddressingMode() | |
860 | : cast<StoreSDNode>(Op)->getAddressingMode(); | |
861 | ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC) | |
862 | ? ARM_AM::add : ARM_AM::sub; | |
863 | int Val; | |
864 | if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) { // 12 bits. | |
865 | Offset = CurDAG->getRegister(0, MVT::i32); | |
866 | Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, Val, | |
867 | ARM_AM::no_shift), | |
868 | MVT::i32); | |
869 | return true; | |
870 | } | |
871 | ||
872 | return false; | |
873 | } | |
874 | ||
875 | bool ARMDAGToDAGISel::SelectAddrOffsetNone(SDValue N, SDValue &Base) { | |
876 | Base = N; | |
877 | return true; | |
878 | } | |
879 | ||
880 | bool ARMDAGToDAGISel::SelectAddrMode3(SDValue N, | |
881 | SDValue &Base, SDValue &Offset, | |
882 | SDValue &Opc) { | |
883 | if (N.getOpcode() == ISD::SUB) { | |
884 | // X - C is canonicalize to X + -C, no need to handle it here. | |
885 | Base = N.getOperand(0); | |
886 | Offset = N.getOperand(1); | |
887 | Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::sub, 0),MVT::i32); | |
888 | return true; | |
889 | } | |
890 | ||
891 | if (!CurDAG->isBaseWithConstantOffset(N)) { | |
892 | Base = N; | |
893 | if (N.getOpcode() == ISD::FrameIndex) { | |
894 | int FI = cast<FrameIndexSDNode>(N)->getIndex(); | |
85aaf69f | 895 | Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy()); |
223e47cc LB |
896 | } |
897 | Offset = CurDAG->getRegister(0, MVT::i32); | |
898 | Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0),MVT::i32); | |
899 | return true; | |
900 | } | |
901 | ||
902 | // If the RHS is +/- imm8, fold into addr mode. | |
903 | int RHSC; | |
904 | if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1, | |
905 | -256 + 1, 256, RHSC)) { // 8 bits. | |
906 | Base = N.getOperand(0); | |
907 | if (Base.getOpcode() == ISD::FrameIndex) { | |
908 | int FI = cast<FrameIndexSDNode>(Base)->getIndex(); | |
85aaf69f | 909 | Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy()); |
223e47cc LB |
910 | } |
911 | Offset = CurDAG->getRegister(0, MVT::i32); | |
912 | ||
913 | ARM_AM::AddrOpc AddSub = ARM_AM::add; | |
914 | if (RHSC < 0) { | |
915 | AddSub = ARM_AM::sub; | |
916 | RHSC = -RHSC; | |
917 | } | |
918 | Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, RHSC),MVT::i32); | |
919 | return true; | |
920 | } | |
921 | ||
922 | Base = N.getOperand(0); | |
923 | Offset = N.getOperand(1); | |
924 | Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0), MVT::i32); | |
925 | return true; | |
926 | } | |
927 | ||
928 | bool ARMDAGToDAGISel::SelectAddrMode3Offset(SDNode *Op, SDValue N, | |
929 | SDValue &Offset, SDValue &Opc) { | |
930 | unsigned Opcode = Op->getOpcode(); | |
931 | ISD::MemIndexedMode AM = (Opcode == ISD::LOAD) | |
932 | ? cast<LoadSDNode>(Op)->getAddressingMode() | |
933 | : cast<StoreSDNode>(Op)->getAddressingMode(); | |
934 | ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC) | |
935 | ? ARM_AM::add : ARM_AM::sub; | |
936 | int Val; | |
937 | if (isScaledConstantInRange(N, /*Scale=*/1, 0, 256, Val)) { // 12 bits. | |
938 | Offset = CurDAG->getRegister(0, MVT::i32); | |
939 | Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, Val), MVT::i32); | |
940 | return true; | |
941 | } | |
942 | ||
943 | Offset = N; | |
944 | Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, 0), MVT::i32); | |
945 | return true; | |
946 | } | |
947 | ||
948 | bool ARMDAGToDAGISel::SelectAddrMode5(SDValue N, | |
949 | SDValue &Base, SDValue &Offset) { | |
950 | if (!CurDAG->isBaseWithConstantOffset(N)) { | |
951 | Base = N; | |
952 | if (N.getOpcode() == ISD::FrameIndex) { | |
953 | int FI = cast<FrameIndexSDNode>(N)->getIndex(); | |
85aaf69f | 954 | Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy()); |
223e47cc | 955 | } else if (N.getOpcode() == ARMISD::Wrapper && |
1a4d82fc | 956 | N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress) { |
223e47cc LB |
957 | Base = N.getOperand(0); |
958 | } | |
959 | Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0), | |
960 | MVT::i32); | |
961 | return true; | |
962 | } | |
963 | ||
964 | // If the RHS is +/- imm8, fold into addr mode. | |
965 | int RHSC; | |
966 | if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/4, | |
967 | -256 + 1, 256, RHSC)) { | |
968 | Base = N.getOperand(0); | |
969 | if (Base.getOpcode() == ISD::FrameIndex) { | |
970 | int FI = cast<FrameIndexSDNode>(Base)->getIndex(); | |
85aaf69f | 971 | Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy()); |
223e47cc LB |
972 | } |
973 | ||
974 | ARM_AM::AddrOpc AddSub = ARM_AM::add; | |
975 | if (RHSC < 0) { | |
976 | AddSub = ARM_AM::sub; | |
977 | RHSC = -RHSC; | |
978 | } | |
979 | Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(AddSub, RHSC), | |
980 | MVT::i32); | |
981 | return true; | |
982 | } | |
983 | ||
984 | Base = N; | |
985 | Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0), | |
986 | MVT::i32); | |
987 | return true; | |
988 | } | |
989 | ||
990 | bool ARMDAGToDAGISel::SelectAddrMode6(SDNode *Parent, SDValue N, SDValue &Addr, | |
991 | SDValue &Align) { | |
992 | Addr = N; | |
993 | ||
994 | unsigned Alignment = 0; | |
995 | if (LSBaseSDNode *LSN = dyn_cast<LSBaseSDNode>(Parent)) { | |
996 | // This case occurs only for VLD1-lane/dup and VST1-lane instructions. | |
997 | // The maximum alignment is equal to the memory size being referenced. | |
998 | unsigned LSNAlign = LSN->getAlignment(); | |
999 | unsigned MemSize = LSN->getMemoryVT().getSizeInBits() / 8; | |
1000 | if (LSNAlign >= MemSize && MemSize > 1) | |
1001 | Alignment = MemSize; | |
1002 | } else { | |
1003 | // All other uses of addrmode6 are for intrinsics. For now just record | |
1004 | // the raw alignment value; it will be refined later based on the legal | |
1005 | // alignment operands for the intrinsic. | |
1006 | Alignment = cast<MemIntrinsicSDNode>(Parent)->getAlignment(); | |
1007 | } | |
1008 | ||
1009 | Align = CurDAG->getTargetConstant(Alignment, MVT::i32); | |
1010 | return true; | |
1011 | } | |
1012 | ||
1013 | bool ARMDAGToDAGISel::SelectAddrMode6Offset(SDNode *Op, SDValue N, | |
1014 | SDValue &Offset) { | |
1015 | LSBaseSDNode *LdSt = cast<LSBaseSDNode>(Op); | |
1016 | ISD::MemIndexedMode AM = LdSt->getAddressingMode(); | |
1017 | if (AM != ISD::POST_INC) | |
1018 | return false; | |
1019 | Offset = N; | |
1020 | if (ConstantSDNode *NC = dyn_cast<ConstantSDNode>(N)) { | |
1021 | if (NC->getZExtValue() * 8 == LdSt->getMemoryVT().getSizeInBits()) | |
1022 | Offset = CurDAG->getRegister(0, MVT::i32); | |
1023 | } | |
1024 | return true; | |
1025 | } | |
1026 | ||
1027 | bool ARMDAGToDAGISel::SelectAddrModePC(SDValue N, | |
1028 | SDValue &Offset, SDValue &Label) { | |
1029 | if (N.getOpcode() == ARMISD::PIC_ADD && N.hasOneUse()) { | |
1030 | Offset = N.getOperand(0); | |
1031 | SDValue N1 = N.getOperand(1); | |
1032 | Label = CurDAG->getTargetConstant(cast<ConstantSDNode>(N1)->getZExtValue(), | |
1033 | MVT::i32); | |
1034 | return true; | |
1035 | } | |
1036 | ||
1037 | return false; | |
1038 | } | |
1039 | ||
1040 | ||
1041 | //===----------------------------------------------------------------------===// | |
1042 | // Thumb Addressing Modes | |
1043 | //===----------------------------------------------------------------------===// | |
1044 | ||
1045 | bool ARMDAGToDAGISel::SelectThumbAddrModeRR(SDValue N, | |
1046 | SDValue &Base, SDValue &Offset){ | |
1047 | if (N.getOpcode() != ISD::ADD && !CurDAG->isBaseWithConstantOffset(N)) { | |
1048 | ConstantSDNode *NC = dyn_cast<ConstantSDNode>(N); | |
1049 | if (!NC || !NC->isNullValue()) | |
1050 | return false; | |
1051 | ||
1052 | Base = Offset = N; | |
1053 | return true; | |
1054 | } | |
1055 | ||
1056 | Base = N.getOperand(0); | |
1057 | Offset = N.getOperand(1); | |
1058 | return true; | |
1059 | } | |
1060 | ||
1061 | bool | |
1062 | ARMDAGToDAGISel::SelectThumbAddrModeRI(SDValue N, SDValue &Base, | |
1063 | SDValue &Offset, unsigned Scale) { | |
1064 | if (Scale == 4) { | |
1065 | SDValue TmpBase, TmpOffImm; | |
1066 | if (SelectThumbAddrModeSP(N, TmpBase, TmpOffImm)) | |
1067 | return false; // We want to select tLDRspi / tSTRspi instead. | |
1068 | ||
1069 | if (N.getOpcode() == ARMISD::Wrapper && | |
1070 | N.getOperand(0).getOpcode() == ISD::TargetConstantPool) | |
1071 | return false; // We want to select tLDRpci instead. | |
1072 | } | |
1073 | ||
1074 | if (!CurDAG->isBaseWithConstantOffset(N)) | |
1075 | return false; | |
1076 | ||
1077 | // Thumb does not have [sp, r] address mode. | |
1078 | RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0)); | |
1079 | RegisterSDNode *RHSR = dyn_cast<RegisterSDNode>(N.getOperand(1)); | |
1080 | if ((LHSR && LHSR->getReg() == ARM::SP) || | |
1081 | (RHSR && RHSR->getReg() == ARM::SP)) | |
1082 | return false; | |
1083 | ||
1084 | // FIXME: Why do we explicitly check for a match here and then return false? | |
1085 | // Presumably to allow something else to match, but shouldn't this be | |
1086 | // documented? | |
1087 | int RHSC; | |
1088 | if (isScaledConstantInRange(N.getOperand(1), Scale, 0, 32, RHSC)) | |
1089 | return false; | |
1090 | ||
1091 | Base = N.getOperand(0); | |
1092 | Offset = N.getOperand(1); | |
1093 | return true; | |
1094 | } | |
1095 | ||
1096 | bool | |
1097 | ARMDAGToDAGISel::SelectThumbAddrModeRI5S1(SDValue N, | |
1098 | SDValue &Base, | |
1099 | SDValue &Offset) { | |
1100 | return SelectThumbAddrModeRI(N, Base, Offset, 1); | |
1101 | } | |
1102 | ||
1103 | bool | |
1104 | ARMDAGToDAGISel::SelectThumbAddrModeRI5S2(SDValue N, | |
1105 | SDValue &Base, | |
1106 | SDValue &Offset) { | |
1107 | return SelectThumbAddrModeRI(N, Base, Offset, 2); | |
1108 | } | |
1109 | ||
1110 | bool | |
1111 | ARMDAGToDAGISel::SelectThumbAddrModeRI5S4(SDValue N, | |
1112 | SDValue &Base, | |
1113 | SDValue &Offset) { | |
1114 | return SelectThumbAddrModeRI(N, Base, Offset, 4); | |
1115 | } | |
1116 | ||
1117 | bool | |
1118 | ARMDAGToDAGISel::SelectThumbAddrModeImm5S(SDValue N, unsigned Scale, | |
1119 | SDValue &Base, SDValue &OffImm) { | |
1120 | if (Scale == 4) { | |
1121 | SDValue TmpBase, TmpOffImm; | |
1122 | if (SelectThumbAddrModeSP(N, TmpBase, TmpOffImm)) | |
1123 | return false; // We want to select tLDRspi / tSTRspi instead. | |
1124 | ||
1125 | if (N.getOpcode() == ARMISD::Wrapper && | |
1126 | N.getOperand(0).getOpcode() == ISD::TargetConstantPool) | |
1127 | return false; // We want to select tLDRpci instead. | |
1128 | } | |
1129 | ||
1130 | if (!CurDAG->isBaseWithConstantOffset(N)) { | |
1131 | if (N.getOpcode() == ARMISD::Wrapper && | |
1a4d82fc | 1132 | N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress) { |
223e47cc LB |
1133 | Base = N.getOperand(0); |
1134 | } else { | |
1135 | Base = N; | |
1136 | } | |
1137 | ||
1138 | OffImm = CurDAG->getTargetConstant(0, MVT::i32); | |
1139 | return true; | |
1140 | } | |
1141 | ||
1142 | RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0)); | |
1143 | RegisterSDNode *RHSR = dyn_cast<RegisterSDNode>(N.getOperand(1)); | |
1144 | if ((LHSR && LHSR->getReg() == ARM::SP) || | |
1145 | (RHSR && RHSR->getReg() == ARM::SP)) { | |
1146 | ConstantSDNode *LHS = dyn_cast<ConstantSDNode>(N.getOperand(0)); | |
1147 | ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1)); | |
1148 | unsigned LHSC = LHS ? LHS->getZExtValue() : 0; | |
1149 | unsigned RHSC = RHS ? RHS->getZExtValue() : 0; | |
1150 | ||
1151 | // Thumb does not have [sp, #imm5] address mode for non-zero imm5. | |
1152 | if (LHSC != 0 || RHSC != 0) return false; | |
1153 | ||
1154 | Base = N; | |
1155 | OffImm = CurDAG->getTargetConstant(0, MVT::i32); | |
1156 | return true; | |
1157 | } | |
1158 | ||
1159 | // If the RHS is + imm5 * scale, fold into addr mode. | |
1160 | int RHSC; | |
1161 | if (isScaledConstantInRange(N.getOperand(1), Scale, 0, 32, RHSC)) { | |
1162 | Base = N.getOperand(0); | |
1163 | OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32); | |
1164 | return true; | |
1165 | } | |
1166 | ||
1167 | Base = N.getOperand(0); | |
1168 | OffImm = CurDAG->getTargetConstant(0, MVT::i32); | |
1169 | return true; | |
1170 | } | |
1171 | ||
1172 | bool | |
1173 | ARMDAGToDAGISel::SelectThumbAddrModeImm5S4(SDValue N, SDValue &Base, | |
1174 | SDValue &OffImm) { | |
1175 | return SelectThumbAddrModeImm5S(N, 4, Base, OffImm); | |
1176 | } | |
1177 | ||
1178 | bool | |
1179 | ARMDAGToDAGISel::SelectThumbAddrModeImm5S2(SDValue N, SDValue &Base, | |
1180 | SDValue &OffImm) { | |
1181 | return SelectThumbAddrModeImm5S(N, 2, Base, OffImm); | |
1182 | } | |
1183 | ||
1184 | bool | |
1185 | ARMDAGToDAGISel::SelectThumbAddrModeImm5S1(SDValue N, SDValue &Base, | |
1186 | SDValue &OffImm) { | |
1187 | return SelectThumbAddrModeImm5S(N, 1, Base, OffImm); | |
1188 | } | |
1189 | ||
1190 | bool ARMDAGToDAGISel::SelectThumbAddrModeSP(SDValue N, | |
1191 | SDValue &Base, SDValue &OffImm) { | |
1192 | if (N.getOpcode() == ISD::FrameIndex) { | |
1193 | int FI = cast<FrameIndexSDNode>(N)->getIndex(); | |
85aaf69f | 1194 | Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy()); |
223e47cc LB |
1195 | OffImm = CurDAG->getTargetConstant(0, MVT::i32); |
1196 | return true; | |
1197 | } | |
1198 | ||
1199 | if (!CurDAG->isBaseWithConstantOffset(N)) | |
1200 | return false; | |
1201 | ||
1202 | RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0)); | |
1203 | if (N.getOperand(0).getOpcode() == ISD::FrameIndex || | |
1204 | (LHSR && LHSR->getReg() == ARM::SP)) { | |
1205 | // If the RHS is + imm8 * scale, fold into addr mode. | |
1206 | int RHSC; | |
1207 | if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/4, 0, 256, RHSC)) { | |
1208 | Base = N.getOperand(0); | |
1209 | if (Base.getOpcode() == ISD::FrameIndex) { | |
1210 | int FI = cast<FrameIndexSDNode>(Base)->getIndex(); | |
85aaf69f | 1211 | Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy()); |
223e47cc LB |
1212 | } |
1213 | OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32); | |
1214 | return true; | |
1215 | } | |
1216 | } | |
1217 | ||
1218 | return false; | |
1219 | } | |
1220 | ||
1221 | ||
1222 | //===----------------------------------------------------------------------===// | |
1223 | // Thumb 2 Addressing Modes | |
1224 | //===----------------------------------------------------------------------===// | |
1225 | ||
1226 | ||
1227 | bool ARMDAGToDAGISel::SelectT2ShifterOperandReg(SDValue N, SDValue &BaseReg, | |
1228 | SDValue &Opc) { | |
1229 | if (DisableShifterOp) | |
1230 | return false; | |
1231 | ||
1232 | ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode()); | |
1233 | ||
1234 | // Don't match base register only case. That is matched to a separate | |
1235 | // lower complexity pattern with explicit register operand. | |
1236 | if (ShOpcVal == ARM_AM::no_shift) return false; | |
1237 | ||
1238 | BaseReg = N.getOperand(0); | |
1239 | unsigned ShImmVal = 0; | |
1240 | if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) { | |
1241 | ShImmVal = RHS->getZExtValue() & 31; | |
1242 | Opc = getI32Imm(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal)); | |
1243 | return true; | |
1244 | } | |
1245 | ||
1246 | return false; | |
1247 | } | |
1248 | ||
1249 | bool ARMDAGToDAGISel::SelectT2AddrModeImm12(SDValue N, | |
1250 | SDValue &Base, SDValue &OffImm) { | |
1251 | // Match simple R + imm12 operands. | |
1252 | ||
1253 | // Base only. | |
1254 | if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB && | |
1255 | !CurDAG->isBaseWithConstantOffset(N)) { | |
1256 | if (N.getOpcode() == ISD::FrameIndex) { | |
1257 | // Match frame index. | |
1258 | int FI = cast<FrameIndexSDNode>(N)->getIndex(); | |
85aaf69f | 1259 | Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy()); |
223e47cc LB |
1260 | OffImm = CurDAG->getTargetConstant(0, MVT::i32); |
1261 | return true; | |
1262 | } | |
1263 | ||
1264 | if (N.getOpcode() == ARMISD::Wrapper && | |
1a4d82fc | 1265 | N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress) { |
223e47cc LB |
1266 | Base = N.getOperand(0); |
1267 | if (Base.getOpcode() == ISD::TargetConstantPool) | |
1268 | return false; // We want to select t2LDRpci instead. | |
1269 | } else | |
1270 | Base = N; | |
1271 | OffImm = CurDAG->getTargetConstant(0, MVT::i32); | |
1272 | return true; | |
1273 | } | |
1274 | ||
1275 | if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) { | |
1276 | if (SelectT2AddrModeImm8(N, Base, OffImm)) | |
1277 | // Let t2LDRi8 handle (R - imm8). | |
1278 | return false; | |
1279 | ||
1280 | int RHSC = (int)RHS->getZExtValue(); | |
1281 | if (N.getOpcode() == ISD::SUB) | |
1282 | RHSC = -RHSC; | |
1283 | ||
1284 | if (RHSC >= 0 && RHSC < 0x1000) { // 12 bits (unsigned) | |
1285 | Base = N.getOperand(0); | |
1286 | if (Base.getOpcode() == ISD::FrameIndex) { | |
1287 | int FI = cast<FrameIndexSDNode>(Base)->getIndex(); | |
85aaf69f | 1288 | Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy()); |
223e47cc LB |
1289 | } |
1290 | OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32); | |
1291 | return true; | |
1292 | } | |
1293 | } | |
1294 | ||
1295 | // Base only. | |
1296 | Base = N; | |
1297 | OffImm = CurDAG->getTargetConstant(0, MVT::i32); | |
1298 | return true; | |
1299 | } | |
1300 | ||
1301 | bool ARMDAGToDAGISel::SelectT2AddrModeImm8(SDValue N, | |
1302 | SDValue &Base, SDValue &OffImm) { | |
1303 | // Match simple R - imm8 operands. | |
1304 | if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB && | |
1305 | !CurDAG->isBaseWithConstantOffset(N)) | |
1306 | return false; | |
1307 | ||
1308 | if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) { | |
1309 | int RHSC = (int)RHS->getSExtValue(); | |
1310 | if (N.getOpcode() == ISD::SUB) | |
1311 | RHSC = -RHSC; | |
1312 | ||
1313 | if ((RHSC >= -255) && (RHSC < 0)) { // 8 bits (always negative) | |
1314 | Base = N.getOperand(0); | |
1315 | if (Base.getOpcode() == ISD::FrameIndex) { | |
1316 | int FI = cast<FrameIndexSDNode>(Base)->getIndex(); | |
85aaf69f | 1317 | Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy()); |
223e47cc LB |
1318 | } |
1319 | OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32); | |
1320 | return true; | |
1321 | } | |
1322 | } | |
1323 | ||
1324 | return false; | |
1325 | } | |
1326 | ||
1327 | bool ARMDAGToDAGISel::SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N, | |
1328 | SDValue &OffImm){ | |
1329 | unsigned Opcode = Op->getOpcode(); | |
1330 | ISD::MemIndexedMode AM = (Opcode == ISD::LOAD) | |
1331 | ? cast<LoadSDNode>(Op)->getAddressingMode() | |
1332 | : cast<StoreSDNode>(Op)->getAddressingMode(); | |
1333 | int RHSC; | |
1334 | if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x100, RHSC)) { // 8 bits. | |
1335 | OffImm = ((AM == ISD::PRE_INC) || (AM == ISD::POST_INC)) | |
1336 | ? CurDAG->getTargetConstant(RHSC, MVT::i32) | |
1337 | : CurDAG->getTargetConstant(-RHSC, MVT::i32); | |
1338 | return true; | |
1339 | } | |
1340 | ||
1341 | return false; | |
1342 | } | |
1343 | ||
1344 | bool ARMDAGToDAGISel::SelectT2AddrModeSoReg(SDValue N, | |
1345 | SDValue &Base, | |
1346 | SDValue &OffReg, SDValue &ShImm) { | |
1347 | // (R - imm8) should be handled by t2LDRi8. The rest are handled by t2LDRi12. | |
1348 | if (N.getOpcode() != ISD::ADD && !CurDAG->isBaseWithConstantOffset(N)) | |
1349 | return false; | |
1350 | ||
1351 | // Leave (R + imm12) for t2LDRi12, (R - imm8) for t2LDRi8. | |
1352 | if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) { | |
1353 | int RHSC = (int)RHS->getZExtValue(); | |
1354 | if (RHSC >= 0 && RHSC < 0x1000) // 12 bits (unsigned) | |
1355 | return false; | |
1356 | else if (RHSC < 0 && RHSC >= -255) // 8 bits | |
1357 | return false; | |
1358 | } | |
1359 | ||
1360 | // Look for (R + R) or (R + (R << [1,2,3])). | |
1361 | unsigned ShAmt = 0; | |
1362 | Base = N.getOperand(0); | |
1363 | OffReg = N.getOperand(1); | |
1364 | ||
1365 | // Swap if it is ((R << c) + R). | |
1366 | ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(OffReg.getOpcode()); | |
1367 | if (ShOpcVal != ARM_AM::lsl) { | |
1368 | ShOpcVal = ARM_AM::getShiftOpcForNode(Base.getOpcode()); | |
1369 | if (ShOpcVal == ARM_AM::lsl) | |
1370 | std::swap(Base, OffReg); | |
1371 | } | |
1372 | ||
1373 | if (ShOpcVal == ARM_AM::lsl) { | |
1374 | // Check to see if the RHS of the shift is a constant, if not, we can't fold | |
1375 | // it. | |
1376 | if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(OffReg.getOperand(1))) { | |
1377 | ShAmt = Sh->getZExtValue(); | |
1378 | if (ShAmt < 4 && isShifterOpProfitable(OffReg, ShOpcVal, ShAmt)) | |
1379 | OffReg = OffReg.getOperand(0); | |
1380 | else { | |
1381 | ShAmt = 0; | |
223e47cc | 1382 | } |
223e47cc LB |
1383 | } |
1384 | } | |
1385 | ||
1386 | ShImm = CurDAG->getTargetConstant(ShAmt, MVT::i32); | |
1387 | ||
1388 | return true; | |
1389 | } | |
1390 | ||
1a4d82fc JJ |
1391 | bool ARMDAGToDAGISel::SelectT2AddrModeExclusive(SDValue N, SDValue &Base, |
1392 | SDValue &OffImm) { | |
1393 | // This *must* succeed since it's used for the irreplaceable ldrex and strex | |
1394 | // instructions. | |
1395 | Base = N; | |
1396 | OffImm = CurDAG->getTargetConstant(0, MVT::i32); | |
1397 | ||
1398 | if (N.getOpcode() != ISD::ADD || !CurDAG->isBaseWithConstantOffset(N)) | |
1399 | return true; | |
1400 | ||
1401 | ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1)); | |
1402 | if (!RHS) | |
1403 | return true; | |
1404 | ||
1405 | uint32_t RHSC = (int)RHS->getZExtValue(); | |
1406 | if (RHSC > 1020 || RHSC % 4 != 0) | |
1407 | return true; | |
1408 | ||
1409 | Base = N.getOperand(0); | |
1410 | if (Base.getOpcode() == ISD::FrameIndex) { | |
1411 | int FI = cast<FrameIndexSDNode>(Base)->getIndex(); | |
85aaf69f | 1412 | Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy()); |
1a4d82fc JJ |
1413 | } |
1414 | ||
1415 | OffImm = CurDAG->getTargetConstant(RHSC / 4, MVT::i32); | |
1416 | return true; | |
1417 | } | |
1418 | ||
223e47cc LB |
1419 | //===--------------------------------------------------------------------===// |
1420 | ||
1421 | /// getAL - Returns a ARMCC::AL immediate node. | |
1422 | static inline SDValue getAL(SelectionDAG *CurDAG) { | |
1423 | return CurDAG->getTargetConstant((uint64_t)ARMCC::AL, MVT::i32); | |
1424 | } | |
1425 | ||
1426 | SDNode *ARMDAGToDAGISel::SelectARMIndexedLoad(SDNode *N) { | |
1427 | LoadSDNode *LD = cast<LoadSDNode>(N); | |
1428 | ISD::MemIndexedMode AM = LD->getAddressingMode(); | |
1429 | if (AM == ISD::UNINDEXED) | |
1a4d82fc | 1430 | return nullptr; |
223e47cc LB |
1431 | |
1432 | EVT LoadedVT = LD->getMemoryVT(); | |
1433 | SDValue Offset, AMOpc; | |
1434 | bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC); | |
1435 | unsigned Opcode = 0; | |
1436 | bool Match = false; | |
1437 | if (LoadedVT == MVT::i32 && isPre && | |
1438 | SelectAddrMode2OffsetImmPre(N, LD->getOffset(), Offset, AMOpc)) { | |
1439 | Opcode = ARM::LDR_PRE_IMM; | |
1440 | Match = true; | |
1441 | } else if (LoadedVT == MVT::i32 && !isPre && | |
1442 | SelectAddrMode2OffsetImm(N, LD->getOffset(), Offset, AMOpc)) { | |
1443 | Opcode = ARM::LDR_POST_IMM; | |
1444 | Match = true; | |
1445 | } else if (LoadedVT == MVT::i32 && | |
1446 | SelectAddrMode2OffsetReg(N, LD->getOffset(), Offset, AMOpc)) { | |
1447 | Opcode = isPre ? ARM::LDR_PRE_REG : ARM::LDR_POST_REG; | |
1448 | Match = true; | |
1449 | ||
1450 | } else if (LoadedVT == MVT::i16 && | |
1451 | SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) { | |
1452 | Match = true; | |
1453 | Opcode = (LD->getExtensionType() == ISD::SEXTLOAD) | |
1454 | ? (isPre ? ARM::LDRSH_PRE : ARM::LDRSH_POST) | |
1455 | : (isPre ? ARM::LDRH_PRE : ARM::LDRH_POST); | |
1456 | } else if (LoadedVT == MVT::i8 || LoadedVT == MVT::i1) { | |
1457 | if (LD->getExtensionType() == ISD::SEXTLOAD) { | |
1458 | if (SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) { | |
1459 | Match = true; | |
1460 | Opcode = isPre ? ARM::LDRSB_PRE : ARM::LDRSB_POST; | |
1461 | } | |
1462 | } else { | |
1463 | if (isPre && | |
1464 | SelectAddrMode2OffsetImmPre(N, LD->getOffset(), Offset, AMOpc)) { | |
1465 | Match = true; | |
1466 | Opcode = ARM::LDRB_PRE_IMM; | |
1467 | } else if (!isPre && | |
1468 | SelectAddrMode2OffsetImm(N, LD->getOffset(), Offset, AMOpc)) { | |
1469 | Match = true; | |
1470 | Opcode = ARM::LDRB_POST_IMM; | |
1471 | } else if (SelectAddrMode2OffsetReg(N, LD->getOffset(), Offset, AMOpc)) { | |
1472 | Match = true; | |
1473 | Opcode = isPre ? ARM::LDRB_PRE_REG : ARM::LDRB_POST_REG; | |
1474 | } | |
1475 | } | |
1476 | } | |
1477 | ||
1478 | if (Match) { | |
1479 | if (Opcode == ARM::LDR_PRE_IMM || Opcode == ARM::LDRB_PRE_IMM) { | |
1480 | SDValue Chain = LD->getChain(); | |
1481 | SDValue Base = LD->getBasePtr(); | |
1482 | SDValue Ops[]= { Base, AMOpc, getAL(CurDAG), | |
1483 | CurDAG->getRegister(0, MVT::i32), Chain }; | |
1a4d82fc JJ |
1484 | return CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32, |
1485 | MVT::i32, MVT::Other, Ops); | |
223e47cc LB |
1486 | } else { |
1487 | SDValue Chain = LD->getChain(); | |
1488 | SDValue Base = LD->getBasePtr(); | |
1489 | SDValue Ops[]= { Base, Offset, AMOpc, getAL(CurDAG), | |
1490 | CurDAG->getRegister(0, MVT::i32), Chain }; | |
1a4d82fc JJ |
1491 | return CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32, |
1492 | MVT::i32, MVT::Other, Ops); | |
223e47cc LB |
1493 | } |
1494 | } | |
1495 | ||
1a4d82fc | 1496 | return nullptr; |
223e47cc LB |
1497 | } |
1498 | ||
1499 | SDNode *ARMDAGToDAGISel::SelectT2IndexedLoad(SDNode *N) { | |
1500 | LoadSDNode *LD = cast<LoadSDNode>(N); | |
1501 | ISD::MemIndexedMode AM = LD->getAddressingMode(); | |
1502 | if (AM == ISD::UNINDEXED) | |
1a4d82fc | 1503 | return nullptr; |
223e47cc LB |
1504 | |
1505 | EVT LoadedVT = LD->getMemoryVT(); | |
1506 | bool isSExtLd = LD->getExtensionType() == ISD::SEXTLOAD; | |
1507 | SDValue Offset; | |
1508 | bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC); | |
1509 | unsigned Opcode = 0; | |
1510 | bool Match = false; | |
1511 | if (SelectT2AddrModeImm8Offset(N, LD->getOffset(), Offset)) { | |
1512 | switch (LoadedVT.getSimpleVT().SimpleTy) { | |
1513 | case MVT::i32: | |
1514 | Opcode = isPre ? ARM::t2LDR_PRE : ARM::t2LDR_POST; | |
1515 | break; | |
1516 | case MVT::i16: | |
1517 | if (isSExtLd) | |
1518 | Opcode = isPre ? ARM::t2LDRSH_PRE : ARM::t2LDRSH_POST; | |
1519 | else | |
1520 | Opcode = isPre ? ARM::t2LDRH_PRE : ARM::t2LDRH_POST; | |
1521 | break; | |
1522 | case MVT::i8: | |
1523 | case MVT::i1: | |
1524 | if (isSExtLd) | |
1525 | Opcode = isPre ? ARM::t2LDRSB_PRE : ARM::t2LDRSB_POST; | |
1526 | else | |
1527 | Opcode = isPre ? ARM::t2LDRB_PRE : ARM::t2LDRB_POST; | |
1528 | break; | |
1529 | default: | |
1a4d82fc | 1530 | return nullptr; |
223e47cc LB |
1531 | } |
1532 | Match = true; | |
1533 | } | |
1534 | ||
1535 | if (Match) { | |
1536 | SDValue Chain = LD->getChain(); | |
1537 | SDValue Base = LD->getBasePtr(); | |
1538 | SDValue Ops[]= { Base, Offset, getAL(CurDAG), | |
1539 | CurDAG->getRegister(0, MVT::i32), Chain }; | |
1a4d82fc JJ |
1540 | return CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32, MVT::i32, |
1541 | MVT::Other, Ops); | |
223e47cc LB |
1542 | } |
1543 | ||
1a4d82fc | 1544 | return nullptr; |
223e47cc LB |
1545 | } |
1546 | ||
970d7e83 LB |
1547 | /// \brief Form a GPRPair pseudo register from a pair of GPR regs. |
1548 | SDNode *ARMDAGToDAGISel::createGPRPairNode(EVT VT, SDValue V0, SDValue V1) { | |
1a4d82fc | 1549 | SDLoc dl(V0.getNode()); |
970d7e83 LB |
1550 | SDValue RegClass = |
1551 | CurDAG->getTargetConstant(ARM::GPRPairRegClassID, MVT::i32); | |
1552 | SDValue SubReg0 = CurDAG->getTargetConstant(ARM::gsub_0, MVT::i32); | |
1553 | SDValue SubReg1 = CurDAG->getTargetConstant(ARM::gsub_1, MVT::i32); | |
1554 | const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 }; | |
1a4d82fc | 1555 | return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops); |
970d7e83 LB |
1556 | } |
1557 | ||
1558 | /// \brief Form a D register from a pair of S registers. | |
1559 | SDNode *ARMDAGToDAGISel::createSRegPairNode(EVT VT, SDValue V0, SDValue V1) { | |
1a4d82fc | 1560 | SDLoc dl(V0.getNode()); |
223e47cc LB |
1561 | SDValue RegClass = |
1562 | CurDAG->getTargetConstant(ARM::DPR_VFP2RegClassID, MVT::i32); | |
1563 | SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, MVT::i32); | |
1564 | SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, MVT::i32); | |
1565 | const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 }; | |
1a4d82fc | 1566 | return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops); |
223e47cc LB |
1567 | } |
1568 | ||
970d7e83 LB |
1569 | /// \brief Form a quad register from a pair of D registers. |
1570 | SDNode *ARMDAGToDAGISel::createDRegPairNode(EVT VT, SDValue V0, SDValue V1) { | |
1a4d82fc | 1571 | SDLoc dl(V0.getNode()); |
223e47cc LB |
1572 | SDValue RegClass = CurDAG->getTargetConstant(ARM::QPRRegClassID, MVT::i32); |
1573 | SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, MVT::i32); | |
1574 | SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, MVT::i32); | |
1575 | const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 }; | |
1a4d82fc | 1576 | return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops); |
223e47cc LB |
1577 | } |
1578 | ||
970d7e83 LB |
1579 | /// \brief Form 4 consecutive D registers from a pair of Q registers. |
1580 | SDNode *ARMDAGToDAGISel::createQRegPairNode(EVT VT, SDValue V0, SDValue V1) { | |
1a4d82fc | 1581 | SDLoc dl(V0.getNode()); |
223e47cc LB |
1582 | SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, MVT::i32); |
1583 | SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, MVT::i32); | |
1584 | SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, MVT::i32); | |
1585 | const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 }; | |
1a4d82fc | 1586 | return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops); |
223e47cc LB |
1587 | } |
1588 | ||
970d7e83 LB |
1589 | /// \brief Form 4 consecutive S registers. |
1590 | SDNode *ARMDAGToDAGISel::createQuadSRegsNode(EVT VT, SDValue V0, SDValue V1, | |
223e47cc | 1591 | SDValue V2, SDValue V3) { |
1a4d82fc | 1592 | SDLoc dl(V0.getNode()); |
223e47cc LB |
1593 | SDValue RegClass = |
1594 | CurDAG->getTargetConstant(ARM::QPR_VFP2RegClassID, MVT::i32); | |
1595 | SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, MVT::i32); | |
1596 | SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, MVT::i32); | |
1597 | SDValue SubReg2 = CurDAG->getTargetConstant(ARM::ssub_2, MVT::i32); | |
1598 | SDValue SubReg3 = CurDAG->getTargetConstant(ARM::ssub_3, MVT::i32); | |
1599 | const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1, | |
1600 | V2, SubReg2, V3, SubReg3 }; | |
1a4d82fc | 1601 | return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops); |
223e47cc LB |
1602 | } |
1603 | ||
970d7e83 LB |
1604 | /// \brief Form 4 consecutive D registers. |
1605 | SDNode *ARMDAGToDAGISel::createQuadDRegsNode(EVT VT, SDValue V0, SDValue V1, | |
223e47cc | 1606 | SDValue V2, SDValue V3) { |
1a4d82fc | 1607 | SDLoc dl(V0.getNode()); |
223e47cc LB |
1608 | SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, MVT::i32); |
1609 | SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, MVT::i32); | |
1610 | SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, MVT::i32); | |
1611 | SDValue SubReg2 = CurDAG->getTargetConstant(ARM::dsub_2, MVT::i32); | |
1612 | SDValue SubReg3 = CurDAG->getTargetConstant(ARM::dsub_3, MVT::i32); | |
1613 | const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1, | |
1614 | V2, SubReg2, V3, SubReg3 }; | |
1a4d82fc | 1615 | return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops); |
223e47cc LB |
1616 | } |
1617 | ||
970d7e83 LB |
1618 | /// \brief Form 4 consecutive Q registers. |
1619 | SDNode *ARMDAGToDAGISel::createQuadQRegsNode(EVT VT, SDValue V0, SDValue V1, | |
223e47cc | 1620 | SDValue V2, SDValue V3) { |
1a4d82fc | 1621 | SDLoc dl(V0.getNode()); |
223e47cc LB |
1622 | SDValue RegClass = CurDAG->getTargetConstant(ARM::QQQQPRRegClassID, MVT::i32); |
1623 | SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, MVT::i32); | |
1624 | SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, MVT::i32); | |
1625 | SDValue SubReg2 = CurDAG->getTargetConstant(ARM::qsub_2, MVT::i32); | |
1626 | SDValue SubReg3 = CurDAG->getTargetConstant(ARM::qsub_3, MVT::i32); | |
1627 | const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1, | |
1628 | V2, SubReg2, V3, SubReg3 }; | |
1a4d82fc | 1629 | return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops); |
223e47cc LB |
1630 | } |
1631 | ||
1632 | /// GetVLDSTAlign - Get the alignment (in bytes) for the alignment operand | |
1633 | /// of a NEON VLD or VST instruction. The supported values depend on the | |
1634 | /// number of registers being loaded. | |
1635 | SDValue ARMDAGToDAGISel::GetVLDSTAlign(SDValue Align, unsigned NumVecs, | |
1636 | bool is64BitVector) { | |
1637 | unsigned NumRegs = NumVecs; | |
1638 | if (!is64BitVector && NumVecs < 3) | |
1639 | NumRegs *= 2; | |
1640 | ||
1641 | unsigned Alignment = cast<ConstantSDNode>(Align)->getZExtValue(); | |
1642 | if (Alignment >= 32 && NumRegs == 4) | |
1643 | Alignment = 32; | |
1644 | else if (Alignment >= 16 && (NumRegs == 2 || NumRegs == 4)) | |
1645 | Alignment = 16; | |
1646 | else if (Alignment >= 8) | |
1647 | Alignment = 8; | |
1648 | else | |
1649 | Alignment = 0; | |
1650 | ||
1651 | return CurDAG->getTargetConstant(Alignment, MVT::i32); | |
1652 | } | |
1653 | ||
1a4d82fc JJ |
1654 | static bool isVLDfixed(unsigned Opc) |
1655 | { | |
1656 | switch (Opc) { | |
1657 | default: return false; | |
1658 | case ARM::VLD1d8wb_fixed : return true; | |
1659 | case ARM::VLD1d16wb_fixed : return true; | |
1660 | case ARM::VLD1d64Qwb_fixed : return true; | |
1661 | case ARM::VLD1d32wb_fixed : return true; | |
1662 | case ARM::VLD1d64wb_fixed : return true; | |
1663 | case ARM::VLD1d64TPseudoWB_fixed : return true; | |
1664 | case ARM::VLD1d64QPseudoWB_fixed : return true; | |
1665 | case ARM::VLD1q8wb_fixed : return true; | |
1666 | case ARM::VLD1q16wb_fixed : return true; | |
1667 | case ARM::VLD1q32wb_fixed : return true; | |
1668 | case ARM::VLD1q64wb_fixed : return true; | |
1669 | case ARM::VLD2d8wb_fixed : return true; | |
1670 | case ARM::VLD2d16wb_fixed : return true; | |
1671 | case ARM::VLD2d32wb_fixed : return true; | |
1672 | case ARM::VLD2q8PseudoWB_fixed : return true; | |
1673 | case ARM::VLD2q16PseudoWB_fixed : return true; | |
1674 | case ARM::VLD2q32PseudoWB_fixed : return true; | |
1675 | case ARM::VLD2DUPd8wb_fixed : return true; | |
1676 | case ARM::VLD2DUPd16wb_fixed : return true; | |
1677 | case ARM::VLD2DUPd32wb_fixed : return true; | |
1678 | } | |
1679 | } | |
1680 | ||
1681 | static bool isVSTfixed(unsigned Opc) | |
1682 | { | |
1683 | switch (Opc) { | |
1684 | default: return false; | |
1685 | case ARM::VST1d8wb_fixed : return true; | |
1686 | case ARM::VST1d16wb_fixed : return true; | |
1687 | case ARM::VST1d32wb_fixed : return true; | |
1688 | case ARM::VST1d64wb_fixed : return true; | |
1689 | case ARM::VST1q8wb_fixed : return true; | |
1690 | case ARM::VST1q16wb_fixed : return true; | |
1691 | case ARM::VST1q32wb_fixed : return true; | |
1692 | case ARM::VST1q64wb_fixed : return true; | |
1693 | case ARM::VST1d64TPseudoWB_fixed : return true; | |
1694 | case ARM::VST1d64QPseudoWB_fixed : return true; | |
1695 | case ARM::VST2d8wb_fixed : return true; | |
1696 | case ARM::VST2d16wb_fixed : return true; | |
1697 | case ARM::VST2d32wb_fixed : return true; | |
1698 | case ARM::VST2q8PseudoWB_fixed : return true; | |
1699 | case ARM::VST2q16PseudoWB_fixed : return true; | |
1700 | case ARM::VST2q32PseudoWB_fixed : return true; | |
1701 | } | |
1702 | } | |
1703 | ||
223e47cc LB |
1704 | // Get the register stride update opcode of a VLD/VST instruction that |
1705 | // is otherwise equivalent to the given fixed stride updating instruction. | |
1706 | static unsigned getVLDSTRegisterUpdateOpcode(unsigned Opc) { | |
1a4d82fc JJ |
1707 | assert((isVLDfixed(Opc) || isVSTfixed(Opc)) |
1708 | && "Incorrect fixed stride updating instruction."); | |
223e47cc LB |
1709 | switch (Opc) { |
1710 | default: break; | |
1711 | case ARM::VLD1d8wb_fixed: return ARM::VLD1d8wb_register; | |
1712 | case ARM::VLD1d16wb_fixed: return ARM::VLD1d16wb_register; | |
1713 | case ARM::VLD1d32wb_fixed: return ARM::VLD1d32wb_register; | |
1714 | case ARM::VLD1d64wb_fixed: return ARM::VLD1d64wb_register; | |
1715 | case ARM::VLD1q8wb_fixed: return ARM::VLD1q8wb_register; | |
1716 | case ARM::VLD1q16wb_fixed: return ARM::VLD1q16wb_register; | |
1717 | case ARM::VLD1q32wb_fixed: return ARM::VLD1q32wb_register; | |
1718 | case ARM::VLD1q64wb_fixed: return ARM::VLD1q64wb_register; | |
1a4d82fc JJ |
1719 | case ARM::VLD1d64Twb_fixed: return ARM::VLD1d64Twb_register; |
1720 | case ARM::VLD1d64Qwb_fixed: return ARM::VLD1d64Qwb_register; | |
1721 | case ARM::VLD1d64TPseudoWB_fixed: return ARM::VLD1d64TPseudoWB_register; | |
1722 | case ARM::VLD1d64QPseudoWB_fixed: return ARM::VLD1d64QPseudoWB_register; | |
223e47cc LB |
1723 | |
1724 | case ARM::VST1d8wb_fixed: return ARM::VST1d8wb_register; | |
1725 | case ARM::VST1d16wb_fixed: return ARM::VST1d16wb_register; | |
1726 | case ARM::VST1d32wb_fixed: return ARM::VST1d32wb_register; | |
1727 | case ARM::VST1d64wb_fixed: return ARM::VST1d64wb_register; | |
1728 | case ARM::VST1q8wb_fixed: return ARM::VST1q8wb_register; | |
1729 | case ARM::VST1q16wb_fixed: return ARM::VST1q16wb_register; | |
1730 | case ARM::VST1q32wb_fixed: return ARM::VST1q32wb_register; | |
1731 | case ARM::VST1q64wb_fixed: return ARM::VST1q64wb_register; | |
1732 | case ARM::VST1d64TPseudoWB_fixed: return ARM::VST1d64TPseudoWB_register; | |
1733 | case ARM::VST1d64QPseudoWB_fixed: return ARM::VST1d64QPseudoWB_register; | |
1734 | ||
1735 | case ARM::VLD2d8wb_fixed: return ARM::VLD2d8wb_register; | |
1736 | case ARM::VLD2d16wb_fixed: return ARM::VLD2d16wb_register; | |
1737 | case ARM::VLD2d32wb_fixed: return ARM::VLD2d32wb_register; | |
1738 | case ARM::VLD2q8PseudoWB_fixed: return ARM::VLD2q8PseudoWB_register; | |
1739 | case ARM::VLD2q16PseudoWB_fixed: return ARM::VLD2q16PseudoWB_register; | |
1740 | case ARM::VLD2q32PseudoWB_fixed: return ARM::VLD2q32PseudoWB_register; | |
1741 | ||
1742 | case ARM::VST2d8wb_fixed: return ARM::VST2d8wb_register; | |
1743 | case ARM::VST2d16wb_fixed: return ARM::VST2d16wb_register; | |
1744 | case ARM::VST2d32wb_fixed: return ARM::VST2d32wb_register; | |
1745 | case ARM::VST2q8PseudoWB_fixed: return ARM::VST2q8PseudoWB_register; | |
1746 | case ARM::VST2q16PseudoWB_fixed: return ARM::VST2q16PseudoWB_register; | |
1747 | case ARM::VST2q32PseudoWB_fixed: return ARM::VST2q32PseudoWB_register; | |
1748 | ||
1749 | case ARM::VLD2DUPd8wb_fixed: return ARM::VLD2DUPd8wb_register; | |
1750 | case ARM::VLD2DUPd16wb_fixed: return ARM::VLD2DUPd16wb_register; | |
1751 | case ARM::VLD2DUPd32wb_fixed: return ARM::VLD2DUPd32wb_register; | |
1752 | } | |
1753 | return Opc; // If not one we handle, return it unchanged. | |
1754 | } | |
1755 | ||
1756 | SDNode *ARMDAGToDAGISel::SelectVLD(SDNode *N, bool isUpdating, unsigned NumVecs, | |
1757 | const uint16_t *DOpcodes, | |
1758 | const uint16_t *QOpcodes0, | |
1759 | const uint16_t *QOpcodes1) { | |
1760 | assert(NumVecs >= 1 && NumVecs <= 4 && "VLD NumVecs out-of-range"); | |
1a4d82fc | 1761 | SDLoc dl(N); |
223e47cc LB |
1762 | |
1763 | SDValue MemAddr, Align; | |
1764 | unsigned AddrOpIdx = isUpdating ? 1 : 2; | |
1765 | if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align)) | |
1a4d82fc | 1766 | return nullptr; |
223e47cc LB |
1767 | |
1768 | SDValue Chain = N->getOperand(0); | |
1769 | EVT VT = N->getValueType(0); | |
1770 | bool is64BitVector = VT.is64BitVector(); | |
1771 | Align = GetVLDSTAlign(Align, NumVecs, is64BitVector); | |
1772 | ||
1773 | unsigned OpcodeIndex; | |
1774 | switch (VT.getSimpleVT().SimpleTy) { | |
1775 | default: llvm_unreachable("unhandled vld type"); | |
1776 | // Double-register operations: | |
1777 | case MVT::v8i8: OpcodeIndex = 0; break; | |
1778 | case MVT::v4i16: OpcodeIndex = 1; break; | |
1779 | case MVT::v2f32: | |
1780 | case MVT::v2i32: OpcodeIndex = 2; break; | |
1781 | case MVT::v1i64: OpcodeIndex = 3; break; | |
1782 | // Quad-register operations: | |
1783 | case MVT::v16i8: OpcodeIndex = 0; break; | |
1784 | case MVT::v8i16: OpcodeIndex = 1; break; | |
1785 | case MVT::v4f32: | |
1786 | case MVT::v4i32: OpcodeIndex = 2; break; | |
85aaf69f | 1787 | case MVT::v2f64: |
223e47cc LB |
1788 | case MVT::v2i64: OpcodeIndex = 3; |
1789 | assert(NumVecs == 1 && "v2i64 type only supported for VLD1"); | |
1790 | break; | |
1791 | } | |
1792 | ||
1793 | EVT ResTy; | |
1794 | if (NumVecs == 1) | |
1795 | ResTy = VT; | |
1796 | else { | |
1797 | unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs; | |
1798 | if (!is64BitVector) | |
1799 | ResTyElts *= 2; | |
1800 | ResTy = EVT::getVectorVT(*CurDAG->getContext(), MVT::i64, ResTyElts); | |
1801 | } | |
1802 | std::vector<EVT> ResTys; | |
1803 | ResTys.push_back(ResTy); | |
1804 | if (isUpdating) | |
1805 | ResTys.push_back(MVT::i32); | |
1806 | ResTys.push_back(MVT::Other); | |
1807 | ||
1808 | SDValue Pred = getAL(CurDAG); | |
1809 | SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); | |
1810 | SDNode *VLd; | |
1811 | SmallVector<SDValue, 7> Ops; | |
1812 | ||
1813 | // Double registers and VLD1/VLD2 quad registers are directly supported. | |
1814 | if (is64BitVector || NumVecs <= 2) { | |
1815 | unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] : | |
1816 | QOpcodes0[OpcodeIndex]); | |
1817 | Ops.push_back(MemAddr); | |
1818 | Ops.push_back(Align); | |
1819 | if (isUpdating) { | |
1820 | SDValue Inc = N->getOperand(AddrOpIdx + 1); | |
1821 | // FIXME: VLD1/VLD2 fixed increment doesn't need Reg0. Remove the reg0 | |
1822 | // case entirely when the rest are updated to that form, too. | |
1a4d82fc | 1823 | if ((NumVecs <= 2) && !isa<ConstantSDNode>(Inc.getNode())) |
223e47cc | 1824 | Opc = getVLDSTRegisterUpdateOpcode(Opc); |
1a4d82fc | 1825 | // FIXME: We use a VLD1 for v1i64 even if the pseudo says vld2/3/4, so |
223e47cc | 1826 | // check for that explicitly too. Horribly hacky, but temporary. |
1a4d82fc | 1827 | if ((NumVecs > 2 && !isVLDfixed(Opc)) || |
223e47cc LB |
1828 | !isa<ConstantSDNode>(Inc.getNode())) |
1829 | Ops.push_back(isa<ConstantSDNode>(Inc.getNode()) ? Reg0 : Inc); | |
1830 | } | |
1831 | Ops.push_back(Pred); | |
1832 | Ops.push_back(Reg0); | |
1833 | Ops.push_back(Chain); | |
1a4d82fc | 1834 | VLd = CurDAG->getMachineNode(Opc, dl, ResTys, Ops); |
223e47cc LB |
1835 | |
1836 | } else { | |
1837 | // Otherwise, quad registers are loaded with two separate instructions, | |
1838 | // where one loads the even registers and the other loads the odd registers. | |
1839 | EVT AddrTy = MemAddr.getValueType(); | |
1840 | ||
1841 | // Load the even subregs. This is always an updating load, so that it | |
1842 | // provides the address to the second load for the odd subregs. | |
1843 | SDValue ImplDef = | |
1844 | SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, ResTy), 0); | |
1845 | const SDValue OpsA[] = { MemAddr, Align, Reg0, ImplDef, Pred, Reg0, Chain }; | |
1846 | SDNode *VLdA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex], dl, | |
1a4d82fc | 1847 | ResTy, AddrTy, MVT::Other, OpsA); |
223e47cc LB |
1848 | Chain = SDValue(VLdA, 2); |
1849 | ||
1850 | // Load the odd subregs. | |
1851 | Ops.push_back(SDValue(VLdA, 1)); | |
1852 | Ops.push_back(Align); | |
1853 | if (isUpdating) { | |
1854 | SDValue Inc = N->getOperand(AddrOpIdx + 1); | |
1855 | assert(isa<ConstantSDNode>(Inc.getNode()) && | |
1856 | "only constant post-increment update allowed for VLD3/4"); | |
1857 | (void)Inc; | |
1858 | Ops.push_back(Reg0); | |
1859 | } | |
1860 | Ops.push_back(SDValue(VLdA, 0)); | |
1861 | Ops.push_back(Pred); | |
1862 | Ops.push_back(Reg0); | |
1863 | Ops.push_back(Chain); | |
1a4d82fc | 1864 | VLd = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys, Ops); |
223e47cc LB |
1865 | } |
1866 | ||
1867 | // Transfer memoperands. | |
1868 | MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); | |
1869 | MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand(); | |
1870 | cast<MachineSDNode>(VLd)->setMemRefs(MemOp, MemOp + 1); | |
1871 | ||
1872 | if (NumVecs == 1) | |
1873 | return VLd; | |
1874 | ||
1875 | // Extract out the subregisters. | |
1876 | SDValue SuperReg = SDValue(VLd, 0); | |
1877 | assert(ARM::dsub_7 == ARM::dsub_0+7 && | |
1878 | ARM::qsub_3 == ARM::qsub_0+3 && "Unexpected subreg numbering"); | |
1879 | unsigned Sub0 = (is64BitVector ? ARM::dsub_0 : ARM::qsub_0); | |
1880 | for (unsigned Vec = 0; Vec < NumVecs; ++Vec) | |
1881 | ReplaceUses(SDValue(N, Vec), | |
1882 | CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg)); | |
1883 | ReplaceUses(SDValue(N, NumVecs), SDValue(VLd, 1)); | |
1884 | if (isUpdating) | |
1885 | ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLd, 2)); | |
1a4d82fc | 1886 | return nullptr; |
223e47cc LB |
1887 | } |
1888 | ||
1889 | SDNode *ARMDAGToDAGISel::SelectVST(SDNode *N, bool isUpdating, unsigned NumVecs, | |
1890 | const uint16_t *DOpcodes, | |
1891 | const uint16_t *QOpcodes0, | |
1892 | const uint16_t *QOpcodes1) { | |
1893 | assert(NumVecs >= 1 && NumVecs <= 4 && "VST NumVecs out-of-range"); | |
1a4d82fc | 1894 | SDLoc dl(N); |
223e47cc LB |
1895 | |
1896 | SDValue MemAddr, Align; | |
1897 | unsigned AddrOpIdx = isUpdating ? 1 : 2; | |
1898 | unsigned Vec0Idx = 3; // AddrOpIdx + (isUpdating ? 2 : 1) | |
1899 | if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align)) | |
1a4d82fc | 1900 | return nullptr; |
223e47cc LB |
1901 | |
1902 | MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); | |
1903 | MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand(); | |
1904 | ||
1905 | SDValue Chain = N->getOperand(0); | |
1906 | EVT VT = N->getOperand(Vec0Idx).getValueType(); | |
1907 | bool is64BitVector = VT.is64BitVector(); | |
1908 | Align = GetVLDSTAlign(Align, NumVecs, is64BitVector); | |
1909 | ||
1910 | unsigned OpcodeIndex; | |
1911 | switch (VT.getSimpleVT().SimpleTy) { | |
1912 | default: llvm_unreachable("unhandled vst type"); | |
1913 | // Double-register operations: | |
1914 | case MVT::v8i8: OpcodeIndex = 0; break; | |
1915 | case MVT::v4i16: OpcodeIndex = 1; break; | |
1916 | case MVT::v2f32: | |
1917 | case MVT::v2i32: OpcodeIndex = 2; break; | |
1918 | case MVT::v1i64: OpcodeIndex = 3; break; | |
1919 | // Quad-register operations: | |
1920 | case MVT::v16i8: OpcodeIndex = 0; break; | |
1921 | case MVT::v8i16: OpcodeIndex = 1; break; | |
1922 | case MVT::v4f32: | |
1923 | case MVT::v4i32: OpcodeIndex = 2; break; | |
85aaf69f | 1924 | case MVT::v2f64: |
223e47cc LB |
1925 | case MVT::v2i64: OpcodeIndex = 3; |
1926 | assert(NumVecs == 1 && "v2i64 type only supported for VST1"); | |
1927 | break; | |
1928 | } | |
1929 | ||
1930 | std::vector<EVT> ResTys; | |
1931 | if (isUpdating) | |
1932 | ResTys.push_back(MVT::i32); | |
1933 | ResTys.push_back(MVT::Other); | |
1934 | ||
1935 | SDValue Pred = getAL(CurDAG); | |
1936 | SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); | |
1937 | SmallVector<SDValue, 7> Ops; | |
1938 | ||
1939 | // Double registers and VST1/VST2 quad registers are directly supported. | |
1940 | if (is64BitVector || NumVecs <= 2) { | |
1941 | SDValue SrcReg; | |
1942 | if (NumVecs == 1) { | |
1943 | SrcReg = N->getOperand(Vec0Idx); | |
1944 | } else if (is64BitVector) { | |
1945 | // Form a REG_SEQUENCE to force register allocation. | |
1946 | SDValue V0 = N->getOperand(Vec0Idx + 0); | |
1947 | SDValue V1 = N->getOperand(Vec0Idx + 1); | |
1948 | if (NumVecs == 2) | |
970d7e83 | 1949 | SrcReg = SDValue(createDRegPairNode(MVT::v2i64, V0, V1), 0); |
223e47cc LB |
1950 | else { |
1951 | SDValue V2 = N->getOperand(Vec0Idx + 2); | |
1952 | // If it's a vst3, form a quad D-register and leave the last part as | |
1953 | // an undef. | |
1954 | SDValue V3 = (NumVecs == 3) | |
1955 | ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,dl,VT), 0) | |
1956 | : N->getOperand(Vec0Idx + 3); | |
970d7e83 | 1957 | SrcReg = SDValue(createQuadDRegsNode(MVT::v4i64, V0, V1, V2, V3), 0); |
223e47cc LB |
1958 | } |
1959 | } else { | |
1960 | // Form a QQ register. | |
1961 | SDValue Q0 = N->getOperand(Vec0Idx); | |
1962 | SDValue Q1 = N->getOperand(Vec0Idx + 1); | |
970d7e83 | 1963 | SrcReg = SDValue(createQRegPairNode(MVT::v4i64, Q0, Q1), 0); |
223e47cc LB |
1964 | } |
1965 | ||
1966 | unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] : | |
1967 | QOpcodes0[OpcodeIndex]); | |
1968 | Ops.push_back(MemAddr); | |
1969 | Ops.push_back(Align); | |
1970 | if (isUpdating) { | |
1971 | SDValue Inc = N->getOperand(AddrOpIdx + 1); | |
1972 | // FIXME: VST1/VST2 fixed increment doesn't need Reg0. Remove the reg0 | |
1973 | // case entirely when the rest are updated to that form, too. | |
1974 | if (NumVecs <= 2 && !isa<ConstantSDNode>(Inc.getNode())) | |
1975 | Opc = getVLDSTRegisterUpdateOpcode(Opc); | |
1a4d82fc | 1976 | // FIXME: We use a VST1 for v1i64 even if the pseudo says vld2/3/4, so |
223e47cc | 1977 | // check for that explicitly too. Horribly hacky, but temporary. |
1a4d82fc JJ |
1978 | if (!isa<ConstantSDNode>(Inc.getNode())) |
1979 | Ops.push_back(Inc); | |
1980 | else if (NumVecs > 2 && !isVSTfixed(Opc)) | |
1981 | Ops.push_back(Reg0); | |
223e47cc LB |
1982 | } |
1983 | Ops.push_back(SrcReg); | |
1984 | Ops.push_back(Pred); | |
1985 | Ops.push_back(Reg0); | |
1986 | Ops.push_back(Chain); | |
1a4d82fc | 1987 | SDNode *VSt = CurDAG->getMachineNode(Opc, dl, ResTys, Ops); |
223e47cc LB |
1988 | |
1989 | // Transfer memoperands. | |
1990 | cast<MachineSDNode>(VSt)->setMemRefs(MemOp, MemOp + 1); | |
1991 | ||
1992 | return VSt; | |
1993 | } | |
1994 | ||
1995 | // Otherwise, quad registers are stored with two separate instructions, | |
1996 | // where one stores the even registers and the other stores the odd registers. | |
1997 | ||
1998 | // Form the QQQQ REG_SEQUENCE. | |
1999 | SDValue V0 = N->getOperand(Vec0Idx + 0); | |
2000 | SDValue V1 = N->getOperand(Vec0Idx + 1); | |
2001 | SDValue V2 = N->getOperand(Vec0Idx + 2); | |
2002 | SDValue V3 = (NumVecs == 3) | |
2003 | ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0) | |
2004 | : N->getOperand(Vec0Idx + 3); | |
970d7e83 | 2005 | SDValue RegSeq = SDValue(createQuadQRegsNode(MVT::v8i64, V0, V1, V2, V3), 0); |
223e47cc LB |
2006 | |
2007 | // Store the even D registers. This is always an updating store, so that it | |
2008 | // provides the address to the second store for the odd subregs. | |
2009 | const SDValue OpsA[] = { MemAddr, Align, Reg0, RegSeq, Pred, Reg0, Chain }; | |
2010 | SDNode *VStA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex], dl, | |
2011 | MemAddr.getValueType(), | |
1a4d82fc | 2012 | MVT::Other, OpsA); |
223e47cc LB |
2013 | cast<MachineSDNode>(VStA)->setMemRefs(MemOp, MemOp + 1); |
2014 | Chain = SDValue(VStA, 1); | |
2015 | ||
2016 | // Store the odd D registers. | |
2017 | Ops.push_back(SDValue(VStA, 0)); | |
2018 | Ops.push_back(Align); | |
2019 | if (isUpdating) { | |
2020 | SDValue Inc = N->getOperand(AddrOpIdx + 1); | |
2021 | assert(isa<ConstantSDNode>(Inc.getNode()) && | |
2022 | "only constant post-increment update allowed for VST3/4"); | |
2023 | (void)Inc; | |
2024 | Ops.push_back(Reg0); | |
2025 | } | |
2026 | Ops.push_back(RegSeq); | |
2027 | Ops.push_back(Pred); | |
2028 | Ops.push_back(Reg0); | |
2029 | Ops.push_back(Chain); | |
2030 | SDNode *VStB = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys, | |
1a4d82fc | 2031 | Ops); |
223e47cc LB |
2032 | cast<MachineSDNode>(VStB)->setMemRefs(MemOp, MemOp + 1); |
2033 | return VStB; | |
2034 | } | |
2035 | ||
2036 | SDNode *ARMDAGToDAGISel::SelectVLDSTLane(SDNode *N, bool IsLoad, | |
2037 | bool isUpdating, unsigned NumVecs, | |
2038 | const uint16_t *DOpcodes, | |
2039 | const uint16_t *QOpcodes) { | |
2040 | assert(NumVecs >=2 && NumVecs <= 4 && "VLDSTLane NumVecs out-of-range"); | |
1a4d82fc | 2041 | SDLoc dl(N); |
223e47cc LB |
2042 | |
2043 | SDValue MemAddr, Align; | |
2044 | unsigned AddrOpIdx = isUpdating ? 1 : 2; | |
2045 | unsigned Vec0Idx = 3; // AddrOpIdx + (isUpdating ? 2 : 1) | |
2046 | if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align)) | |
1a4d82fc | 2047 | return nullptr; |
223e47cc LB |
2048 | |
2049 | MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); | |
2050 | MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand(); | |
2051 | ||
2052 | SDValue Chain = N->getOperand(0); | |
2053 | unsigned Lane = | |
2054 | cast<ConstantSDNode>(N->getOperand(Vec0Idx + NumVecs))->getZExtValue(); | |
2055 | EVT VT = N->getOperand(Vec0Idx).getValueType(); | |
2056 | bool is64BitVector = VT.is64BitVector(); | |
2057 | ||
2058 | unsigned Alignment = 0; | |
2059 | if (NumVecs != 3) { | |
2060 | Alignment = cast<ConstantSDNode>(Align)->getZExtValue(); | |
2061 | unsigned NumBytes = NumVecs * VT.getVectorElementType().getSizeInBits()/8; | |
2062 | if (Alignment > NumBytes) | |
2063 | Alignment = NumBytes; | |
2064 | if (Alignment < 8 && Alignment < NumBytes) | |
2065 | Alignment = 0; | |
2066 | // Alignment must be a power of two; make sure of that. | |
2067 | Alignment = (Alignment & -Alignment); | |
2068 | if (Alignment == 1) | |
2069 | Alignment = 0; | |
2070 | } | |
2071 | Align = CurDAG->getTargetConstant(Alignment, MVT::i32); | |
2072 | ||
2073 | unsigned OpcodeIndex; | |
2074 | switch (VT.getSimpleVT().SimpleTy) { | |
2075 | default: llvm_unreachable("unhandled vld/vst lane type"); | |
2076 | // Double-register operations: | |
2077 | case MVT::v8i8: OpcodeIndex = 0; break; | |
2078 | case MVT::v4i16: OpcodeIndex = 1; break; | |
2079 | case MVT::v2f32: | |
2080 | case MVT::v2i32: OpcodeIndex = 2; break; | |
2081 | // Quad-register operations: | |
2082 | case MVT::v8i16: OpcodeIndex = 0; break; | |
2083 | case MVT::v4f32: | |
2084 | case MVT::v4i32: OpcodeIndex = 1; break; | |
2085 | } | |
2086 | ||
2087 | std::vector<EVT> ResTys; | |
2088 | if (IsLoad) { | |
2089 | unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs; | |
2090 | if (!is64BitVector) | |
2091 | ResTyElts *= 2; | |
2092 | ResTys.push_back(EVT::getVectorVT(*CurDAG->getContext(), | |
2093 | MVT::i64, ResTyElts)); | |
2094 | } | |
2095 | if (isUpdating) | |
2096 | ResTys.push_back(MVT::i32); | |
2097 | ResTys.push_back(MVT::Other); | |
2098 | ||
2099 | SDValue Pred = getAL(CurDAG); | |
2100 | SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); | |
2101 | ||
2102 | SmallVector<SDValue, 8> Ops; | |
2103 | Ops.push_back(MemAddr); | |
2104 | Ops.push_back(Align); | |
2105 | if (isUpdating) { | |
2106 | SDValue Inc = N->getOperand(AddrOpIdx + 1); | |
2107 | Ops.push_back(isa<ConstantSDNode>(Inc.getNode()) ? Reg0 : Inc); | |
2108 | } | |
2109 | ||
2110 | SDValue SuperReg; | |
2111 | SDValue V0 = N->getOperand(Vec0Idx + 0); | |
2112 | SDValue V1 = N->getOperand(Vec0Idx + 1); | |
2113 | if (NumVecs == 2) { | |
2114 | if (is64BitVector) | |
970d7e83 | 2115 | SuperReg = SDValue(createDRegPairNode(MVT::v2i64, V0, V1), 0); |
223e47cc | 2116 | else |
970d7e83 | 2117 | SuperReg = SDValue(createQRegPairNode(MVT::v4i64, V0, V1), 0); |
223e47cc LB |
2118 | } else { |
2119 | SDValue V2 = N->getOperand(Vec0Idx + 2); | |
2120 | SDValue V3 = (NumVecs == 3) | |
2121 | ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0) | |
2122 | : N->getOperand(Vec0Idx + 3); | |
2123 | if (is64BitVector) | |
970d7e83 | 2124 | SuperReg = SDValue(createQuadDRegsNode(MVT::v4i64, V0, V1, V2, V3), 0); |
223e47cc | 2125 | else |
970d7e83 | 2126 | SuperReg = SDValue(createQuadQRegsNode(MVT::v8i64, V0, V1, V2, V3), 0); |
223e47cc LB |
2127 | } |
2128 | Ops.push_back(SuperReg); | |
2129 | Ops.push_back(getI32Imm(Lane)); | |
2130 | Ops.push_back(Pred); | |
2131 | Ops.push_back(Reg0); | |
2132 | Ops.push_back(Chain); | |
2133 | ||
2134 | unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] : | |
2135 | QOpcodes[OpcodeIndex]); | |
1a4d82fc | 2136 | SDNode *VLdLn = CurDAG->getMachineNode(Opc, dl, ResTys, Ops); |
223e47cc LB |
2137 | cast<MachineSDNode>(VLdLn)->setMemRefs(MemOp, MemOp + 1); |
2138 | if (!IsLoad) | |
2139 | return VLdLn; | |
2140 | ||
2141 | // Extract the subregisters. | |
2142 | SuperReg = SDValue(VLdLn, 0); | |
2143 | assert(ARM::dsub_7 == ARM::dsub_0+7 && | |
2144 | ARM::qsub_3 == ARM::qsub_0+3 && "Unexpected subreg numbering"); | |
2145 | unsigned Sub0 = is64BitVector ? ARM::dsub_0 : ARM::qsub_0; | |
2146 | for (unsigned Vec = 0; Vec < NumVecs; ++Vec) | |
2147 | ReplaceUses(SDValue(N, Vec), | |
2148 | CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg)); | |
2149 | ReplaceUses(SDValue(N, NumVecs), SDValue(VLdLn, 1)); | |
2150 | if (isUpdating) | |
2151 | ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLdLn, 2)); | |
1a4d82fc | 2152 | return nullptr; |
223e47cc LB |
2153 | } |
2154 | ||
2155 | SDNode *ARMDAGToDAGISel::SelectVLDDup(SDNode *N, bool isUpdating, | |
2156 | unsigned NumVecs, | |
2157 | const uint16_t *Opcodes) { | |
2158 | assert(NumVecs >=2 && NumVecs <= 4 && "VLDDup NumVecs out-of-range"); | |
1a4d82fc | 2159 | SDLoc dl(N); |
223e47cc LB |
2160 | |
2161 | SDValue MemAddr, Align; | |
2162 | if (!SelectAddrMode6(N, N->getOperand(1), MemAddr, Align)) | |
1a4d82fc | 2163 | return nullptr; |
223e47cc LB |
2164 | |
2165 | MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); | |
2166 | MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand(); | |
2167 | ||
2168 | SDValue Chain = N->getOperand(0); | |
2169 | EVT VT = N->getValueType(0); | |
2170 | ||
2171 | unsigned Alignment = 0; | |
2172 | if (NumVecs != 3) { | |
2173 | Alignment = cast<ConstantSDNode>(Align)->getZExtValue(); | |
2174 | unsigned NumBytes = NumVecs * VT.getVectorElementType().getSizeInBits()/8; | |
2175 | if (Alignment > NumBytes) | |
2176 | Alignment = NumBytes; | |
2177 | if (Alignment < 8 && Alignment < NumBytes) | |
2178 | Alignment = 0; | |
2179 | // Alignment must be a power of two; make sure of that. | |
2180 | Alignment = (Alignment & -Alignment); | |
2181 | if (Alignment == 1) | |
2182 | Alignment = 0; | |
2183 | } | |
2184 | Align = CurDAG->getTargetConstant(Alignment, MVT::i32); | |
2185 | ||
2186 | unsigned OpcodeIndex; | |
2187 | switch (VT.getSimpleVT().SimpleTy) { | |
2188 | default: llvm_unreachable("unhandled vld-dup type"); | |
2189 | case MVT::v8i8: OpcodeIndex = 0; break; | |
2190 | case MVT::v4i16: OpcodeIndex = 1; break; | |
2191 | case MVT::v2f32: | |
2192 | case MVT::v2i32: OpcodeIndex = 2; break; | |
2193 | } | |
2194 | ||
2195 | SDValue Pred = getAL(CurDAG); | |
2196 | SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); | |
2197 | SDValue SuperReg; | |
2198 | unsigned Opc = Opcodes[OpcodeIndex]; | |
2199 | SmallVector<SDValue, 6> Ops; | |
2200 | Ops.push_back(MemAddr); | |
2201 | Ops.push_back(Align); | |
2202 | if (isUpdating) { | |
2203 | // fixed-stride update instructions don't have an explicit writeback | |
2204 | // operand. It's implicit in the opcode itself. | |
2205 | SDValue Inc = N->getOperand(2); | |
2206 | if (!isa<ConstantSDNode>(Inc.getNode())) | |
2207 | Ops.push_back(Inc); | |
2208 | // FIXME: VLD3 and VLD4 haven't been updated to that form yet. | |
2209 | else if (NumVecs > 2) | |
2210 | Ops.push_back(Reg0); | |
2211 | } | |
2212 | Ops.push_back(Pred); | |
2213 | Ops.push_back(Reg0); | |
2214 | Ops.push_back(Chain); | |
2215 | ||
2216 | unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs; | |
2217 | std::vector<EVT> ResTys; | |
2218 | ResTys.push_back(EVT::getVectorVT(*CurDAG->getContext(), MVT::i64,ResTyElts)); | |
2219 | if (isUpdating) | |
2220 | ResTys.push_back(MVT::i32); | |
2221 | ResTys.push_back(MVT::Other); | |
1a4d82fc | 2222 | SDNode *VLdDup = CurDAG->getMachineNode(Opc, dl, ResTys, Ops); |
223e47cc LB |
2223 | cast<MachineSDNode>(VLdDup)->setMemRefs(MemOp, MemOp + 1); |
2224 | SuperReg = SDValue(VLdDup, 0); | |
2225 | ||
2226 | // Extract the subregisters. | |
2227 | assert(ARM::dsub_7 == ARM::dsub_0+7 && "Unexpected subreg numbering"); | |
2228 | unsigned SubIdx = ARM::dsub_0; | |
2229 | for (unsigned Vec = 0; Vec < NumVecs; ++Vec) | |
2230 | ReplaceUses(SDValue(N, Vec), | |
2231 | CurDAG->getTargetExtractSubreg(SubIdx+Vec, dl, VT, SuperReg)); | |
2232 | ReplaceUses(SDValue(N, NumVecs), SDValue(VLdDup, 1)); | |
2233 | if (isUpdating) | |
2234 | ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLdDup, 2)); | |
1a4d82fc | 2235 | return nullptr; |
223e47cc LB |
2236 | } |
2237 | ||
2238 | SDNode *ARMDAGToDAGISel::SelectVTBL(SDNode *N, bool IsExt, unsigned NumVecs, | |
2239 | unsigned Opc) { | |
2240 | assert(NumVecs >= 2 && NumVecs <= 4 && "VTBL NumVecs out-of-range"); | |
1a4d82fc | 2241 | SDLoc dl(N); |
223e47cc LB |
2242 | EVT VT = N->getValueType(0); |
2243 | unsigned FirstTblReg = IsExt ? 2 : 1; | |
2244 | ||
2245 | // Form a REG_SEQUENCE to force register allocation. | |
2246 | SDValue RegSeq; | |
2247 | SDValue V0 = N->getOperand(FirstTblReg + 0); | |
2248 | SDValue V1 = N->getOperand(FirstTblReg + 1); | |
2249 | if (NumVecs == 2) | |
970d7e83 | 2250 | RegSeq = SDValue(createDRegPairNode(MVT::v16i8, V0, V1), 0); |
223e47cc LB |
2251 | else { |
2252 | SDValue V2 = N->getOperand(FirstTblReg + 2); | |
2253 | // If it's a vtbl3, form a quad D-register and leave the last part as | |
2254 | // an undef. | |
2255 | SDValue V3 = (NumVecs == 3) | |
2256 | ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0) | |
2257 | : N->getOperand(FirstTblReg + 3); | |
970d7e83 | 2258 | RegSeq = SDValue(createQuadDRegsNode(MVT::v4i64, V0, V1, V2, V3), 0); |
223e47cc LB |
2259 | } |
2260 | ||
2261 | SmallVector<SDValue, 6> Ops; | |
2262 | if (IsExt) | |
2263 | Ops.push_back(N->getOperand(1)); | |
2264 | Ops.push_back(RegSeq); | |
2265 | Ops.push_back(N->getOperand(FirstTblReg + NumVecs)); | |
2266 | Ops.push_back(getAL(CurDAG)); // predicate | |
2267 | Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // predicate register | |
1a4d82fc | 2268 | return CurDAG->getMachineNode(Opc, dl, VT, Ops); |
223e47cc LB |
2269 | } |
2270 | ||
2271 | SDNode *ARMDAGToDAGISel::SelectV6T2BitfieldExtractOp(SDNode *N, | |
2272 | bool isSigned) { | |
2273 | if (!Subtarget->hasV6T2Ops()) | |
1a4d82fc | 2274 | return nullptr; |
223e47cc | 2275 | |
970d7e83 LB |
2276 | unsigned Opc = isSigned |
2277 | ? (Subtarget->isThumb() ? ARM::t2SBFX : ARM::SBFX) | |
223e47cc LB |
2278 | : (Subtarget->isThumb() ? ARM::t2UBFX : ARM::UBFX); |
2279 | ||
223e47cc LB |
2280 | // For unsigned extracts, check for a shift right and mask |
2281 | unsigned And_imm = 0; | |
2282 | if (N->getOpcode() == ISD::AND) { | |
2283 | if (isOpcWithIntImmediate(N, ISD::AND, And_imm)) { | |
2284 | ||
2285 | // The immediate is a mask of the low bits iff imm & (imm+1) == 0 | |
2286 | if (And_imm & (And_imm + 1)) | |
1a4d82fc | 2287 | return nullptr; |
223e47cc LB |
2288 | |
2289 | unsigned Srl_imm = 0; | |
2290 | if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SRL, | |
2291 | Srl_imm)) { | |
2292 | assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!"); | |
2293 | ||
2294 | // Note: The width operand is encoded as width-1. | |
2295 | unsigned Width = CountTrailingOnes_32(And_imm) - 1; | |
2296 | unsigned LSB = Srl_imm; | |
970d7e83 | 2297 | |
223e47cc | 2298 | SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); |
970d7e83 LB |
2299 | |
2300 | if ((LSB + Width + 1) == N->getValueType(0).getSizeInBits()) { | |
2301 | // It's cheaper to use a right shift to extract the top bits. | |
2302 | if (Subtarget->isThumb()) { | |
2303 | Opc = isSigned ? ARM::t2ASRri : ARM::t2LSRri; | |
2304 | SDValue Ops[] = { N->getOperand(0).getOperand(0), | |
2305 | CurDAG->getTargetConstant(LSB, MVT::i32), | |
2306 | getAL(CurDAG), Reg0, Reg0 }; | |
1a4d82fc | 2307 | return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops); |
970d7e83 LB |
2308 | } |
2309 | ||
2310 | // ARM models shift instructions as MOVsi with shifter operand. | |
2311 | ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(ISD::SRL); | |
2312 | SDValue ShOpc = | |
2313 | CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, LSB), | |
2314 | MVT::i32); | |
2315 | SDValue Ops[] = { N->getOperand(0).getOperand(0), ShOpc, | |
2316 | getAL(CurDAG), Reg0, Reg0 }; | |
1a4d82fc | 2317 | return CurDAG->SelectNodeTo(N, ARM::MOVsi, MVT::i32, Ops); |
970d7e83 LB |
2318 | } |
2319 | ||
223e47cc LB |
2320 | SDValue Ops[] = { N->getOperand(0).getOperand(0), |
2321 | CurDAG->getTargetConstant(LSB, MVT::i32), | |
2322 | CurDAG->getTargetConstant(Width, MVT::i32), | |
1a4d82fc JJ |
2323 | getAL(CurDAG), Reg0 }; |
2324 | return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops); | |
223e47cc LB |
2325 | } |
2326 | } | |
1a4d82fc | 2327 | return nullptr; |
223e47cc LB |
2328 | } |
2329 | ||
2330 | // Otherwise, we're looking for a shift of a shift | |
2331 | unsigned Shl_imm = 0; | |
2332 | if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SHL, Shl_imm)) { | |
2333 | assert(Shl_imm > 0 && Shl_imm < 32 && "bad amount in shift node!"); | |
2334 | unsigned Srl_imm = 0; | |
2335 | if (isInt32Immediate(N->getOperand(1), Srl_imm)) { | |
2336 | assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!"); | |
2337 | // Note: The width operand is encoded as width-1. | |
2338 | unsigned Width = 32 - Srl_imm - 1; | |
2339 | int LSB = Srl_imm - Shl_imm; | |
2340 | if (LSB < 0) | |
1a4d82fc | 2341 | return nullptr; |
223e47cc LB |
2342 | SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); |
2343 | SDValue Ops[] = { N->getOperand(0).getOperand(0), | |
2344 | CurDAG->getTargetConstant(LSB, MVT::i32), | |
2345 | CurDAG->getTargetConstant(Width, MVT::i32), | |
2346 | getAL(CurDAG), Reg0 }; | |
1a4d82fc | 2347 | return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops); |
223e47cc LB |
2348 | } |
2349 | } | |
223e47cc | 2350 | |
1a4d82fc JJ |
2351 | if (N->getOpcode() == ISD::SIGN_EXTEND_INREG) { |
2352 | unsigned Width = cast<VTSDNode>(N->getOperand(1))->getVT().getSizeInBits(); | |
2353 | unsigned LSB = 0; | |
2354 | if (!isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SRL, LSB) && | |
2355 | !isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SRA, LSB)) | |
2356 | return nullptr; | |
2357 | ||
2358 | if (LSB + Width > 32) | |
2359 | return nullptr; | |
2360 | ||
2361 | SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); | |
2362 | SDValue Ops[] = { N->getOperand(0).getOperand(0), | |
2363 | CurDAG->getTargetConstant(LSB, MVT::i32), | |
2364 | CurDAG->getTargetConstant(Width - 1, MVT::i32), | |
2365 | getAL(CurDAG), Reg0 }; | |
2366 | return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops); | |
223e47cc | 2367 | } |
1a4d82fc JJ |
2368 | |
2369 | return nullptr; | |
223e47cc LB |
2370 | } |
2371 | ||
2372 | /// Target-specific DAG combining for ISD::XOR. | |
2373 | /// Target-independent combining lowers SELECT_CC nodes of the form | |
2374 | /// select_cc setg[ge] X, 0, X, -X | |
2375 | /// select_cc setgt X, -1, X, -X | |
2376 | /// select_cc setl[te] X, 0, -X, X | |
2377 | /// select_cc setlt X, 1, -X, X | |
2378 | /// which represent Integer ABS into: | |
2379 | /// Y = sra (X, size(X)-1); xor (add (X, Y), Y) | |
2380 | /// ARM instruction selection detects the latter and matches it to | |
2381 | /// ARM::ABS or ARM::t2ABS machine node. | |
2382 | SDNode *ARMDAGToDAGISel::SelectABSOp(SDNode *N){ | |
2383 | SDValue XORSrc0 = N->getOperand(0); | |
2384 | SDValue XORSrc1 = N->getOperand(1); | |
2385 | EVT VT = N->getValueType(0); | |
2386 | ||
2387 | if (Subtarget->isThumb1Only()) | |
1a4d82fc | 2388 | return nullptr; |
223e47cc LB |
2389 | |
2390 | if (XORSrc0.getOpcode() != ISD::ADD || XORSrc1.getOpcode() != ISD::SRA) | |
1a4d82fc | 2391 | return nullptr; |
223e47cc LB |
2392 | |
2393 | SDValue ADDSrc0 = XORSrc0.getOperand(0); | |
2394 | SDValue ADDSrc1 = XORSrc0.getOperand(1); | |
2395 | SDValue SRASrc0 = XORSrc1.getOperand(0); | |
2396 | SDValue SRASrc1 = XORSrc1.getOperand(1); | |
2397 | ConstantSDNode *SRAConstant = dyn_cast<ConstantSDNode>(SRASrc1); | |
2398 | EVT XType = SRASrc0.getValueType(); | |
2399 | unsigned Size = XType.getSizeInBits() - 1; | |
2400 | ||
2401 | if (ADDSrc1 == XORSrc1 && ADDSrc0 == SRASrc0 && | |
1a4d82fc | 2402 | XType.isInteger() && SRAConstant != nullptr && |
223e47cc LB |
2403 | Size == SRAConstant->getZExtValue()) { |
2404 | unsigned Opcode = Subtarget->isThumb2() ? ARM::t2ABS : ARM::ABS; | |
2405 | return CurDAG->SelectNodeTo(N, Opcode, VT, ADDSrc0); | |
2406 | } | |
2407 | ||
1a4d82fc | 2408 | return nullptr; |
223e47cc LB |
2409 | } |
2410 | ||
2411 | SDNode *ARMDAGToDAGISel::SelectConcatVector(SDNode *N) { | |
2412 | // The only time a CONCAT_VECTORS operation can have legal types is when | |
2413 | // two 64-bit vectors are concatenated to a 128-bit vector. | |
2414 | EVT VT = N->getValueType(0); | |
2415 | if (!VT.is128BitVector() || N->getNumOperands() != 2) | |
2416 | llvm_unreachable("unexpected CONCAT_VECTORS"); | |
970d7e83 | 2417 | return createDRegPairNode(VT, N->getOperand(0), N->getOperand(1)); |
223e47cc LB |
2418 | } |
2419 | ||
223e47cc | 2420 | SDNode *ARMDAGToDAGISel::Select(SDNode *N) { |
1a4d82fc | 2421 | SDLoc dl(N); |
223e47cc | 2422 | |
1a4d82fc JJ |
2423 | if (N->isMachineOpcode()) { |
2424 | N->setNodeId(-1); | |
2425 | return nullptr; // Already selected. | |
2426 | } | |
223e47cc LB |
2427 | |
2428 | switch (N->getOpcode()) { | |
2429 | default: break; | |
970d7e83 LB |
2430 | case ISD::INLINEASM: { |
2431 | SDNode *ResNode = SelectInlineAsm(N); | |
2432 | if (ResNode) | |
2433 | return ResNode; | |
2434 | break; | |
2435 | } | |
223e47cc LB |
2436 | case ISD::XOR: { |
2437 | // Select special operations if XOR node forms integer ABS pattern | |
2438 | SDNode *ResNode = SelectABSOp(N); | |
2439 | if (ResNode) | |
2440 | return ResNode; | |
2441 | // Other cases are autogenerated. | |
2442 | break; | |
2443 | } | |
2444 | case ISD::Constant: { | |
2445 | unsigned Val = cast<ConstantSDNode>(N)->getZExtValue(); | |
2446 | bool UseCP = true; | |
1a4d82fc | 2447 | if (Subtarget->useMovt(*MF)) |
223e47cc LB |
2448 | // Thumb2-aware targets have the MOVT instruction, so all immediates can |
2449 | // be done with MOV + MOVT, at worst. | |
1a4d82fc | 2450 | UseCP = false; |
223e47cc LB |
2451 | else { |
2452 | if (Subtarget->isThumb()) { | |
1a4d82fc JJ |
2453 | UseCP = (Val > 255 && // MOV |
2454 | ~Val > 255 && // MOV + MVN | |
2455 | !ARM_AM::isThumbImmShiftedVal(Val) && // MOV + LSL | |
2456 | !(Subtarget->hasV6T2Ops() && Val <= 0xffff)); // MOVW | |
223e47cc | 2457 | } else |
1a4d82fc JJ |
2458 | UseCP = (ARM_AM::getSOImmVal(Val) == -1 && // MOV |
2459 | ARM_AM::getSOImmVal(~Val) == -1 && // MVN | |
2460 | !ARM_AM::isSOImmTwoPartVal(Val) && // two instrs. | |
2461 | !(Subtarget->hasV6T2Ops() && Val <= 0xffff)); // MOVW | |
223e47cc LB |
2462 | } |
2463 | ||
2464 | if (UseCP) { | |
85aaf69f SL |
2465 | SDValue CPIdx = CurDAG->getTargetConstantPool( |
2466 | ConstantInt::get(Type::getInt32Ty(*CurDAG->getContext()), Val), | |
2467 | TLI->getPointerTy()); | |
223e47cc LB |
2468 | |
2469 | SDNode *ResNode; | |
1a4d82fc | 2470 | if (Subtarget->isThumb()) { |
223e47cc LB |
2471 | SDValue Pred = getAL(CurDAG); |
2472 | SDValue PredReg = CurDAG->getRegister(0, MVT::i32); | |
2473 | SDValue Ops[] = { CPIdx, Pred, PredReg, CurDAG->getEntryNode() }; | |
2474 | ResNode = CurDAG->getMachineNode(ARM::tLDRpci, dl, MVT::i32, MVT::Other, | |
1a4d82fc | 2475 | Ops); |
223e47cc LB |
2476 | } else { |
2477 | SDValue Ops[] = { | |
2478 | CPIdx, | |
2479 | CurDAG->getTargetConstant(0, MVT::i32), | |
2480 | getAL(CurDAG), | |
2481 | CurDAG->getRegister(0, MVT::i32), | |
2482 | CurDAG->getEntryNode() | |
2483 | }; | |
2484 | ResNode=CurDAG->getMachineNode(ARM::LDRcp, dl, MVT::i32, MVT::Other, | |
1a4d82fc | 2485 | Ops); |
223e47cc LB |
2486 | } |
2487 | ReplaceUses(SDValue(N, 0), SDValue(ResNode, 0)); | |
1a4d82fc | 2488 | return nullptr; |
223e47cc LB |
2489 | } |
2490 | ||
2491 | // Other cases are autogenerated. | |
2492 | break; | |
2493 | } | |
2494 | case ISD::FrameIndex: { | |
2495 | // Selects to ADDri FI, 0 which in turn will become ADDri SP, imm. | |
2496 | int FI = cast<FrameIndexSDNode>(N)->getIndex(); | |
85aaf69f | 2497 | SDValue TFI = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy()); |
223e47cc | 2498 | if (Subtarget->isThumb1Only()) { |
85aaf69f SL |
2499 | return CurDAG->SelectNodeTo(N, ARM::tADDframe, MVT::i32, TFI, |
2500 | CurDAG->getTargetConstant(0, MVT::i32)); | |
223e47cc LB |
2501 | } else { |
2502 | unsigned Opc = ((Subtarget->isThumb() && Subtarget->hasThumb2()) ? | |
2503 | ARM::t2ADDri : ARM::ADDri); | |
2504 | SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, MVT::i32), | |
2505 | getAL(CurDAG), CurDAG->getRegister(0, MVT::i32), | |
2506 | CurDAG->getRegister(0, MVT::i32) }; | |
1a4d82fc | 2507 | return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops); |
223e47cc LB |
2508 | } |
2509 | } | |
2510 | case ISD::SRL: | |
2511 | if (SDNode *I = SelectV6T2BitfieldExtractOp(N, false)) | |
2512 | return I; | |
2513 | break; | |
1a4d82fc | 2514 | case ISD::SIGN_EXTEND_INREG: |
223e47cc LB |
2515 | case ISD::SRA: |
2516 | if (SDNode *I = SelectV6T2BitfieldExtractOp(N, true)) | |
2517 | return I; | |
2518 | break; | |
2519 | case ISD::MUL: | |
2520 | if (Subtarget->isThumb1Only()) | |
2521 | break; | |
2522 | if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1))) { | |
2523 | unsigned RHSV = C->getZExtValue(); | |
2524 | if (!RHSV) break; | |
2525 | if (isPowerOf2_32(RHSV-1)) { // 2^n+1? | |
2526 | unsigned ShImm = Log2_32(RHSV-1); | |
2527 | if (ShImm >= 32) | |
2528 | break; | |
2529 | SDValue V = N->getOperand(0); | |
2530 | ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm); | |
2531 | SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32); | |
2532 | SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); | |
2533 | if (Subtarget->isThumb()) { | |
2534 | SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0, Reg0 }; | |
1a4d82fc | 2535 | return CurDAG->SelectNodeTo(N, ARM::t2ADDrs, MVT::i32, Ops); |
223e47cc LB |
2536 | } else { |
2537 | SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 }; | |
1a4d82fc | 2538 | return CurDAG->SelectNodeTo(N, ARM::ADDrsi, MVT::i32, Ops); |
223e47cc LB |
2539 | } |
2540 | } | |
2541 | if (isPowerOf2_32(RHSV+1)) { // 2^n-1? | |
2542 | unsigned ShImm = Log2_32(RHSV+1); | |
2543 | if (ShImm >= 32) | |
2544 | break; | |
2545 | SDValue V = N->getOperand(0); | |
2546 | ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm); | |
2547 | SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32); | |
2548 | SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); | |
2549 | if (Subtarget->isThumb()) { | |
2550 | SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0, Reg0 }; | |
1a4d82fc | 2551 | return CurDAG->SelectNodeTo(N, ARM::t2RSBrs, MVT::i32, Ops); |
223e47cc LB |
2552 | } else { |
2553 | SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 }; | |
1a4d82fc | 2554 | return CurDAG->SelectNodeTo(N, ARM::RSBrsi, MVT::i32, Ops); |
223e47cc LB |
2555 | } |
2556 | } | |
2557 | } | |
2558 | break; | |
2559 | case ISD::AND: { | |
2560 | // Check for unsigned bitfield extract | |
2561 | if (SDNode *I = SelectV6T2BitfieldExtractOp(N, false)) | |
2562 | return I; | |
2563 | ||
2564 | // (and (or x, c2), c1) and top 16-bits of c1 and c2 match, lower 16-bits | |
2565 | // of c1 are 0xffff, and lower 16-bit of c2 are 0. That is, the top 16-bits | |
2566 | // are entirely contributed by c2 and lower 16-bits are entirely contributed | |
2567 | // by x. That's equal to (or (and x, 0xffff), (and c1, 0xffff0000)). | |
2568 | // Select it to: "movt x, ((c1 & 0xffff) >> 16) | |
2569 | EVT VT = N->getValueType(0); | |
2570 | if (VT != MVT::i32) | |
2571 | break; | |
2572 | unsigned Opc = (Subtarget->isThumb() && Subtarget->hasThumb2()) | |
2573 | ? ARM::t2MOVTi16 | |
2574 | : (Subtarget->hasV6T2Ops() ? ARM::MOVTi16 : 0); | |
2575 | if (!Opc) | |
2576 | break; | |
2577 | SDValue N0 = N->getOperand(0), N1 = N->getOperand(1); | |
2578 | ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1); | |
2579 | if (!N1C) | |
2580 | break; | |
2581 | if (N0.getOpcode() == ISD::OR && N0.getNode()->hasOneUse()) { | |
2582 | SDValue N2 = N0.getOperand(1); | |
2583 | ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2); | |
2584 | if (!N2C) | |
2585 | break; | |
2586 | unsigned N1CVal = N1C->getZExtValue(); | |
2587 | unsigned N2CVal = N2C->getZExtValue(); | |
2588 | if ((N1CVal & 0xffff0000U) == (N2CVal & 0xffff0000U) && | |
2589 | (N1CVal & 0xffffU) == 0xffffU && | |
2590 | (N2CVal & 0xffffU) == 0x0U) { | |
2591 | SDValue Imm16 = CurDAG->getTargetConstant((N2CVal & 0xFFFF0000U) >> 16, | |
2592 | MVT::i32); | |
2593 | SDValue Ops[] = { N0.getOperand(0), Imm16, | |
2594 | getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) }; | |
1a4d82fc | 2595 | return CurDAG->getMachineNode(Opc, dl, VT, Ops); |
223e47cc LB |
2596 | } |
2597 | } | |
2598 | break; | |
2599 | } | |
2600 | case ARMISD::VMOVRRD: | |
2601 | return CurDAG->getMachineNode(ARM::VMOVRRD, dl, MVT::i32, MVT::i32, | |
2602 | N->getOperand(0), getAL(CurDAG), | |
2603 | CurDAG->getRegister(0, MVT::i32)); | |
2604 | case ISD::UMUL_LOHI: { | |
2605 | if (Subtarget->isThumb1Only()) | |
2606 | break; | |
2607 | if (Subtarget->isThumb()) { | |
2608 | SDValue Ops[] = { N->getOperand(0), N->getOperand(1), | |
1a4d82fc JJ |
2609 | getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) }; |
2610 | return CurDAG->getMachineNode(ARM::t2UMULL, dl, MVT::i32, MVT::i32, Ops); | |
223e47cc LB |
2611 | } else { |
2612 | SDValue Ops[] = { N->getOperand(0), N->getOperand(1), | |
2613 | getAL(CurDAG), CurDAG->getRegister(0, MVT::i32), | |
2614 | CurDAG->getRegister(0, MVT::i32) }; | |
2615 | return CurDAG->getMachineNode(Subtarget->hasV6Ops() ? | |
2616 | ARM::UMULL : ARM::UMULLv5, | |
1a4d82fc | 2617 | dl, MVT::i32, MVT::i32, Ops); |
223e47cc LB |
2618 | } |
2619 | } | |
2620 | case ISD::SMUL_LOHI: { | |
2621 | if (Subtarget->isThumb1Only()) | |
2622 | break; | |
2623 | if (Subtarget->isThumb()) { | |
2624 | SDValue Ops[] = { N->getOperand(0), N->getOperand(1), | |
2625 | getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) }; | |
1a4d82fc | 2626 | return CurDAG->getMachineNode(ARM::t2SMULL, dl, MVT::i32, MVT::i32, Ops); |
223e47cc LB |
2627 | } else { |
2628 | SDValue Ops[] = { N->getOperand(0), N->getOperand(1), | |
2629 | getAL(CurDAG), CurDAG->getRegister(0, MVT::i32), | |
2630 | CurDAG->getRegister(0, MVT::i32) }; | |
2631 | return CurDAG->getMachineNode(Subtarget->hasV6Ops() ? | |
2632 | ARM::SMULL : ARM::SMULLv5, | |
1a4d82fc | 2633 | dl, MVT::i32, MVT::i32, Ops); |
223e47cc LB |
2634 | } |
2635 | } | |
2636 | case ARMISD::UMLAL:{ | |
2637 | if (Subtarget->isThumb()) { | |
2638 | SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2), | |
2639 | N->getOperand(3), getAL(CurDAG), | |
2640 | CurDAG->getRegister(0, MVT::i32)}; | |
1a4d82fc | 2641 | return CurDAG->getMachineNode(ARM::t2UMLAL, dl, MVT::i32, MVT::i32, Ops); |
223e47cc LB |
2642 | }else{ |
2643 | SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2), | |
2644 | N->getOperand(3), getAL(CurDAG), | |
2645 | CurDAG->getRegister(0, MVT::i32), | |
2646 | CurDAG->getRegister(0, MVT::i32) }; | |
2647 | return CurDAG->getMachineNode(Subtarget->hasV6Ops() ? | |
2648 | ARM::UMLAL : ARM::UMLALv5, | |
1a4d82fc | 2649 | dl, MVT::i32, MVT::i32, Ops); |
223e47cc LB |
2650 | } |
2651 | } | |
2652 | case ARMISD::SMLAL:{ | |
2653 | if (Subtarget->isThumb()) { | |
2654 | SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2), | |
2655 | N->getOperand(3), getAL(CurDAG), | |
2656 | CurDAG->getRegister(0, MVT::i32)}; | |
1a4d82fc | 2657 | return CurDAG->getMachineNode(ARM::t2SMLAL, dl, MVT::i32, MVT::i32, Ops); |
223e47cc LB |
2658 | }else{ |
2659 | SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2), | |
2660 | N->getOperand(3), getAL(CurDAG), | |
2661 | CurDAG->getRegister(0, MVT::i32), | |
2662 | CurDAG->getRegister(0, MVT::i32) }; | |
2663 | return CurDAG->getMachineNode(Subtarget->hasV6Ops() ? | |
2664 | ARM::SMLAL : ARM::SMLALv5, | |
1a4d82fc | 2665 | dl, MVT::i32, MVT::i32, Ops); |
223e47cc LB |
2666 | } |
2667 | } | |
2668 | case ISD::LOAD: { | |
1a4d82fc | 2669 | SDNode *ResNode = nullptr; |
223e47cc LB |
2670 | if (Subtarget->isThumb() && Subtarget->hasThumb2()) |
2671 | ResNode = SelectT2IndexedLoad(N); | |
2672 | else | |
2673 | ResNode = SelectARMIndexedLoad(N); | |
2674 | if (ResNode) | |
2675 | return ResNode; | |
2676 | // Other cases are autogenerated. | |
2677 | break; | |
2678 | } | |
2679 | case ARMISD::BRCOND: { | |
2680 | // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc) | |
2681 | // Emits: (Bcc:void (bb:Other):$dst, (imm:i32):$cc) | |
2682 | // Pattern complexity = 6 cost = 1 size = 0 | |
2683 | ||
2684 | // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc) | |
2685 | // Emits: (tBcc:void (bb:Other):$dst, (imm:i32):$cc) | |
2686 | // Pattern complexity = 6 cost = 1 size = 0 | |
2687 | ||
2688 | // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc) | |
2689 | // Emits: (t2Bcc:void (bb:Other):$dst, (imm:i32):$cc) | |
2690 | // Pattern complexity = 6 cost = 1 size = 0 | |
2691 | ||
2692 | unsigned Opc = Subtarget->isThumb() ? | |
2693 | ((Subtarget->hasThumb2()) ? ARM::t2Bcc : ARM::tBcc) : ARM::Bcc; | |
2694 | SDValue Chain = N->getOperand(0); | |
2695 | SDValue N1 = N->getOperand(1); | |
2696 | SDValue N2 = N->getOperand(2); | |
2697 | SDValue N3 = N->getOperand(3); | |
2698 | SDValue InFlag = N->getOperand(4); | |
2699 | assert(N1.getOpcode() == ISD::BasicBlock); | |
2700 | assert(N2.getOpcode() == ISD::Constant); | |
2701 | assert(N3.getOpcode() == ISD::Register); | |
2702 | ||
2703 | SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned) | |
2704 | cast<ConstantSDNode>(N2)->getZExtValue()), | |
2705 | MVT::i32); | |
2706 | SDValue Ops[] = { N1, Tmp2, N3, Chain, InFlag }; | |
2707 | SDNode *ResNode = CurDAG->getMachineNode(Opc, dl, MVT::Other, | |
1a4d82fc | 2708 | MVT::Glue, Ops); |
223e47cc LB |
2709 | Chain = SDValue(ResNode, 0); |
2710 | if (N->getNumValues() == 2) { | |
2711 | InFlag = SDValue(ResNode, 1); | |
2712 | ReplaceUses(SDValue(N, 1), InFlag); | |
2713 | } | |
2714 | ReplaceUses(SDValue(N, 0), | |
2715 | SDValue(Chain.getNode(), Chain.getResNo())); | |
1a4d82fc | 2716 | return nullptr; |
223e47cc | 2717 | } |
223e47cc LB |
2718 | case ARMISD::VZIP: { |
2719 | unsigned Opc = 0; | |
2720 | EVT VT = N->getValueType(0); | |
2721 | switch (VT.getSimpleVT().SimpleTy) { | |
1a4d82fc | 2722 | default: return nullptr; |
223e47cc LB |
2723 | case MVT::v8i8: Opc = ARM::VZIPd8; break; |
2724 | case MVT::v4i16: Opc = ARM::VZIPd16; break; | |
2725 | case MVT::v2f32: | |
2726 | // vzip.32 Dd, Dm is a pseudo-instruction expanded to vtrn.32 Dd, Dm. | |
2727 | case MVT::v2i32: Opc = ARM::VTRNd32; break; | |
2728 | case MVT::v16i8: Opc = ARM::VZIPq8; break; | |
2729 | case MVT::v8i16: Opc = ARM::VZIPq16; break; | |
2730 | case MVT::v4f32: | |
2731 | case MVT::v4i32: Opc = ARM::VZIPq32; break; | |
2732 | } | |
2733 | SDValue Pred = getAL(CurDAG); | |
2734 | SDValue PredReg = CurDAG->getRegister(0, MVT::i32); | |
2735 | SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg }; | |
1a4d82fc | 2736 | return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops); |
223e47cc LB |
2737 | } |
2738 | case ARMISD::VUZP: { | |
2739 | unsigned Opc = 0; | |
2740 | EVT VT = N->getValueType(0); | |
2741 | switch (VT.getSimpleVT().SimpleTy) { | |
1a4d82fc | 2742 | default: return nullptr; |
223e47cc LB |
2743 | case MVT::v8i8: Opc = ARM::VUZPd8; break; |
2744 | case MVT::v4i16: Opc = ARM::VUZPd16; break; | |
2745 | case MVT::v2f32: | |
2746 | // vuzp.32 Dd, Dm is a pseudo-instruction expanded to vtrn.32 Dd, Dm. | |
2747 | case MVT::v2i32: Opc = ARM::VTRNd32; break; | |
2748 | case MVT::v16i8: Opc = ARM::VUZPq8; break; | |
2749 | case MVT::v8i16: Opc = ARM::VUZPq16; break; | |
2750 | case MVT::v4f32: | |
2751 | case MVT::v4i32: Opc = ARM::VUZPq32; break; | |
2752 | } | |
2753 | SDValue Pred = getAL(CurDAG); | |
2754 | SDValue PredReg = CurDAG->getRegister(0, MVT::i32); | |
2755 | SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg }; | |
1a4d82fc | 2756 | return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops); |
223e47cc LB |
2757 | } |
2758 | case ARMISD::VTRN: { | |
2759 | unsigned Opc = 0; | |
2760 | EVT VT = N->getValueType(0); | |
2761 | switch (VT.getSimpleVT().SimpleTy) { | |
1a4d82fc | 2762 | default: return nullptr; |
223e47cc LB |
2763 | case MVT::v8i8: Opc = ARM::VTRNd8; break; |
2764 | case MVT::v4i16: Opc = ARM::VTRNd16; break; | |
2765 | case MVT::v2f32: | |
2766 | case MVT::v2i32: Opc = ARM::VTRNd32; break; | |
2767 | case MVT::v16i8: Opc = ARM::VTRNq8; break; | |
2768 | case MVT::v8i16: Opc = ARM::VTRNq16; break; | |
2769 | case MVT::v4f32: | |
2770 | case MVT::v4i32: Opc = ARM::VTRNq32; break; | |
2771 | } | |
2772 | SDValue Pred = getAL(CurDAG); | |
2773 | SDValue PredReg = CurDAG->getRegister(0, MVT::i32); | |
2774 | SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg }; | |
1a4d82fc | 2775 | return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops); |
223e47cc LB |
2776 | } |
2777 | case ARMISD::BUILD_VECTOR: { | |
2778 | EVT VecVT = N->getValueType(0); | |
2779 | EVT EltVT = VecVT.getVectorElementType(); | |
2780 | unsigned NumElts = VecVT.getVectorNumElements(); | |
2781 | if (EltVT == MVT::f64) { | |
2782 | assert(NumElts == 2 && "unexpected type for BUILD_VECTOR"); | |
970d7e83 | 2783 | return createDRegPairNode(VecVT, N->getOperand(0), N->getOperand(1)); |
223e47cc LB |
2784 | } |
2785 | assert(EltVT == MVT::f32 && "unexpected type for BUILD_VECTOR"); | |
2786 | if (NumElts == 2) | |
970d7e83 | 2787 | return createSRegPairNode(VecVT, N->getOperand(0), N->getOperand(1)); |
223e47cc | 2788 | assert(NumElts == 4 && "unexpected type for BUILD_VECTOR"); |
970d7e83 | 2789 | return createQuadSRegsNode(VecVT, N->getOperand(0), N->getOperand(1), |
223e47cc LB |
2790 | N->getOperand(2), N->getOperand(3)); |
2791 | } | |
2792 | ||
2793 | case ARMISD::VLD2DUP: { | |
2794 | static const uint16_t Opcodes[] = { ARM::VLD2DUPd8, ARM::VLD2DUPd16, | |
2795 | ARM::VLD2DUPd32 }; | |
2796 | return SelectVLDDup(N, false, 2, Opcodes); | |
2797 | } | |
2798 | ||
2799 | case ARMISD::VLD3DUP: { | |
2800 | static const uint16_t Opcodes[] = { ARM::VLD3DUPd8Pseudo, | |
2801 | ARM::VLD3DUPd16Pseudo, | |
2802 | ARM::VLD3DUPd32Pseudo }; | |
2803 | return SelectVLDDup(N, false, 3, Opcodes); | |
2804 | } | |
2805 | ||
2806 | case ARMISD::VLD4DUP: { | |
2807 | static const uint16_t Opcodes[] = { ARM::VLD4DUPd8Pseudo, | |
2808 | ARM::VLD4DUPd16Pseudo, | |
2809 | ARM::VLD4DUPd32Pseudo }; | |
2810 | return SelectVLDDup(N, false, 4, Opcodes); | |
2811 | } | |
2812 | ||
2813 | case ARMISD::VLD2DUP_UPD: { | |
2814 | static const uint16_t Opcodes[] = { ARM::VLD2DUPd8wb_fixed, | |
2815 | ARM::VLD2DUPd16wb_fixed, | |
2816 | ARM::VLD2DUPd32wb_fixed }; | |
2817 | return SelectVLDDup(N, true, 2, Opcodes); | |
2818 | } | |
2819 | ||
2820 | case ARMISD::VLD3DUP_UPD: { | |
2821 | static const uint16_t Opcodes[] = { ARM::VLD3DUPd8Pseudo_UPD, | |
2822 | ARM::VLD3DUPd16Pseudo_UPD, | |
2823 | ARM::VLD3DUPd32Pseudo_UPD }; | |
2824 | return SelectVLDDup(N, true, 3, Opcodes); | |
2825 | } | |
2826 | ||
2827 | case ARMISD::VLD4DUP_UPD: { | |
2828 | static const uint16_t Opcodes[] = { ARM::VLD4DUPd8Pseudo_UPD, | |
2829 | ARM::VLD4DUPd16Pseudo_UPD, | |
2830 | ARM::VLD4DUPd32Pseudo_UPD }; | |
2831 | return SelectVLDDup(N, true, 4, Opcodes); | |
2832 | } | |
2833 | ||
2834 | case ARMISD::VLD1_UPD: { | |
2835 | static const uint16_t DOpcodes[] = { ARM::VLD1d8wb_fixed, | |
2836 | ARM::VLD1d16wb_fixed, | |
2837 | ARM::VLD1d32wb_fixed, | |
2838 | ARM::VLD1d64wb_fixed }; | |
2839 | static const uint16_t QOpcodes[] = { ARM::VLD1q8wb_fixed, | |
2840 | ARM::VLD1q16wb_fixed, | |
2841 | ARM::VLD1q32wb_fixed, | |
2842 | ARM::VLD1q64wb_fixed }; | |
1a4d82fc | 2843 | return SelectVLD(N, true, 1, DOpcodes, QOpcodes, nullptr); |
223e47cc LB |
2844 | } |
2845 | ||
2846 | case ARMISD::VLD2_UPD: { | |
2847 | static const uint16_t DOpcodes[] = { ARM::VLD2d8wb_fixed, | |
2848 | ARM::VLD2d16wb_fixed, | |
2849 | ARM::VLD2d32wb_fixed, | |
2850 | ARM::VLD1q64wb_fixed}; | |
2851 | static const uint16_t QOpcodes[] = { ARM::VLD2q8PseudoWB_fixed, | |
2852 | ARM::VLD2q16PseudoWB_fixed, | |
2853 | ARM::VLD2q32PseudoWB_fixed }; | |
1a4d82fc | 2854 | return SelectVLD(N, true, 2, DOpcodes, QOpcodes, nullptr); |
223e47cc LB |
2855 | } |
2856 | ||
2857 | case ARMISD::VLD3_UPD: { | |
2858 | static const uint16_t DOpcodes[] = { ARM::VLD3d8Pseudo_UPD, | |
2859 | ARM::VLD3d16Pseudo_UPD, | |
2860 | ARM::VLD3d32Pseudo_UPD, | |
1a4d82fc | 2861 | ARM::VLD1d64TPseudoWB_fixed}; |
223e47cc LB |
2862 | static const uint16_t QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD, |
2863 | ARM::VLD3q16Pseudo_UPD, | |
2864 | ARM::VLD3q32Pseudo_UPD }; | |
2865 | static const uint16_t QOpcodes1[] = { ARM::VLD3q8oddPseudo_UPD, | |
2866 | ARM::VLD3q16oddPseudo_UPD, | |
2867 | ARM::VLD3q32oddPseudo_UPD }; | |
2868 | return SelectVLD(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1); | |
2869 | } | |
2870 | ||
2871 | case ARMISD::VLD4_UPD: { | |
2872 | static const uint16_t DOpcodes[] = { ARM::VLD4d8Pseudo_UPD, | |
2873 | ARM::VLD4d16Pseudo_UPD, | |
2874 | ARM::VLD4d32Pseudo_UPD, | |
1a4d82fc | 2875 | ARM::VLD1d64QPseudoWB_fixed}; |
223e47cc LB |
2876 | static const uint16_t QOpcodes0[] = { ARM::VLD4q8Pseudo_UPD, |
2877 | ARM::VLD4q16Pseudo_UPD, | |
2878 | ARM::VLD4q32Pseudo_UPD }; | |
2879 | static const uint16_t QOpcodes1[] = { ARM::VLD4q8oddPseudo_UPD, | |
2880 | ARM::VLD4q16oddPseudo_UPD, | |
2881 | ARM::VLD4q32oddPseudo_UPD }; | |
2882 | return SelectVLD(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1); | |
2883 | } | |
2884 | ||
2885 | case ARMISD::VLD2LN_UPD: { | |
2886 | static const uint16_t DOpcodes[] = { ARM::VLD2LNd8Pseudo_UPD, | |
2887 | ARM::VLD2LNd16Pseudo_UPD, | |
2888 | ARM::VLD2LNd32Pseudo_UPD }; | |
2889 | static const uint16_t QOpcodes[] = { ARM::VLD2LNq16Pseudo_UPD, | |
2890 | ARM::VLD2LNq32Pseudo_UPD }; | |
2891 | return SelectVLDSTLane(N, true, true, 2, DOpcodes, QOpcodes); | |
2892 | } | |
2893 | ||
2894 | case ARMISD::VLD3LN_UPD: { | |
2895 | static const uint16_t DOpcodes[] = { ARM::VLD3LNd8Pseudo_UPD, | |
2896 | ARM::VLD3LNd16Pseudo_UPD, | |
2897 | ARM::VLD3LNd32Pseudo_UPD }; | |
2898 | static const uint16_t QOpcodes[] = { ARM::VLD3LNq16Pseudo_UPD, | |
2899 | ARM::VLD3LNq32Pseudo_UPD }; | |
2900 | return SelectVLDSTLane(N, true, true, 3, DOpcodes, QOpcodes); | |
2901 | } | |
2902 | ||
2903 | case ARMISD::VLD4LN_UPD: { | |
2904 | static const uint16_t DOpcodes[] = { ARM::VLD4LNd8Pseudo_UPD, | |
2905 | ARM::VLD4LNd16Pseudo_UPD, | |
2906 | ARM::VLD4LNd32Pseudo_UPD }; | |
2907 | static const uint16_t QOpcodes[] = { ARM::VLD4LNq16Pseudo_UPD, | |
2908 | ARM::VLD4LNq32Pseudo_UPD }; | |
2909 | return SelectVLDSTLane(N, true, true, 4, DOpcodes, QOpcodes); | |
2910 | } | |
2911 | ||
2912 | case ARMISD::VST1_UPD: { | |
2913 | static const uint16_t DOpcodes[] = { ARM::VST1d8wb_fixed, | |
2914 | ARM::VST1d16wb_fixed, | |
2915 | ARM::VST1d32wb_fixed, | |
2916 | ARM::VST1d64wb_fixed }; | |
2917 | static const uint16_t QOpcodes[] = { ARM::VST1q8wb_fixed, | |
2918 | ARM::VST1q16wb_fixed, | |
2919 | ARM::VST1q32wb_fixed, | |
2920 | ARM::VST1q64wb_fixed }; | |
1a4d82fc | 2921 | return SelectVST(N, true, 1, DOpcodes, QOpcodes, nullptr); |
223e47cc LB |
2922 | } |
2923 | ||
2924 | case ARMISD::VST2_UPD: { | |
2925 | static const uint16_t DOpcodes[] = { ARM::VST2d8wb_fixed, | |
2926 | ARM::VST2d16wb_fixed, | |
2927 | ARM::VST2d32wb_fixed, | |
2928 | ARM::VST1q64wb_fixed}; | |
2929 | static const uint16_t QOpcodes[] = { ARM::VST2q8PseudoWB_fixed, | |
2930 | ARM::VST2q16PseudoWB_fixed, | |
2931 | ARM::VST2q32PseudoWB_fixed }; | |
1a4d82fc | 2932 | return SelectVST(N, true, 2, DOpcodes, QOpcodes, nullptr); |
223e47cc LB |
2933 | } |
2934 | ||
2935 | case ARMISD::VST3_UPD: { | |
2936 | static const uint16_t DOpcodes[] = { ARM::VST3d8Pseudo_UPD, | |
2937 | ARM::VST3d16Pseudo_UPD, | |
2938 | ARM::VST3d32Pseudo_UPD, | |
2939 | ARM::VST1d64TPseudoWB_fixed}; | |
2940 | static const uint16_t QOpcodes0[] = { ARM::VST3q8Pseudo_UPD, | |
2941 | ARM::VST3q16Pseudo_UPD, | |
2942 | ARM::VST3q32Pseudo_UPD }; | |
2943 | static const uint16_t QOpcodes1[] = { ARM::VST3q8oddPseudo_UPD, | |
2944 | ARM::VST3q16oddPseudo_UPD, | |
2945 | ARM::VST3q32oddPseudo_UPD }; | |
2946 | return SelectVST(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1); | |
2947 | } | |
2948 | ||
2949 | case ARMISD::VST4_UPD: { | |
2950 | static const uint16_t DOpcodes[] = { ARM::VST4d8Pseudo_UPD, | |
2951 | ARM::VST4d16Pseudo_UPD, | |
2952 | ARM::VST4d32Pseudo_UPD, | |
2953 | ARM::VST1d64QPseudoWB_fixed}; | |
2954 | static const uint16_t QOpcodes0[] = { ARM::VST4q8Pseudo_UPD, | |
2955 | ARM::VST4q16Pseudo_UPD, | |
2956 | ARM::VST4q32Pseudo_UPD }; | |
2957 | static const uint16_t QOpcodes1[] = { ARM::VST4q8oddPseudo_UPD, | |
2958 | ARM::VST4q16oddPseudo_UPD, | |
2959 | ARM::VST4q32oddPseudo_UPD }; | |
2960 | return SelectVST(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1); | |
2961 | } | |
2962 | ||
2963 | case ARMISD::VST2LN_UPD: { | |
2964 | static const uint16_t DOpcodes[] = { ARM::VST2LNd8Pseudo_UPD, | |
2965 | ARM::VST2LNd16Pseudo_UPD, | |
2966 | ARM::VST2LNd32Pseudo_UPD }; | |
2967 | static const uint16_t QOpcodes[] = { ARM::VST2LNq16Pseudo_UPD, | |
2968 | ARM::VST2LNq32Pseudo_UPD }; | |
2969 | return SelectVLDSTLane(N, false, true, 2, DOpcodes, QOpcodes); | |
2970 | } | |
2971 | ||
2972 | case ARMISD::VST3LN_UPD: { | |
2973 | static const uint16_t DOpcodes[] = { ARM::VST3LNd8Pseudo_UPD, | |
2974 | ARM::VST3LNd16Pseudo_UPD, | |
2975 | ARM::VST3LNd32Pseudo_UPD }; | |
2976 | static const uint16_t QOpcodes[] = { ARM::VST3LNq16Pseudo_UPD, | |
2977 | ARM::VST3LNq32Pseudo_UPD }; | |
2978 | return SelectVLDSTLane(N, false, true, 3, DOpcodes, QOpcodes); | |
2979 | } | |
2980 | ||
2981 | case ARMISD::VST4LN_UPD: { | |
2982 | static const uint16_t DOpcodes[] = { ARM::VST4LNd8Pseudo_UPD, | |
2983 | ARM::VST4LNd16Pseudo_UPD, | |
2984 | ARM::VST4LNd32Pseudo_UPD }; | |
2985 | static const uint16_t QOpcodes[] = { ARM::VST4LNq16Pseudo_UPD, | |
2986 | ARM::VST4LNq32Pseudo_UPD }; | |
2987 | return SelectVLDSTLane(N, false, true, 4, DOpcodes, QOpcodes); | |
2988 | } | |
2989 | ||
2990 | case ISD::INTRINSIC_VOID: | |
2991 | case ISD::INTRINSIC_W_CHAIN: { | |
2992 | unsigned IntNo = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue(); | |
2993 | switch (IntNo) { | |
2994 | default: | |
2995 | break; | |
2996 | ||
1a4d82fc | 2997 | case Intrinsic::arm_ldaexd: |
223e47cc | 2998 | case Intrinsic::arm_ldrexd: { |
1a4d82fc | 2999 | SDLoc dl(N); |
223e47cc | 3000 | SDValue Chain = N->getOperand(0); |
1a4d82fc | 3001 | SDValue MemAddr = N->getOperand(2); |
970d7e83 | 3002 | bool isThumb = Subtarget->isThumb() && Subtarget->hasThumb2(); |
1a4d82fc JJ |
3003 | |
3004 | bool IsAcquire = IntNo == Intrinsic::arm_ldaexd; | |
3005 | unsigned NewOpc = isThumb ? (IsAcquire ? ARM::t2LDAEXD : ARM::t2LDREXD) | |
3006 | : (IsAcquire ? ARM::LDAEXD : ARM::LDREXD); | |
223e47cc LB |
3007 | |
3008 | // arm_ldrexd returns a i64 value in {i32, i32} | |
3009 | std::vector<EVT> ResTys; | |
970d7e83 LB |
3010 | if (isThumb) { |
3011 | ResTys.push_back(MVT::i32); | |
3012 | ResTys.push_back(MVT::i32); | |
3013 | } else | |
3014 | ResTys.push_back(MVT::Untyped); | |
223e47cc LB |
3015 | ResTys.push_back(MVT::Other); |
3016 | ||
970d7e83 | 3017 | // Place arguments in the right order. |
223e47cc LB |
3018 | SmallVector<SDValue, 7> Ops; |
3019 | Ops.push_back(MemAddr); | |
3020 | Ops.push_back(getAL(CurDAG)); | |
3021 | Ops.push_back(CurDAG->getRegister(0, MVT::i32)); | |
3022 | Ops.push_back(Chain); | |
1a4d82fc | 3023 | SDNode *Ld = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops); |
223e47cc LB |
3024 | // Transfer memoperands. |
3025 | MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); | |
3026 | MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand(); | |
3027 | cast<MachineSDNode>(Ld)->setMemRefs(MemOp, MemOp + 1); | |
3028 | ||
223e47cc | 3029 | // Remap uses. |
970d7e83 | 3030 | SDValue OutChain = isThumb ? SDValue(Ld, 2) : SDValue(Ld, 1); |
223e47cc | 3031 | if (!SDValue(N, 0).use_empty()) { |
970d7e83 LB |
3032 | SDValue Result; |
3033 | if (isThumb) | |
3034 | Result = SDValue(Ld, 0); | |
3035 | else { | |
3036 | SDValue SubRegIdx = CurDAG->getTargetConstant(ARM::gsub_0, MVT::i32); | |
3037 | SDNode *ResNode = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, | |
3038 | dl, MVT::i32, SDValue(Ld, 0), SubRegIdx); | |
3039 | Result = SDValue(ResNode,0); | |
3040 | } | |
223e47cc LB |
3041 | ReplaceUses(SDValue(N, 0), Result); |
3042 | } | |
3043 | if (!SDValue(N, 1).use_empty()) { | |
970d7e83 LB |
3044 | SDValue Result; |
3045 | if (isThumb) | |
3046 | Result = SDValue(Ld, 1); | |
3047 | else { | |
3048 | SDValue SubRegIdx = CurDAG->getTargetConstant(ARM::gsub_1, MVT::i32); | |
3049 | SDNode *ResNode = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, | |
3050 | dl, MVT::i32, SDValue(Ld, 0), SubRegIdx); | |
3051 | Result = SDValue(ResNode,0); | |
3052 | } | |
223e47cc LB |
3053 | ReplaceUses(SDValue(N, 1), Result); |
3054 | } | |
970d7e83 | 3055 | ReplaceUses(SDValue(N, 2), OutChain); |
1a4d82fc | 3056 | return nullptr; |
223e47cc | 3057 | } |
1a4d82fc | 3058 | case Intrinsic::arm_stlexd: |
223e47cc | 3059 | case Intrinsic::arm_strexd: { |
1a4d82fc | 3060 | SDLoc dl(N); |
223e47cc LB |
3061 | SDValue Chain = N->getOperand(0); |
3062 | SDValue Val0 = N->getOperand(2); | |
3063 | SDValue Val1 = N->getOperand(3); | |
3064 | SDValue MemAddr = N->getOperand(4); | |
3065 | ||
223e47cc LB |
3066 | // Store exclusive double return a i32 value which is the return status |
3067 | // of the issued store. | |
970d7e83 | 3068 | EVT ResTys[] = { MVT::i32, MVT::Other }; |
223e47cc | 3069 | |
970d7e83 LB |
3070 | bool isThumb = Subtarget->isThumb() && Subtarget->hasThumb2(); |
3071 | // Place arguments in the right order. | |
223e47cc | 3072 | SmallVector<SDValue, 7> Ops; |
970d7e83 LB |
3073 | if (isThumb) { |
3074 | Ops.push_back(Val0); | |
3075 | Ops.push_back(Val1); | |
3076 | } else | |
3077 | // arm_strexd uses GPRPair. | |
3078 | Ops.push_back(SDValue(createGPRPairNode(MVT::Untyped, Val0, Val1), 0)); | |
223e47cc LB |
3079 | Ops.push_back(MemAddr); |
3080 | Ops.push_back(getAL(CurDAG)); | |
3081 | Ops.push_back(CurDAG->getRegister(0, MVT::i32)); | |
3082 | Ops.push_back(Chain); | |
3083 | ||
1a4d82fc JJ |
3084 | bool IsRelease = IntNo == Intrinsic::arm_stlexd; |
3085 | unsigned NewOpc = isThumb ? (IsRelease ? ARM::t2STLEXD : ARM::t2STREXD) | |
3086 | : (IsRelease ? ARM::STLEXD : ARM::STREXD); | |
223e47cc | 3087 | |
1a4d82fc | 3088 | SDNode *St = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops); |
223e47cc LB |
3089 | // Transfer memoperands. |
3090 | MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); | |
3091 | MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand(); | |
3092 | cast<MachineSDNode>(St)->setMemRefs(MemOp, MemOp + 1); | |
3093 | ||
3094 | return St; | |
3095 | } | |
3096 | ||
3097 | case Intrinsic::arm_neon_vld1: { | |
3098 | static const uint16_t DOpcodes[] = { ARM::VLD1d8, ARM::VLD1d16, | |
3099 | ARM::VLD1d32, ARM::VLD1d64 }; | |
3100 | static const uint16_t QOpcodes[] = { ARM::VLD1q8, ARM::VLD1q16, | |
3101 | ARM::VLD1q32, ARM::VLD1q64}; | |
1a4d82fc | 3102 | return SelectVLD(N, false, 1, DOpcodes, QOpcodes, nullptr); |
223e47cc LB |
3103 | } |
3104 | ||
3105 | case Intrinsic::arm_neon_vld2: { | |
3106 | static const uint16_t DOpcodes[] = { ARM::VLD2d8, ARM::VLD2d16, | |
3107 | ARM::VLD2d32, ARM::VLD1q64 }; | |
3108 | static const uint16_t QOpcodes[] = { ARM::VLD2q8Pseudo, ARM::VLD2q16Pseudo, | |
3109 | ARM::VLD2q32Pseudo }; | |
1a4d82fc | 3110 | return SelectVLD(N, false, 2, DOpcodes, QOpcodes, nullptr); |
223e47cc LB |
3111 | } |
3112 | ||
3113 | case Intrinsic::arm_neon_vld3: { | |
3114 | static const uint16_t DOpcodes[] = { ARM::VLD3d8Pseudo, | |
3115 | ARM::VLD3d16Pseudo, | |
3116 | ARM::VLD3d32Pseudo, | |
3117 | ARM::VLD1d64TPseudo }; | |
3118 | static const uint16_t QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD, | |
3119 | ARM::VLD3q16Pseudo_UPD, | |
3120 | ARM::VLD3q32Pseudo_UPD }; | |
3121 | static const uint16_t QOpcodes1[] = { ARM::VLD3q8oddPseudo, | |
3122 | ARM::VLD3q16oddPseudo, | |
3123 | ARM::VLD3q32oddPseudo }; | |
3124 | return SelectVLD(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1); | |
3125 | } | |
3126 | ||
3127 | case Intrinsic::arm_neon_vld4: { | |
3128 | static const uint16_t DOpcodes[] = { ARM::VLD4d8Pseudo, | |
3129 | ARM::VLD4d16Pseudo, | |
3130 | ARM::VLD4d32Pseudo, | |
3131 | ARM::VLD1d64QPseudo }; | |
3132 | static const uint16_t QOpcodes0[] = { ARM::VLD4q8Pseudo_UPD, | |
3133 | ARM::VLD4q16Pseudo_UPD, | |
3134 | ARM::VLD4q32Pseudo_UPD }; | |
3135 | static const uint16_t QOpcodes1[] = { ARM::VLD4q8oddPseudo, | |
3136 | ARM::VLD4q16oddPseudo, | |
3137 | ARM::VLD4q32oddPseudo }; | |
3138 | return SelectVLD(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1); | |
3139 | } | |
3140 | ||
3141 | case Intrinsic::arm_neon_vld2lane: { | |
3142 | static const uint16_t DOpcodes[] = { ARM::VLD2LNd8Pseudo, | |
3143 | ARM::VLD2LNd16Pseudo, | |
3144 | ARM::VLD2LNd32Pseudo }; | |
3145 | static const uint16_t QOpcodes[] = { ARM::VLD2LNq16Pseudo, | |
3146 | ARM::VLD2LNq32Pseudo }; | |
3147 | return SelectVLDSTLane(N, true, false, 2, DOpcodes, QOpcodes); | |
3148 | } | |
3149 | ||
3150 | case Intrinsic::arm_neon_vld3lane: { | |
3151 | static const uint16_t DOpcodes[] = { ARM::VLD3LNd8Pseudo, | |
3152 | ARM::VLD3LNd16Pseudo, | |
3153 | ARM::VLD3LNd32Pseudo }; | |
3154 | static const uint16_t QOpcodes[] = { ARM::VLD3LNq16Pseudo, | |
3155 | ARM::VLD3LNq32Pseudo }; | |
3156 | return SelectVLDSTLane(N, true, false, 3, DOpcodes, QOpcodes); | |
3157 | } | |
3158 | ||
3159 | case Intrinsic::arm_neon_vld4lane: { | |
3160 | static const uint16_t DOpcodes[] = { ARM::VLD4LNd8Pseudo, | |
3161 | ARM::VLD4LNd16Pseudo, | |
3162 | ARM::VLD4LNd32Pseudo }; | |
3163 | static const uint16_t QOpcodes[] = { ARM::VLD4LNq16Pseudo, | |
3164 | ARM::VLD4LNq32Pseudo }; | |
3165 | return SelectVLDSTLane(N, true, false, 4, DOpcodes, QOpcodes); | |
3166 | } | |
3167 | ||
3168 | case Intrinsic::arm_neon_vst1: { | |
3169 | static const uint16_t DOpcodes[] = { ARM::VST1d8, ARM::VST1d16, | |
3170 | ARM::VST1d32, ARM::VST1d64 }; | |
3171 | static const uint16_t QOpcodes[] = { ARM::VST1q8, ARM::VST1q16, | |
3172 | ARM::VST1q32, ARM::VST1q64 }; | |
1a4d82fc | 3173 | return SelectVST(N, false, 1, DOpcodes, QOpcodes, nullptr); |
223e47cc LB |
3174 | } |
3175 | ||
3176 | case Intrinsic::arm_neon_vst2: { | |
3177 | static const uint16_t DOpcodes[] = { ARM::VST2d8, ARM::VST2d16, | |
3178 | ARM::VST2d32, ARM::VST1q64 }; | |
3179 | static uint16_t QOpcodes[] = { ARM::VST2q8Pseudo, ARM::VST2q16Pseudo, | |
3180 | ARM::VST2q32Pseudo }; | |
1a4d82fc | 3181 | return SelectVST(N, false, 2, DOpcodes, QOpcodes, nullptr); |
223e47cc LB |
3182 | } |
3183 | ||
3184 | case Intrinsic::arm_neon_vst3: { | |
3185 | static const uint16_t DOpcodes[] = { ARM::VST3d8Pseudo, | |
3186 | ARM::VST3d16Pseudo, | |
3187 | ARM::VST3d32Pseudo, | |
3188 | ARM::VST1d64TPseudo }; | |
3189 | static const uint16_t QOpcodes0[] = { ARM::VST3q8Pseudo_UPD, | |
3190 | ARM::VST3q16Pseudo_UPD, | |
3191 | ARM::VST3q32Pseudo_UPD }; | |
3192 | static const uint16_t QOpcodes1[] = { ARM::VST3q8oddPseudo, | |
3193 | ARM::VST3q16oddPseudo, | |
3194 | ARM::VST3q32oddPseudo }; | |
3195 | return SelectVST(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1); | |
3196 | } | |
3197 | ||
3198 | case Intrinsic::arm_neon_vst4: { | |
3199 | static const uint16_t DOpcodes[] = { ARM::VST4d8Pseudo, | |
3200 | ARM::VST4d16Pseudo, | |
3201 | ARM::VST4d32Pseudo, | |
3202 | ARM::VST1d64QPseudo }; | |
3203 | static const uint16_t QOpcodes0[] = { ARM::VST4q8Pseudo_UPD, | |
3204 | ARM::VST4q16Pseudo_UPD, | |
3205 | ARM::VST4q32Pseudo_UPD }; | |
3206 | static const uint16_t QOpcodes1[] = { ARM::VST4q8oddPseudo, | |
3207 | ARM::VST4q16oddPseudo, | |
3208 | ARM::VST4q32oddPseudo }; | |
3209 | return SelectVST(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1); | |
3210 | } | |
3211 | ||
3212 | case Intrinsic::arm_neon_vst2lane: { | |
3213 | static const uint16_t DOpcodes[] = { ARM::VST2LNd8Pseudo, | |
3214 | ARM::VST2LNd16Pseudo, | |
3215 | ARM::VST2LNd32Pseudo }; | |
3216 | static const uint16_t QOpcodes[] = { ARM::VST2LNq16Pseudo, | |
3217 | ARM::VST2LNq32Pseudo }; | |
3218 | return SelectVLDSTLane(N, false, false, 2, DOpcodes, QOpcodes); | |
3219 | } | |
3220 | ||
3221 | case Intrinsic::arm_neon_vst3lane: { | |
3222 | static const uint16_t DOpcodes[] = { ARM::VST3LNd8Pseudo, | |
3223 | ARM::VST3LNd16Pseudo, | |
3224 | ARM::VST3LNd32Pseudo }; | |
3225 | static const uint16_t QOpcodes[] = { ARM::VST3LNq16Pseudo, | |
3226 | ARM::VST3LNq32Pseudo }; | |
3227 | return SelectVLDSTLane(N, false, false, 3, DOpcodes, QOpcodes); | |
3228 | } | |
3229 | ||
3230 | case Intrinsic::arm_neon_vst4lane: { | |
3231 | static const uint16_t DOpcodes[] = { ARM::VST4LNd8Pseudo, | |
3232 | ARM::VST4LNd16Pseudo, | |
3233 | ARM::VST4LNd32Pseudo }; | |
3234 | static const uint16_t QOpcodes[] = { ARM::VST4LNq16Pseudo, | |
3235 | ARM::VST4LNq32Pseudo }; | |
3236 | return SelectVLDSTLane(N, false, false, 4, DOpcodes, QOpcodes); | |
3237 | } | |
3238 | } | |
3239 | break; | |
3240 | } | |
3241 | ||
3242 | case ISD::INTRINSIC_WO_CHAIN: { | |
3243 | unsigned IntNo = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue(); | |
3244 | switch (IntNo) { | |
3245 | default: | |
3246 | break; | |
3247 | ||
3248 | case Intrinsic::arm_neon_vtbl2: | |
3249 | return SelectVTBL(N, false, 2, ARM::VTBL2); | |
3250 | case Intrinsic::arm_neon_vtbl3: | |
3251 | return SelectVTBL(N, false, 3, ARM::VTBL3Pseudo); | |
3252 | case Intrinsic::arm_neon_vtbl4: | |
3253 | return SelectVTBL(N, false, 4, ARM::VTBL4Pseudo); | |
3254 | ||
3255 | case Intrinsic::arm_neon_vtbx2: | |
3256 | return SelectVTBL(N, true, 2, ARM::VTBX2); | |
3257 | case Intrinsic::arm_neon_vtbx3: | |
3258 | return SelectVTBL(N, true, 3, ARM::VTBX3Pseudo); | |
3259 | case Intrinsic::arm_neon_vtbx4: | |
3260 | return SelectVTBL(N, true, 4, ARM::VTBX4Pseudo); | |
3261 | } | |
3262 | break; | |
3263 | } | |
3264 | ||
3265 | case ARMISD::VTBL1: { | |
1a4d82fc | 3266 | SDLoc dl(N); |
223e47cc LB |
3267 | EVT VT = N->getValueType(0); |
3268 | SmallVector<SDValue, 6> Ops; | |
3269 | ||
3270 | Ops.push_back(N->getOperand(0)); | |
3271 | Ops.push_back(N->getOperand(1)); | |
3272 | Ops.push_back(getAL(CurDAG)); // Predicate | |
3273 | Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // Predicate Register | |
1a4d82fc | 3274 | return CurDAG->getMachineNode(ARM::VTBL1, dl, VT, Ops); |
223e47cc LB |
3275 | } |
3276 | case ARMISD::VTBL2: { | |
1a4d82fc | 3277 | SDLoc dl(N); |
223e47cc LB |
3278 | EVT VT = N->getValueType(0); |
3279 | ||
3280 | // Form a REG_SEQUENCE to force register allocation. | |
3281 | SDValue V0 = N->getOperand(0); | |
3282 | SDValue V1 = N->getOperand(1); | |
970d7e83 | 3283 | SDValue RegSeq = SDValue(createDRegPairNode(MVT::v16i8, V0, V1), 0); |
223e47cc LB |
3284 | |
3285 | SmallVector<SDValue, 6> Ops; | |
3286 | Ops.push_back(RegSeq); | |
3287 | Ops.push_back(N->getOperand(2)); | |
3288 | Ops.push_back(getAL(CurDAG)); // Predicate | |
3289 | Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // Predicate Register | |
1a4d82fc | 3290 | return CurDAG->getMachineNode(ARM::VTBL2, dl, VT, Ops); |
223e47cc LB |
3291 | } |
3292 | ||
3293 | case ISD::CONCAT_VECTORS: | |
3294 | return SelectConcatVector(N); | |
223e47cc LB |
3295 | } |
3296 | ||
3297 | return SelectCode(N); | |
3298 | } | |
3299 | ||
970d7e83 LB |
3300 | SDNode *ARMDAGToDAGISel::SelectInlineAsm(SDNode *N){ |
3301 | std::vector<SDValue> AsmNodeOperands; | |
3302 | unsigned Flag, Kind; | |
3303 | bool Changed = false; | |
3304 | unsigned NumOps = N->getNumOperands(); | |
3305 | ||
970d7e83 LB |
3306 | // Normally, i64 data is bounded to two arbitrary GRPs for "%r" constraint. |
3307 | // However, some instrstions (e.g. ldrexd/strexd in ARM mode) require | |
3308 | // (even/even+1) GPRs and use %n and %Hn to refer to the individual regs | |
3309 | // respectively. Since there is no constraint to explicitly specify a | |
1a4d82fc JJ |
3310 | // reg pair, we use GPRPair reg class for "%r" for 64-bit data. For Thumb, |
3311 | // the 64-bit data may be referred by H, Q, R modifiers, so we still pack | |
3312 | // them into a GPRPair. | |
970d7e83 | 3313 | |
1a4d82fc JJ |
3314 | SDLoc dl(N); |
3315 | SDValue Glue = N->getGluedNode() ? N->getOperand(NumOps-1) | |
3316 | : SDValue(nullptr,0); | |
970d7e83 | 3317 | |
1a4d82fc | 3318 | SmallVector<bool, 8> OpChanged; |
970d7e83 | 3319 | // Glue node will be appended late. |
1a4d82fc | 3320 | for(unsigned i = 0, e = N->getGluedNode() ? NumOps - 1 : NumOps; i < e; ++i) { |
970d7e83 LB |
3321 | SDValue op = N->getOperand(i); |
3322 | AsmNodeOperands.push_back(op); | |
3323 | ||
3324 | if (i < InlineAsm::Op_FirstOperand) | |
3325 | continue; | |
3326 | ||
3327 | if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(i))) { | |
3328 | Flag = C->getZExtValue(); | |
3329 | Kind = InlineAsm::getKind(Flag); | |
3330 | } | |
3331 | else | |
3332 | continue; | |
3333 | ||
1a4d82fc JJ |
3334 | // Immediate operands to inline asm in the SelectionDAG are modeled with |
3335 | // two operands. The first is a constant of value InlineAsm::Kind_Imm, and | |
3336 | // the second is a constant with the value of the immediate. If we get here | |
3337 | // and we have a Kind_Imm, skip the next operand, and continue. | |
3338 | if (Kind == InlineAsm::Kind_Imm) { | |
3339 | SDValue op = N->getOperand(++i); | |
3340 | AsmNodeOperands.push_back(op); | |
3341 | continue; | |
3342 | } | |
3343 | ||
3344 | unsigned NumRegs = InlineAsm::getNumOperandRegisters(Flag); | |
3345 | if (NumRegs) | |
3346 | OpChanged.push_back(false); | |
3347 | ||
3348 | unsigned DefIdx = 0; | |
3349 | bool IsTiedToChangedOp = false; | |
3350 | // If it's a use that is tied with a previous def, it has no | |
3351 | // reg class constraint. | |
3352 | if (Changed && InlineAsm::isUseOperandTiedToDef(Flag, DefIdx)) | |
3353 | IsTiedToChangedOp = OpChanged[DefIdx]; | |
3354 | ||
970d7e83 LB |
3355 | if (Kind != InlineAsm::Kind_RegUse && Kind != InlineAsm::Kind_RegDef |
3356 | && Kind != InlineAsm::Kind_RegDefEarlyClobber) | |
3357 | continue; | |
3358 | ||
970d7e83 LB |
3359 | unsigned RC; |
3360 | bool HasRC = InlineAsm::hasRegClassConstraint(Flag, RC); | |
1a4d82fc JJ |
3361 | if ((!IsTiedToChangedOp && (!HasRC || RC != ARM::GPRRegClassID)) |
3362 | || NumRegs != 2) | |
970d7e83 LB |
3363 | continue; |
3364 | ||
1a4d82fc | 3365 | assert((i+2 < NumOps) && "Invalid number of operands in inline asm"); |
970d7e83 LB |
3366 | SDValue V0 = N->getOperand(i+1); |
3367 | SDValue V1 = N->getOperand(i+2); | |
3368 | unsigned Reg0 = cast<RegisterSDNode>(V0)->getReg(); | |
3369 | unsigned Reg1 = cast<RegisterSDNode>(V1)->getReg(); | |
3370 | SDValue PairedReg; | |
3371 | MachineRegisterInfo &MRI = MF->getRegInfo(); | |
3372 | ||
3373 | if (Kind == InlineAsm::Kind_RegDef || | |
3374 | Kind == InlineAsm::Kind_RegDefEarlyClobber) { | |
3375 | // Replace the two GPRs with 1 GPRPair and copy values from GPRPair to | |
3376 | // the original GPRs. | |
3377 | ||
3378 | unsigned GPVR = MRI.createVirtualRegister(&ARM::GPRPairRegClass); | |
3379 | PairedReg = CurDAG->getRegister(GPVR, MVT::Untyped); | |
3380 | SDValue Chain = SDValue(N,0); | |
3381 | ||
3382 | SDNode *GU = N->getGluedUser(); | |
3383 | SDValue RegCopy = CurDAG->getCopyFromReg(Chain, dl, GPVR, MVT::Untyped, | |
3384 | Chain.getValue(1)); | |
3385 | ||
3386 | // Extract values from a GPRPair reg and copy to the original GPR reg. | |
3387 | SDValue Sub0 = CurDAG->getTargetExtractSubreg(ARM::gsub_0, dl, MVT::i32, | |
3388 | RegCopy); | |
3389 | SDValue Sub1 = CurDAG->getTargetExtractSubreg(ARM::gsub_1, dl, MVT::i32, | |
3390 | RegCopy); | |
3391 | SDValue T0 = CurDAG->getCopyToReg(Sub0, dl, Reg0, Sub0, | |
3392 | RegCopy.getValue(1)); | |
3393 | SDValue T1 = CurDAG->getCopyToReg(Sub1, dl, Reg1, Sub1, T0.getValue(1)); | |
3394 | ||
3395 | // Update the original glue user. | |
3396 | std::vector<SDValue> Ops(GU->op_begin(), GU->op_end()-1); | |
3397 | Ops.push_back(T1.getValue(1)); | |
1a4d82fc | 3398 | CurDAG->UpdateNodeOperands(GU, Ops); |
970d7e83 LB |
3399 | } |
3400 | else { | |
3401 | // For Kind == InlineAsm::Kind_RegUse, we first copy two GPRs into a | |
3402 | // GPRPair and then pass the GPRPair to the inline asm. | |
3403 | SDValue Chain = AsmNodeOperands[InlineAsm::Op_InputChain]; | |
3404 | ||
3405 | // As REG_SEQ doesn't take RegisterSDNode, we copy them first. | |
3406 | SDValue T0 = CurDAG->getCopyFromReg(Chain, dl, Reg0, MVT::i32, | |
3407 | Chain.getValue(1)); | |
3408 | SDValue T1 = CurDAG->getCopyFromReg(Chain, dl, Reg1, MVT::i32, | |
3409 | T0.getValue(1)); | |
3410 | SDValue Pair = SDValue(createGPRPairNode(MVT::Untyped, T0, T1), 0); | |
3411 | ||
3412 | // Copy REG_SEQ into a GPRPair-typed VR and replace the original two | |
3413 | // i32 VRs of inline asm with it. | |
3414 | unsigned GPVR = MRI.createVirtualRegister(&ARM::GPRPairRegClass); | |
3415 | PairedReg = CurDAG->getRegister(GPVR, MVT::Untyped); | |
3416 | Chain = CurDAG->getCopyToReg(T1, dl, GPVR, Pair, T1.getValue(1)); | |
3417 | ||
3418 | AsmNodeOperands[InlineAsm::Op_InputChain] = Chain; | |
3419 | Glue = Chain.getValue(1); | |
3420 | } | |
3421 | ||
3422 | Changed = true; | |
3423 | ||
3424 | if(PairedReg.getNode()) { | |
1a4d82fc | 3425 | OpChanged[OpChanged.size() -1 ] = true; |
970d7e83 | 3426 | Flag = InlineAsm::getFlagWord(Kind, 1 /* RegNum*/); |
1a4d82fc JJ |
3427 | if (IsTiedToChangedOp) |
3428 | Flag = InlineAsm::getFlagWordForMatchingOp(Flag, DefIdx); | |
3429 | else | |
3430 | Flag = InlineAsm::getFlagWordForRegClass(Flag, ARM::GPRPairRegClassID); | |
970d7e83 LB |
3431 | // Replace the current flag. |
3432 | AsmNodeOperands[AsmNodeOperands.size() -1] = CurDAG->getTargetConstant( | |
3433 | Flag, MVT::i32); | |
3434 | // Add the new register node and skip the original two GPRs. | |
3435 | AsmNodeOperands.push_back(PairedReg); | |
3436 | // Skip the next two GPRs. | |
3437 | i += 2; | |
3438 | } | |
3439 | } | |
3440 | ||
1a4d82fc JJ |
3441 | if (Glue.getNode()) |
3442 | AsmNodeOperands.push_back(Glue); | |
970d7e83 | 3443 | if (!Changed) |
1a4d82fc | 3444 | return nullptr; |
970d7e83 | 3445 | |
1a4d82fc JJ |
3446 | SDValue New = CurDAG->getNode(ISD::INLINEASM, SDLoc(N), |
3447 | CurDAG->getVTList(MVT::Other, MVT::Glue), AsmNodeOperands); | |
970d7e83 LB |
3448 | New->setNodeId(-1); |
3449 | return New.getNode(); | |
3450 | } | |
3451 | ||
3452 | ||
223e47cc LB |
3453 | bool ARMDAGToDAGISel:: |
3454 | SelectInlineAsmMemoryOperand(const SDValue &Op, char ConstraintCode, | |
3455 | std::vector<SDValue> &OutOps) { | |
3456 | assert(ConstraintCode == 'm' && "unexpected asm memory constraint"); | |
3457 | // Require the address to be in a register. That is safe for all ARM | |
3458 | // variants and it is hard to do anything much smarter without knowing | |
3459 | // how the operand is used. | |
3460 | OutOps.push_back(Op); | |
3461 | return false; | |
3462 | } | |
3463 | ||
3464 | /// createARMISelDag - This pass converts a legalized DAG into a | |
3465 | /// ARM-specific DAG, ready for instruction scheduling. | |
3466 | /// | |
3467 | FunctionPass *llvm::createARMISelDag(ARMBaseTargetMachine &TM, | |
3468 | CodeGenOpt::Level OptLevel) { | |
3469 | return new ARMDAGToDAGISel(TM, OptLevel); | |
3470 | } |