[rustc.git] / src / llvm / lib / Target / X86 / X86SelectionDAGInfo.cpp

//===-- X86SelectionDAGInfo.cpp - X86 SelectionDAG Info -------------------===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// This file implements the X86SelectionDAGInfo class.
//
//===----------------------------------------------------------------------===//

#define DEBUG_TYPE "x86-selectiondag-info"
#include "X86TargetMachine.h"
#include "llvm/CodeGen/SelectionDAG.h"
#include "llvm/IR/DerivedTypes.h"
using namespace llvm;

X86SelectionDAGInfo::X86SelectionDAGInfo(const X86TargetMachine &TM) :
  TargetSelectionDAGInfo(TM),
  Subtarget(&TM.getSubtarget<X86Subtarget>()),
  TLI(*TM.getTargetLowering()) {
}

X86SelectionDAGInfo::~X86SelectionDAGInfo() {
}

SDValue
X86SelectionDAGInfo::EmitTargetCodeForMemset(SelectionDAG &DAG, DebugLoc dl,
                                             SDValue Chain,
                                             SDValue Dst, SDValue Src,
                                             SDValue Size, unsigned Align,
                                             bool isVolatile,
                                         MachinePointerInfo DstPtrInfo) const {
  ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);

  // If to a segment-relative address space, use the default lowering.
  if (DstPtrInfo.getAddrSpace() >= 256)
    return SDValue();

  // If not DWORD aligned or size is more than the threshold, call the library.
  // The libc version is likely to be faster for these cases. It can use the
  // address value and run time information about the CPU.
  if ((Align & 3) != 0 ||
      !ConstantSize ||
      ConstantSize->getZExtValue() >
        Subtarget->getMaxInlineSizeThreshold()) {
    SDValue InFlag(0, 0);

    // Check to see if there is a specialized entry-point for memory zeroing.
    ConstantSDNode *V = dyn_cast<ConstantSDNode>(Src);

    if (const char *bzeroEntry =  V &&
        V->isNullValue() ? Subtarget->getBZeroEntry() : 0) {
      EVT IntPtr = TLI.getPointerTy();
      Type *IntPtrTy = getDataLayout()->getIntPtrType(*DAG.getContext());
      TargetLowering::ArgListTy Args;
      TargetLowering::ArgListEntry Entry;
      Entry.Node = Dst;
      Entry.Ty = IntPtrTy;
      Args.push_back(Entry);
      Entry.Node = Size;
      Args.push_back(Entry);
      TargetLowering::
      CallLoweringInfo CLI(Chain, Type::getVoidTy(*DAG.getContext()),
                        false, false, false, false,
                        0, CallingConv::C, /*isTailCall=*/false,
                        /*doesNotRet=*/false, /*isReturnValueUsed=*/false,
                        DAG.getExternalSymbol(bzeroEntry, IntPtr), Args,
                        DAG, dl);
      std::pair<SDValue,SDValue> CallResult =
        TLI.LowerCallTo(CLI);
      return CallResult.second;
    }

    // Otherwise have the target-independent code call memset.
    return SDValue();
  }

  uint64_t SizeVal = ConstantSize->getZExtValue();
  SDValue InFlag(0, 0);
  EVT AVT;
  SDValue Count;
  ConstantSDNode *ValC = dyn_cast<ConstantSDNode>(Src);
  unsigned BytesLeft = 0;
  bool TwoRepStos = false;
  if (ValC) {
    unsigned ValReg;
    uint64_t Val = ValC->getZExtValue() & 255;

    // If the value is a constant, then we can potentially use larger sets.
    switch (Align & 3) {
    case 2:   // WORD aligned
      AVT = MVT::i16;
      ValReg = X86::AX;
      Val = (Val << 8) | Val;
      break;
    case 0:  // DWORD aligned
      AVT = MVT::i32;
      ValReg = X86::EAX;
      Val = (Val << 8)  | Val;
      Val = (Val << 16) | Val;
      if (Subtarget->is64Bit() && ((Align & 0x7) == 0)) {  // QWORD aligned
        AVT = MVT::i64;
        ValReg = X86::RAX;
        Val = (Val << 32) | Val;
      }
      break;
    default:  // Byte aligned
      AVT = MVT::i8;
      ValReg = X86::AL;
      Count = DAG.getIntPtrConstant(SizeVal);
      break;
    }

    if (AVT.bitsGT(MVT::i8)) {
      unsigned UBytes = AVT.getSizeInBits() / 8;
      Count = DAG.getIntPtrConstant(SizeVal / UBytes);
      BytesLeft = SizeVal % UBytes;
    }

    Chain  = DAG.getCopyToReg(Chain, dl, ValReg, DAG.getConstant(Val, AVT),
                              InFlag);
    InFlag = Chain.getValue(1);
  } else {
    AVT = MVT::i8;
    Count  = DAG.getIntPtrConstant(SizeVal);
    Chain  = DAG.getCopyToReg(Chain, dl, X86::AL, Src, InFlag);
    InFlag = Chain.getValue(1);
  }

  Chain  = DAG.getCopyToReg(Chain, dl, Subtarget->is64Bit() ? X86::RCX :
                                                              X86::ECX,
                            Count, InFlag);
  InFlag = Chain.getValue(1);
  Chain  = DAG.getCopyToReg(Chain, dl, Subtarget->is64Bit() ? X86::RDI :
                                                              X86::EDI,
                            Dst, InFlag);
  InFlag = Chain.getValue(1);

  SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
  SDValue Ops[] = { Chain, DAG.getValueType(AVT), InFlag };
  Chain = DAG.getNode(X86ISD::REP_STOS, dl, Tys, Ops, array_lengthof(Ops));

  if (TwoRepStos) {
    InFlag = Chain.getValue(1);
    Count  = Size;
    EVT CVT = Count.getValueType();
    SDValue Left = DAG.getNode(ISD::AND, dl, CVT, Count,
                               DAG.getConstant((AVT == MVT::i64) ? 7 : 3, CVT));
    Chain  = DAG.getCopyToReg(Chain, dl, (CVT == MVT::i64) ? X86::RCX :
                                                             X86::ECX,
                              Left, InFlag);
    InFlag = Chain.getValue(1);
    Tys = DAG.getVTList(MVT::Other, MVT::Glue);
    SDValue Ops[] = { Chain, DAG.getValueType(MVT::i8), InFlag };
    Chain = DAG.getNode(X86ISD::REP_STOS, dl, Tys, Ops, array_lengthof(Ops));
  } else if (BytesLeft) {
    // Handle the last 1 - 7 bytes.
    unsigned Offset = SizeVal - BytesLeft;
    EVT AddrVT = Dst.getValueType();
    EVT SizeVT = Size.getValueType();

    Chain = DAG.getMemset(Chain, dl,
                          DAG.getNode(ISD::ADD, dl, AddrVT, Dst,
                                      DAG.getConstant(Offset, AddrVT)),
                          Src,
                          DAG.getConstant(BytesLeft, SizeVT),
                          Align, isVolatile, DstPtrInfo.getWithOffset(Offset));
  }

  // TODO: Use a Tokenfactor, as in memcpy, instead of a single chain.
  return Chain;
}

SDValue
X86SelectionDAGInfo::EmitTargetCodeForMemcpy(SelectionDAG &DAG, DebugLoc dl,
                                        SDValue Chain, SDValue Dst, SDValue Src,
                                        SDValue Size, unsigned Align,
                                        bool isVolatile, bool AlwaysInline,
                                         MachinePointerInfo DstPtrInfo,
                                         MachinePointerInfo SrcPtrInfo) const {
  // This requires the copy size to be a constant, preferably
  // within a subtarget-specific limit.
  ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
  if (!ConstantSize)
    return SDValue();
  uint64_t SizeVal = ConstantSize->getZExtValue();
  if (!AlwaysInline && SizeVal > Subtarget->getMaxInlineSizeThreshold())
    return SDValue();

  /// If not DWORD aligned, it is more efficient to call the library.  However
  /// if calling the library is not allowed (AlwaysInline), then soldier on as
  /// the code generated here is better than the long load-store sequence we
  /// would otherwise get.
  if (!AlwaysInline && (Align & 3) != 0)
    return SDValue();

  // If to a segment-relative address space, use the default lowering.
  if (DstPtrInfo.getAddrSpace() >= 256 ||
      SrcPtrInfo.getAddrSpace() >= 256)
    return SDValue();

  // ESI might be used as a base pointer, in that case we can't simply overwrite
  // the register.  Fall back to generic code.
  const X86RegisterInfo *TRI =
      static_cast<const X86RegisterInfo *>(DAG.getTarget().getRegisterInfo());
  if (TRI->hasBasePointer(DAG.getMachineFunction()) &&
      TRI->getBaseRegister() == X86::ESI)
    return SDValue();

  MVT AVT;
  if (Align & 1)
    AVT = MVT::i8;
  else if (Align & 2)
    AVT = MVT::i16;
  else if (Align & 4)
    // DWORD aligned
    AVT = MVT::i32;
  else
    // QWORD aligned
    AVT = Subtarget->is64Bit() ? MVT::i64 : MVT::i32;

  unsigned UBytes = AVT.getSizeInBits() / 8;
  unsigned CountVal = SizeVal / UBytes;
  SDValue Count = DAG.getIntPtrConstant(CountVal);
  unsigned BytesLeft = SizeVal % UBytes;

  SDValue InFlag(0, 0);
  Chain  = DAG.getCopyToReg(Chain, dl, Subtarget->is64Bit() ? X86::RCX :
                                                              X86::ECX,
                            Count, InFlag);
  InFlag = Chain.getValue(1);
  Chain  = DAG.getCopyToReg(Chain, dl, Subtarget->is64Bit() ? X86::RDI :
                                                              X86::EDI,
                            Dst, InFlag);
  InFlag = Chain.getValue(1);
  Chain  = DAG.getCopyToReg(Chain, dl, Subtarget->is64Bit() ? X86::RSI :
                                                              X86::ESI,
                            Src, InFlag);
  InFlag = Chain.getValue(1);

  SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
  SDValue Ops[] = { Chain, DAG.getValueType(AVT), InFlag };
  SDValue RepMovs = DAG.getNode(X86ISD::REP_MOVS, dl, Tys, Ops,
                                array_lengthof(Ops));

  SmallVector<SDValue, 4> Results;
  Results.push_back(RepMovs);
  if (BytesLeft) {
    // Handle the last 1 - 7 bytes.
    unsigned Offset = SizeVal - BytesLeft;
    EVT DstVT = Dst.getValueType();
    EVT SrcVT = Src.getValueType();
    EVT SizeVT = Size.getValueType();
    Results.push_back(DAG.getMemcpy(Chain, dl,
                                    DAG.getNode(ISD::ADD, dl, DstVT, Dst,
                                                DAG.getConstant(Offset, DstVT)),
                                    DAG.getNode(ISD::ADD, dl, SrcVT, Src,
                                                DAG.getConstant(Offset, SrcVT)),
                                    DAG.getConstant(BytesLeft, SizeVT),
                                    Align, isVolatile, AlwaysInline,
                                    DstPtrInfo.getWithOffset(Offset),
                                    SrcPtrInfo.getWithOffset(Offset)));
  }

  return DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
                     &Results[0], Results.size());
}
Commit	Line	Data
223e47cc LB	1	//===-- X86SelectionDAGInfo.cpp - X86 SelectionDAG Info -------------------===//
	2	//
	3	// The LLVM Compiler Infrastructure
	4	//
	5	// This file is distributed under the University of Illinois Open Source
	6	// License. See LICENSE.TXT for details.
	7	//
	8	//===----------------------------------------------------------------------===//
	9	//
	10	// This file implements the X86SelectionDAGInfo class.
	11	//
	12	//===----------------------------------------------------------------------===//
	13
	14	#define DEBUG_TYPE "x86-selectiondag-info"
	15	#include "X86TargetMachine.h"
223e47cc	16	#include "llvm/CodeGen/SelectionDAG.h"
970d7e83	17	#include "llvm/IR/DerivedTypes.h"
223e47cc LB	18	using namespace llvm;
	19
	20	X86SelectionDAGInfo::X86SelectionDAGInfo(const X86TargetMachine &TM) :
	21	TargetSelectionDAGInfo(TM),
	22	Subtarget(&TM.getSubtarget<X86Subtarget>()),
	23	TLI(*TM.getTargetLowering()) {
	24	}
	25
	26	X86SelectionDAGInfo::~X86SelectionDAGInfo() {
	27	}
	28
	29	SDValue
	30	X86SelectionDAGInfo::EmitTargetCodeForMemset(SelectionDAG &DAG, DebugLoc dl,
	31	SDValue Chain,
	32	SDValue Dst, SDValue Src,
	33	SDValue Size, unsigned Align,
	34	bool isVolatile,
	35	MachinePointerInfo DstPtrInfo) const {
	36	ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
	37
	38	// If to a segment-relative address space, use the default lowering.
	39	if (DstPtrInfo.getAddrSpace() >= 256)
	40	return SDValue();
	41
	42	// If not DWORD aligned or size is more than the threshold, call the library.
	43	// The libc version is likely to be faster for these cases. It can use the
	44	// address value and run time information about the CPU.
	45	if ((Align & 3) != 0 \|\|
	46	!ConstantSize \|\|
	47	ConstantSize->getZExtValue() >
	48	Subtarget->getMaxInlineSizeThreshold()) {
	49	SDValue InFlag(0, 0);
	50
	51	// Check to see if there is a specialized entry-point for memory zeroing.
	52	ConstantSDNode *V = dyn_cast<ConstantSDNode>(Src);
	53
	54	if (const char *bzeroEntry = V &&
	55	V->isNullValue() ? Subtarget->getBZeroEntry() : 0) {
	56	EVT IntPtr = TLI.getPointerTy();
970d7e83	57	Type IntPtrTy = getDataLayout()->getIntPtrType(DAG.getContext());
223e47cc LB	58	TargetLowering::ArgListTy Args;
	59	TargetLowering::ArgListEntry Entry;
	60	Entry.Node = Dst;
	61	Entry.Ty = IntPtrTy;
	62	Args.push_back(Entry);
	63	Entry.Node = Size;
	64	Args.push_back(Entry);
	65	TargetLowering::
	66	CallLoweringInfo CLI(Chain, Type::getVoidTy(*DAG.getContext()),
	67	false, false, false, false,
	68	0, CallingConv::C, /isTailCall=/false,
	69	/doesNotRet=/false, /isReturnValueUsed=/false,
	70	DAG.getExternalSymbol(bzeroEntry, IntPtr), Args,
	71	DAG, dl);
	72	std::pair<SDValue,SDValue> CallResult =
	73	TLI.LowerCallTo(CLI);
	74	return CallResult.second;
	75	}
	76
	77	// Otherwise have the target-independent code call memset.
	78	return SDValue();
	79	}
	80
	81	uint64_t SizeVal = ConstantSize->getZExtValue();
	82	SDValue InFlag(0, 0);
	83	EVT AVT;
	84	SDValue Count;
	85	ConstantSDNode *ValC = dyn_cast<ConstantSDNode>(Src);
	86	unsigned BytesLeft = 0;
	87	bool TwoRepStos = false;
	88	if (ValC) {
	89	unsigned ValReg;
	90	uint64_t Val = ValC->getZExtValue() & 255;
	91
	92	// If the value is a constant, then we can potentially use larger sets.
	93	switch (Align & 3) {
	94	case 2: // WORD aligned
	95	AVT = MVT::i16;
	96	ValReg = X86::AX;
	97	Val = (Val << 8) \| Val;
	98	break;
	99	case 0: // DWORD aligned
	100	AVT = MVT::i32;
	101	ValReg = X86::EAX;
	102	Val = (Val << 8) \| Val;
	103	Val = (Val << 16) \| Val;
	104	if (Subtarget->is64Bit() && ((Align & 0x7) == 0)) { // QWORD aligned
	105	AVT = MVT::i64;
	106	ValReg = X86::RAX;
	107	Val = (Val << 32) \| Val;
	108	}
	109	break;
	110	default: // Byte aligned
	111	AVT = MVT::i8;
	112	ValReg = X86::AL;
	113	Count = DAG.getIntPtrConstant(SizeVal);
	114	break;
	115	}
	116
	117	if (AVT.bitsGT(MVT::i8)) {
	118	unsigned UBytes = AVT.getSizeInBits() / 8;
	119	Count = DAG.getIntPtrConstant(SizeVal / UBytes);
	120	BytesLeft = SizeVal % UBytes;
	121	}
122
123	Chain = DAG.getCopyToReg(Chain, dl, ValReg, DAG.getConstant(Val, AVT),
124	InFlag);
125	InFlag = Chain.getValue(1);
126	} else {
127	AVT = MVT::i8;
128	Count = DAG.getIntPtrConstant(SizeVal);
129	Chain = DAG.getCopyToReg(Chain, dl, X86::AL, Src, InFlag);
130	InFlag = Chain.getValue(1);
131	}
132
133	Chain = DAG.getCopyToReg(Chain, dl, Subtarget->is64Bit() ? X86::RCX :
134	X86::ECX,
135	Count, InFlag);
136	InFlag = Chain.getValue(1);
137	Chain = DAG.getCopyToReg(Chain, dl, Subtarget->is64Bit() ? X86::RDI :
138	X86::EDI,
139	Dst, InFlag);
140	InFlag = Chain.getValue(1);
141
142	SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
143	SDValue Ops[] = { Chain, DAG.getValueType(AVT), InFlag };
144	Chain = DAG.getNode(X86ISD::REP_STOS, dl, Tys, Ops, array_lengthof(Ops));
145
146	if (TwoRepStos) {
147	InFlag = Chain.getValue(1);
148	Count = Size;
149	EVT CVT = Count.getValueType();
150	SDValue Left = DAG.getNode(ISD::AND, dl, CVT, Count,
151	DAG.getConstant((AVT == MVT::i64) ? 7 : 3, CVT));
152	Chain = DAG.getCopyToReg(Chain, dl, (CVT == MVT::i64) ? X86::RCX :
153	X86::ECX,
154	Left, InFlag);
155	InFlag = Chain.getValue(1);
156	Tys = DAG.getVTList(MVT::Other, MVT::Glue);
157	SDValue Ops[] = { Chain, DAG.getValueType(MVT::i8), InFlag };
158	Chain = DAG.getNode(X86ISD::REP_STOS, dl, Tys, Ops, array_lengthof(Ops));
159	} else if (BytesLeft) {
160	// Handle the last 1 - 7 bytes.
161	unsigned Offset = SizeVal - BytesLeft;
162	EVT AddrVT = Dst.getValueType();
163	EVT SizeVT = Size.getValueType();
164
165	Chain = DAG.getMemset(Chain, dl,
166	DAG.getNode(ISD::ADD, dl, AddrVT, Dst,
167	DAG.getConstant(Offset, AddrVT)),
168	Src,
169	DAG.getConstant(BytesLeft, SizeVT),
170	Align, isVolatile, DstPtrInfo.getWithOffset(Offset));
171	}
172
173	// TODO: Use a Tokenfactor, as in memcpy, instead of a single chain.
174	return Chain;
175	}
176
177	SDValue
178	X86SelectionDAGInfo::EmitTargetCodeForMemcpy(SelectionDAG &DAG, DebugLoc dl,
179	SDValue Chain, SDValue Dst, SDValue Src,
180	SDValue Size, unsigned Align,
181	bool isVolatile, bool AlwaysInline,
182	MachinePointerInfo DstPtrInfo,
183	MachinePointerInfo SrcPtrInfo) const {
184	// This requires the copy size to be a constant, preferably
185	// within a subtarget-specific limit.
186	ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
187	if (!ConstantSize)
188	return SDValue();
189	uint64_t SizeVal = ConstantSize->getZExtValue();
190	if (!AlwaysInline && SizeVal > Subtarget->getMaxInlineSizeThreshold())
191	return SDValue();
192
193	/// If not DWORD aligned, it is more efficient to call the library. However
194	/// if calling the library is not allowed (AlwaysInline), then soldier on as
195	/// the code generated here is better than the long load-store sequence we
196	/// would otherwise get.
197	if (!AlwaysInline && (Align & 3) != 0)
198	return SDValue();
199
200	// If to a segment-relative address space, use the default lowering.
201	if (DstPtrInfo.getAddrSpace() >= 256 \|\|
202	SrcPtrInfo.getAddrSpace() >= 256)
203	return SDValue();
204
970d7e83 LB	205	// ESI might be used as a base pointer, in that case we can't simply overwrite
	206	// the register. Fall back to generic code.
	207	const X86RegisterInfo *TRI =
	208	static_cast<const X86RegisterInfo *>(DAG.getTarget().getRegisterInfo());
	209	if (TRI->hasBasePointer(DAG.getMachineFunction()) &&
	210	TRI->getBaseRegister() == X86::ESI)
	211	return SDValue();
	212
223e47cc LB	213	MVT AVT;
	214	if (Align & 1)
	215	AVT = MVT::i8;
	216	else if (Align & 2)
	217	AVT = MVT::i16;
	218	else if (Align & 4)
	219	// DWORD aligned
	220	AVT = MVT::i32;
	221	else
	222	// QWORD aligned
	223	AVT = Subtarget->is64Bit() ? MVT::i64 : MVT::i32;
	224
	225	unsigned UBytes = AVT.getSizeInBits() / 8;
	226	unsigned CountVal = SizeVal / UBytes;
	227	SDValue Count = DAG.getIntPtrConstant(CountVal);
	228	unsigned BytesLeft = SizeVal % UBytes;
	229
	230	SDValue InFlag(0, 0);
	231	Chain = DAG.getCopyToReg(Chain, dl, Subtarget->is64Bit() ? X86::RCX :
	232	X86::ECX,
	233	Count, InFlag);
	234	InFlag = Chain.getValue(1);
	235	Chain = DAG.getCopyToReg(Chain, dl, Subtarget->is64Bit() ? X86::RDI :
	236	X86::EDI,
	237	Dst, InFlag);
	238	InFlag = Chain.getValue(1);
	239	Chain = DAG.getCopyToReg(Chain, dl, Subtarget->is64Bit() ? X86::RSI :
	240	X86::ESI,
	241	Src, InFlag);
	242	InFlag = Chain.getValue(1);
	243
	244	SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
	245	SDValue Ops[] = { Chain, DAG.getValueType(AVT), InFlag };
	246	SDValue RepMovs = DAG.getNode(X86ISD::REP_MOVS, dl, Tys, Ops,
	247	array_lengthof(Ops));
	248
	249	SmallVector<SDValue, 4> Results;
	250	Results.push_back(RepMovs);
	251	if (BytesLeft) {
	252	// Handle the last 1 - 7 bytes.
	253	unsigned Offset = SizeVal - BytesLeft;
	254	EVT DstVT = Dst.getValueType();
	255	EVT SrcVT = Src.getValueType();
	256	EVT SizeVT = Size.getValueType();
	257	Results.push_back(DAG.getMemcpy(Chain, dl,
	258	DAG.getNode(ISD::ADD, dl, DstVT, Dst,
	259	DAG.getConstant(Offset, DstVT)),
	260	DAG.getNode(ISD::ADD, dl, SrcVT, Src,
	261	DAG.getConstant(Offset, SrcVT)),
	262	DAG.getConstant(BytesLeft, SizeVT),
	263	Align, isVolatile, AlwaysInline,
	264	DstPtrInfo.getWithOffset(Offset),
	265	SrcPtrInfo.getWithOffset(Offset)));
	266	}
	267
	268	return DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
	269	&Results[0], Results.size());
	270	}