[rustc.git] / src / llvm / lib / Target / X86 / X86SelectionDAGInfo.cpp

//===-- X86SelectionDAGInfo.cpp - X86 SelectionDAG Info -------------------===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// This file implements the X86SelectionDAGInfo class.
//
//===----------------------------------------------------------------------===//

#include "X86InstrInfo.h"
#include "X86ISelLowering.h"
#include "X86RegisterInfo.h"
#include "X86Subtarget.h"
#include "X86SelectionDAGInfo.h"
#include "llvm/CodeGen/SelectionDAG.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/Target/TargetLowering.h"

using namespace llvm;

#define DEBUG_TYPE "x86-selectiondag-info"

X86SelectionDAGInfo::X86SelectionDAGInfo(const DataLayout &DL)
    : TargetSelectionDAGInfo(&DL) {}

X86SelectionDAGInfo::~X86SelectionDAGInfo() {}

bool X86SelectionDAGInfo::isBaseRegConflictPossible(
    SelectionDAG &DAG, ArrayRef<unsigned> ClobberSet) const {
  // We cannot use TRI->hasBasePointer() until *after* we select all basic
  // blocks.  Legalization may introduce new stack temporaries with large
  // alignment requirements.  Fall back to generic code if there are any
  // dynamic stack adjustments (hopefully rare) and the base pointer would
  // conflict if we had to use it.
  MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo();
  if (!MFI->hasVarSizedObjects() && !MFI->hasInlineAsmWithSPAdjust())
    return false;

  const X86RegisterInfo *TRI = static_cast<const X86RegisterInfo *>(
      DAG.getSubtarget().getRegisterInfo());
  unsigned BaseReg = TRI->getBaseRegister();
  for (unsigned R : ClobberSet)
    if (BaseReg == R)
      return true;
  return false;
}

SDValue
X86SelectionDAGInfo::EmitTargetCodeForMemset(SelectionDAG &DAG, SDLoc dl,
                                             SDValue Chain,
                                             SDValue Dst, SDValue Src,
                                             SDValue Size, unsigned Align,
                                             bool isVolatile,
                                         MachinePointerInfo DstPtrInfo) const {
  ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
  const X86Subtarget &Subtarget = DAG.getTarget().getSubtarget<X86Subtarget>();

#ifndef NDEBUG
  // If the base register might conflict with our physical registers, bail out.
  unsigned ClobberSet[] = {X86::RCX, X86::RAX, X86::RDI,
                           X86::ECX, X86::EAX, X86::EDI};
  assert(!isBaseRegConflictPossible(DAG, ClobberSet));
#endif

  // If to a segment-relative address space, use the default lowering.
  if (DstPtrInfo.getAddrSpace() >= 256)
    return SDValue();

  // If not DWORD aligned or size is more than the threshold, call the library.
  // The libc version is likely to be faster for these cases. It can use the
  // address value and run time information about the CPU.
  if ((Align & 3) != 0 || !ConstantSize ||
      ConstantSize->getZExtValue() > Subtarget.getMaxInlineSizeThreshold()) {
    // Check to see if there is a specialized entry-point for memory zeroing.
    ConstantSDNode *V = dyn_cast<ConstantSDNode>(Src);

    if (const char *bzeroEntry =  V &&
        V->isNullValue() ? Subtarget.getBZeroEntry() : nullptr) {
      EVT IntPtr = DAG.getTargetLoweringInfo().getPointerTy();
      Type *IntPtrTy = getDataLayout()->getIntPtrType(*DAG.getContext());
      TargetLowering::ArgListTy Args;
      TargetLowering::ArgListEntry Entry;
      Entry.Node = Dst;
      Entry.Ty = IntPtrTy;
      Args.push_back(Entry);
      Entry.Node = Size;
      Args.push_back(Entry);

      TargetLowering::CallLoweringInfo CLI(DAG);
      CLI.setDebugLoc(dl).setChain(Chain)
        .setCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()),
                   DAG.getExternalSymbol(bzeroEntry, IntPtr), std::move(Args),
                   0)
        .setDiscardResult();

      std::pair<SDValue,SDValue> CallResult = DAG.getTargetLoweringInfo().LowerCallTo(CLI);
      return CallResult.second;
    }

    // Otherwise have the target-independent code call memset.
    return SDValue();
  }

  uint64_t SizeVal = ConstantSize->getZExtValue();
  SDValue InFlag;
  EVT AVT;
  SDValue Count;
  ConstantSDNode *ValC = dyn_cast<ConstantSDNode>(Src);
  unsigned BytesLeft = 0;
  bool TwoRepStos = false;
  if (ValC) {
    unsigned ValReg;
    uint64_t Val = ValC->getZExtValue() & 255;

    // If the value is a constant, then we can potentially use larger sets.
    switch (Align & 3) {
    case 2:   // WORD aligned
      AVT = MVT::i16;
      ValReg = X86::AX;
      Val = (Val << 8) | Val;
      break;
    case 0:  // DWORD aligned
      AVT = MVT::i32;
      ValReg = X86::EAX;
      Val = (Val << 8)  | Val;
      Val = (Val << 16) | Val;
      if (Subtarget.is64Bit() && ((Align & 0x7) == 0)) {  // QWORD aligned
        AVT = MVT::i64;
        ValReg = X86::RAX;
        Val = (Val << 32) | Val;
      }
      break;
    default:  // Byte aligned
      AVT = MVT::i8;
      ValReg = X86::AL;
      Count = DAG.getIntPtrConstant(SizeVal);
      break;
    }

    if (AVT.bitsGT(MVT::i8)) {
      unsigned UBytes = AVT.getSizeInBits() / 8;
      Count = DAG.getIntPtrConstant(SizeVal / UBytes);
      BytesLeft = SizeVal % UBytes;
    }

    Chain  = DAG.getCopyToReg(Chain, dl, ValReg, DAG.getConstant(Val, AVT),
                              InFlag);
    InFlag = Chain.getValue(1);
  } else {
    AVT = MVT::i8;
    Count  = DAG.getIntPtrConstant(SizeVal);
    Chain  = DAG.getCopyToReg(Chain, dl, X86::AL, Src, InFlag);
    InFlag = Chain.getValue(1);
  }

  Chain = DAG.getCopyToReg(Chain, dl, Subtarget.is64Bit() ? X86::RCX : X86::ECX,
                           Count, InFlag);
  InFlag = Chain.getValue(1);
  Chain = DAG.getCopyToReg(Chain, dl, Subtarget.is64Bit() ? X86::RDI : X86::EDI,
                           Dst, InFlag);
  InFlag = Chain.getValue(1);

  SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
  SDValue Ops[] = { Chain, DAG.getValueType(AVT), InFlag };
  Chain = DAG.getNode(X86ISD::REP_STOS, dl, Tys, Ops);

  if (TwoRepStos) {
    InFlag = Chain.getValue(1);
    Count  = Size;
    EVT CVT = Count.getValueType();
    SDValue Left = DAG.getNode(ISD::AND, dl, CVT, Count,
                               DAG.getConstant((AVT == MVT::i64) ? 7 : 3, CVT));
    Chain  = DAG.getCopyToReg(Chain, dl, (CVT == MVT::i64) ? X86::RCX :
                                                             X86::ECX,
                              Left, InFlag);
    InFlag = Chain.getValue(1);
    Tys = DAG.getVTList(MVT::Other, MVT::Glue);
    SDValue Ops[] = { Chain, DAG.getValueType(MVT::i8), InFlag };
    Chain = DAG.getNode(X86ISD::REP_STOS, dl, Tys, Ops);
  } else if (BytesLeft) {
    // Handle the last 1 - 7 bytes.
    unsigned Offset = SizeVal - BytesLeft;
    EVT AddrVT = Dst.getValueType();
    EVT SizeVT = Size.getValueType();

    Chain = DAG.getMemset(Chain, dl,
                          DAG.getNode(ISD::ADD, dl, AddrVT, Dst,
                                      DAG.getConstant(Offset, AddrVT)),
                          Src,
                          DAG.getConstant(BytesLeft, SizeVT),
                          Align, isVolatile, DstPtrInfo.getWithOffset(Offset));
  }

  // TODO: Use a Tokenfactor, as in memcpy, instead of a single chain.
  return Chain;
}

SDValue
X86SelectionDAGInfo::EmitTargetCodeForMemcpy(SelectionDAG &DAG, SDLoc dl,
                                        SDValue Chain, SDValue Dst, SDValue Src,
                                        SDValue Size, unsigned Align,
                                        bool isVolatile, bool AlwaysInline,
                                         MachinePointerInfo DstPtrInfo,
                                         MachinePointerInfo SrcPtrInfo) const {
  // This requires the copy size to be a constant, preferably
  // within a subtarget-specific limit.
  ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
  const X86Subtarget &Subtarget = DAG.getTarget().getSubtarget<X86Subtarget>();
  if (!ConstantSize)
    return SDValue();
  uint64_t SizeVal = ConstantSize->getZExtValue();
  if (!AlwaysInline && SizeVal > Subtarget.getMaxInlineSizeThreshold())
    return SDValue();

  /// If not DWORD aligned, it is more efficient to call the library.  However
  /// if calling the library is not allowed (AlwaysInline), then soldier on as
  /// the code generated here is better than the long load-store sequence we
  /// would otherwise get.
  if (!AlwaysInline && (Align & 3) != 0)
    return SDValue();

  // If to a segment-relative address space, use the default lowering.
  if (DstPtrInfo.getAddrSpace() >= 256 ||
      SrcPtrInfo.getAddrSpace() >= 256)
    return SDValue();

  // If the base register might conflict with our physical registers, bail out.
  unsigned ClobberSet[] = {X86::RCX, X86::RSI, X86::RDI,
                           X86::ECX, X86::ESI, X86::EDI};
  if (isBaseRegConflictPossible(DAG, ClobberSet))
    return SDValue();

  MVT AVT;
  if (Align & 1)
    AVT = MVT::i8;
  else if (Align & 2)
    AVT = MVT::i16;
  else if (Align & 4)
    // DWORD aligned
    AVT = MVT::i32;
  else
    // QWORD aligned
    AVT = Subtarget.is64Bit() ? MVT::i64 : MVT::i32;

  unsigned UBytes = AVT.getSizeInBits() / 8;
  unsigned CountVal = SizeVal / UBytes;
  SDValue Count = DAG.getIntPtrConstant(CountVal);
  unsigned BytesLeft = SizeVal % UBytes;

  SDValue InFlag;
  Chain  = DAG.getCopyToReg(Chain, dl, Subtarget.is64Bit() ? X86::RCX :
                                                              X86::ECX,
                            Count, InFlag);
  InFlag = Chain.getValue(1);
  Chain  = DAG.getCopyToReg(Chain, dl, Subtarget.is64Bit() ? X86::RDI :
                                                              X86::EDI,
                            Dst, InFlag);
  InFlag = Chain.getValue(1);
  Chain  = DAG.getCopyToReg(Chain, dl, Subtarget.is64Bit() ? X86::RSI :
                                                              X86::ESI,
                            Src, InFlag);
  InFlag = Chain.getValue(1);

  SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
  SDValue Ops[] = { Chain, DAG.getValueType(AVT), InFlag };
  SDValue RepMovs = DAG.getNode(X86ISD::REP_MOVS, dl, Tys, Ops);

  SmallVector<SDValue, 4> Results;
  Results.push_back(RepMovs);
  if (BytesLeft) {
    // Handle the last 1 - 7 bytes.
    unsigned Offset = SizeVal - BytesLeft;
    EVT DstVT = Dst.getValueType();
    EVT SrcVT = Src.getValueType();
    EVT SizeVT = Size.getValueType();
    Results.push_back(DAG.getMemcpy(Chain, dl,
                                    DAG.getNode(ISD::ADD, dl, DstVT, Dst,
                                                DAG.getConstant(Offset, DstVT)),
                                    DAG.getNode(ISD::ADD, dl, SrcVT, Src,
                                                DAG.getConstant(Offset, SrcVT)),
                                    DAG.getConstant(BytesLeft, SizeVT),
                                    Align, isVolatile, AlwaysInline,
                                    DstPtrInfo.getWithOffset(Offset),
                                    SrcPtrInfo.getWithOffset(Offset)));
  }

  return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Results);
}
Commit	Line	Data
223e47cc LB	1	//===-- X86SelectionDAGInfo.cpp - X86 SelectionDAG Info -------------------===//
	2	//
	3	// The LLVM Compiler Infrastructure
	4	//
	5	// This file is distributed under the University of Illinois Open Source
	6	// License. See LICENSE.TXT for details.
	7	//
	8	//===----------------------------------------------------------------------===//
	9	//
	10	// This file implements the X86SelectionDAGInfo class.
	11	//
	12	//===----------------------------------------------------------------------===//
	13
1a4d82fc JJ	14	#include "X86InstrInfo.h"
	15	#include "X86ISelLowering.h"
	16	#include "X86RegisterInfo.h"
	17	#include "X86Subtarget.h"
	18	#include "X86SelectionDAGInfo.h"
223e47cc	19	#include "llvm/CodeGen/SelectionDAG.h"
970d7e83	20	#include "llvm/IR/DerivedTypes.h"
1a4d82fc JJ	21	#include "llvm/Target/TargetLowering.h"
1a4d82fc JJ	22
223e47cc LB	23	using namespace llvm;
223e47cc LB	24
1a4d82fc	25	#define DEBUG_TYPE "x86-selectiondag-info"
223e47cc	26
1a4d82fc JJ	27	X86SelectionDAGInfo::X86SelectionDAGInfo(const DataLayout &DL)
	28	: TargetSelectionDAGInfo(&DL) {}
	29
	30	X86SelectionDAGInfo::~X86SelectionDAGInfo() {}
	31
	32	bool X86SelectionDAGInfo::isBaseRegConflictPossible(
	33	SelectionDAG &DAG, ArrayRef<unsigned> ClobberSet) const {
	34	// We cannot use TRI->hasBasePointer() until after we select all basic
	35	// blocks. Legalization may introduce new stack temporaries with large
	36	// alignment requirements. Fall back to generic code if there are any
	37	// dynamic stack adjustments (hopefully rare) and the base pointer would
	38	// conflict if we had to use it.
	39	MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo();
	40	if (!MFI->hasVarSizedObjects() && !MFI->hasInlineAsmWithSPAdjust())
	41	return false;
	42
	43	const X86RegisterInfo TRI = static_cast<const X86RegisterInfo >(
	44	DAG.getSubtarget().getRegisterInfo());
	45	unsigned BaseReg = TRI->getBaseRegister();
	46	for (unsigned R : ClobberSet)
	47	if (BaseReg == R)
	48	return true;
	49	return false;
223e47cc LB	50	}
	51
	52	SDValue
1a4d82fc	53	X86SelectionDAGInfo::EmitTargetCodeForMemset(SelectionDAG &DAG, SDLoc dl,
223e47cc LB	54	SDValue Chain,
	55	SDValue Dst, SDValue Src,
	56	SDValue Size, unsigned Align,
	57	bool isVolatile,
	58	MachinePointerInfo DstPtrInfo) const {
	59	ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
1a4d82fc JJ	60	const X86Subtarget &Subtarget = DAG.getTarget().getSubtarget<X86Subtarget>();
	61
	62	#ifndef NDEBUG
	63	// If the base register might conflict with our physical registers, bail out.
	64	unsigned ClobberSet[] = {X86::RCX, X86::RAX, X86::RDI,
	65	X86::ECX, X86::EAX, X86::EDI};
	66	assert(!isBaseRegConflictPossible(DAG, ClobberSet));
	67	#endif
223e47cc LB	68
	69	// If to a segment-relative address space, use the default lowering.
	70	if (DstPtrInfo.getAddrSpace() >= 256)
	71	return SDValue();
	72
	73	// If not DWORD aligned or size is more than the threshold, call the library.
	74	// The libc version is likely to be faster for these cases. It can use the
	75	// address value and run time information about the CPU.
1a4d82fc JJ	76	if ((Align & 3) != 0 \|\| !ConstantSize \|\|
1a4d82fc JJ	77	ConstantSize->getZExtValue() > Subtarget.getMaxInlineSizeThreshold()) {
223e47cc LB	78	// Check to see if there is a specialized entry-point for memory zeroing.
	79	ConstantSDNode *V = dyn_cast<ConstantSDNode>(Src);
	80
	81	if (const char *bzeroEntry = V &&
1a4d82fc JJ	82	V->isNullValue() ? Subtarget.getBZeroEntry() : nullptr) {
1a4d82fc JJ	83	EVT IntPtr = DAG.getTargetLoweringInfo().getPointerTy();
970d7e83	84	Type IntPtrTy = getDataLayout()->getIntPtrType(DAG.getContext());
223e47cc LB	85	TargetLowering::ArgListTy Args;
	86	TargetLowering::ArgListEntry Entry;
	87	Entry.Node = Dst;
	88	Entry.Ty = IntPtrTy;
	89	Args.push_back(Entry);
	90	Entry.Node = Size;
	91	Args.push_back(Entry);
1a4d82fc JJ	92
	93	TargetLowering::CallLoweringInfo CLI(DAG);
	94	CLI.setDebugLoc(dl).setChain(Chain)
	95	.setCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()),
	96	DAG.getExternalSymbol(bzeroEntry, IntPtr), std::move(Args),
	97	0)
	98	.setDiscardResult();
	99
	100	std::pair<SDValue,SDValue> CallResult = DAG.getTargetLoweringInfo().LowerCallTo(CLI);
223e47cc LB	101	return CallResult.second;
	102	}
	103
	104	// Otherwise have the target-independent code call memset.
	105	return SDValue();
	106	}
	107
	108	uint64_t SizeVal = ConstantSize->getZExtValue();
1a4d82fc	109	SDValue InFlag;
223e47cc LB	110	EVT AVT;
	111	SDValue Count;
	112	ConstantSDNode *ValC = dyn_cast<ConstantSDNode>(Src);
	113	unsigned BytesLeft = 0;
	114	bool TwoRepStos = false;
	115	if (ValC) {
	116	unsigned ValReg;
	117	uint64_t Val = ValC->getZExtValue() & 255;
	118
	119	// If the value is a constant, then we can potentially use larger sets.
	120	switch (Align & 3) {
	121	case 2: // WORD aligned
	122	AVT = MVT::i16;
	123	ValReg = X86::AX;
	124	Val = (Val << 8) \| Val;
	125	break;
	126	case 0: // DWORD aligned
	127	AVT = MVT::i32;
	128	ValReg = X86::EAX;
	129	Val = (Val << 8) \| Val;
	130	Val = (Val << 16) \| Val;
1a4d82fc	131	if (Subtarget.is64Bit() && ((Align & 0x7) == 0)) { // QWORD aligned
223e47cc LB	132	AVT = MVT::i64;
	133	ValReg = X86::RAX;
	134	Val = (Val << 32) \| Val;
	135	}
	136	break;
	137	default: // Byte aligned
	138	AVT = MVT::i8;
	139	ValReg = X86::AL;
	140	Count = DAG.getIntPtrConstant(SizeVal);
	141	break;
	142	}
	143
	144	if (AVT.bitsGT(MVT::i8)) {
	145	unsigned UBytes = AVT.getSizeInBits() / 8;
	146	Count = DAG.getIntPtrConstant(SizeVal / UBytes);
	147	BytesLeft = SizeVal % UBytes;
	148	}
	149
	150	Chain = DAG.getCopyToReg(Chain, dl, ValReg, DAG.getConstant(Val, AVT),
	151	InFlag);
	152	InFlag = Chain.getValue(1);
	153	} else {
	154	AVT = MVT::i8;
	155	Count = DAG.getIntPtrConstant(SizeVal);
	156	Chain = DAG.getCopyToReg(Chain, dl, X86::AL, Src, InFlag);
	157	InFlag = Chain.getValue(1);
	158	}
	159
1a4d82fc JJ	160	Chain = DAG.getCopyToReg(Chain, dl, Subtarget.is64Bit() ? X86::RCX : X86::ECX,
1a4d82fc JJ	161	Count, InFlag);
223e47cc	162	InFlag = Chain.getValue(1);
1a4d82fc JJ	163	Chain = DAG.getCopyToReg(Chain, dl, Subtarget.is64Bit() ? X86::RDI : X86::EDI,
1a4d82fc JJ	164	Dst, InFlag);
223e47cc LB	165	InFlag = Chain.getValue(1);
	166
	167	SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
	168	SDValue Ops[] = { Chain, DAG.getValueType(AVT), InFlag };
1a4d82fc	169	Chain = DAG.getNode(X86ISD::REP_STOS, dl, Tys, Ops);
223e47cc LB	170
	171	if (TwoRepStos) {
	172	InFlag = Chain.getValue(1);
	173	Count = Size;
	174	EVT CVT = Count.getValueType();
	175	SDValue Left = DAG.getNode(ISD::AND, dl, CVT, Count,
	176	DAG.getConstant((AVT == MVT::i64) ? 7 : 3, CVT));
	177	Chain = DAG.getCopyToReg(Chain, dl, (CVT == MVT::i64) ? X86::RCX :
	178	X86::ECX,
	179	Left, InFlag);
	180	InFlag = Chain.getValue(1);
	181	Tys = DAG.getVTList(MVT::Other, MVT::Glue);
	182	SDValue Ops[] = { Chain, DAG.getValueType(MVT::i8), InFlag };
1a4d82fc	183	Chain = DAG.getNode(X86ISD::REP_STOS, dl, Tys, Ops);
223e47cc LB	184	} else if (BytesLeft) {
	185	// Handle the last 1 - 7 bytes.
	186	unsigned Offset = SizeVal - BytesLeft;
	187	EVT AddrVT = Dst.getValueType();
	188	EVT SizeVT = Size.getValueType();
	189
	190	Chain = DAG.getMemset(Chain, dl,
	191	DAG.getNode(ISD::ADD, dl, AddrVT, Dst,
	192	DAG.getConstant(Offset, AddrVT)),
	193	Src,
	194	DAG.getConstant(BytesLeft, SizeVT),
	195	Align, isVolatile, DstPtrInfo.getWithOffset(Offset));
	196	}
	197
	198	// TODO: Use a Tokenfactor, as in memcpy, instead of a single chain.
	199	return Chain;
	200	}
	201
	202	SDValue
1a4d82fc	203	X86SelectionDAGInfo::EmitTargetCodeForMemcpy(SelectionDAG &DAG, SDLoc dl,
223e47cc LB	204	SDValue Chain, SDValue Dst, SDValue Src,
	205	SDValue Size, unsigned Align,
	206	bool isVolatile, bool AlwaysInline,
	207	MachinePointerInfo DstPtrInfo,
	208	MachinePointerInfo SrcPtrInfo) const {
	209	// This requires the copy size to be a constant, preferably
	210	// within a subtarget-specific limit.
	211	ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
1a4d82fc	212	const X86Subtarget &Subtarget = DAG.getTarget().getSubtarget<X86Subtarget>();
223e47cc LB	213	if (!ConstantSize)
	214	return SDValue();
	215	uint64_t SizeVal = ConstantSize->getZExtValue();
1a4d82fc	216	if (!AlwaysInline && SizeVal > Subtarget.getMaxInlineSizeThreshold())
223e47cc LB	217	return SDValue();
	218
	219	/// If not DWORD aligned, it is more efficient to call the library. However
	220	/// if calling the library is not allowed (AlwaysInline), then soldier on as
	221	/// the code generated here is better than the long load-store sequence we
	222	/// would otherwise get.
	223	if (!AlwaysInline && (Align & 3) != 0)
	224	return SDValue();
	225
	226	// If to a segment-relative address space, use the default lowering.
	227	if (DstPtrInfo.getAddrSpace() >= 256 \|\|
	228	SrcPtrInfo.getAddrSpace() >= 256)
	229	return SDValue();
	230
1a4d82fc JJ	231	// If the base register might conflict with our physical registers, bail out.
	232	unsigned ClobberSet[] = {X86::RCX, X86::RSI, X86::RDI,
	233	X86::ECX, X86::ESI, X86::EDI};
	234	if (isBaseRegConflictPossible(DAG, ClobberSet))
970d7e83 LB	235	return SDValue();
970d7e83 LB	236
223e47cc LB	237	MVT AVT;
	238	if (Align & 1)
	239	AVT = MVT::i8;
	240	else if (Align & 2)
	241	AVT = MVT::i16;
	242	else if (Align & 4)
	243	// DWORD aligned
	244	AVT = MVT::i32;
	245	else
	246	// QWORD aligned
1a4d82fc	247	AVT = Subtarget.is64Bit() ? MVT::i64 : MVT::i32;
223e47cc LB	248
	249	unsigned UBytes = AVT.getSizeInBits() / 8;
	250	unsigned CountVal = SizeVal / UBytes;
	251	SDValue Count = DAG.getIntPtrConstant(CountVal);
	252	unsigned BytesLeft = SizeVal % UBytes;
	253
1a4d82fc JJ	254	SDValue InFlag;
1a4d82fc JJ	255	Chain = DAG.getCopyToReg(Chain, dl, Subtarget.is64Bit() ? X86::RCX :
223e47cc LB	256	X86::ECX,
	257	Count, InFlag);
	258	InFlag = Chain.getValue(1);
1a4d82fc	259	Chain = DAG.getCopyToReg(Chain, dl, Subtarget.is64Bit() ? X86::RDI :
223e47cc LB	260	X86::EDI,
	261	Dst, InFlag);
	262	InFlag = Chain.getValue(1);
1a4d82fc	263	Chain = DAG.getCopyToReg(Chain, dl, Subtarget.is64Bit() ? X86::RSI :
223e47cc LB	264	X86::ESI,
	265	Src, InFlag);
	266	InFlag = Chain.getValue(1);
	267
	268	SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
	269	SDValue Ops[] = { Chain, DAG.getValueType(AVT), InFlag };
1a4d82fc	270	SDValue RepMovs = DAG.getNode(X86ISD::REP_MOVS, dl, Tys, Ops);
223e47cc LB	271
	272	SmallVector<SDValue, 4> Results;
	273	Results.push_back(RepMovs);
	274	if (BytesLeft) {
	275	// Handle the last 1 - 7 bytes.
	276	unsigned Offset = SizeVal - BytesLeft;
	277	EVT DstVT = Dst.getValueType();
	278	EVT SrcVT = Src.getValueType();
	279	EVT SizeVT = Size.getValueType();
	280	Results.push_back(DAG.getMemcpy(Chain, dl,
	281	DAG.getNode(ISD::ADD, dl, DstVT, Dst,
	282	DAG.getConstant(Offset, DstVT)),
	283	DAG.getNode(ISD::ADD, dl, SrcVT, Src,
	284	DAG.getConstant(Offset, SrcVT)),
	285	DAG.getConstant(BytesLeft, SizeVT),
	286	Align, isVolatile, AlwaysInline,
	287	DstPtrInfo.getWithOffset(Offset),
	288	SrcPtrInfo.getWithOffset(Offset)));
	289	}
	290
1a4d82fc	291	return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Results);
223e47cc	292	}