]>
Commit | Line | Data |
---|---|---|
1d09f67e TL |
1 | // Licensed to the Apache Software Foundation (ASF) under one |
2 | // or more contributor license agreements. See the NOTICE file | |
3 | // distributed with this work for additional information | |
4 | // regarding copyright ownership. The ASF licenses this file | |
5 | // to you under the Apache License, Version 2.0 (the | |
6 | // "License"); you may not use this file except in compliance | |
7 | // with the License. You may obtain a copy of the License at | |
8 | // | |
9 | // http://www.apache.org/licenses/LICENSE-2.0 | |
10 | // | |
11 | // Unless required by applicable law or agreed to in writing, | |
12 | // software distributed under the License is distributed on an | |
13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
14 | // KIND, either express or implied. See the License for the | |
15 | // specific language governing permissions and limitations | |
16 | // under the License. | |
17 | ||
18 | #include <sstream> | |
19 | #include <unordered_set> | |
20 | #include <utility> | |
21 | ||
22 | #include "arrow/status.h" | |
23 | #include "gandiva/decimal_ir.h" | |
24 | #include "gandiva/decimal_type_util.h" | |
25 | ||
26 | // Algorithms adapted from Apache Impala | |
27 | ||
28 | namespace gandiva { | |
29 | ||
30 | #define ADD_TRACE_32(msg, value) \ | |
31 | if (enable_ir_traces_) { \ | |
32 | AddTrace32(msg, value); \ | |
33 | } | |
34 | #define ADD_TRACE_128(msg, value) \ | |
35 | if (enable_ir_traces_) { \ | |
36 | AddTrace128(msg, value); \ | |
37 | } | |
38 | ||
39 | // These are the functions defined in this file. The rest are in precompiled folder, | |
40 | // and the i128 needs to be dis-assembled for those. | |
41 | static const char* kAddFunction = "add_decimal128_decimal128"; | |
42 | static const char* kSubtractFunction = "subtract_decimal128_decimal128"; | |
43 | static const char* kEQFunction = "equal_decimal128_decimal128"; | |
44 | static const char* kNEFunction = "not_equal_decimal128_decimal128"; | |
45 | static const char* kLTFunction = "less_than_decimal128_decimal128"; | |
46 | static const char* kLEFunction = "less_than_or_equal_to_decimal128_decimal128"; | |
47 | static const char* kGTFunction = "greater_than_decimal128_decimal128"; | |
48 | static const char* kGEFunction = "greater_than_or_equal_to_decimal128_decimal128"; | |
49 | ||
50 | static const std::unordered_set<std::string> kDecimalIRBuilderFunctions{ | |
51 | kAddFunction, kSubtractFunction, kEQFunction, kNEFunction, | |
52 | kLTFunction, kLEFunction, kGTFunction, kGEFunction}; | |
53 | ||
54 | const char* DecimalIR::kScaleMultipliersName = "gandivaScaleMultipliers"; | |
55 | ||
56 | /// Populate globals required by decimal IR. | |
57 | /// TODO: can this be done just once ? | |
58 | void DecimalIR::AddGlobals(Engine* engine) { | |
59 | auto types = engine->types(); | |
60 | ||
61 | // populate vector : [ 1, 10, 100, 1000, ..] | |
62 | std::string value = "1"; | |
63 | std::vector<llvm::Constant*> scale_multipliers; | |
64 | for (int i = 0; i < DecimalTypeUtil::kMaxPrecision + 1; ++i) { | |
65 | auto multiplier = | |
66 | llvm::ConstantInt::get(llvm::Type::getInt128Ty(*engine->context()), value, 10); | |
67 | scale_multipliers.push_back(multiplier); | |
68 | value.append("0"); | |
69 | } | |
70 | ||
71 | auto array_type = | |
72 | llvm::ArrayType::get(types->i128_type(), DecimalTypeUtil::kMaxPrecision + 1); | |
73 | auto initializer = llvm::ConstantArray::get( | |
74 | array_type, llvm::ArrayRef<llvm::Constant*>(scale_multipliers)); | |
75 | ||
76 | auto globalScaleMultipliers = new llvm::GlobalVariable( | |
77 | *engine->module(), array_type, true /*constant*/, | |
78 | llvm::GlobalValue::LinkOnceAnyLinkage, initializer, kScaleMultipliersName); | |
79 | globalScaleMultipliers->setAlignment(LLVM_ALIGN(16)); | |
80 | } | |
81 | ||
82 | // Lookup intrinsic functions | |
83 | void DecimalIR::InitializeIntrinsics() { | |
84 | sadd_with_overflow_fn_ = llvm::Intrinsic::getDeclaration( | |
85 | module(), llvm::Intrinsic::sadd_with_overflow, types()->i128_type()); | |
86 | DCHECK_NE(sadd_with_overflow_fn_, nullptr); | |
87 | ||
88 | smul_with_overflow_fn_ = llvm::Intrinsic::getDeclaration( | |
89 | module(), llvm::Intrinsic::smul_with_overflow, types()->i128_type()); | |
90 | DCHECK_NE(smul_with_overflow_fn_, nullptr); | |
91 | ||
92 | i128_with_overflow_struct_type_ = | |
93 | sadd_with_overflow_fn_->getFunctionType()->getReturnType(); | |
94 | } | |
95 | ||
96 | // CPP: return kScaleMultipliers[scale] | |
97 | llvm::Value* DecimalIR::GetScaleMultiplier(llvm::Value* scale) { | |
98 | auto const_array = module()->getGlobalVariable(kScaleMultipliersName); | |
99 | auto ptr = CreateGEP(ir_builder(), const_array, {types()->i32_constant(0), scale}); | |
100 | return CreateLoad(ir_builder(), ptr); | |
101 | } | |
102 | ||
103 | // CPP: x <= y ? y : x | |
104 | llvm::Value* DecimalIR::GetHigherScale(llvm::Value* x_scale, llvm::Value* y_scale) { | |
105 | llvm::Value* le = ir_builder()->CreateICmpSLE(x_scale, y_scale); | |
106 | return ir_builder()->CreateSelect(le, y_scale, x_scale); | |
107 | } | |
108 | ||
109 | // CPP: return (increase_scale_by <= 0) ? | |
110 | // in_value : in_value * GetScaleMultiplier(increase_scale_by) | |
111 | llvm::Value* DecimalIR::IncreaseScale(llvm::Value* in_value, | |
112 | llvm::Value* increase_scale_by) { | |
113 | llvm::Value* le_zero = | |
114 | ir_builder()->CreateICmpSLE(increase_scale_by, types()->i32_constant(0)); | |
115 | // then block | |
116 | auto then_lambda = [&] { return in_value; }; | |
117 | ||
118 | // else block | |
119 | auto else_lambda = [&] { | |
120 | llvm::Value* multiplier = GetScaleMultiplier(increase_scale_by); | |
121 | return ir_builder()->CreateMul(in_value, multiplier); | |
122 | }; | |
123 | ||
124 | return BuildIfElse(le_zero, types()->i128_type(), then_lambda, else_lambda); | |
125 | } | |
126 | ||
127 | // CPP: return (increase_scale_by <= 0) ? | |
128 | // {in_value,false} : {in_value * GetScaleMultiplier(increase_scale_by),true} | |
129 | // | |
130 | // The return value also indicates if there was an overflow while increasing the scale. | |
131 | DecimalIR::ValueWithOverflow DecimalIR::IncreaseScaleWithOverflowCheck( | |
132 | llvm::Value* in_value, llvm::Value* increase_scale_by) { | |
133 | llvm::Value* le_zero = | |
134 | ir_builder()->CreateICmpSLE(increase_scale_by, types()->i32_constant(0)); | |
135 | ||
136 | // then block | |
137 | auto then_lambda = [&] { | |
138 | ValueWithOverflow ret{in_value, types()->false_constant()}; | |
139 | return ret.AsStruct(this); | |
140 | }; | |
141 | ||
142 | // else block | |
143 | auto else_lambda = [&] { | |
144 | llvm::Value* multiplier = GetScaleMultiplier(increase_scale_by); | |
145 | return ir_builder()->CreateCall(smul_with_overflow_fn_, {in_value, multiplier}); | |
146 | }; | |
147 | ||
148 | auto ir_struct = | |
149 | BuildIfElse(le_zero, i128_with_overflow_struct_type_, then_lambda, else_lambda); | |
150 | return ValueWithOverflow::MakeFromStruct(this, ir_struct); | |
151 | } | |
152 | ||
153 | // CPP: return (reduce_scale_by <= 0) ? | |
154 | // in_value : in_value / GetScaleMultiplier(reduce_scale_by) | |
155 | // | |
156 | // ReduceScale cannot cause an overflow. | |
157 | llvm::Value* DecimalIR::ReduceScale(llvm::Value* in_value, llvm::Value* reduce_scale_by) { | |
158 | auto le_zero = ir_builder()->CreateICmpSLE(reduce_scale_by, types()->i32_constant(0)); | |
159 | // then block | |
160 | auto then_lambda = [&] { return in_value; }; | |
161 | ||
162 | // else block | |
163 | auto else_lambda = [&] { | |
164 | // TODO : handle rounding. | |
165 | llvm::Value* multiplier = GetScaleMultiplier(reduce_scale_by); | |
166 | return ir_builder()->CreateSDiv(in_value, multiplier); | |
167 | }; | |
168 | ||
169 | return BuildIfElse(le_zero, types()->i128_type(), then_lambda, else_lambda); | |
170 | } | |
171 | ||
172 | /// @brief Fast-path for add | |
173 | /// Adjust x and y to the same scale, and add them. | |
174 | llvm::Value* DecimalIR::AddFastPath(const ValueFull& x, const ValueFull& y) { | |
175 | auto higher_scale = GetHigherScale(x.scale(), y.scale()); | |
176 | ADD_TRACE_32("AddFastPath : higher_scale", higher_scale); | |
177 | ||
178 | // CPP : x_scaled = IncreaseScale(x_value, higher_scale - x_scale) | |
179 | auto x_delta = ir_builder()->CreateSub(higher_scale, x.scale()); | |
180 | auto x_scaled = IncreaseScale(x.value(), x_delta); | |
181 | ADD_TRACE_128("AddFastPath : x_scaled", x_scaled); | |
182 | ||
183 | // CPP : y_scaled = IncreaseScale(y_value, higher_scale - y_scale) | |
184 | auto y_delta = ir_builder()->CreateSub(higher_scale, y.scale()); | |
185 | auto y_scaled = IncreaseScale(y.value(), y_delta); | |
186 | ADD_TRACE_128("AddFastPath : y_scaled", y_scaled); | |
187 | ||
188 | auto sum = ir_builder()->CreateAdd(x_scaled, y_scaled); | |
189 | ADD_TRACE_128("AddFastPath : sum", sum); | |
190 | return sum; | |
191 | } | |
192 | ||
193 | // @brief Add with overflow check. | |
194 | /// Adjust x and y to the same scale, add them, and reduce sum to output scale. | |
195 | /// If there is an overflow, the sum is set to 0. | |
196 | DecimalIR::ValueWithOverflow DecimalIR::AddWithOverflowCheck(const ValueFull& x, | |
197 | const ValueFull& y, | |
198 | const ValueFull& out) { | |
199 | auto higher_scale = GetHigherScale(x.scale(), y.scale()); | |
200 | ADD_TRACE_32("AddWithOverflowCheck : higher_scale", higher_scale); | |
201 | ||
202 | // CPP : x_scaled = IncreaseScale(x_value, higher_scale - x.scale()) | |
203 | auto x_delta = ir_builder()->CreateSub(higher_scale, x.scale()); | |
204 | auto x_scaled = IncreaseScaleWithOverflowCheck(x.value(), x_delta); | |
205 | ADD_TRACE_128("AddWithOverflowCheck : x_scaled", x_scaled.value()); | |
206 | ||
207 | // CPP : y_scaled = IncreaseScale(y_value, higher_scale - y_scale) | |
208 | auto y_delta = ir_builder()->CreateSub(higher_scale, y.scale()); | |
209 | auto y_scaled = IncreaseScaleWithOverflowCheck(y.value(), y_delta); | |
210 | ADD_TRACE_128("AddWithOverflowCheck : y_scaled", y_scaled.value()); | |
211 | ||
212 | // CPP : sum = x_scaled + y_scaled | |
213 | auto sum_ir_struct = ir_builder()->CreateCall(sadd_with_overflow_fn_, | |
214 | {x_scaled.value(), y_scaled.value()}); | |
215 | auto sum = ValueWithOverflow::MakeFromStruct(this, sum_ir_struct); | |
216 | ADD_TRACE_128("AddWithOverflowCheck : sum", sum.value()); | |
217 | ||
218 | // CPP : overflow ? 0 : sum / GetScaleMultiplier(max_scale - out_scale) | |
219 | auto overflow = GetCombinedOverflow({x_scaled, y_scaled, sum}); | |
220 | ADD_TRACE_32("AddWithOverflowCheck : overflow", overflow); | |
221 | auto then_lambda = [&] { | |
222 | // if there is an overflow, the value returned won't be used. so, save the division. | |
223 | return types()->i128_constant(0); | |
224 | }; | |
225 | auto else_lambda = [&] { | |
226 | auto reduce_scale_by = ir_builder()->CreateSub(higher_scale, out.scale()); | |
227 | return ReduceScale(sum.value(), reduce_scale_by); | |
228 | }; | |
229 | auto sum_descaled = | |
230 | BuildIfElse(overflow, types()->i128_type(), then_lambda, else_lambda); | |
231 | return ValueWithOverflow(sum_descaled, overflow); | |
232 | } | |
233 | ||
234 | // This is pretty complex, so use CPP fns. | |
235 | llvm::Value* DecimalIR::AddLarge(const ValueFull& x, const ValueFull& y, | |
236 | const ValueFull& out) { | |
237 | auto block = ir_builder()->GetInsertBlock(); | |
238 | auto out_high_ptr = new llvm::AllocaInst(types()->i64_type(), 0, "out_hi", block); | |
239 | auto out_low_ptr = new llvm::AllocaInst(types()->i64_type(), 0, "out_low", block); | |
240 | auto x_split = ValueSplit::MakeFromInt128(this, x.value()); | |
241 | auto y_split = ValueSplit::MakeFromInt128(this, y.value()); | |
242 | ||
243 | std::vector<llvm::Value*> args = { | |
244 | x_split.high(), x_split.low(), x.precision(), x.scale(), | |
245 | y_split.high(), y_split.low(), y.precision(), y.scale(), | |
246 | out.precision(), out.scale(), out_high_ptr, out_low_ptr, | |
247 | }; | |
248 | ir_builder()->CreateCall(module()->getFunction("add_large_decimal128_decimal128"), | |
249 | args); | |
250 | ||
251 | auto out_high = CreateLoad(ir_builder(), out_high_ptr); | |
252 | auto out_low = CreateLoad(ir_builder(), out_low_ptr); | |
253 | auto sum = ValueSplit(out_high, out_low).AsInt128(this); | |
254 | ADD_TRACE_128("AddLarge : sum", sum); | |
255 | return sum; | |
256 | } | |
257 | ||
258 | /// The output scale/precision cannot be arbitrary values. The algo here depends on them | |
259 | /// to be the same as computed in DecimalTypeSql. | |
260 | /// TODO: enforce this. | |
261 | Status DecimalIR::BuildAdd() { | |
262 | // Create fn prototype : | |
263 | // int128_t | |
264 | // add_decimal128_decimal128(int128_t x_value, int32_t x_precision, int32_t x_scale, | |
265 | // int128_t y_value, int32_t y_precision, int32_t y_scale | |
266 | // int32_t out_precision, int32_t out_scale) | |
267 | auto i32 = types()->i32_type(); | |
268 | auto i128 = types()->i128_type(); | |
269 | auto function = BuildFunction(kAddFunction, i128, | |
270 | { | |
271 | {"x_value", i128}, | |
272 | {"x_precision", i32}, | |
273 | {"x_scale", i32}, | |
274 | {"y_value", i128}, | |
275 | {"y_precision", i32}, | |
276 | {"y_scale", i32}, | |
277 | {"out_precision", i32}, | |
278 | {"out_scale", i32}, | |
279 | }); | |
280 | ||
281 | auto arg_iter = function->arg_begin(); | |
282 | ValueFull x(&arg_iter[0], &arg_iter[1], &arg_iter[2]); | |
283 | ValueFull y(&arg_iter[3], &arg_iter[4], &arg_iter[5]); | |
284 | ValueFull out(nullptr, &arg_iter[6], &arg_iter[7]); | |
285 | ||
286 | auto entry = llvm::BasicBlock::Create(*context(), "entry", function); | |
287 | ir_builder()->SetInsertPoint(entry); | |
288 | ||
289 | // CPP : | |
290 | // if (out_precision < 38) { | |
291 | // return AddFastPath(x, y) | |
292 | // } else { | |
293 | // ret = AddWithOverflowCheck(x, y) | |
294 | // if (ret.overflow) | |
295 | // return AddLarge(x, y) | |
296 | // else | |
297 | // return ret.value; | |
298 | // } | |
299 | llvm::Value* lt_max_precision = ir_builder()->CreateICmpSLT( | |
300 | out.precision(), types()->i32_constant(DecimalTypeUtil::kMaxPrecision)); | |
301 | auto then_lambda = [&] { | |
302 | // fast-path add | |
303 | return AddFastPath(x, y); | |
304 | }; | |
305 | auto else_lambda = [&] { | |
306 | if (kUseOverflowIntrinsics) { | |
307 | // do the add and check if there was overflow | |
308 | auto ret = AddWithOverflowCheck(x, y, out); | |
309 | ||
310 | // if there is an overflow, switch to the AddLarge codepath. | |
311 | return BuildIfElse( | |
312 | ret.overflow(), types()->i128_type(), [&] { return AddLarge(x, y, out); }, | |
313 | [&] { return ret.value(); }); | |
314 | } else { | |
315 | return AddLarge(x, y, out); | |
316 | } | |
317 | }; | |
318 | auto value = | |
319 | BuildIfElse(lt_max_precision, types()->i128_type(), then_lambda, else_lambda); | |
320 | ||
321 | // store result to out | |
322 | ir_builder()->CreateRet(value); | |
323 | return Status::OK(); | |
324 | } | |
325 | ||
326 | Status DecimalIR::BuildSubtract() { | |
327 | // Create fn prototype : | |
328 | // int128_t | |
329 | // subtract_decimal128_decimal128(int128_t x_value, int32_t x_precision, int32_t | |
330 | // x_scale, | |
331 | // int128_t y_value, int32_t y_precision, int32_t y_scale | |
332 | // int32_t out_precision, int32_t out_scale) | |
333 | auto i32 = types()->i32_type(); | |
334 | auto i128 = types()->i128_type(); | |
335 | auto function = BuildFunction(kSubtractFunction, i128, | |
336 | { | |
337 | {"x_value", i128}, | |
338 | {"x_precision", i32}, | |
339 | {"x_scale", i32}, | |
340 | {"y_value", i128}, | |
341 | {"y_precision", i32}, | |
342 | {"y_scale", i32}, | |
343 | {"out_precision", i32}, | |
344 | {"out_scale", i32}, | |
345 | }); | |
346 | ||
347 | auto entry = llvm::BasicBlock::Create(*context(), "entry", function); | |
348 | ir_builder()->SetInsertPoint(entry); | |
349 | ||
350 | // reuse add function after negating y_value. i.e | |
351 | // add(x_value, x_precision, x_scale, -y_value, y_precision, y_scale, | |
352 | // out_precision, out_scale) | |
353 | std::vector<llvm::Value*> args; | |
354 | int i = 0; | |
355 | for (auto& in_arg : function->args()) { | |
356 | if (i == 3) { | |
357 | auto y_neg_value = ir_builder()->CreateNeg(&in_arg); | |
358 | args.push_back(y_neg_value); | |
359 | } else { | |
360 | args.push_back(&in_arg); | |
361 | } | |
362 | ++i; | |
363 | } | |
364 | auto value = ir_builder()->CreateCall(module()->getFunction(kAddFunction), args); | |
365 | ||
366 | // store result to out | |
367 | ir_builder()->CreateRet(value); | |
368 | return Status::OK(); | |
369 | } | |
370 | ||
371 | Status DecimalIR::BuildCompare(const std::string& function_name, | |
372 | llvm::ICmpInst::Predicate cmp_instruction) { | |
373 | // Create fn prototype : | |
374 | // bool | |
375 | // function_name(int128_t x_value, int32_t x_precision, int32_t x_scale, | |
376 | // int128_t y_value, int32_t y_precision, int32_t y_scale) | |
377 | ||
378 | auto i32 = types()->i32_type(); | |
379 | auto i128 = types()->i128_type(); | |
380 | auto function = BuildFunction(function_name, types()->i1_type(), | |
381 | { | |
382 | {"x_value", i128}, | |
383 | {"x_precision", i32}, | |
384 | {"x_scale", i32}, | |
385 | {"y_value", i128}, | |
386 | {"y_precision", i32}, | |
387 | {"y_scale", i32}, | |
388 | }); | |
389 | ||
390 | auto arg_iter = function->arg_begin(); | |
391 | ValueFull x(&arg_iter[0], &arg_iter[1], &arg_iter[2]); | |
392 | ValueFull y(&arg_iter[3], &arg_iter[4], &arg_iter[5]); | |
393 | ||
394 | auto entry = llvm::BasicBlock::Create(*context(), "entry", function); | |
395 | ir_builder()->SetInsertPoint(entry); | |
396 | ||
397 | // Make call to pre-compiled IR function. | |
398 | auto x_split = ValueSplit::MakeFromInt128(this, x.value()); | |
399 | auto y_split = ValueSplit::MakeFromInt128(this, y.value()); | |
400 | ||
401 | std::vector<llvm::Value*> args = { | |
402 | x_split.high(), x_split.low(), x.precision(), x.scale(), | |
403 | y_split.high(), y_split.low(), y.precision(), y.scale(), | |
404 | }; | |
405 | auto cmp_value = ir_builder()->CreateCall( | |
406 | module()->getFunction("compare_decimal128_decimal128_internal"), args); | |
407 | auto result = | |
408 | ir_builder()->CreateICmp(cmp_instruction, cmp_value, types()->i32_constant(0)); | |
409 | ir_builder()->CreateRet(result); | |
410 | return Status::OK(); | |
411 | } | |
412 | ||
413 | llvm::Value* DecimalIR::CallDecimalFunction(const std::string& function_name, | |
414 | llvm::Type* return_type, | |
415 | const std::vector<llvm::Value*>& params) { | |
416 | if (kDecimalIRBuilderFunctions.count(function_name) != 0) { | |
417 | // this is fn built with the irbuilder. | |
418 | return ir_builder()->CreateCall(module()->getFunction(function_name), params); | |
419 | } | |
420 | ||
421 | // ppre-compiler fn : disassemble i128 to two i64s and re-assemble. | |
422 | auto i128 = types()->i128_type(); | |
423 | auto i64 = types()->i64_type(); | |
424 | std::vector<llvm::Value*> dis_assembled_args; | |
425 | for (auto& arg : params) { | |
426 | if (arg->getType() == i128) { | |
427 | // split i128 arg into two int64s. | |
428 | auto split = ValueSplit::MakeFromInt128(this, arg); | |
429 | dis_assembled_args.push_back(split.high()); | |
430 | dis_assembled_args.push_back(split.low()); | |
431 | } else { | |
432 | dis_assembled_args.push_back(arg); | |
433 | } | |
434 | } | |
435 | ||
436 | llvm::Value* result = nullptr; | |
437 | if (return_type == i128) { | |
438 | // for i128 ret, replace with two int64* args, and join them. | |
439 | auto block = ir_builder()->GetInsertBlock(); | |
440 | auto out_high_ptr = new llvm::AllocaInst(i64, 0, "out_hi", block); | |
441 | auto out_low_ptr = new llvm::AllocaInst(i64, 0, "out_low", block); | |
442 | dis_assembled_args.push_back(out_high_ptr); | |
443 | dis_assembled_args.push_back(out_low_ptr); | |
444 | ||
445 | // Make call to pre-compiled IR function. | |
446 | ir_builder()->CreateCall(module()->getFunction(function_name), dis_assembled_args); | |
447 | ||
448 | auto out_high = CreateLoad(ir_builder(), out_high_ptr); | |
449 | auto out_low = CreateLoad(ir_builder(), out_low_ptr); | |
450 | result = ValueSplit(out_high, out_low).AsInt128(this); | |
451 | } else { | |
452 | DCHECK_NE(return_type, types()->void_type()); | |
453 | ||
454 | // Make call to pre-compiled IR function. | |
455 | result = ir_builder()->CreateCall(module()->getFunction(function_name), | |
456 | dis_assembled_args); | |
457 | } | |
458 | return result; | |
459 | } | |
460 | ||
461 | Status DecimalIR::AddFunctions(Engine* engine) { | |
462 | auto decimal_ir = std::make_shared<DecimalIR>(engine); | |
463 | ||
464 | // Populate global variables used by decimal operations. | |
465 | decimal_ir->AddGlobals(engine); | |
466 | ||
467 | // Lookup intrinsic functions | |
468 | decimal_ir->InitializeIntrinsics(); | |
469 | ||
470 | ARROW_RETURN_NOT_OK(decimal_ir->BuildAdd()); | |
471 | ARROW_RETURN_NOT_OK(decimal_ir->BuildSubtract()); | |
472 | ARROW_RETURN_NOT_OK(decimal_ir->BuildCompare(kEQFunction, llvm::ICmpInst::ICMP_EQ)); | |
473 | ARROW_RETURN_NOT_OK(decimal_ir->BuildCompare(kNEFunction, llvm::ICmpInst::ICMP_NE)); | |
474 | ARROW_RETURN_NOT_OK(decimal_ir->BuildCompare(kLTFunction, llvm::ICmpInst::ICMP_SLT)); | |
475 | ARROW_RETURN_NOT_OK(decimal_ir->BuildCompare(kLEFunction, llvm::ICmpInst::ICMP_SLE)); | |
476 | ARROW_RETURN_NOT_OK(decimal_ir->BuildCompare(kGTFunction, llvm::ICmpInst::ICMP_SGT)); | |
477 | ARROW_RETURN_NOT_OK(decimal_ir->BuildCompare(kGEFunction, llvm::ICmpInst::ICMP_SGE)); | |
478 | return Status::OK(); | |
479 | } | |
480 | ||
481 | // Do an bitwise-or of all the overflow bits. | |
482 | llvm::Value* DecimalIR::GetCombinedOverflow( | |
483 | std::vector<DecimalIR::ValueWithOverflow> vec) { | |
484 | llvm::Value* res = types()->false_constant(); | |
485 | for (auto& val : vec) { | |
486 | res = ir_builder()->CreateOr(res, val.overflow()); | |
487 | } | |
488 | return res; | |
489 | } | |
490 | ||
491 | DecimalIR::ValueSplit DecimalIR::ValueSplit::MakeFromInt128(DecimalIR* decimal_ir, | |
492 | llvm::Value* in) { | |
493 | auto builder = decimal_ir->ir_builder(); | |
494 | auto types = decimal_ir->types(); | |
495 | ||
496 | auto high = builder->CreateLShr(in, types->i128_constant(64)); | |
497 | high = builder->CreateTrunc(high, types->i64_type()); | |
498 | auto low = builder->CreateTrunc(in, types->i64_type()); | |
499 | return ValueSplit(high, low); | |
500 | } | |
501 | ||
502 | /// Convert IR struct {%i64, %i64} to cpp class ValueSplit | |
503 | DecimalIR::ValueSplit DecimalIR::ValueSplit::MakeFromStruct(DecimalIR* decimal_ir, | |
504 | llvm::Value* dstruct) { | |
505 | auto builder = decimal_ir->ir_builder(); | |
506 | auto high = builder->CreateExtractValue(dstruct, 0); | |
507 | auto low = builder->CreateExtractValue(dstruct, 1); | |
508 | return DecimalIR::ValueSplit(high, low); | |
509 | } | |
510 | ||
511 | llvm::Value* DecimalIR::ValueSplit::AsInt128(DecimalIR* decimal_ir) const { | |
512 | auto builder = decimal_ir->ir_builder(); | |
513 | auto types = decimal_ir->types(); | |
514 | ||
515 | auto value = builder->CreateSExt(high_, types->i128_type()); | |
516 | value = builder->CreateShl(value, types->i128_constant(64)); | |
517 | value = builder->CreateAdd(value, builder->CreateZExt(low_, types->i128_type())); | |
518 | return value; | |
519 | } | |
520 | ||
521 | /// Convert IR struct {%i128, %i1} to cpp class ValueWithOverflow | |
522 | DecimalIR::ValueWithOverflow DecimalIR::ValueWithOverflow::MakeFromStruct( | |
523 | DecimalIR* decimal_ir, llvm::Value* dstruct) { | |
524 | auto builder = decimal_ir->ir_builder(); | |
525 | auto value = builder->CreateExtractValue(dstruct, 0); | |
526 | auto overflow = builder->CreateExtractValue(dstruct, 1); | |
527 | return DecimalIR::ValueWithOverflow(value, overflow); | |
528 | } | |
529 | ||
530 | /// Convert to IR struct {%i128, %i1} | |
531 | llvm::Value* DecimalIR::ValueWithOverflow::AsStruct(DecimalIR* decimal_ir) const { | |
532 | auto builder = decimal_ir->ir_builder(); | |
533 | ||
534 | auto undef = llvm::UndefValue::get(decimal_ir->i128_with_overflow_struct_type_); | |
535 | auto struct_val = builder->CreateInsertValue(undef, value(), 0); | |
536 | return builder->CreateInsertValue(struct_val, overflow(), 1); | |
537 | } | |
538 | ||
539 | /// debug traces | |
540 | void DecimalIR::AddTrace(const std::string& fmt, std::vector<llvm::Value*> args) { | |
541 | DCHECK(enable_ir_traces_); | |
542 | ||
543 | auto ir_str = ir_builder()->CreateGlobalStringPtr(fmt); | |
544 | args.insert(args.begin(), ir_str); | |
545 | ir_builder()->CreateCall(module()->getFunction("printf"), args, "trace"); | |
546 | } | |
547 | ||
548 | void DecimalIR::AddTrace32(const std::string& msg, llvm::Value* value) { | |
549 | AddTrace("DECIMAL_IR_TRACE:: " + msg + " %d\n", {value}); | |
550 | } | |
551 | ||
552 | void DecimalIR::AddTrace128(const std::string& msg, llvm::Value* value) { | |
553 | // convert i128 into two i64s for printing | |
554 | auto split = ValueSplit::MakeFromInt128(this, value); | |
555 | AddTrace("DECIMAL_IR_TRACE:: " + msg + " %llx:%llx (%lld:%llu)\n", | |
556 | {split.high(), split.low(), split.high(), split.low()}); | |
557 | } | |
558 | ||
559 | } // namespace gandiva |