1 # Licensed to the Apache Software Foundation (ASF) under one
2 # or more contributor license agreements. See the NOTICE file
3 # distributed with this work for additional information
4 # regarding copyright ownership. The ASF licenses this file
5 # to you under the Apache License, Version 2.0 (the
6 # "License"); you may not use this file except in compliance
7 # with the License. You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing,
12 # software distributed under the License is distributed on an
13 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14 # KIND, either express or implied. See the License for the
15 # specific language governing permissions and limitations
18 from itertools
import filterfalse
, groupby
, tee
21 from tempfile
import NamedTemporaryFile
23 from .core
import Benchmark
24 from ..utils
.command
import Command
25 from ..utils
.maven
import Maven
28 def partition(pred
, iterable
):
29 # adapted from python's examples
30 t1
, t2
= tee(iterable
)
31 return list(filter(pred
, t1
)), list(filterfalse(pred
, t2
))
34 class JavaMicrobenchmarkHarnessCommand(Command
):
35 """ Run a Java Micro Benchmark Harness
37 This assumes the binary supports the standard command line options,
38 notably `-Dbenchmark_filter`
41 def __init__(self
, build
, benchmark_filter
=None):
42 self
.benchmark_filter
= benchmark_filter
46 """ Extract benchmark names from output between "Benchmarks:" and "[INFO]".
47 Assume the following output:
50 org.apache.arrow.vector.IntBenchmarks.setIntDirectly
52 org.apache.arrow.vector.IntBenchmarks.setWithValueHolder
53 org.apache.arrow.vector.IntBenchmarks.setWithWriter
58 def list_benchmarks(self
):
60 if self
.benchmark_filter
:
61 argv
.append("-Dbenchmark.filter={}".format(self
.benchmark_filter
))
62 result
= self
.build
.list(
63 *argv
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
)
67 for line
in str.splitlines(result
.stdout
.decode("utf-8")):
69 if line
.startswith("Benchmarks:"):
72 if line
.startswith("org.apache.arrow"):
74 if line
.startswith("[INFO]"):
78 def results(self
, repetitions
):
79 with
NamedTemporaryFile(suffix
=".json") as out
:
80 argv
= ["-Dbenchmark.runs={}".format(repetitions
),
81 "-Dbenchmark.resultfile={}".format(out
.name
),
82 "-Dbenchmark.resultformat=json"]
83 if self
.benchmark_filter
:
85 "-Dbenchmark.filter={}".format(self
.benchmark_filter
)
88 self
.build
.benchmark(*argv
, check
=True)
92 class JavaMicrobenchmarkHarnessObservation
:
93 """ Represents one run of a single Java Microbenchmark Harness
96 def __init__(self
, benchmark
, primaryMetric
,
97 forks
, warmupIterations
, measurementIterations
, **counters
):
99 self
.primaryMetric
= primaryMetric
100 self
.score
= primaryMetric
["score"]
101 self
.score_unit
= primaryMetric
["scoreUnit"]
103 self
.warmups
= warmupIterations
104 self
.runs
= measurementIterations
106 "mode": counters
["mode"],
107 "threads": counters
["threads"],
108 "warmups": warmupIterations
,
109 "warmupTime": counters
["warmupTime"],
110 "measurements": measurementIterations
,
111 "measurementTime": counters
["measurementTime"],
112 "jvmArgs": counters
["jvmArgs"]
114 self
.reciprocal_value
= True if self
.score_unit
.endswith(
116 if self
.score_unit
.startswith("ops/"):
117 idx
= self
.score_unit
.find("/")
118 self
.normalizePerSec(self
.score_unit
[idx
+1:])
119 elif self
.score_unit
.endswith("/op"):
120 idx
= self
.score_unit
.find("/")
121 self
.normalizePerSec(self
.score_unit
[:idx
])
123 self
.normalizeFactor
= 1
127 """ Return the benchmark value."""
128 val
= 1 / self
.score
if self
.reciprocal_value
else self
.score
129 return val
* self
.normalizeFactor
131 def normalizePerSec(self
, unit
):
133 self
.normalizeFactor
= 1000 * 1000 * 1000
135 self
.normalizeFactor
= 1000 * 1000
137 self
.normalizeFactor
= 1000
139 self
.normalizeFactor
= 1 / 60
141 self
.normalizeFactor
= 1 / (60 * 60)
143 self
.normalizeFactor
= 1 / (60 * 60 * 24)
145 self
.normalizeFactor
= 1
149 if self
.score_unit
.startswith("ops/"):
150 return "items_per_second"
151 elif self
.score_unit
.endswith("/op"):
152 return "items_per_second"
157 return str(self
.value
)
160 class JavaMicrobenchmarkHarness(Benchmark
):
161 """ A set of JavaMicrobenchmarkHarnessObservations. """
163 def __init__(self
, name
, runs
):
164 """ Initialize a JavaMicrobenchmarkHarness.
169 Name of the benchmark
173 runs: list(JavaMicrobenchmarkHarnessObservation)
174 Repetitions of JavaMicrobenchmarkHarnessObservation run.
178 self
.runs
= sorted(runs
, key
=lambda b
: b
.value
)
179 unit
= self
.runs
[0].unit
181 less_is_better
= not unit
.endswith("per_second")
182 values
= [b
.value
for b
in self
.runs
]
184 # Slight kludge to extract the UserCounters for each benchmark
185 counters
= self
.runs
[0].counters
186 super().__init
__(name
, unit
, less_is_better
, values
, time_unit
, times
,
190 return "JavaMicrobenchmark[name={},runs={}]".format(
191 self
.name
, self
.runs
)
194 def from_json(cls
, payload
):
199 lambda x
: JavaMicrobenchmarkHarnessObservation(**x
), payload
)
200 groups
= groupby(sorted(benchmarks
, key
=group_key
), group_key
)
201 return [cls(k
, list(bs
)) for k
, bs
in groups
]