projects / libgit2.git / blame
| Commit | Line | Data |
|---|---|---|
| ad5611d8 TR |
1 | #!/bin/bash |
| 2 | ||
| 3 | set -eo pipefail | |
| 4 | ||
| 5 | # | |
| 6 | # parse the command line | |
| 7 | # | |
| 8 | ||
| 9 | usage() { echo "usage: $(basename "$0") [--cli <path>] [--baseline-cli <path>] [--suite <suite>] [--json <path>] [--zip <path>] [--verbose] [--debug]"; } | |
| 10 | ||
| 11 | TEST_CLI="git" | |
| 12 | BASELINE_CLI= | |
| 13 | SUITE= | |
| 14 | JSON_RESULT= | |
| 15 | ZIP_RESULT= | |
| 16 | OUTPUT_DIR= | |
| 17 | VERBOSE= | |
| 18 | DEBUG= | |
| 19 | NEXT= | |
| 20 | ||
| 21 | for a in "$@"; do | |
| 22 | if [ "${NEXT}" = "cli" ]; then | |
| 23 | TEST_CLI="${a}" | |
| 24 | NEXT= | |
| 25 | elif [ "${NEXT}" = "baseline-cli" ]; then | |
| 26 | BASELINE_CLI="${a}" | |
| 27 | NEXT= | |
| 28 | elif [ "${NEXT}" = "suite" ]; then | |
| 29 | SUITE="${a}" | |
| 30 | NEXT= | |
| 31 | elif [ "${NEXT}" = "json" ]; then | |
| 32 | JSON_RESULT="${a}" | |
| 33 | NEXT= | |
| 34 | elif [ "${NEXT}" = "zip" ]; then | |
| 35 | ZIP_RESULT="${a}" | |
| 36 | NEXT= | |
| 37 | elif [ "${NEXT}" = "output-dir" ]; then | |
| 38 | OUTPUT_DIR="${a}" | |
| 39 | NEXT= | |
| 40 | elif [ "${a}" = "c" ] || [ "${a}" = "--cli" ]; then | |
| 41 | NEXT="cli" | |
| 42 | elif [[ "${a}" == "-c"* ]]; then | |
| 43 | TEST_CLI="${a/-c/}" | |
| 44 | elif [ "${a}" = "b" ] || [ "${a}" = "--baseline-cli" ]; then | |
| 45 | NEXT="baseline-cli" | |
| 46 | elif [[ "${a}" == "-b"* ]]; then | |
| 47 | BASELINE_CLI="${a/-b/}" | |
| 48 | elif [ "${a}" = "-s" ] || [ "${a}" = "--suite" ]; then | |
| 49 | NEXT="suite" | |
| 50 | elif [[ "${a}" == "-s"* ]]; then | |
| 51 | SUITE="${a/-s/}" | |
| 52 | elif [ "${a}" = "-v" ] || [ "${a}" == "--verbose" ]; then | |
| 53 | VERBOSE=1 | |
| 54 | elif [ "${a}" == "--debug" ]; then | |
| 55 | VERBOSE=1 | |
| 56 | DEBUG=1 | |
| 57 | elif [ "${a}" = "-j" ] || [ "${a}" == "--json" ]; then | |
| 58 | NEXT="json" | |
| 59 | elif [[ "${a}" == "-j"* ]]; then | |
| 60 | JSON_RESULT="${a/-j/}" | |
| 61 | elif [ "${a}" = "-z" ] || [ "${a}" == "--zip" ]; then | |
| 62 | NEXT="zip" | |
| 63 | elif [[ "${a}" == "-z"* ]]; then | |
| 64 | ZIP_RESULT="${a/-z/}" | |
| 65 | elif [ "${a}" = "--output-dir" ]; then | |
| 66 | NEXT="output-dir" | |
| 67 | else | |
| 68 | echo "$(basename "$0"): unknown option: ${a}" 1>&2 | |
| 69 | usage 1>&2 | |
| 70 | exit 1 | |
| 71 | fi | |
| 72 | done | |
| 73 | ||
| 74 | if [ "${NEXT}" != "" ]; then | |
| 75 | usage 1>&2 | |
| 76 | exit 1 | |
| 77 | fi | |
| 78 | ||
| 79 | if [ "${OUTPUT_DIR}" = "" ]; then | |
| 80 | OUTPUT_DIR=${OUTPUT_DIR:="$(mktemp -d)"} | |
| 81 | CLEANUP_DIR=1 | |
| 82 | fi | |
| 83 | ||
| 84 | # | |
| 85 | # collect some information about the test environment | |
| 86 | # | |
| 87 | ||
| 88 | SYSTEM_OS=$(uname -s) | |
| 89 | if [ "${SYSTEM_OS}" = "Darwin" ]; then SYSTEM_OS="macOS"; fi | |
| 90 | ||
| 91 | SYSTEM_KERNEL=$(uname -v) | |
| 92 | ||
| 93 | fullpath() { | |
| 94 | if [[ "$(uname -s)" == "MINGW"* && $(cygpath -u "${TEST_CLI}") == "/"* ]]; then | |
| 95 | echo "${TEST_CLI}" | |
| 96 | elif [[ "${TEST_CLI}" == "/"* ]]; then | |
| 97 | echo "${TEST_CLI}" | |
| 98 | else | |
| 99 | which "${TEST_CLI}" | |
| 100 | fi | |
| 101 | } | |
| 102 | ||
| 103 | cli_version() { | |
| 104 | if [[ "$(uname -s)" == "MINGW"* ]]; then | |
| 105 | $(cygpath -u "$1") --version | |
| 106 | else | |
| 107 | "$1" --version | |
| 108 | fi | |
| 109 | } | |
| 110 | ||
| 111 | TEST_CLI_NAME=$(basename "${TEST_CLI}") | |
| 112 | TEST_CLI_PATH=$(fullpath "${TEST_CLI}") | |
| 113 | TEST_CLI_VERSION=$(cli_version "${TEST_CLI}") | |
| 114 | ||
| 115 | if [ "${BASELINE_CLI}" != "" ]; then | |
| 116 | if [[ "${BASELINE_CLI}" == "/"* ]]; then | |
| 117 | BASELINE_CLI_PATH="${BASELINE_CLI}" | |
| 118 | else | |
| 119 | BASELINE_CLI_PATH=$(which "${BASELINE_CLI}") | |
| 120 | fi | |
| 121 | ||
| 122 | BASELINE_CLI_NAME=$(basename "${BASELINE_CLI}") | |
| 123 | BASELINE_CLI_PATH=$(fullpath "${BASELINE_CLI}") | |
| 124 | BASELINE_CLI_VERSION=$(cli_version "${BASELINE_CLI}") | |
| 125 | fi | |
| 126 | ||
| 127 | # | |
| 128 | # run the benchmarks | |
| 129 | # | |
| 130 | ||
| 131 | echo "##############################################################################" | |
| 132 | if [ "${SUITE}" != "" ]; then | |
| 133 | SUITE_PREFIX="${SUITE/::/__}" | |
| 134 | echo "## Running ${SUITE} benchmarks" | |
| 135 | else | |
| 136 | echo "## Running all benchmarks" | |
| 137 | fi | |
| 138 | echo "##############################################################################" | |
| 139 | echo "" | |
| 140 | ||
| 141 | if [ "${BASELINE_CLI}" != "" ]; then | |
| 142 | echo "# Baseline CLI: ${BASELINE_CLI} (${BASELINE_CLI_VERSION})" | |
| 143 | fi | |
| 144 | echo "# Test CLI: ${TEST_CLI} (${TEST_CLI_VERSION})" | |
| 145 | echo "" | |
| 146 | ||
| 147 | BENCHMARK_DIR=${BENCHMARK_DIR:=$(dirname "$0")} | |
| 148 | ANY_FOUND= | |
| 149 | ANY_FAILED= | |
| 150 | ||
| 151 | indent() { sed "s/^/ /"; } | |
| 152 | time_in_ms() { if [ "$(uname -s)" = "Darwin" ]; then date "+%s000"; else date "+%s%N" ; fi; } | |
| 153 | humanize_secs() { | |
| 154 | units=('s' 'ms' 'us' 'ns') | |
| 155 | unit=0 | |
| 156 | time="${1}" | |
| 157 | ||
| 158 | if [ "${time}" = "" ]; then | |
| 159 | echo "" | |
| 160 | return | |
| 161 | fi | |
| 162 | ||
| 163 | # bash doesn't do floating point arithmetic. ick. | |
| 164 | while [[ "${time}" == "0."* ]] && [ "$((unit+1))" != "${#units[*]}" ]; do | |
| 165 | time="$(echo | awk "{ print ${time} * 1000 }")" | |
| 166 | unit=$((unit+1)) | |
| 167 | done | |
| 168 | ||
| 169 | echo "${time} ${units[$unit]}" | |
| 170 | } | |
| 171 | ||
| 172 | TIME_START=$(time_in_ms) | |
| 173 | ||
| 174 | for TEST_PATH in "${BENCHMARK_DIR}"/*; do | |
| 175 | TEST_FILE=$(basename "${TEST_PATH}") | |
| 176 | ||
| 177 | if [ ! -f "${TEST_PATH}" ] || [ ! -x "${TEST_PATH}" ]; then | |
| 178 | continue | |
| 179 | fi | |
| 180 | ||
| 181 | if [[ "${TEST_FILE}" != *"__"* ]]; then | |
| 182 | continue | |
| 183 | fi | |
| 184 | ||
| 185 | if [[ "${TEST_FILE}" != "${SUITE_PREFIX}"* ]]; then | |
| 186 | continue | |
| 187 | fi | |
| 188 | ||
| 189 | ANY_FOUND=1 | |
| 190 | TEST_NAME="${TEST_FILE/__/::}" | |
| 191 | ||
| 192 | echo -n "${TEST_NAME}:" | |
| 193 | if [ "${VERBOSE}" = "1" ]; then | |
| 194 | echo "" | |
| 195 | else | |
| 196 | echo -n " " | |
| 197 | fi | |
| 198 | ||
| 199 | if [ "${DEBUG}" = "1" ]; then | |
| 200 | SHOW_OUTPUT="--show-output" | |
| 201 | fi | |
| 202 | ||
| 203 | OUTPUT_FILE="${OUTPUT_DIR}/${TEST_FILE}.out" | |
| 204 | JSON_FILE="${OUTPUT_DIR}/${TEST_FILE}.json" | |
| 205 | ERROR_FILE="${OUTPUT_DIR}/${TEST_FILE}.err" | |
| 206 | ||
| 207 | FAILED= | |
| 208 | ${TEST_PATH} --cli "${TEST_CLI}" --baseline-cli "${BASELINE_CLI}" --json "${JSON_FILE}" ${SHOW_OUTPUT} >"${OUTPUT_FILE}" 2>"${ERROR_FILE}" || FAILED=1 | |
| 209 | ||
| 210 | if [ "${FAILED}" = "1" ]; then | |
| 211 | if [ "${VERBOSE}" != "1" ]; then | |
| 212 | echo "failed!" | |
| 213 | fi | |
| 214 | ||
| 215 | indent < "${ERROR_FILE}" | |
| 216 | ANY_FAILED=1 | |
| 217 | continue | |
| 218 | fi | |
| 219 | ||
| 220 | # in verbose mode, just print the hyperfine results; otherwise, | |
| 221 | # pull the useful information out of its json and summarize it | |
| 222 | if [ "${VERBOSE}" = "1" ]; then | |
| 223 | indent < "${OUTPUT_FILE}" | |
| 224 | else | |
| 225 | jq -r '[ .results[0].mean, .results[0].stddev, .results[1].mean, .results[1].stddev ] | @tsv' < "${JSON_FILE}" | while IFS=$'\t' read -r one_mean one_stddev two_mean two_stddev; do | |
| 226 | one_mean=$(humanize_secs "${one_mean}") | |
| 227 | one_stddev=$(humanize_secs "${one_stddev}") | |
| 228 | ||
| 229 | if [ "${two_mean}" != "" ]; then | |
| 230 | two_mean=$(humanize_secs "${two_mean}") | |
| 231 | two_stddev=$(humanize_secs "${two_stddev}") | |
| 232 | ||
| 233 | echo "${one_mean} ± ${one_stddev} vs ${two_mean} ± ${two_stddev}" | |
| 234 | else | |
| 235 | echo "${one_mean} ± ${one_stddev}" | |
| 236 | fi | |
| 237 | done | |
| 238 | fi | |
| 239 | ||
| 240 | # add our metadata to the hyperfine json result | |
| 241 | jq ". |= { \"name\": \"${TEST_NAME}\" } + ." < "${JSON_FILE}" > "${JSON_FILE}.new" && mv "${JSON_FILE}.new" "${JSON_FILE}" | |
| 242 | done | |
| 243 | ||
| 244 | TIME_END=$(time_in_ms) | |
| 245 | ||
| 246 | if [ "$ANY_FOUND" != "1" ]; then | |
| 247 | echo "" | |
| 248 | echo "error: no benchmark suite \"${SUITE}\"." | |
| 249 | echo "" | |
| 250 | exit 1 | |
| 251 | fi | |
| 252 | ||
| 253 | escape() { | |
| 254 | echo "${1//\\/\\\\}" | |
| 255 | } | |
| 256 | ||
| 257 | # combine all the individual benchmark results into a single json file | |
| 258 | if [ "${JSON_RESULT}" != "" ]; then | |
| 259 | if [ "${VERBOSE}" = "1" ]; then | |
| 260 | echo "" | |
| 261 | echo "# Writing JSON results: ${JSON_RESULT}" | |
| 262 | fi | |
| 263 | ||
| 264 | SYSTEM_JSON="{ \"os\": \"${SYSTEM_OS}\", \"kernel\": \"${SYSTEM_KERNEL}\" }" | |
| 265 | TIME_JSON="{ \"start\": ${TIME_START}, \"end\": ${TIME_END} }" | |
| 266 | TEST_CLI_JSON="{ \"name\": \"${TEST_CLI_NAME}\", \"path\": \"$(escape "${TEST_CLI_PATH}")\", \"version\": \"${TEST_CLI_VERSION}\" }" | |
| 267 | BASELINE_CLI_JSON="{ \"name\": \"${BASELINE_CLI_NAME}\", \"path\": \"$(escape "${BASELINE_CLI_PATH}")\", \"version\": \"${BASELINE_CLI_VERSION}\" }" | |
| 268 | ||
| 269 | if [ "${BASELINE_CLI}" != "" ]; then | |
| 270 | EXECUTOR_JSON="{ \"baseline\": ${BASELINE_CLI_JSON}, \"cli\": ${TEST_CLI_JSON} }" | |
| 271 | else | |
| 272 | EXECUTOR_JSON="{ \"cli\": ${TEST_CLI_JSON} }" | |
| 273 | fi | |
| 274 | ||
| 275 | # add our metadata to all the test results | |
| 276 | jq -n "{ \"system\": ${SYSTEM_JSON}, \"time\": ${TIME_JSON}, \"executor\": ${EXECUTOR_JSON}, \"tests\": [inputs] }" "${OUTPUT_DIR}"/*.json > "${JSON_RESULT}" | |
| 277 | fi | |
| 278 | ||
| 279 | # combine all the data into a zip if requested | |
| 280 | if [ "${ZIP_RESULT}" != "" ]; then | |
| 281 | if [ "${VERBOSE}" = "1" ]; then | |
| 282 | if [ "${JSON_RESULT}" = "" ]; then echo ""; fi | |
| 283 | echo "# Writing ZIP results: ${ZIP_RESULT}" | |
| 284 | fi | |
| 285 | ||
| 286 | zip -jr "${ZIP_RESULT}" "${OUTPUT_DIR}" >/dev/null | |
| 287 | fi | |
| 288 | ||
| 289 | if [ "$CLEANUP_DIR" = "1" ]; then | |
| 290 | rm -f "${OUTPUT_DIR}"/*.out | |
| 291 | rm -f "${OUTPUT_DIR}"/*.err | |
| 292 | rm -f "${OUTPUT_DIR}"/*.json | |
| 293 | rmdir "${OUTPUT_DIR}" | |
| 294 | fi | |
| 295 | ||
| 296 | if [ "$ANY_FAILED" = "1" ]; then | |
| 297 | exit 1 | |
| 298 | fi |