test/TEST-36-NUMAPOLICY/testsuite.sh

   1 #!/usr/bin/env bash
   2 set -ex
   3 set -o pipefail
   4
   5 at_exit() {
   6     if [ $? -ne 0 ]; then
   7         # We're exiting with a non-zero EC, let's dump test artifacts
   8         # for easier debugging
   9         [ -f "$straceLog" ] && cat "$straceLog"
  10         [ -f "$journalLog" ] && cat "$journalLog"
  11     fi
  12 }
  13
  14 trap at_exit EXIT
  15
  16 systemd-analyze log-level debug
  17 systemd-analyze log-target journal
  18
  19 # Log files
  20 straceLog='strace.log'
  21 journalLog='journal.log'
  22
  23 # Systemd config files
  24 testUnit='numa-test.service'
  25 testUnitFile="/etc/systemd/system/$testUnit"
  26 testUnitNUMAConf="$testUnitFile.d/numa.conf"
  27
  28 # Sleep constants (we should probably figure out something better but nothing comes to mind)
  29 journalSleep=5
  30 sleepAfterStart=1
  31
  32 # Journal cursor for easier navigation
  33 journalCursorFile="jounalCursorFile"
  34
  35 startStrace() {
  36     coproc strace -qq -p 1 -o $straceLog -e set_mempolicy -s 1024 $1
  37     # Wait for strace to properly "initialize"
  38     sleep $sleepAfterStart
  39 }
  40
  41 stopStrace() {
  42     kill -s TERM $COPROC_PID
  43     # Make sure the strace process is indeed dead
  44     while kill -0 $COPROC_PID 2>/dev/null; do sleep 0.1; done
  45 }
  46
  47 startJournalctl() {
  48     # Save journal's cursor for later navigation
  49     journalctl --no-pager --cursor-file="$journalCursorFile" -n0 -ocat
  50 }
  51
  52 stopJournalctl() {
  53     local unit="${1:-init.scope}"
  54     # Using journalctl --sync should be better than using SIGRTMIN+1, as
  55     # the --sync wait until the synchronization is complete
  56     echo "Force journald to write all queued messages"
  57     journalctl --sync
  58     journalctl -u $unit --cursor-file="$journalCursorFile" > "$journalLog"
  59 }
  60
  61 checkNUMA() {
  62     # NUMA enabled system should have at least NUMA node0
  63     test -e /sys/devices/system/node/node0
  64 }
  65
  66 writePID1NUMAPolicy() {
  67     echo [Manager] > $confDir/numa.conf
  68     echo NUMAPolicy=$1 >> $confDir/numa.conf
  69     echo NUMAMask=$2>> $confDir/numa.conf
  70 }
  71
  72 writeTestUnit() {
  73     echo [Service] > $testUnitFile
  74     echo ExecStart=/bin/sleep 3600 >> $testUnitFile
  75     mkdir -p $testUnitFile.d/
  76 }
  77
  78 writeTestUnitNUMAPolicy() {
  79     echo [Service] > $testUnitNUMAConf
  80     echo NUMAPolicy=$1 >> $testUnitNUMAConf
  81     echo NUMAMask=$2>> $testUnitNUMAConf
  82     systemctl daemon-reload
  83 }
  84
  85 pid1ReloadWithStrace() {
  86     startStrace
  87     systemctl daemon-reload
  88     sleep $sleepAfterStart
  89     stopStrace
  90 }
  91
  92 pid1ReloadWithJournal() {
  93     startJournalctl
  94     systemctl daemon-reload
  95     stopJournalctl
  96 }
  97
  98 pid1StartUnitWithStrace() {
  99     startStrace '-f'
 100     systemctl start $1
 101     sleep $sleepAfterStart
 102     stopStrace
 103 }
 104
 105 pid1StartUnitWithJournal() {
 106     startJournalctl
 107     systemctl start $1
 108     sleep $sleepAfterStart
 109     stopJournalctl
 110 }
 111
 112 pid1StopUnit() {
 113     systemctl stop $1
 114 }
 115
 116 systemctlCheckNUMAProperties() {
 117     local LOGFILE="$(mktemp)"
 118     systemctl show -p NUMAPolicy $1 > "$LOGFILE"
 119     grep "NUMAPolicy=$2" "$LOGFILE"
 120
 121     > "$LOGFILE"
 122
 123     if [ -n "$3" ]; then
 124         systemctl show -p NUMAMask $1 > "$LOGFILE"
 125         grep "NUMAMask=$3" "$LOGFILE"
 126     fi
 127 }
 128
 129 writeTestUnit
 130
 131 # Create systemd config drop-in directory
 132 confDir="/etc/systemd/system.conf.d/"
 133 mkdir -p "$confDir"
 134
 135 if ! checkNUMA; then
 136     echo >&2 "NUMA is not supported on this machine, switching to a simple sanity check"
 137
 138     echo "PID1 NUMAPolicy=default && NUMAMask=0 check without NUMA support"
 139     writePID1NUMAPolicy "default" "0"
 140     startJournalctl
 141     systemctl daemon-reload
 142     stopJournalctl
 143     grep "NUMA support not available, ignoring" "$journalLog"
 144
 145     echo "systemd-run NUMAPolicy=default && NUMAMask=0 check without NUMA support"
 146     runUnit='numa-systemd-run-test.service'
 147     startJournalctl
 148     systemd-run -p NUMAPolicy=default -p NUMAMask=0 --unit $runUnit sleep 1000
 149     sleep $sleepAfterStart
 150     pid1StopUnit $runUnit
 151     stopJournalctl $runUnit
 152     grep "NUMA support not available, ignoring" "$journalLog"
 153
 154 else
 155     echo "PID1 NUMAPolicy support - Default policy w/o mask"
 156     writePID1NUMAPolicy "default"
 157     pid1ReloadWithStrace
 158     # Kernel requires that nodemask argument is set to NULL when setting default policy
 159     grep "set_mempolicy(MPOL_DEFAULT, NULL" $straceLog
 160
 161     echo "PID1 NUMAPolicy support - Default policy w/ mask"
 162     writePID1NUMAPolicy "default" "0"
 163     pid1ReloadWithStrace
 164     grep "set_mempolicy(MPOL_DEFAULT, NULL" $straceLog
 165
 166     echo "PID1 NUMAPolicy support - Bind policy w/o mask"
 167     writePID1NUMAPolicy "bind"
 168     pid1ReloadWithJournal
 169     grep "Failed to set NUMA memory policy: Invalid argument" $journalLog
 170
 171     echo "PID1 NUMAPolicy support - Bind policy w/ mask"
 172     writePID1NUMAPolicy "bind" "0"
 173     pid1ReloadWithStrace
 174     grep -P "set_mempolicy\(MPOL_BIND, \[0x0*1\]" $straceLog
 175
 176     echo "PID1 NUMAPolicy support - Interleave policy w/o mask"
 177     writePID1NUMAPolicy "interleave"
 178     pid1ReloadWithJournal
 179     grep "Failed to set NUMA memory policy: Invalid argument" $journalLog
 180
 181     echo "PID1 NUMAPolicy support - Interleave policy w/ mask"
 182     writePID1NUMAPolicy "interleave" "0"
 183     pid1ReloadWithStrace
 184     grep -P "set_mempolicy\(MPOL_INTERLEAVE, \[0x0*1\]" $straceLog
 185
 186     echo "PID1 NUMAPolicy support - Preferred policy w/o mask"
 187     writePID1NUMAPolicy "preferred"
 188     pid1ReloadWithJournal
 189     # Preferred policy with empty node mask is actually allowed and should reset allocation policy to default
 190     ! grep "Failed to set NUMA memory policy: Invalid argument" $journalLog
 191
 192     echo "PID1 NUMAPolicy support - Preferred policy w/ mask"
 193     writePID1NUMAPolicy "preferred" "0"
 194     pid1ReloadWithStrace
 195     grep -P "set_mempolicy\(MPOL_PREFERRED, \[0x0*1\]" $straceLog
 196
 197     echo "PID1 NUMAPolicy support - Local policy w/o mask"
 198     writePID1NUMAPolicy "local"
 199     pid1ReloadWithStrace
 200     # Kernel requires that nodemask argument is set to NULL when setting default policy
 201     # The unpatched versions of strace don't recognize the MPOL_LOCAL constant and
 202     # return a numerical constant instead (with a comment):
 203     #   set_mempolicy(0x4 /* MPOL_??? */, NULL, 0) = 0
 204     # Let's cover this scenario as well
 205     grep -E "set_mempolicy\((MPOL_LOCAL|0x4 [^,]*), NULL" $straceLog
 206
 207     echo "PID1 NUMAPolicy support - Local policy w/ mask"
 208     writePID1NUMAPolicy "local" "0"
 209     pid1ReloadWithStrace
 210     grep -E "set_mempolicy\((MPOL_LOCAL|0x4 [^,]*), NULL" $straceLog
 211
 212     echo "Unit file NUMAPolicy support - Default policy w/o mask"
 213     writeTestUnitNUMAPolicy "default"
 214     pid1StartUnitWithStrace $testUnit
 215     systemctlCheckNUMAProperties $testUnit "default"
 216     pid1StopUnit $testUnit
 217     grep "set_mempolicy(MPOL_DEFAULT, NULL" $straceLog
 218
 219     echo "Unit file NUMAPolicy support - Default policy w/ mask"
 220     writeTestUnitNUMAPolicy "default" "0"
 221     pid1StartUnitWithStrace $testUnit
 222     systemctlCheckNUMAProperties $testUnit "default" "0"
 223     pid1StopUnit $testUnit
 224     # Maks must be ignored
 225     grep "set_mempolicy(MPOL_DEFAULT, NULL" $straceLog
 226
 227     echo "Unit file NUMAPolicy support - Bind policy w/o mask"
 228     writeTestUnitNUMAPolicy "bind"
 229     pid1StartUnitWithJournal $testUnit
 230     pid1StopUnit $testUnit
 231     grep "numa-test.service: Main process exited, code=exited, status=242/NUMA" $journalLog
 232
 233     echo "Unit file NUMAPolicy support - Bind policy w/ mask"
 234     writeTestUnitNUMAPolicy "bind" "0"
 235     pid1StartUnitWithStrace $testUnit
 236     systemctlCheckNUMAProperties $testUnit "bind" "0"
 237     pid1StopUnit $testUnit
 238     grep -P "set_mempolicy\(MPOL_BIND, \[0x0*1\]" $straceLog
 239
 240     echo "Unit file NUMAPolicy support - Interleave policy w/o mask"
 241     writeTestUnitNUMAPolicy "interleave"
 242     pid1StartUnitWithStrace $testUnit
 243     pid1StopUnit $testUnit
 244     grep "numa-test.service: Main process exited, code=exited, status=242/NUMA" $journalLog
 245
 246     echo "Unit file NUMAPolicy support - Interleave policy w/ mask"
 247     writeTestUnitNUMAPolicy "interleave" "0"
 248     pid1StartUnitWithStrace $testUnit
 249     systemctlCheckNUMAProperties $testUnit "interleave" "0"
 250     pid1StopUnit $testUnit
 251     grep -P "set_mempolicy\(MPOL_INTERLEAVE, \[0x0*1\]" $straceLog
 252
 253     echo "Unit file NUMAPolicy support - Preferred policy w/o mask"
 254     writeTestUnitNUMAPolicy "preferred"
 255     pid1StartUnitWithJournal $testUnit
 256     systemctlCheckNUMAProperties $testUnit "preferred"
 257     pid1StopUnit $testUnit
 258     ! grep "numa-test.service: Main process exited, code=exited, status=242/NUMA" $journalLog
 259
 260     echo "Unit file NUMAPolicy support - Preferred policy w/ mask"
 261     writeTestUnitNUMAPolicy "preferred" "0"
 262     pid1StartUnitWithStrace $testUnit
 263     systemctlCheckNUMAProperties $testUnit "preferred" "0"
 264     pid1StopUnit $testUnit
 265     grep -P "set_mempolicy\(MPOL_PREFERRED, \[0x0*1\]" $straceLog
 266
 267     echo "Unit file NUMAPolicy support - Local policy w/o mask"
 268     writeTestUnitNUMAPolicy "local"
 269     pid1StartUnitWithStrace $testUnit
 270     systemctlCheckNUMAProperties $testUnit "local"
 271     pid1StopUnit $testUnit
 272     grep -E "set_mempolicy\((MPOL_LOCAL|0x4 [^,]*), NULL" $straceLog
 273
 274     echo "Unit file NUMAPolicy support - Local policy w/ mask"
 275     writeTestUnitNUMAPolicy "local" "0"
 276     pid1StartUnitWithStrace $testUnit
 277     systemctlCheckNUMAProperties $testUnit "local" "0"
 278     pid1StopUnit $testUnit
 279     # Maks must be ignored
 280     grep -E "set_mempolicy\((MPOL_LOCAL|0x4 [^,]*), NULL" $straceLog
 281
 282     echo "Unit file CPUAffinity=NUMA support"
 283     writeTestUnitNUMAPolicy "bind" "0"
 284     echo "CPUAffinity=numa" >> $testUnitNUMAConf
 285     systemctl daemon-reload
 286     systemctl start $testUnit
 287     systemctlCheckNUMAProperties $testUnit "bind" "0"
 288     pid=$(systemctl show --value -p MainPID $testUnit)
 289     cpulist=$(cat /sys/devices/system/node/node0/cpulist)
 290     affinity_systemd=$(systemctl show --value -p CPUAffinity $testUnit)
 291     [ $cpulist = $affinity_systemd ]
 292     pid1StopUnit $testUnit
 293
 294     echo "systemd-run NUMAPolicy support"
 295     runUnit='numa-systemd-run-test.service'
 296
 297     systemd-run -p NUMAPolicy=default --unit $runUnit sleep 1000
 298     systemctlCheckNUMAProperties $runUnit "default"
 299     pid1StopUnit $runUnit
 300
 301     systemd-run -p NUMAPolicy=default -p NUMAMask=0 --unit $runUnit sleep 1000
 302     systemctlCheckNUMAProperties $runUnit "default" ""
 303     pid1StopUnit $runUnit
 304
 305     systemd-run -p NUMAPolicy=bind -p NUMAMask=0 --unit $runUnit sleep 1000
 306     systemctlCheckNUMAProperties $runUnit "bind" "0"
 307     pid1StopUnit $runUnit
 308
 309     systemd-run -p NUMAPolicy=interleave -p NUMAMask=0 --unit $runUnit sleep 1000
 310     systemctlCheckNUMAProperties $runUnit "interleave" "0"
 311     pid1StopUnit $runUnit
 312
 313     systemd-run -p NUMAPolicy=preferred -p NUMAMask=0 --unit $runUnit sleep 1000
 314     systemctlCheckNUMAProperties $runUnit "preferred" "0"
 315     pid1StopUnit $runUnit
 316
 317     systemd-run -p NUMAPolicy=local --unit $runUnit sleep 1000
 318     systemctlCheckNUMAProperties $runUnit "local"
 319     pid1StopUnit $runUnit
 320
 321     systemd-run -p NUMAPolicy=local -p NUMAMask=0 --unit $runUnit sleep 1000
 322     systemctlCheckNUMAProperties $runUnit "local" ""
 323     pid1StopUnit $runUnit
 324
 325     systemd-run -p NUMAPolicy=local -p NUMAMask=0 -p CPUAffinity=numa --unit $runUnit sleep 1000
 326     systemctlCheckNUMAProperties $runUnit "local" ""
 327     systemctl cat $runUnit | grep -q 'CPUAffinity=numa'
 328     pid1StopUnit $runUnit
 329
 330 fi
 331
 332 # Cleanup
 333 rm -rf $testDir
 334 rm -rf $confDir
 335 systemctl daemon-reload
 336
 337 systemd-analyze log-level info
 338
 339 echo OK > /testok
 340
 341 exit 0