From 3db380570af7052620ace20c29e244938610ca71 Mon Sep 17 00:00:00 2001 From: Po-Hsu Lin Date: Mon, 28 Dec 2020 12:34:59 +0800 Subject: selftests/powerpc: Make the test check in eeh-basic.sh posix compliant The == operand is a bash extension, thus this will fail on Ubuntu with: ./eeh-basic.sh: 89: test: 2: unexpected operator As the /bin/sh on Ubuntu is pointed to DASH. Use -eq to fix this posix compatibility issue. Fixes: 996f9e0f93f162 ("selftests/powerpc: Fix eeh-basic.sh exit codes") Signed-off-by: Po-Hsu Lin Reviewed-by: Frederic Barrat Signed-off-by: Michael Ellerman Link: https://lore.kernel.org/r/20201228043459.14281-1-po-hsu.lin@canonical.com --- tools/testing/selftests/powerpc/eeh/eeh-basic.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tools') diff --git a/tools/testing/selftests/powerpc/eeh/eeh-basic.sh b/tools/testing/selftests/powerpc/eeh/eeh-basic.sh index 0d783e1065c8..64779f073e17 100755 --- a/tools/testing/selftests/powerpc/eeh/eeh-basic.sh +++ b/tools/testing/selftests/powerpc/eeh/eeh-basic.sh @@ -86,5 +86,5 @@ echo "$failed devices failed to recover ($dev_count tested)" lspci | diff -u $pre_lspci - rm -f $pre_lspci -test "$failed" == 0 +test "$failed" -eq 0 exit $? -- cgit v1.2.3 From db82f7097c265776c22ad866511074836f17665e Mon Sep 17 00:00:00 2001 From: Oliver O'Halloran Date: Tue, 3 Nov 2020 15:45:01 +1100 Subject: selftests/powerpc: Hoist helper code out of eeh-basic Hoist some of the useful test environment checking and prep code into eeh-functions.sh so they can be reused in other tests. Signed-off-by: Oliver O'Halloran Signed-off-by: Michael Ellerman Link: https://lore.kernel.org/r/20201103044503.917128-1-oohall@gmail.com --- tools/testing/selftests/powerpc/eeh/eeh-basic.sh | 39 ++---------------- .../testing/selftests/powerpc/eeh/eeh-functions.sh | 48 ++++++++++++++++++++++ 2 files changed, 51 insertions(+), 36 deletions(-) mode change 100755 => 100644 tools/testing/selftests/powerpc/eeh/eeh-functions.sh (limited to 'tools') diff --git a/tools/testing/selftests/powerpc/eeh/eeh-basic.sh b/tools/testing/selftests/powerpc/eeh/eeh-basic.sh index 64779f073e17..442b666ccdb5 100755 --- a/tools/testing/selftests/powerpc/eeh/eeh-basic.sh +++ b/tools/testing/selftests/powerpc/eeh/eeh-basic.sh @@ -1,28 +1,13 @@ #!/bin/sh # SPDX-License-Identifier: GPL-2.0-only -KSELFTESTS_SKIP=4 - . ./eeh-functions.sh -if ! eeh_supported ; then - echo "EEH not supported on this system, skipping" - exit $KSELFTESTS_SKIP; -fi - -if [ ! -e "/sys/kernel/debug/powerpc/eeh_dev_check" ] && \ - [ ! -e "/sys/kernel/debug/powerpc/eeh_dev_break" ] ; then - echo "debugfs EEH testing files are missing. Is debugfs mounted?" - exit $KSELFTESTS_SKIP; -fi +eeh_test_prep # NB: may exit pre_lspci=`mktemp` lspci > $pre_lspci -# Bump the max freeze count to something absurd so we don't -# trip over it while breaking things. -echo 5000 > /sys/kernel/debug/powerpc/eeh_max_freezes - # record the devices that we break in here. Assuming everything # goes to plan we should get them back once the recover process # is finished. @@ -30,34 +15,16 @@ devices="" # Build up a list of candidate devices. for dev in `ls -1 /sys/bus/pci/devices/ | grep '\.0$'` ; do - # skip bridges since we can't recover them (yet...) - if [ -e "/sys/bus/pci/devices/$dev/pci_bus" ] ; then - echo "$dev, Skipped: bridge" + if ! eeh_can_break $dev ; then continue; fi - # Skip VFs for now since we don't have a reliable way - # to break them. + # Skip VFs for now since we don't have a reliable way to break them. if [ -e "/sys/bus/pci/devices/$dev/physfn" ] ; then echo "$dev, Skipped: virtfn" continue; fi - if [ "ahci" = "$(basename $(realpath /sys/bus/pci/devices/$dev/driver))" ] ; then - echo "$dev, Skipped: ahci doesn't support recovery" - continue - fi - - # Don't inject errosr into an already-frozen PE. This happens with - # PEs that contain multiple PCI devices (e.g. multi-function cards) - # and injecting new errors during the recovery process will probably - # result in the recovery failing and the device being marked as - # failed. - if ! pe_ok $dev ; then - echo "$dev, Skipped: Bad initial PE state" - continue; - fi - echo "$dev, Added" # Add to this list of device to check diff --git a/tools/testing/selftests/powerpc/eeh/eeh-functions.sh b/tools/testing/selftests/powerpc/eeh/eeh-functions.sh old mode 100755 new mode 100644 index 00dc32c0ed75..9b1bcc1fd4ad --- a/tools/testing/selftests/powerpc/eeh/eeh-functions.sh +++ b/tools/testing/selftests/powerpc/eeh/eeh-functions.sh @@ -1,6 +1,8 @@ #!/bin/sh # SPDX-License-Identifier: GPL-2.0-only +export KSELFTESTS_SKIP=4 + pe_ok() { local dev="$1" local path="/sys/bus/pci/devices/$dev/eeh_pe_state" @@ -39,6 +41,52 @@ eeh_supported() { grep -q 'EEH Subsystem is enabled' /proc/powerpc/eeh } +eeh_test_prep() { + if ! eeh_supported ; then + echo "EEH not supported on this system, skipping" + exit $KSELFTESTS_SKIP; + fi + + if [ ! -e "/sys/kernel/debug/powerpc/eeh_dev_check" ] && \ + [ ! -e "/sys/kernel/debug/powerpc/eeh_dev_break" ] ; then + echo "debugfs EEH testing files are missing. Is debugfs mounted?" + exit $KSELFTESTS_SKIP; + fi + + # Bump the max freeze count to something absurd so we don't + # trip over it while breaking things. + echo 5000 > /sys/kernel/debug/powerpc/eeh_max_freezes +} + +eeh_can_break() { + # skip bridges since we can't recover them (yet...) + if [ -e "/sys/bus/pci/devices/$dev/pci_bus" ] ; then + echo "$dev, Skipped: bridge" + return 1; + fi + + # The ahci driver doesn't support error recovery. If the ahci device + # happens to be hosting the root filesystem, and then we go and break + # it the system will generally go down. We should probably fix that + # at some point + if [ "ahci" = "$(basename $(realpath /sys/bus/pci/devices/$dev/driver))" ] ; then + echo "$dev, Skipped: ahci doesn't support recovery" + return 1; + fi + + # Don't inject errosr into an already-frozen PE. This happens with + # PEs that contain multiple PCI devices (e.g. multi-function cards) + # and injecting new errors during the recovery process will probably + # result in the recovery failing and the device being marked as + # failed. + if ! pe_ok $dev ; then + echo "$dev, Skipped: Bad initial PE state" + return 1; + fi + + return 0 +} + eeh_one_dev() { local dev="$1" -- cgit v1.2.3 From d6749ccba7ff86f99b4672e50db871487ba69f19 Mon Sep 17 00:00:00 2001 From: Oliver O'Halloran Date: Tue, 3 Nov 2020 15:45:02 +1100 Subject: selftests/powerpc: Use stderr for debug messages in eeh-functions We want to use stdout to return lists of devices, etc so log debug / status messages to stderr rather than stdout. Signed-off-by: Oliver O'Halloran Signed-off-by: Michael Ellerman Link: https://lore.kernel.org/r/20201103044503.917128-2-oohall@gmail.com --- tools/testing/selftests/powerpc/eeh/eeh-functions.sh | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) (limited to 'tools') diff --git a/tools/testing/selftests/powerpc/eeh/eeh-functions.sh b/tools/testing/selftests/powerpc/eeh/eeh-functions.sh index 9b1bcc1fd4ad..32e5b7fbf18a 100644 --- a/tools/testing/selftests/powerpc/eeh/eeh-functions.sh +++ b/tools/testing/selftests/powerpc/eeh/eeh-functions.sh @@ -3,6 +3,10 @@ export KSELFTESTS_SKIP=4 +log() { + echo >/dev/stderr $* +} + pe_ok() { local dev="$1" local path="/sys/bus/pci/devices/$dev/eeh_pe_state" @@ -49,7 +53,7 @@ eeh_test_prep() { if [ ! -e "/sys/kernel/debug/powerpc/eeh_dev_check" ] && \ [ ! -e "/sys/kernel/debug/powerpc/eeh_dev_break" ] ; then - echo "debugfs EEH testing files are missing. Is debugfs mounted?" + log "debugfs EEH testing files are missing. Is debugfs mounted?" exit $KSELFTESTS_SKIP; fi @@ -61,7 +65,7 @@ eeh_test_prep() { eeh_can_break() { # skip bridges since we can't recover them (yet...) if [ -e "/sys/bus/pci/devices/$dev/pci_bus" ] ; then - echo "$dev, Skipped: bridge" + log "$dev, Skipped: bridge" return 1; fi @@ -70,7 +74,7 @@ eeh_can_break() { # it the system will generally go down. We should probably fix that # at some point if [ "ahci" = "$(basename $(realpath /sys/bus/pci/devices/$dev/driver))" ] ; then - echo "$dev, Skipped: ahci doesn't support recovery" + log "$dev, Skipped: ahci doesn't support recovery" return 1; fi @@ -80,7 +84,7 @@ eeh_can_break() { # result in the recovery failing and the device being marked as # failed. if ! pe_ok $dev ; then - echo "$dev, Skipped: Bad initial PE state" + log "$dev, Skipped: Bad initial PE state" return 1; fi @@ -94,7 +98,7 @@ eeh_one_dev() { # testing so check that the argument is a well-formed sysfs device # name. if ! test -e /sys/bus/pci/devices/$dev/ ; then - echo "Error: '$dev' must be a sysfs device name (DDDD:BB:DD.F)" + log "Error: '$dev' must be a sysfs device name (DDDD:BB:DD.F)" return 1; fi @@ -118,16 +122,16 @@ eeh_one_dev() { if pe_ok $dev ; then break; fi - echo "$dev, waited $i/${max_wait}" + log "$dev, waited $i/${max_wait}" sleep 1 done if ! pe_ok $dev ; then - echo "$dev, Failed to recover!" + log "$dev, Failed to recover!" return 1; fi - echo "$dev, Recovered after $i seconds" + log "$dev, Recovered after $i seconds" return 0; } -- cgit v1.2.3 From 38132cc0e5a6b22b04fac2e4df25c59435fcd6de Mon Sep 17 00:00:00 2001 From: Oliver O'Halloran Date: Tue, 3 Nov 2020 15:45:03 +1100 Subject: selftests/powerpc: Add VF recovery tests The basic EEH test ignores VFs since we the way the eeh_dev_break debugfs interface works means that if multiple VFs are enabled we may cause errors on all them them. However, we can work around that by only enabling a single VF at a time. This patch adds some infrastructure for finding SR-IOV capable devices and enabling / disabling VFs so we can exercise the VF specific EEH recovery paths. Two new tests are added, one for testing EEH aware devices and one for EEH un-aware VFs. Signed-off-by: Oliver O'Halloran Signed-off-by: Michael Ellerman Link: https://lore.kernel.org/r/20201103044503.917128-3-oohall@gmail.com --- .../testing/selftests/powerpc/eeh/eeh-functions.sh | 108 +++++++++++++++++++++ .../testing/selftests/powerpc/eeh/eeh-vf-aware.sh | 45 +++++++++ .../selftests/powerpc/eeh/eeh-vf-unaware.sh | 35 +++++++ 3 files changed, 188 insertions(+) create mode 100755 tools/testing/selftests/powerpc/eeh/eeh-vf-aware.sh create mode 100755 tools/testing/selftests/powerpc/eeh/eeh-vf-unaware.sh (limited to 'tools') diff --git a/tools/testing/selftests/powerpc/eeh/eeh-functions.sh b/tools/testing/selftests/powerpc/eeh/eeh-functions.sh index 32e5b7fbf18a..70daa3925dcb 100644 --- a/tools/testing/selftests/powerpc/eeh/eeh-functions.sh +++ b/tools/testing/selftests/powerpc/eeh/eeh-functions.sh @@ -135,3 +135,111 @@ eeh_one_dev() { return 0; } +eeh_has_driver() { + test -e /sys/bus/pci/devices/$1/driver; + return $? +} + +eeh_can_recover() { + # we'll get an IO error if the device's current driver doesn't support + # error recovery + echo $1 > '/sys/kernel/debug/powerpc/eeh_dev_can_recover' 2>/dev/null + + return $? +} + +eeh_find_all_pfs() { + devices="" + + # SR-IOV on pseries requires hypervisor support, so check for that + is_pseries="" + if grep -q pSeries /proc/cpuinfo ; then + if [ ! -f /proc/device-tree/rtas/ibm,open-sriov-allow-unfreeze ] || + [ ! -f /proc/device-tree/rtas/ibm,open-sriov-map-pe-number ] ; then + return 1; + fi + + is_pseries="true" + fi + + for dev in `ls -1 /sys/bus/pci/devices/` ; do + sysfs="/sys/bus/pci/devices/$dev" + if [ ! -e "$sysfs/sriov_numvfs" ] ; then + continue + fi + + # skip unsupported PFs on pseries + if [ -z "$is_pseries" ] && + [ ! -f "$sysfs/of_node/ibm,is-open-sriov-pf" ] && + [ ! -f "$sysfs/of_node/ibm,open-sriov-vf-bar-info" ] ; then + continue; + fi + + # no driver, no vfs + if ! eeh_has_driver $dev ; then + continue + fi + + devices="$devices $dev" + done + + if [ -z "$devices" ] ; then + return 1; + fi + + echo $devices + return 0; +} + +# attempts to enable one VF on each PF so we can do VF specific tests. +# stdout: list of enabled VFs, one per line +# return code: 0 if vfs are found, 1 otherwise +eeh_enable_vfs() { + pf_list="$(eeh_find_all_pfs)" + + vfs=0 + for dev in $pf_list ; do + pf_sysfs="/sys/bus/pci/devices/$dev" + + # make sure we have a single VF + echo 0 > "$pf_sysfs/sriov_numvfs" + echo 1 > "$pf_sysfs/sriov_numvfs" + if [ "$?" != 0 ] ; then + log "Unable to enable VFs on $pf, skipping" + continue; + fi + + vf="$(basename $(realpath "$pf_sysfs/virtfn0"))" + if [ $? != 0 ] ; then + log "unable to find enabled vf on $pf" + echo 0 > "$pf_sysfs/sriov_numvfs" + continue; + fi + + if ! eeh_can_break $vf ; then + log "skipping " + + echo 0 > "$pf_sysfs/sriov_numvfs" + continue; + fi + + vfs="$((vfs + 1))" + echo $vf + done + + test "$vfs" != 0 + return $? +} + +eeh_disable_vfs() { + pf_list="$(eeh_find_all_pfs)" + if [ -z "$pf_list" ] ; then + return 1; + fi + + for dev in $pf_list ; do + echo 0 > "/sys/bus/pci/devices/$dev/sriov_numvfs" + done + + return 0; +} diff --git a/tools/testing/selftests/powerpc/eeh/eeh-vf-aware.sh b/tools/testing/selftests/powerpc/eeh/eeh-vf-aware.sh new file mode 100755 index 000000000000..874c11953bb6 --- /dev/null +++ b/tools/testing/selftests/powerpc/eeh/eeh-vf-aware.sh @@ -0,0 +1,45 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only + +. ./eeh-functions.sh + +eeh_test_prep # NB: may exit + +vf_list="$(eeh_enable_vfs)"; +if $? != 0 ; then + log "No usable VFs found. Skipping EEH unaware VF test" + exit $KSELFTESTS_SKIP; +fi + +log "Enabled VFs: $vf_list" + +tested=0 +passed=0 +for vf in $vf_list ; do + log "Testing $vf" + + if ! eeh_can_recover $vf ; then + log "Driver for $vf doesn't support error recovery, skipping" + continue; + fi + + tested="$((tested + 1))" + + log "Breaking $vf..." + if ! eeh_one_dev $vf ; then + log "$vf failed to recover" + continue; + fi + + passed="$((passed + 1))" +done + +eeh_disable_vfs + +if [ "$tested" == 0 ] ; then + echo "No VFs with EEH aware drivers found, skipping" + exit $KSELFTESTS_SKIP +fi + +test "$failed" != 0 +exit $?; diff --git a/tools/testing/selftests/powerpc/eeh/eeh-vf-unaware.sh b/tools/testing/selftests/powerpc/eeh/eeh-vf-unaware.sh new file mode 100755 index 000000000000..8a4c147b9d43 --- /dev/null +++ b/tools/testing/selftests/powerpc/eeh/eeh-vf-unaware.sh @@ -0,0 +1,35 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only + +. ./eeh-functions.sh + +eeh_test_prep # NB: may exit + +vf_list="$(eeh_enable_vfs)"; +if $? != 0 ; then + log "No usable VFs found. Skipping EEH unaware VF test" + exit $KSELFTESTS_SKIP; +fi + +log "Enabled VFs: $vf_list" + +failed=0 +for vf in $vf_list ; do + log "Testing $vf" + + if eeh_can_recover $vf ; then + log "Driver for $vf supports error recovery. Unbinding..." + echo "$vf" > /sys/bus/pci/devices/$vf/driver/unbind + fi + + log "Breaking $vf..." + if ! eeh_one_dev $vf ; then + log "$vf failed to recover" + failed="$((failed + 1))" + fi +done + +eeh_disable_vfs + +test "$failed" != 0 +exit $?; -- cgit v1.2.3