diff options
Diffstat (limited to 'tests/run-debuginfod-find.sh')
| -rwxr-xr-x | tests/run-debuginfod-find.sh | 264 |
1 files changed, 226 insertions, 38 deletions
diff --git a/tests/run-debuginfod-find.sh b/tests/run-debuginfod-find.sh index 0ade03b7..7faad331 100755 --- a/tests/run-debuginfod-find.sh +++ b/tests/run-debuginfod-find.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright (C) 2019 Red Hat, Inc. +# Copyright (C) 2019-2020 Red Hat, Inc. # This file is part of elfutils. # # This file is free software; you can redistribute it and/or modify @@ -18,19 +18,31 @@ . $srcdir/test-subr.sh # includes set -e +type curl 2>/dev/null || (echo "need curl"; exit 77) +type rpm2cpio 2>/dev/null || (echo "need rpm2cpio"; exit 77) +type bzcat 2>/dev/null || (echo "need bzcat"; exit 77) +bsdtar --version | grep -q zstd && zstd=true || zstd=false +echo "zstd=$zstd bsdtar=`bsdtar --version`" + +# for test case debugging, uncomment: +#set -x +#VERBOSE=-vvvv + DB=${PWD}/.debuginfod_tmp.sqlite tempfiles $DB export DEBUGINFOD_CACHE_PATH=${PWD}/.client_cache PID1=0 PID2=0 +PID3=0 cleanup() { if [ $PID1 -ne 0 ]; then kill $PID1; wait $PID1; fi if [ $PID2 -ne 0 ]; then kill $PID2; wait $PID2; fi + if [ $PID3 -ne 0 ]; then kill $PID3; wait $PID3; fi - rm -rf F R L ${PWD}/.client_cache* + rm -rf F R D L Z ${PWD}/foobar ${PWD}/mocktree ${PWD}/.client_cache* ${PWD}/tmp* exit_cleanup } @@ -52,8 +64,8 @@ done # So we gather the LD_LIBRARY_PATH with this cunning trick: ldpath=`testrun sh -c 'echo $LD_LIBRARY_PATH'` -mkdir F R L -# not tempfiles F R L - they are directories which we clean up manually +mkdir F R L D Z +# not tempfiles F R L D Z - they are directories which we clean up manually ln -s ${abs_builddir}/dwfllines L/foo # any program not used elsewhere in this test wait_ready() @@ -77,13 +89,15 @@ wait_ready() done; if [ $timeout -eq 0 ]; then - echo "metric $what never changed to $value on port $port" + echo "metric $what never changed to $value on port $port" + curl -s http://127.0.0.1:$port/metrics exit 1; fi } -env LD_LIBRARY_PATH=$ldpath DEBUGINFOD_URLS= ${abs_builddir}/../debuginfod/debuginfod -F -R -d $DB -p $PORT1 -t0 -g0 R F L & +env LD_LIBRARY_PATH=$ldpath DEBUGINFOD_URLS= ${abs_builddir}/../debuginfod/debuginfod $VERBOSE -F -R -d $DB -p $PORT1 -t0 -g0 --fdcache-fds 1 --fdcache-mbs 2 -Z .tar.xz -Z .tar.bz2=bzcat -v R F Z L > vlog4 2>&1 & PID1=$! +tempfiles vlog4 # Server must become ready wait_ready $PORT1 'ready' 1 export DEBUGINFOD_URLS=http://127.0.0.1:$PORT1/ # or without trailing / @@ -101,16 +115,21 @@ export DEBUGINFOD_TIMEOUT=10 # cannot find it without debuginfod. echo "int main() { return 0; }" > ${PWD}/prog.c tempfiles prog.c -gcc -g -o prog ${PWD}/prog.c - ${abs_top_builddir}/src/strip -g -f prog.debug ${PWD}/prog +# Create a subdirectory to confound source path names +mkdir foobar +gcc -Wl,--build-id -g -o prog ${PWD}/foobar///./../prog.c +testrun ${abs_top_builddir}/src/strip -g -f prog.debug ${PWD}/prog BUILDID=`env LD_LIBRARY_PATH=$ldpath ${abs_builddir}/../src/readelf \ -a prog | grep 'Build ID' | cut -d ' ' -f 7` +wait_ready $PORT1 'thread_work_total{role="traverse"}' 1 mv prog F mv prog.debug F kill -USR1 $PID1 # Wait till both files are in the index. -wait_ready $PORT1 'thread_work_total{file="F"}' 2 +wait_ready $PORT1 'thread_work_total{role="traverse"}' 2 +wait_ready $PORT1 'thread_work_pending{role="scan"}' 0 +wait_ready $PORT1 'thread_busy{role="scan"}' 0 ######################################################################## @@ -125,12 +144,62 @@ rm -rf $DEBUGINFOD_CACHE_PATH # clean it from previous tests filename=`testrun ${abs_top_builddir}/debuginfod/debuginfod-find debuginfo $BUILDID` cmp $filename F/prog.debug -filename=`testrun ${abs_top_builddir}/debuginfod/debuginfod-find executable $BUILDID` +filename=`testrun ${abs_top_builddir}/debuginfod/debuginfod-find executable F/prog` cmp $filename F/prog +# raw source filename +filename=`testrun ${abs_top_builddir}/debuginfod/debuginfod-find source $BUILDID ${PWD}/foobar///./../prog.c` +cmp $filename ${PWD}/prog.c + +# and also the canonicalized one filename=`testrun ${abs_top_builddir}/debuginfod/debuginfod-find source $BUILDID ${PWD}/prog.c` cmp $filename ${PWD}/prog.c + +######################################################################## + +# Test whether the cache default locations are correct + +mkdir tmphome + +# $HOME/.cache should be created. +testrun env HOME=$PWD/tmphome XDG_CACHE_HOME= DEBUGINFOD_CACHE_PATH= ${abs_top_builddir}/debuginfod/debuginfod-find debuginfo $BUILDID +if [ ! -f $PWD/tmphome/.cache/debuginfod_client/$BUILDID/debuginfo ]; then + echo "could not find cache in $PWD/tmphome/.cache" + exit 1 +fi + +# $HOME/.cache should be found. +testrun env HOME=$PWD/tmphome XDG_CACHE_HOME= DEBUGINFOD_CACHE_PATH= ${abs_top_builddir}/debuginfod/debuginfod-find executable $BUILDID +if [ ! -f $PWD/tmphome/.cache/debuginfod_client/$BUILDID/executable ]; then + echo "could not find cache in $PWD/tmphome/.cache" + exit 1 +fi + +# $XDG_CACHE_HOME should take priority over $HOME.cache. +testrun env HOME=$PWD/tmphome XDG_CACHE_HOME=$PWD/tmpxdg DEBUGINFOD_CACHE_PATH= ${abs_top_builddir}/debuginfod/debuginfod-find debuginfo $BUILDID +if [ ! -f $PWD/tmpxdg/debuginfod_client/$BUILDID/debuginfo ]; then + echo "could not find cache in $PWD/tmpxdg/" + exit 1 +fi + +# A cache at the old default location ($HOME/.debuginfod_client_cache) should take +# priority over $HOME/.cache, $XDG_CACHE_HOME. +cp -r $DEBUGINFOD_CACHE_PATH tmphome/.debuginfod_client_cache + +# Add a file that doesn't exist in $HOME/.cache, $XDG_CACHE_HOME. +mkdir tmphome/.debuginfod_client_cache/deadbeef +echo ELF... > tmphome/.debuginfod_client_cache/deadbeef/debuginfo +filename=`testrun env HOME=$PWD/tmphome XDG_CACHE_HOME=$PWD/tmpxdg DEBUGINFOD_CACHE_PATH= ${abs_top_builddir}/debuginfod/debuginfod-find debuginfo deadbeef` +cmp $filename tmphome/.debuginfod_client_cache/deadbeef/debuginfo + +# $DEBUGINFO_CACHE_PATH should take priority over all else. +testrun env HOME=$PWD/tmphome XDG_CACHE_HOME=$PWD/tmpxdg DEBUGINFOD_CACHE_PATH=$PWD/tmpcache ${abs_top_builddir}/debuginfod/debuginfod-find debuginfo $BUILDID +if [ ! -f $PWD/tmpcache/$BUILDID/debuginfo ]; then + echo "could not find cache in $PWD/tmpcache/" + exit 1 +fi + ######################################################################## # Add artifacts to the search paths and test whether debuginfod finds them while already running. @@ -138,31 +207,46 @@ cmp $filename ${PWD}/prog.c # Build another, non-stripped binary echo "int main() { return 0; }" > ${PWD}/prog2.c tempfiles prog2.c -gcc -g -o prog2 ${PWD}/prog2.c +gcc -Wl,--build-id -g -o prog2 ${PWD}/prog2.c BUILDID2=`env LD_LIBRARY_PATH=$ldpath ${abs_builddir}/../src/readelf \ -a prog2 | grep 'Build ID' | cut -d ' ' -f 7` mv prog2 F kill -USR1 $PID1 # Now there should be 3 files in the index -wait_ready $PORT1 'thread_work_total{file="F"}' 3 +wait_ready $PORT1 'thread_work_total{role="traverse"}' 3 +wait_ready $PORT1 'thread_work_pending{role="scan"}' 0 +wait_ready $PORT1 'thread_busy{role="scan"}' 0 # Rerun same tests for the prog2 binary filename=`testrun ${abs_top_builddir}/debuginfod/debuginfod-find -v debuginfo $BUILDID2 2>vlog` cmp $filename F/prog2 cat vlog grep -q Progress vlog +grep -q Downloaded.from vlog tempfiles vlog -filename=`testrun ${abs_top_builddir}/debuginfod/debuginfod-find executable $BUILDID2` +filename=`testrun env DEBUGINFOD_PROGRESS=1 ${abs_top_builddir}/debuginfod/debuginfod-find executable $BUILDID2 2>vlog2` cmp $filename F/prog2 +cat vlog2 +grep -q 'Downloading.*http' vlog2 +tempfiles vlog2 filename=`testrun ${abs_top_builddir}/debuginfod/debuginfod-find source $BUILDID2 ${PWD}/prog2.c` cmp $filename ${PWD}/prog2.c -cp -rp ${abs_srcdir}/debuginfod-rpms R +cp -rvp ${abs_srcdir}/debuginfod-rpms R +if [ "$zstd" = "false" ]; then # nuke the zstd fedora 31 ones + rm -vrf R/debuginfod-rpms/fedora31 +fi + +cp -rvp ${abs_srcdir}/debuginfod-tars Z kill -USR1 $PID1 # All rpms need to be in the index rpms=$(find R -name \*rpm | wc -l) -wait_ready $PORT1 'scanned_total{source="rpm"}' $rpms +wait_ready $PORT1 'scanned_total{source=".rpm archive"}' $rpms +txz=$(find Z -name \*tar.xz | wc -l) +wait_ready $PORT1 'scanned_total{source=".tar.xz archive"}' $txz +tb2=$(find Z -name \*tar.bz2 | wc -l) +wait_ready $PORT1 'scanned_total{source=".tar.bz2 archive"}' $tb2 kill -USR1 $PID1 # two hits of SIGUSR1 may be needed to resolve .debug->dwz->srefs # Expect all source files found in the rpms (they are all called hello.c :) @@ -177,7 +261,7 @@ for i in $newrpms; do mkdir $subdir; cd $subdir; ls -lah ../$i - rpm2cpio ../$i | cpio -id; + rpm2cpio ../$i | cpio -ivd; cd ..; done sourcefiles=$(find -name \*\\.debug \ @@ -187,11 +271,11 @@ sourcefiles=$(find -name \*\\.debug \ cd .. rm -rf extracted -wait_ready $PORT1 'found_sourcerefs_total{source="rpm"}' $sourcefiles +wait_ready $PORT1 'found_sourcerefs_total{source=".rpm archive"}' $sourcefiles -# Run a bank of queries against the debuginfod-rpms test cases +# Run a bank of queries against the debuginfod-rpms / debuginfod-debs test cases -rpm_test() { +archive_test() { __BUILDID=$1 __SOURCEPATH=$2 __SOURCESHA1=$3 @@ -200,29 +284,51 @@ rpm_test() { buildid=`env LD_LIBRARY_PATH=$ldpath ${abs_builddir}/../src/readelf \ -a $filename | grep 'Build ID' | cut -d ' ' -f 7` test $__BUILDID = $buildid + # check that timestamps are plausible - older than the near-present (tmpdir mtime) + test $filename -ot `pwd` + + # run again to assure that fdcache is being enjoyed + filename=`testrun ${abs_top_builddir}/debuginfod/debuginfod-find executable $__BUILDID` + buildid=`env LD_LIBRARY_PATH=$ldpath ${abs_builddir}/../src/readelf \ + -a $filename | grep 'Build ID' | cut -d ' ' -f 7` + test $__BUILDID = $buildid + test $filename -ot `pwd` filename=`testrun ${abs_top_builddir}/debuginfod/debuginfod-find debuginfo $__BUILDID` buildid=`env LD_LIBRARY_PATH=$ldpath ${abs_builddir}/../src/readelf \ -a $filename | grep 'Build ID' | cut -d ' ' -f 7` test $__BUILDID = $buildid - - filename=`testrun ${abs_top_builddir}/debuginfod/debuginfod-find source $__BUILDID $__SOURCEPATH` - hash=`cat $filename | sha1sum | awk '{print $1}'` - test $__SOURCESHA1 = $hash + test $filename -ot `pwd` + + if test "x$__SOURCEPATH" != "x"; then + filename=`testrun ${abs_top_builddir}/debuginfod/debuginfod-find source $__BUILDID $__SOURCEPATH` + hash=`cat $filename | sha1sum | awk '{print $1}'` + test $__SOURCESHA1 = $hash + test $filename -ot `pwd` + fi } # common source file sha1 SHA=f4a1a8062be998ae93b8f1cd744a398c6de6dbb1 +# fedora31 +if [ $zstd = true ]; then + # fedora31 uses zstd compression on rpms, older rpm2cpio/libarchive can't handle it + # and we're not using the fancy -Z '.rpm=(rpm2cpio|zstdcat)<' workaround in this testsuite + archive_test 420e9e3308971f4b817cc5bf83928b41a6909d88 /usr/src/debug/hello3-1.0-2.x86_64/foobar////./../hello.c $SHA + archive_test 87c08d12c78174f1082b7c888b3238219b0eb265 /usr/src/debug/hello3-1.0-2.x86_64///foobar/./..//hello.c $SHA +fi # fedora30 -rpm_test c36708a78618d597dee15d0dc989f093ca5f9120 /usr/src/debug/hello2-1.0-2.x86_64/hello.c $SHA -rpm_test 41a236eb667c362a1c4196018cc4581e09722b1b /usr/src/debug/hello2-1.0-2.x86_64/hello.c $SHA +archive_test c36708a78618d597dee15d0dc989f093ca5f9120 /usr/src/debug/hello2-1.0-2.x86_64/hello.c $SHA +archive_test 41a236eb667c362a1c4196018cc4581e09722b1b /usr/src/debug/hello2-1.0-2.x86_64/hello.c $SHA # rhel7 -rpm_test bc1febfd03ca05e030f0d205f7659db29f8a4b30 /usr/src/debug/hello-1.0/hello.c $SHA -rpm_test f0aa15b8aba4f3c28cac3c2a73801fefa644a9f2 /usr/src/debug/hello-1.0/hello.c $SHA +archive_test bc1febfd03ca05e030f0d205f7659db29f8a4b30 /usr/src/debug/hello-1.0/hello.c $SHA +archive_test f0aa15b8aba4f3c28cac3c2a73801fefa644a9f2 /usr/src/debug/hello-1.0/hello.c $SHA # rhel6 -rpm_test bbbf92ebee5228310e398609c23c2d7d53f6e2f9 /usr/src/debug/hello-1.0/hello.c $SHA -rpm_test d44d42cbd7d915bc938c81333a21e355a6022fb7 /usr/src/debug/hello-1.0/hello.c $SHA +archive_test bbbf92ebee5228310e398609c23c2d7d53f6e2f9 /usr/src/debug/hello-1.0/hello.c $SHA +archive_test d44d42cbd7d915bc938c81333a21e355a6022fb7 /usr/src/debug/hello-1.0/hello.c $SHA +# arch +archive_test cee13b2ea505a7f37bd20d271c6bc7e5f8d2dfcb /usr/src/debug/hello.c 7a1334e086b97e5f124003a6cfb3ed792d10cdf4 RPM_BUILDID=d44d42cbd7d915bc938c81333a21e355a6022fb7 # in rhel6/ subdir, for a later test @@ -235,6 +341,8 @@ RPM_BUILDID=d44d42cbd7d915bc938c81333a21e355a6022fb7 # in rhel6/ subdir, for a l rm -r R/debuginfod-rpms/rhel6/* kill -USR2 $PID1 # groom cycle # Expect 3 rpms to be deleted by the groom +# 1 groom already took place at/soon-after startup, so -USR2 makes 2 +wait_ready $PORT1 'thread_work_total{role="groom"}' 2 wait_ready $PORT1 'groom{statistic="file d/e"}' 3 rm -rf $DEBUGINFOD_CACHE_PATH # clean it from previous tests @@ -257,16 +365,43 @@ export DEBUGINFOD_CACHE_PATH=${PWD}/.client_cache2 mkdir -p $DEBUGINFOD_CACHE_PATH # NB: inherits the DEBUGINFOD_URLS to the first server # NB: run in -L symlink-following mode for the L subdir -env LD_LIBRARY_PATH=$ldpath ${abs_builddir}/../debuginfod/debuginfod -F -d ${DB}_2 -p $PORT2 -L L & +env LD_LIBRARY_PATH=$ldpath ${abs_builddir}/../debuginfod/debuginfod $VERBOSE -F -U -d ${DB}_2 -p $PORT2 -L L D > vlog3 2>&1 & PID2=$! +tempfiles vlog3 tempfiles ${DB}_2 wait_ready $PORT2 'ready' 1 +wait_ready $PORT2 'thread_work_total{role="traverse"}' 1 +wait_ready $PORT2 'thread_work_pending{role="scan"}' 0 +wait_ready $PORT2 'thread_busy{role="scan"}' 0 # have clients contact the new server export DEBUGINFOD_URLS=http://127.0.0.1:$PORT2 + +if type bsdtar 2>/dev/null; then + # copy in the deb files + cp -rvp ${abs_srcdir}/debuginfod-debs/*deb D + kill -USR1 $PID2 + # All debs need to be in the index + debs=$(find D -name \*.deb | wc -l) + wait_ready $PORT2 'scanned_total{source=".deb archive"}' `expr $debs` + ddebs=$(find D -name \*.ddeb | wc -l) + wait_ready $PORT2 'scanned_total{source=".ddeb archive"}' `expr $ddebs` + + # ubuntu + archive_test f17a29b5a25bd4960531d82aa6b07c8abe84fa66 "" "" +fi + rm -rf $DEBUGINFOD_CACHE_PATH testrun ${abs_top_builddir}/debuginfod/debuginfod-find debuginfo $BUILDID +# send a request to stress XFF and User-Agent federation relay; +# we'll grep for the two patterns in vlog4 +curl -s -H 'User-Agent: TESTCURL' -H 'X-Forwarded-For: TESTXFF' $DEBUGINFOD_URLS/buildid/deaddeadbeef00000000/debuginfo -o /dev/null || true + +grep UA:TESTCURL vlog4 +grep XFF:TESTXFF vlog4 + + # confirm that first server can't resolve symlinked info in L/ but second can BUILDID=`env LD_LIBRARY_PATH=$ldpath ${abs_builddir}/../src/readelf \ -a L/foo | grep 'Build ID' | cut -d ' ' -f 7` @@ -288,14 +423,22 @@ testrun ${abs_builddir}/debuginfod_build_id_find -e F/prog2 1 ######################################################################## -# Fetch some metrics, if curl program is installed -if type curl 2>/dev/null; then - curl http://127.0.0.1:$PORT1/badapi - curl http://127.0.0.1:$PORT1/metrics - curl http://127.0.0.1:$PORT2/metrics - curl http://127.0.0.1:$PORT1/metrics | grep -q 'http_responses_total.*result.*error' - curl http://127.0.0.1:$PORT2/metrics | grep -q 'http_responses_total.*result.*upstream' -fi +# Fetch some metrics +curl -s http://127.0.0.1:$PORT1/badapi +curl -s http://127.0.0.1:$PORT1/metrics +curl -s http://127.0.0.1:$PORT2/metrics +curl -s http://127.0.0.1:$PORT1/metrics | grep -q 'http_responses_total.*result.*error' +curl -s http://127.0.0.1:$PORT1/metrics | grep -q 'http_responses_total.*result.*fdcache' +curl -s http://127.0.0.1:$PORT2/metrics | grep -q 'http_responses_total.*result.*upstream' +curl -s http://127.0.0.1:$PORT1/metrics | grep 'http_responses_duration_milliseconds_count' +curl -s http://127.0.0.1:$PORT1/metrics | grep 'http_responses_duration_milliseconds_sum' +curl -s http://127.0.0.1:$PORT1/metrics | grep 'http_responses_transfer_bytes_count' +curl -s http://127.0.0.1:$PORT1/metrics | grep 'http_responses_transfer_bytes_sum' + +# And generate a few errors into the second debuginfod's logs, for analysis just below +curl -s http://127.0.0.1:$PORT2/badapi > /dev/null || true +curl -s http://127.0.0.1:$PORT2/buildid/deadbeef/debuginfo > /dev/null || true + ######################################################################## @@ -310,8 +453,21 @@ tempfiles .debuginfod_* testrun ${abs_builddir}/debuginfod_build_id_find -e F/prog2 1 +# check out the debuginfod logs for the new style status lines +# cat vlog3 +grep -q 'UA:.*XFF:.*GET /buildid/.* 200 ' vlog3 +grep -q 'UA:.*XFF:.*GET /metrics 200 ' vlog3 +grep -q 'UA:.*XFF:.*GET /badapi 503 ' vlog3 +grep -q 'UA:.*XFF:.*GET /buildid/deadbeef.* 404 ' vlog3 + ######################################################################## +# Add some files to the cache that do not fit its naming format. +# They should survive cache cleaning. +mkdir $DEBUGINFOD_CACHE_PATH/malformed +touch $DEBUGINFOD_CACHE_PATH/malformed0 +touch $DEBUGINFOD_CACHE_PATH/malformed/malformed1 + # Trigger a cache clean and run the tests again. The clients should be unable to # find the target. echo 0 > $DEBUGINFOD_CACHE_PATH/cache_clean_interval_s @@ -321,4 +477,36 @@ testrun ${abs_builddir}/debuginfod_build_id_find -e F/prog 1 testrun ${abs_top_builddir}/debuginfod/debuginfod-find debuginfo $BUILDID2 && false || true +if [ ! -f $DEBUGINFOD_CACHE_PATH/malformed0 ] \ + || [ ! -f $DEBUGINFOD_CACHE_PATH/malformed/malformed1 ]; then + echo "unrelated files did not survive cache cleaning" + exit 1 +fi + +# Test debuginfod without a path list; reuse $PORT1 +env LD_LIBRARY_PATH=$ldpath ${abs_builddir}/../debuginfod/debuginfod $VERBOSE -F -U -d :memory: -p $PORT1 -L -F & +PID3=$! +wait_ready $PORT1 'thread_work_total{role="traverse"}' 1 +wait_ready $PORT1 'thread_work_pending{role="scan"}' 0 +wait_ready $PORT1 'thread_busy{role="scan"}' 0 +kill -int $PID3 +wait $PID3 +PID3=0 + +######################################################################## +# Test fetching a file using file:// . No debuginfod server needs to be run for +# this test. +local_dir=${PWD}/mocktree/buildid/aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd/source/my/path +mkdir -p ${local_dir} +echo "int main() { return 0; }" > ${local_dir}/main.c + +# first test that is doesn't work, when no DEBUGINFOD_URLS is set +DEBUGINFOD_URLS="" +testrun ${abs_top_builddir}/debuginfod/debuginfod-find source aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd /my/path/main.c && false || true + +# Now test is with proper DEBUGINFOD_URLS +DEBUGINFOD_URLS="file://${PWD}/mocktree/" +filename=`testrun ${abs_top_builddir}/debuginfod/debuginfod-find source aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd /my/path/main.c` +cmp $filename ${local_dir}/main.c + exit 0 |