source: ThirdParty/JobMarket/tests/regression/testsuite-checkalive.at@ be848d

### fragment the molecule and check the number of configs

AT_SETUP([Fragmentation - Automation: Dead Worker is removed])
AT_KEYWORDS([fragmentation automation controller poolworker server checkalive])

WORKERPORT=11075
CONTROLLERPORT=11076
WORKERLISTENPORT=11077

TIMEOUT=3

# start service in background
${AUTOTEST_PATH}/JobMarketServer -v 5 --workerport $WORKERPORT --controllerport $CONTROLLERPORT --timeout $TIMEOUT &
server_pid=$!
AT_CHECK([sleep 1], 0, [ignore], [ignore], [kill $server_pid])

# let worker enroll in pool
${AUTOTEST_PATH}/JobMarketPoolWorker -v 5 --signal 2 --server 127.0.0.1:${WORKERPORT} --hostname 127.0.0.1 --listen $WORKERLISTENPORT &
worker_pid=$!
AT_CHECK([sleep 1], 0, [ignore], [ignore], [kill $server_pid $worker_pid])

# create script to call
AT_DATA([sleep.sh], [#!/bin/bash
sleep `cat <$1`
])
AT_CHECK([chmod +x sleep.sh], 0, [ignore], [ignore], [kill $server_pid $worker_pid])
# add one jobs via JobAdder that needs at least TIMEOUT time
AT_CHECK([${AUTOTEST_PATH}/JobMarketController --server 127.0.0.1:$CONTROLLERPORT --command createjobs --executable "./sleep.sh" --jobcommand "3"], 0, [stdout], [ignore], [kill $server_pid])
AT_CHECK([fgrep "Sending 1 jobs ..." stdout], 0, [ignore], [ignore], [kill $server_pid])
AT_CHECK([sleep 1], 0, [ignore], [ignore], [kill $server_pid $worker_pid])

# send kill signal to worker such that it shuts down _not_ properly
AT_CHECK([kill -9 $worker_pid], 0, [stdout], [ignore], [kill $server_pid $worker_pid])

# let more than timeout time pass
let timetopass=2*$TIMEOUT
AT_CHECK([sleep $timetopass], 0, [ignore], [ignore], [kill $server_pid $worker_pid])

# clear resubmit job from queue
AT_CHECK([${AUTOTEST_PATH}/JobMarketController --server 127.0.0.1:$CONTROLLERPORT --command removealljobs], 0, [stdout], [ignore], [kill $server_pid])
AT_CHECK([sleep 1], 0, [ignore], [ignore], [kill $server_pid])

# shutdown server, should be without glitch as no worker is left
AT_CHECK([${AUTOTEST_PATH}/JobMarketController --server 127.0.0.1:$CONTROLLERPORT --command shutdown], 0, [ignore], [ignore], [kill $server_pid $worker_pid])

# check that Server is truely down
AT_CHECK([sleep 1], 0, [ignore], [ignore], [kill $server_pid $worker_pid])
AT_CHECK([kill $server_pid $worker_pid], 1, [ignore], [ignore])

AT_CLEANUP