source: ThirdParty/JobMarket/tests/regression/testsuite-resubmitjobs.at@ be848d

### fragment the molecule and check the number of configs

AT_SETUP([Fragmentation - Automation: Resubmitting failed jobs])
AT_KEYWORDS([fragmentation automation server worker jobadder])

# check that ports are unique over all tests such that they may run in parallel
WORKERPORT=11070
CONTROLLERPORT=11071
WORKERLISTENPORT=11072

# start service in background
${AUTOTEST_PATH}/JobMarketServer --workerport $WORKERPORT --controllerport $CONTROLLERPORT &
server_pid=$!
AT_CHECK([sleep 1], 0, [ignore], [ignore], [kill $server_pid])

# add one always failing job via JobAdder
AT_CHECK([${AUTOTEST_PATH}/JobMarketController --server 127.0.0.1:$CONTROLLERPORT --command createjobs --executable "/bin/false" --jobcommand "Nothing"], 0, [stdout], [ignore], [kill $server_pid])
AT_CHECK([fgrep "Sending 1 jobs ..." stdout], 0, [ignore], [ignore], [kill $server_pid])

# enlist a worker to work on jobs
${AUTOTEST_PATH}/JobMarketPoolWorker -v 5 --signal 2 --server 127.0.0.1:${WORKERPORT} --hostname 127.0.0.1 --listen $WORKERLISTENPORT 2>workererr &
worker_pid=$!
AT_CHECK([sleep 2], 0, [ignore], [ignore], [kill $server_pid $worker_pid])

# send kill signal to worker such that it shuts down
AT_CHECK([kill -INT $worker_pid], 0, [ignore], [ignore], [kill $server_pid $worker_pid])

# check that it truely failed for four times
AT_CHECK([fgrep -c "Job 1 failed on executing" workererr], 0, [stdout], [ignore], [kill $server_pid])
AT_CHECK([grep "4" stdout], 0, [ignore], [ignore])

# shutdown
AT_CHECK([${AUTOTEST_PATH}/JobMarketController --server 127.0.0.1:$CONTROLLERPORT --command shutdown], 0, [ignore], [ignore], [kill $server_pid])

# check that server is truely down
AT_CHECK([sleep 1], 0, [ignore], [ignore], [kill $server_pid])
AT_CHECK([kill $server_pid], 1, [ignore], [ignore])

AT_CHECK([rm -f workererr], 0, [ignore], [ignore])

AT_CLEANUP