USER = jtomczak HOMEDIR = /home/users/jtomczak JOB_ID = 15984 JOB_NAME = FS-LOW QUEUE = wp12 NSLOTS = 78 NHOSTS = 8 SUB_HOST = rupc04 SUB_DIR = /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta200/nf6/NEW EXE_HOST = n182 EXE_DIR = /mnt/n182 OUT_DIR = /scratch TMP_DIR = /tmp/15984.1.wp12 INFO = Files larger than 100MB will be compressed to reduce the network load. INFO2 = Do NOT remove this file while job is running! PORT = 20984 rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta200/nf6/NEW/FIX_MU: not a plain file rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta200/nf6/NEW/imp.0: not a plain file Failure to rcp files! Your job started at Wed Oct 19 12:58:25 EDT 2011 on host n182 in /mnt/n182/15984jtomczak Submission directory is /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta200/nf6/NEW on host rupc04 ----------------------------------------------------------------------------------------- + echo -catch_rsh /opt/sge6/spool/n182/active_jobs/15984.1/pe_hostfile /opt/mpich2/intel/11.1/ -catch_rsh /opt/sge6/spool/n182/active_jobs/15984.1/pe_hostfile /opt/mpich2/intel/11.1/ + SLEEPTIME=5 + RETRIES=10 + catch_rsh=0 + catch_hostname=0 + '[' 3 -gt 0 ']' + case "$1" in + catch_rsh=1 + shift + '[' 2 -gt 0 ']' + case "$1" in + break ++ basename /opt/sge6/mpich2_smpd/startmpich2.sh + me=startmpich2.sh + '[' 2 -lt 2 ']' + pe_hostfile=/opt/sge6/spool/n182/active_jobs/15984.1/pe_hostfile + MPICH2_ROOT=/opt/mpich2/intel/11.1/ + export MPICH2_ROOT + '[' '!' -x /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 ']' + '[' '!' -x /opt/mpich2/intel/11.1//bin/smpd ']' + '[' '!' -r /opt/sge6/spool/n182/active_jobs/15984.1/pe_hostfile ']' + machines=/tmp/15984.1.wp12/machines + PeHostfile2MachineFile /opt/sge6/spool/n182/active_jobs/15984.1/pe_hostfile + cat /opt/sge6/spool/n182/active_jobs/15984.1/pe_hostfile + read line ++ echo n182 8 wp12@n182 UNDEFINED ++ cut -f1 '-d ' ++ cut -f1 -d. + host=n182 ++ echo n182 8 wp12@n182 UNDEFINED ++ cut -f2 '-d ' + nslots=8 + i=1 + '[' 1 -le 8 ']' + echo n182 ++ expr 1 + 1 + i=2 + '[' 2 -le 8 ']' + echo n182 ++ expr 2 + 1 + i=3 + '[' 3 -le 8 ']' + echo n182 ++ expr 3 + 1 + i=4 + '[' 4 -le 8 ']' + echo n182 ++ expr 4 + 1 + i=5 + '[' 5 -le 8 ']' + echo n182 ++ expr 5 + 1 + i=6 + '[' 6 -le 8 ']' + echo n182 ++ expr 6 + 1 + i=7 + '[' 7 -le 8 ']' + echo n182 ++ expr 7 + 1 + i=8 + '[' 8 -le 8 ']' + echo n182 ++ expr 8 + 1 + i=9 + '[' 9 -le 8 ']' + read line ++ echo n172 12 wp12@n172 UNDEFINED ++ cut -f1 '-d ' ++ cut -f1 -d. + host=n172 ++ echo n172 12 wp12@n172 UNDEFINED ++ cut -f2 '-d ' + nslots=12 + i=1 + '[' 1 -le 12 ']' + echo n172 ++ expr 1 + 1 + i=2 + '[' 2 -le 12 ']' + echo n172 ++ expr 2 + 1 + i=3 + '[' 3 -le 12 ']' + echo n172 ++ expr 3 + 1 + i=4 + '[' 4 -le 12 ']' + echo n172 ++ expr 4 + 1 + i=5 + '[' 5 -le 12 ']' + echo n172 ++ expr 5 + 1 + i=6 + '[' 6 -le 12 ']' + echo n172 ++ expr 6 + 1 + i=7 + '[' 7 -le 12 ']' + echo n172 ++ expr 7 + 1 + i=8 + '[' 8 -le 12 ']' + echo n172 ++ expr 8 + 1 + i=9 + '[' 9 -le 12 ']' + echo n172 ++ expr 9 + 1 + i=10 + '[' 10 -le 12 ']' + echo n172 ++ expr 10 + 1 + i=11 + '[' 11 -le 12 ']' + echo n172 ++ expr 11 + 1 + i=12 + '[' 12 -le 12 ']' + echo n172 ++ expr 12 + 1 + i=13 + '[' 13 -le 12 ']' + read line ++ echo n173 4 wp12@n173 UNDEFINED ++ cut -f1 '-d ' ++ cut -f1 -d. + host=n173 ++ echo n173 4 wp12@n173 UNDEFINED ++ cut -f2 '-d ' + nslots=4 + i=1 + '[' 1 -le 4 ']' + echo n173 ++ expr 1 + 1 + i=2 + '[' 2 -le 4 ']' + echo n173 ++ expr 2 + 1 + i=3 + '[' 3 -le 4 ']' + echo n173 ++ expr 3 + 1 + i=4 + '[' 4 -le 4 ']' + echo n173 ++ expr 4 + 1 + i=5 + '[' 5 -le 4 ']' + read line ++ echo n183 12 wp12@n183 UNDEFINED ++ cut -f1 '-d ' ++ cut -f1 -d. + host=n183 ++ echo n183 12 wp12@n183 UNDEFINED ++ cut -f2 '-d ' + nslots=12 + i=1 + '[' 1 -le 12 ']' + echo n183 ++ expr 1 + 1 + i=2 + '[' 2 -le 12 ']' + echo n183 ++ expr 2 + 1 + i=3 + '[' 3 -le 12 ']' + echo n183 ++ expr 3 + 1 + i=4 + '[' 4 -le 12 ']' + echo n183 ++ expr 4 + 1 + i=5 + '[' 5 -le 12 ']' + echo n183 ++ expr 5 + 1 + i=6 + '[' 6 -le 12 ']' + echo n183 ++ expr 6 + 1 + i=7 + '[' 7 -le 12 ']' + echo n183 ++ expr 7 + 1 + i=8 + '[' 8 -le 12 ']' + echo n183 ++ expr 8 + 1 + i=9 + '[' 9 -le 12 ']' + echo n183 ++ expr 9 + 1 + i=10 + '[' 10 -le 12 ']' + echo n183 ++ expr 10 + 1 + i=11 + '[' 11 -le 12 ']' + echo n183 ++ expr 11 + 1 + i=12 + '[' 12 -le 12 ']' + echo n183 ++ expr 12 + 1 + i=13 + '[' 13 -le 12 ']' + read line ++ echo n179 12 wp12@n179 UNDEFINED ++ cut -f1 '-d ' ++ cut -f1 -d. + host=n179 ++ echo n179 12 wp12@n179 UNDEFINED ++ cut -f2 '-d ' + nslots=12 + i=1 + '[' 1 -le 12 ']' + echo n179 ++ expr 1 + 1 + i=2 + '[' 2 -le 12 ']' + echo n179 ++ expr 2 + 1 + i=3 + '[' 3 -le 12 ']' + echo n179 ++ expr 3 + 1 + i=4 + '[' 4 -le 12 ']' + echo n179 ++ expr 4 + 1 + i=5 + '[' 5 -le 12 ']' + echo n179 ++ expr 5 + 1 + i=6 + '[' 6 -le 12 ']' + echo n179 ++ expr 6 + 1 + i=7 + '[' 7 -le 12 ']' + echo n179 ++ expr 7 + 1 + i=8 + '[' 8 -le 12 ']' + echo n179 ++ expr 8 + 1 + i=9 + '[' 9 -le 12 ']' + echo n179 ++ expr 9 + 1 + i=10 + '[' 10 -le 12 ']' + echo n179 ++ expr 10 + 1 + i=11 + '[' 11 -le 12 ']' + echo n179 ++ expr 11 + 1 + i=12 + '[' 12 -le 12 ']' + echo n179 ++ expr 12 + 1 + i=13 + '[' 13 -le 12 ']' + read line ++ echo n180 12 wp12@n180 UNDEFINED ++ cut -f1 '-d ' ++ cut -f1 -d. + host=n180 ++ echo n180 12 wp12@n180 UNDEFINED ++ cut -f2 '-d ' + nslots=12 + i=1 + '[' 1 -le 12 ']' + echo n180 ++ expr 1 + 1 + i=2 + '[' 2 -le 12 ']' + echo n180 ++ expr 2 + 1 + i=3 + '[' 3 -le 12 ']' + echo n180 ++ expr 3 + 1 + i=4 + '[' 4 -le 12 ']' + echo n180 ++ expr 4 + 1 + i=5 + '[' 5 -le 12 ']' + echo n180 ++ expr 5 + 1 + i=6 + '[' 6 -le 12 ']' + echo n180 ++ expr 6 + 1 + i=7 + '[' 7 -le 12 ']' + echo n180 ++ expr 7 + 1 + i=8 + '[' 8 -le 12 ']' + echo n180 ++ expr 8 + 1 + i=9 + '[' 9 -le 12 ']' + echo n180 ++ expr 9 + 1 + i=10 + '[' 10 -le 12 ']' + echo n180 ++ expr 10 + 1 + i=11 + '[' 11 -le 12 ']' + echo n180 ++ expr 11 + 1 + i=12 + '[' 12 -le 12 ']' + echo n180 ++ expr 12 + 1 + i=13 + '[' 13 -le 12 ']' + read line ++ echo n176 6 wp12@n176 UNDEFINED ++ cut -f1 '-d ' ++ cut -f1 -d. + host=n176 ++ echo n176 6 wp12@n176 UNDEFINED ++ cut -f2 '-d ' + nslots=6 + i=1 + '[' 1 -le 6 ']' + echo n176 ++ expr 1 + 1 + i=2 + '[' 2 -le 6 ']' + echo n176 ++ expr 2 + 1 + i=3 + '[' 3 -le 6 ']' + echo n176 ++ expr 3 + 1 + i=4 + '[' 4 -le 6 ']' + echo n176 ++ expr 4 + 1 + i=5 + '[' 5 -le 6 ']' + echo n176 ++ expr 5 + 1 + i=6 + '[' 6 -le 6 ']' + echo n176 ++ expr 6 + 1 + i=7 + '[' 7 -le 6 ']' + read line ++ echo n174 12 wp12@n174 UNDEFINED ++ cut -f1 '-d ' ++ cut -f1 -d. + host=n174 ++ echo n174 12 wp12@n174 UNDEFINED ++ cut -f2 '-d ' + nslots=12 + i=1 + '[' 1 -le 12 ']' + echo n174 ++ expr 1 + 1 + i=2 + '[' 2 -le 12 ']' + echo n174 ++ expr 2 + 1 + i=3 + '[' 3 -le 12 ']' + echo n174 ++ expr 3 + 1 + i=4 + '[' 4 -le 12 ']' + echo n174 ++ expr 4 + 1 + i=5 + '[' 5 -le 12 ']' + echo n174 ++ expr 5 + 1 + i=6 + '[' 6 -le 12 ']' + echo n174 ++ expr 6 + 1 + i=7 + '[' 7 -le 12 ']' + echo n174 ++ expr 7 + 1 + i=8 + '[' 8 -le 12 ']' + echo n174 ++ expr 8 + 1 + i=9 + '[' 9 -le 12 ']' + echo n174 ++ expr 9 + 1 + i=10 + '[' 10 -le 12 ']' + echo n174 ++ expr 10 + 1 + i=11 + '[' 11 -le 12 ']' + echo n174 ++ expr 11 + 1 + i=12 + '[' 12 -le 12 ']' + echo n174 ++ expr 12 + 1 + i=13 + '[' 13 -le 12 ']' + read line + cat /tmp/15984.1.wp12/machines n182 n182 n182 n182 n182 n182 n182 n182 n172 n172 n172 n172 n172 n172 n172 n172 n172 n172 n172 n172 n173 n173 n173 n173 n183 n183 n183 n183 n183 n183 n183 n183 n183 n183 n183 n183 n179 n179 n179 n179 n179 n179 n179 n179 n179 n179 n179 n179 n180 n180 n180 n180 n180 n180 n180 n180 n180 n180 n180 n180 n176 n176 n176 n176 n176 n176 n174 n174 n174 n174 n174 n174 n174 n174 n174 n174 n174 n174 + '[' 1 = 1 ']' + rsh_wrapper=/opt/sge6/mpich2_smpd/rsh + '[' '!' -x /opt/sge6/mpich2_smpd/rsh ']' + rshcmd=rsh + case "$ARC" in + ln -s /opt/sge6/mpich2_smpd/rsh /tmp/15984.1.wp12/rsh + hash -r + '[' 0 = 1 ']' + port=20984 ++ cat /tmp/15984.1.wp12/machines ++ uniq + for host in '`cat $machines | uniq`' + /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n182 /opt/mpich2/intel/11.1//bin/smpd 20984 + for host in '`cat $machines | uniq`' + /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n172 /opt/mpich2/intel/11.1//bin/smpd 20984 + for host in '`cat $machines | uniq`' + /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n173 /opt/mpich2/intel/11.1//bin/smpd 20984 + for host in '`cat $machines | uniq`' + /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n183 /opt/mpich2/intel/11.1//bin/smpd 20984 + for host in '`cat $machines | uniq`' + /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n179 /opt/mpich2/intel/11.1//bin/smpd 20984 + for host in '`cat $machines | uniq`' + /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n180 /opt/mpich2/intel/11.1//bin/smpd 20984 + for host in '`cat $machines | uniq`' + /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n176 /opt/mpich2/intel/11.1//bin/smpd 20984 + for host in '`cat $machines | uniq`' + /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n174 /opt/mpich2/intel/11.1//bin/smpd 20984 + actual_retry=1 + '[' 1 -le 10 ']' + echo 'startmpich2.sh: check for smpd daemons (1 of 10)' startmpich2.sh: check for smpd daemons (1 of 10) + numhosts=0 ++ cat /tmp/15984.1.wp12/machines ++ uniq + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 20984 -status n182 ++ head -n 1 ++ cut -f 1 -d ' ' Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1 internal ABORT - process 0 + '[' '' == smpd ']' + echo 'startmpich2.sh: missing smpd on n182' startmpich2.sh: missing smpd on n182 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 20984 -status n172 ++ head -n 1 ++ cut -f 1 -d ' ' rsh n183 /opt/mpich2/intel/11.1//bin/smpd -port 20984 -d 0 rsh n180 /opt/mpich2/intel/11.1//bin/smpd -port 20984 -d 0 rsh n173 /opt/mpich2/intel/11.1//bin/smpd -port 20984 -d 0 rsh n179 /opt/mpich2/intel/11.1//bin/smpd -port 20984 -d 0 rsh n176 /opt/mpich2/intel/11.1//bin/smpd -port 20984 -d 0 rsh n182 /opt/mpich2/intel/11.1//bin/smpd -port 20984 -d 0 rsh n174 /opt/mpich2/intel/11.1//bin/smpd -port 20984 -d 0 rsh n172 /opt/mpich2/intel/11.1//bin/smpd -port 20984 -d 0 Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1 internal ABORT - process 0 + '[' '' == smpd ']' + echo 'startmpich2.sh: missing smpd on n172' startmpich2.sh: missing smpd on n172 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 20984 -status n173 ++ head -n 1 ++ cut -f 1 -d ' ' Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1 internal ABORT - process 0 + '[' '' == smpd ']' + echo 'startmpich2.sh: missing smpd on n173' startmpich2.sh: missing smpd on n173 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 20984 -status n183 ++ head -n 1 ++ cut -f 1 -d ' ' Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1 internal ABORT - process 0 + '[' '' == smpd ']' + echo 'startmpich2.sh: missing smpd on n183' startmpich2.sh: missing smpd on n183 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 20984 -status n179 ++ head -n 1 ++ cut -f 1 -d ' ' Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1 internal ABORT - process 0 + '[' '' == smpd ']' + echo 'startmpich2.sh: missing smpd on n179' startmpich2.sh: missing smpd on n179 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 20984 -status n180 ++ head -n 1 ++ cut -f 1 -d ' ' Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1 internal ABORT - process 0 + '[' '' == smpd ']' + echo 'startmpich2.sh: missing smpd on n180' startmpich2.sh: missing smpd on n180 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 20984 -status n176 ++ head -n 1 ++ cut -f 1 -d ' ' Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1 internal ABORT - process 0 + '[' '' == smpd ']' + echo 'startmpich2.sh: missing smpd on n176' startmpich2.sh: missing smpd on n176 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 20984 -status n174 ++ head -n 1 ++ cut -f 1 -d ' ' Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1 internal ABORT - process 0 + '[' '' == smpd ']' + echo 'startmpich2.sh: missing smpd on n174' startmpich2.sh: missing smpd on n174 + '[' 8 -eq 0 ']' + sleep 5 + (( actual_retry += 1 )) + '[' 2 -le 10 ']' + echo 'startmpich2.sh: check for smpd daemons (2 of 10)' startmpich2.sh: check for smpd daemons (2 of 10) + numhosts=0 ++ cat /tmp/15984.1.wp12/machines ++ uniq + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 20984 -status n182 ++ head -n 1 ++ cut -f 1 -d ' ' + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n182' startmpich2.sh: found running smpd on n182 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 20984 -status n172 ++ head -n 1 ++ cut -f 1 -d ' ' + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n172' startmpich2.sh: found running smpd on n172 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 20984 -status n173 ++ head -n 1 ++ cut -f 1 -d ' ' + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n173' startmpich2.sh: found running smpd on n173 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 20984 -status n183 ++ head -n 1 ++ cut -f 1 -d ' ' + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n183' startmpich2.sh: found running smpd on n183 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 20984 -status n179 ++ head -n 1 ++ cut -f 1 -d ' ' + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n179' startmpich2.sh: found running smpd on n179 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 20984 -status n180 ++ head -n 1 ++ cut -f 1 -d ' ' + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n180' startmpich2.sh: found running smpd on n180 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 20984 -status n176 ++ head -n 1 ++ cut -f 1 -d ' ' + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n176' startmpich2.sh: found running smpd on n176 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 20984 -status n174 ++ head -n 1 ++ cut -f 1 -d ' ' + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n174' startmpich2.sh: found running smpd on n174 + '[' 8 -eq 8 ']' + echo 'startmpich2.sh: got all 8 of 8 nodes' startmpich2.sh: got all 8 of 8 nodes + exit 0 START END Sun Oct 23 13:04:44 EDT 2011 -catch_rsh /opt/mpich2/intel/11.1/ Your job finished at Sun Oct 23 13:04:49 EDT 2011