USER = jtomczak HOMEDIR = /home/users/jtomczak JOB_ID = 5282 JOB_NAME = FS6-80 QUEUE = wp10 NSLOTS = 60 NHOSTS = 6 SUB_HOST = rupc04 SUB_DIR = /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta80/nf6/NEW EXE_HOST = n146 NFS_HOST = caip02s EXE_DIR = /mnt/p06 OUT_DIR = /tmp TMP_DIR = /tmp/5282.1.wp10 INFO = Files larger than 100MB will be compressed to reduce the network load. INFO2 = Do NOT remove this file while job is running! PORT = 20282 rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta80/nf6/NEW/imp.0: not a plain file Failure to rcp files! Your job started at Sat Apr 23 13:55:23 EDT 2011 on host n146 in /mnt/p06/5282jtomczak Submission directory is /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta80/nf6/NEW on host rupc04 ----------------------------------------------------------------------------------------- + echo -catch_rsh /opt/sge6/spool/n146/active_jobs/5282.1/pe_hostfile /opt/mpich2/intel/11.1/ -catch_rsh /opt/sge6/spool/n146/active_jobs/5282.1/pe_hostfile /opt/mpich2/intel/11.1/ + SLEEPTIME=5 + RETRIES=10 + catch_rsh=0 + catch_hostname=0 + '[' 3 -gt 0 ']' + case "$1" in + catch_rsh=1 + shift + '[' 2 -gt 0 ']' + case "$1" in + break ++ basename /opt/sge6/mpich2_smpd/startmpich2.sh + me=startmpich2.sh + '[' 2 -lt 2 ']' + pe_hostfile=/opt/sge6/spool/n146/active_jobs/5282.1/pe_hostfile + MPICH2_ROOT=/opt/mpich2/intel/11.1/ + export MPICH2_ROOT + '[' '!' -x /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 ']' + '[' '!' -x /opt/mpich2/intel/11.1//bin/smpd ']' + '[' '!' -r /opt/sge6/spool/n146/active_jobs/5282.1/pe_hostfile ']' + machines=/tmp/5282.1.wp10/machines + PeHostfile2MachineFile /opt/sge6/spool/n146/active_jobs/5282.1/pe_hostfile + cat /opt/sge6/spool/n146/active_jobs/5282.1/pe_hostfile + read line ++ echo n146 4 wp10@n146 UNDEFINED ++ cut -f1 '-d ' ++ cut -f1 -d. + host=n146 ++ echo n146 4 wp10@n146 UNDEFINED ++ cut -f2 '-d ' + nslots=4 + i=1 + '[' 1 -le 4 ']' + echo n146 ++ expr 1 + 1 + i=2 + '[' 2 -le 4 ']' + echo n146 ++ expr 2 + 1 + i=3 + '[' 3 -le 4 ']' + echo n146 ++ expr 3 + 1 + i=4 + '[' 4 -le 4 ']' + echo n146 ++ expr 4 + 1 + i=5 + '[' 5 -le 4 ']' + read line ++ echo n141 12 wp10@n141 UNDEFINED ++ cut -f1 '-d ' ++ cut -f1 -d. + host=n141 ++ echo n141 12 wp10@n141 UNDEFINED ++ cut -f2 '-d ' + nslots=12 + i=1 + '[' 1 -le 12 ']' + echo n141 ++ expr 1 + 1 + i=2 + '[' 2 -le 12 ']' + echo n141 ++ expr 2 + 1 + i=3 + '[' 3 -le 12 ']' + echo n141 ++ expr 3 + 1 + i=4 + '[' 4 -le 12 ']' + echo n141 ++ expr 4 + 1 + i=5 + '[' 5 -le 12 ']' + echo n141 ++ expr 5 + 1 + i=6 + '[' 6 -le 12 ']' + echo n141 ++ expr 6 + 1 + i=7 + '[' 7 -le 12 ']' + echo n141 ++ expr 7 + 1 + i=8 + '[' 8 -le 12 ']' + echo n141 ++ expr 8 + 1 + i=9 + '[' 9 -le 12 ']' + echo n141 ++ expr 9 + 1 + i=10 + '[' 10 -le 12 ']' + echo n141 ++ expr 10 + 1 + i=11 + '[' 11 -le 12 ']' + echo n141 ++ expr 11 + 1 + i=12 + '[' 12 -le 12 ']' + echo n141 ++ expr 12 + 1 + i=13 + '[' 13 -le 12 ']' + read line ++ echo n142 12 wp10@n142 UNDEFINED ++ cut -f1 '-d ' ++ cut -f1 -d. + host=n142 ++ echo n142 12 wp10@n142 UNDEFINED ++ cut -f2 '-d ' + nslots=12 + i=1 + '[' 1 -le 12 ']' + echo n142 ++ expr 1 + 1 + i=2 + '[' 2 -le 12 ']' + echo n142 ++ expr 2 + 1 + i=3 + '[' 3 -le 12 ']' + echo n142 ++ expr 3 + 1 + i=4 + '[' 4 -le 12 ']' + echo n142 ++ expr 4 + 1 + i=5 + '[' 5 -le 12 ']' + echo n142 ++ expr 5 + 1 + i=6 + '[' 6 -le 12 ']' + echo n142 ++ expr 6 + 1 + i=7 + '[' 7 -le 12 ']' + echo n142 ++ expr 7 + 1 + i=8 + '[' 8 -le 12 ']' + echo n142 ++ expr 8 + 1 + i=9 + '[' 9 -le 12 ']' + echo n142 ++ expr 9 + 1 + i=10 + '[' 10 -le 12 ']' + echo n142 ++ expr 10 + 1 + i=11 + '[' 11 -le 12 ']' + echo n142 ++ expr 11 + 1 + i=12 + '[' 12 -le 12 ']' + echo n142 ++ expr 12 + 1 + i=13 + '[' 13 -le 12 ']' + read line ++ echo n143 12 wp10@n143 UNDEFINED ++ cut -f1 '-d ' ++ cut -f1 -d. + host=n143 ++ echo n143 12 wp10@n143 UNDEFINED ++ cut -f2 '-d ' + nslots=12 + i=1 + '[' 1 -le 12 ']' + echo n143 ++ expr 1 + 1 + i=2 + '[' 2 -le 12 ']' + echo n143 ++ expr 2 + 1 + i=3 + '[' 3 -le 12 ']' + echo n143 ++ expr 3 + 1 + i=4 + '[' 4 -le 12 ']' + echo n143 ++ expr 4 + 1 + i=5 + '[' 5 -le 12 ']' + echo n143 ++ expr 5 + 1 + i=6 + '[' 6 -le 12 ']' + echo n143 ++ expr 6 + 1 + i=7 + '[' 7 -le 12 ']' + echo n143 ++ expr 7 + 1 + i=8 + '[' 8 -le 12 ']' + echo n143 ++ expr 8 + 1 + i=9 + '[' 9 -le 12 ']' + echo n143 ++ expr 9 + 1 + i=10 + '[' 10 -le 12 ']' + echo n143 ++ expr 10 + 1 + i=11 + '[' 11 -le 12 ']' + echo n143 ++ expr 11 + 1 + i=12 + '[' 12 -le 12 ']' + echo n143 ++ expr 12 + 1 + i=13 + '[' 13 -le 12 ']' + read line ++ echo n144 12 wp10@n144 UNDEFINED ++ cut -f1 -d. ++ cut -f1 '-d ' + host=n144 ++ echo n144 12 wp10@n144 UNDEFINED ++ cut -f2 '-d ' + nslots=12 + i=1 + '[' 1 -le 12 ']' + echo n144 ++ expr 1 + 1 + i=2 + '[' 2 -le 12 ']' + echo n144 ++ expr 2 + 1 + i=3 + '[' 3 -le 12 ']' + echo n144 ++ expr 3 + 1 + i=4 + '[' 4 -le 12 ']' + echo n144 ++ expr 4 + 1 + i=5 + '[' 5 -le 12 ']' + echo n144 ++ expr 5 + 1 + i=6 + '[' 6 -le 12 ']' + echo n144 ++ expr 6 + 1 + i=7 + '[' 7 -le 12 ']' + echo n144 ++ expr 7 + 1 + i=8 + '[' 8 -le 12 ']' + echo n144 ++ expr 8 + 1 + i=9 + '[' 9 -le 12 ']' + echo n144 ++ expr 9 + 1 + i=10 + '[' 10 -le 12 ']' + echo n144 ++ expr 10 + 1 + i=11 + '[' 11 -le 12 ']' + echo n144 ++ expr 11 + 1 + i=12 + '[' 12 -le 12 ']' + echo n144 ++ expr 12 + 1 + i=13 + '[' 13 -le 12 ']' + read line ++ echo n145 8 wp10@n145 UNDEFINED ++ cut -f1 '-d ' ++ cut -f1 -d. + host=n145 ++ echo n145 8 wp10@n145 UNDEFINED ++ cut -f2 '-d ' + nslots=8 + i=1 + '[' 1 -le 8 ']' + echo n145 ++ expr 1 + 1 + i=2 + '[' 2 -le 8 ']' + echo n145 ++ expr 2 + 1 + i=3 + '[' 3 -le 8 ']' + echo n145 ++ expr 3 + 1 + i=4 + '[' 4 -le 8 ']' + echo n145 ++ expr 4 + 1 + i=5 + '[' 5 -le 8 ']' + echo n145 ++ expr 5 + 1 + i=6 + '[' 6 -le 8 ']' + echo n145 ++ expr 6 + 1 + i=7 + '[' 7 -le 8 ']' + echo n145 ++ expr 7 + 1 + i=8 + '[' 8 -le 8 ']' + echo n145 ++ expr 8 + 1 + i=9 + '[' 9 -le 8 ']' + read line + cat /tmp/5282.1.wp10/machines n146 n146 n146 n146 n141 n141 n141 n141 n141 n141 n141 n141 n141 n141 n141 n141 n142 n142 n142 n142 n142 n142 n142 n142 n142 n142 n142 n142 n143 n143 n143 n143 n143 n143 n143 n143 n143 n143 n143 n143 n144 n144 n144 n144 n144 n144 n144 n144 n144 n144 n144 n144 n145 n145 n145 n145 n145 n145 n145 n145 + '[' 1 = 1 ']' + rsh_wrapper=/opt/sge6/mpich2_smpd/rsh + '[' '!' -x /opt/sge6/mpich2_smpd/rsh ']' + rshcmd=rsh + case "$ARC" in + ln -s /opt/sge6/mpich2_smpd/rsh /tmp/5282.1.wp10/rsh + hash -r + '[' 0 = 1 ']' + port=20282 ++ cat /tmp/5282.1.wp10/machines ++ uniq + for host in '`cat $machines | uniq`' + /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n146 /opt/mpich2/intel/11.1//bin/smpd 20282 + for host in '`cat $machines | uniq`' + /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n141 /opt/mpich2/intel/11.1//bin/smpd 20282 + for host in '`cat $machines | uniq`' + /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n142 /opt/mpich2/intel/11.1//bin/smpd 20282 + for host in '`cat $machines | uniq`' + /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n143 /opt/mpich2/intel/11.1//bin/smpd 20282 + for host in '`cat $machines | uniq`' + /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n144 /opt/mpich2/intel/11.1//bin/smpd 20282 + for host in '`cat $machines | uniq`' + /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n145 /opt/mpich2/intel/11.1//bin/smpd 20282 + actual_retry=1 + '[' 1 -le 10 ']' + echo 'startmpich2.sh: check for smpd daemons (1 of 10)' startmpich2.sh: check for smpd daemons (1 of 10) + numhosts=0 rsh n143 /opt/mpich2/intel/11.1//bin/smpd -port 20282 -d 0 ++ cat /tmp/5282.1.wp10/machines ++ uniq + for host in '`cat $machines | uniq`' ++ head -n 1 ++ cut -f 1 -d ' ' rsh n142 /opt/mpich2/intel/11.1//bin/smpd -port 20282 -d 0 rsh n144 /opt/mpich2/intel/11.1//bin/smpd -port 20282 -d 0 ++ /opt/mpich2/intel/11.1//bin/smpd -port 20282 -status n146 Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1 internal ABORT - process 0 + '[' '' == smpd ']' + echo 'startmpich2.sh: missing smpd on n146' startmpich2.sh: missing smpd on n146 + for host in '`cat $machines | uniq`' rsh n146 /opt/mpich2/intel/11.1//bin/smpd -port 20282 -d 0 rsh n141 /opt/mpich2/intel/11.1//bin/smpd -port 20282 -d 0 ++ head -n 1 ++ /opt/mpich2/intel/11.1//bin/smpd -port 20282 -status n141 ++ cut -f 1 -d ' ' rsh n145 /opt/mpich2/intel/11.1//bin/smpd -port 20282 -d 0 Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1 internal ABORT - process 0 + '[' '' == smpd ']' + echo 'startmpich2.sh: missing smpd on n141' startmpich2.sh: missing smpd on n141 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 20282 -status n142 Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1 internal ABORT - process 0 ++ head -n 1 ++ cut -f 1 -d ' ' + '[' '' == smpd ']' + echo 'startmpich2.sh: missing smpd on n142' startmpich2.sh: missing smpd on n142 + for host in '`cat $machines | uniq`' ++ head -n 1 ++ /opt/mpich2/intel/11.1//bin/smpd -port 20282 -status n143 ++ cut -f 1 -d ' ' Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1 internal ABORT - process 0 + '[' '' == smpd ']' + echo 'startmpich2.sh: missing smpd on n143' startmpich2.sh: missing smpd on n143 + for host in '`cat $machines | uniq`' ++ head -n 1 ++ /opt/mpich2/intel/11.1//bin/smpd -port 20282 -status n144 Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1 internal ABORT - process 0 ++ cut -f 1 -d ' ' + '[' '' == smpd ']' + echo 'startmpich2.sh: missing smpd on n144' startmpich2.sh: missing smpd on n144 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 20282 -status n145 ++ head -n 1 ++ cut -f 1 -d ' ' + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n145' startmpich2.sh: found running smpd on n145 + '[' 6 -eq 1 ']' + sleep 5 + (( actual_retry += 1 )) + '[' 2 -le 10 ']' + echo 'startmpich2.sh: check for smpd daemons (2 of 10)' startmpich2.sh: check for smpd daemons (2 of 10) + numhosts=0 ++ cat /tmp/5282.1.wp10/machines ++ uniq + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 20282 -status n146 ++ cut -f 1 -d ' ' ++ head -n 1 + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n146' startmpich2.sh: found running smpd on n146 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 20282 -status n141 ++ head -n 1 ++ cut -f 1 -d ' ' + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n141' startmpich2.sh: found running smpd on n141 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 20282 -status n142 ++ head -n 1 ++ cut -f 1 -d ' ' + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n142' startmpich2.sh: found running smpd on n142 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 20282 -status n143 ++ head -n 1 ++ cut -f 1 -d ' ' + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n143' startmpich2.sh: found running smpd on n143 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 20282 -status n144 ++ head -n 1 ++ cut -f 1 -d ' ' + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n144' startmpich2.sh: found running smpd on n144 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 20282 -status n145 ++ head -n 1 ++ cut -f 1 -d ' ' + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n145' startmpich2.sh: found running smpd on n145 + '[' 6 -eq 6 ']' + echo 'startmpich2.sh: got all 6 of 6 nodes' startmpich2.sh: got all 6 of 6 nodes + exit 0 START END Sat Apr 23 16:14:11 EDT 2011 -catch_rsh /opt/mpich2/intel/11.1/ Your job finished at Sat Apr 23 16:14:16 EDT 2011