USER = jtomczak HOMEDIR = /home/users/jtomczak JOB_ID = 598 JOB_NAME = FS-K-100-6 QUEUE = ibm NSLOTS = 64 NHOSTS = 4 SUB_HOST = rupc06 SUB_DIR = /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta100/nf6/NEW/GO/KERNEL EXE_HOST = n207 NFS_HOST = rupc-caip-04 EXE_DIR = /mnt/p06 OUT_DIR = /tmp TMP_DIR = /tmp/598.1.ibm INFO = Files larger than 100MB will be compressed to reduce the network load. INFO2 = Do NOT remove this file while job is running! PORT = 20598 rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta100/nf6/NEW/GO/KERNEL/595jtomczak: not a plain file rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta100/nf6/NEW/GO/KERNEL/596jtomczak: not a plain file rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta100/nf6/NEW/GO/KERNEL/597jtomczak: not a plain file Failure to rcp files! Your job started at Sun May 22 20:42:06 EDT 2011 on host n207 in /mnt/p06/598jtomczak Submission directory is /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta100/nf6/NEW/GO/KERNEL on host rupc06 ----------------------------------------------------------------------------------------- + echo -catch_rsh /opt/sge6/spool/n207/active_jobs/598.1/pe_hostfile /opt/mpich2/gnu/ -catch_rsh /opt/sge6/spool/n207/active_jobs/598.1/pe_hostfile /opt/mpich2/gnu/ + SLEEPTIME=5 + RETRIES=10 + catch_rsh=0 + catch_hostname=0 + '[' 3 -gt 0 ']' + case "$1" in + catch_rsh=1 + shift + '[' 2 -gt 0 ']' + case "$1" in + break ++ basename /opt/sge6/mpich2_smpd/startmpich2.sh + me=startmpich2.sh + '[' 2 -lt 2 ']' + pe_hostfile=/opt/sge6/spool/n207/active_jobs/598.1/pe_hostfile + MPICH2_ROOT=/opt/mpich2/gnu/ + export MPICH2_ROOT + '[' '!' -x /opt/sge6/mpich2_smpd/bin/lx26-ppc64/start_mpich2 ']' + '[' '!' -x /opt/mpich2/gnu//bin/smpd ']' + '[' '!' -r /opt/sge6/spool/n207/active_jobs/598.1/pe_hostfile ']' + machines=/tmp/598.1.ibm/machines + PeHostfile2MachineFile /opt/sge6/spool/n207/active_jobs/598.1/pe_hostfile + cat /opt/sge6/spool/n207/active_jobs/598.1/pe_hostfile + read line ++ echo n207 16 ibm@n207 UNDEFINED ++ cut -f1 '-d ' ++ cut -f1 -d. + host=n207 ++ echo n207 16 ibm@n207 UNDEFINED ++ cut -f2 '-d ' + nslots=16 + i=1 + '[' 1 -le 16 ']' + echo n207 ++ expr 1 + 1 + i=2 + '[' 2 -le 16 ']' + echo n207 ++ expr 2 + 1 + i=3 + '[' 3 -le 16 ']' + echo n207 ++ expr 3 + 1 + i=4 + '[' 4 -le 16 ']' + echo n207 ++ expr 4 + 1 + i=5 + '[' 5 -le 16 ']' + echo n207 ++ expr 5 + 1 + i=6 + '[' 6 -le 16 ']' + echo n207 ++ expr 6 + 1 + i=7 + '[' 7 -le 16 ']' + echo n207 ++ expr 7 + 1 + i=8 + '[' 8 -le 16 ']' + echo n207 ++ expr 8 + 1 + i=9 + '[' 9 -le 16 ']' + echo n207 ++ expr 9 + 1 + i=10 + '[' 10 -le 16 ']' + echo n207 ++ expr 10 + 1 + i=11 + '[' 11 -le 16 ']' + echo n207 ++ expr 11 + 1 + i=12 + '[' 12 -le 16 ']' + echo n207 ++ expr 12 + 1 + i=13 + '[' 13 -le 16 ']' + echo n207 ++ expr 13 + 1 + i=14 + '[' 14 -le 16 ']' + echo n207 ++ expr 14 + 1 + i=15 + '[' 15 -le 16 ']' + echo n207 ++ expr 15 + 1 + i=16 + '[' 16 -le 16 ']' + echo n207 ++ expr 16 + 1 + i=17 + '[' 17 -le 16 ']' + read line ++ echo n201 16 ibm@n201 UNDEFINED ++ cut -f1 '-d ' ++ cut -f1 -d. + host=n201 ++ echo n201 16 ibm@n201 UNDEFINED ++ cut -f2 '-d ' + nslots=16 + i=1 + '[' 1 -le 16 ']' + echo n201 ++ expr 1 + 1 + i=2 + '[' 2 -le 16 ']' + echo n201 ++ expr 2 + 1 + i=3 + '[' 3 -le 16 ']' + echo n201 ++ expr 3 + 1 + i=4 + '[' 4 -le 16 ']' + echo n201 ++ expr 4 + 1 + i=5 + '[' 5 -le 16 ']' + echo n201 ++ expr 5 + 1 + i=6 + '[' 6 -le 16 ']' + echo n201 ++ expr 6 + 1 + i=7 + '[' 7 -le 16 ']' + echo n201 ++ expr 7 + 1 + i=8 + '[' 8 -le 16 ']' + echo n201 ++ expr 8 + 1 + i=9 + '[' 9 -le 16 ']' + echo n201 ++ expr 9 + 1 + i=10 + '[' 10 -le 16 ']' + echo n201 ++ expr 10 + 1 + i=11 + '[' 11 -le 16 ']' + echo n201 ++ expr 11 + 1 + i=12 + '[' 12 -le 16 ']' + echo n201 ++ expr 12 + 1 + i=13 + '[' 13 -le 16 ']' + echo n201 ++ expr 13 + 1 + i=14 + '[' 14 -le 16 ']' + echo n201 ++ expr 14 + 1 + i=15 + '[' 15 -le 16 ']' + echo n201 ++ expr 15 + 1 + i=16 + '[' 16 -le 16 ']' + echo n201 ++ expr 16 + 1 + i=17 + '[' 17 -le 16 ']' + read line ++ echo n206 16 ibm@n206 UNDEFINED ++ cut -f1 '-d ' ++ cut -f1 -d. + host=n206 ++ echo n206 16 ibm@n206 UNDEFINED ++ cut -f2 '-d ' + nslots=16 + i=1 + '[' 1 -le 16 ']' + echo n206 ++ expr 1 + 1 + i=2 + '[' 2 -le 16 ']' + echo n206 ++ expr 2 + 1 + i=3 + '[' 3 -le 16 ']' + echo n206 ++ expr 3 + 1 + i=4 + '[' 4 -le 16 ']' + echo n206 ++ expr 4 + 1 + i=5 + '[' 5 -le 16 ']' + echo n206 ++ expr 5 + 1 + i=6 + '[' 6 -le 16 ']' + echo n206 ++ expr 6 + 1 + i=7 + '[' 7 -le 16 ']' + echo n206 ++ expr 7 + 1 + i=8 + '[' 8 -le 16 ']' + echo n206 ++ expr 8 + 1 + i=9 + '[' 9 -le 16 ']' + echo n206 ++ expr 9 + 1 + i=10 + '[' 10 -le 16 ']' + echo n206 ++ expr 10 + 1 + i=11 + '[' 11 -le 16 ']' + echo n206 ++ expr 11 + 1 + i=12 + '[' 12 -le 16 ']' + echo n206 ++ expr 12 + 1 + i=13 + '[' 13 -le 16 ']' + echo n206 ++ expr 13 + 1 + i=14 + '[' 14 -le 16 ']' + echo n206 ++ expr 14 + 1 + i=15 + '[' 15 -le 16 ']' + echo n206 ++ expr 15 + 1 + i=16 + '[' 16 -le 16 ']' + echo n206 ++ expr 16 + 1 + i=17 + '[' 17 -le 16 ']' + read line ++ echo n202 16 ibm@n202 UNDEFINED ++ cut -f1 '-d ' ++ cut -f1 -d. + host=n202 ++ echo n202 16 ibm@n202 UNDEFINED ++ cut -f2 '-d ' + nslots=16 + i=1 + '[' 1 -le 16 ']' + echo n202 ++ expr 1 + 1 + i=2 + '[' 2 -le 16 ']' + echo n202 ++ expr 2 + 1 + i=3 + '[' 3 -le 16 ']' + echo n202 ++ expr 3 + 1 + i=4 + '[' 4 -le 16 ']' + echo n202 ++ expr 4 + 1 + i=5 + '[' 5 -le 16 ']' + echo n202 ++ expr 5 + 1 + i=6 + '[' 6 -le 16 ']' + echo n202 ++ expr 6 + 1 + i=7 + '[' 7 -le 16 ']' + echo n202 ++ expr 7 + 1 + i=8 + '[' 8 -le 16 ']' + echo n202 ++ expr 8 + 1 + i=9 + '[' 9 -le 16 ']' + echo n202 ++ expr 9 + 1 + i=10 + '[' 10 -le 16 ']' + echo n202 ++ expr 10 + 1 + i=11 + '[' 11 -le 16 ']' + echo n202 ++ expr 11 + 1 + i=12 + '[' 12 -le 16 ']' + echo n202 ++ expr 12 + 1 + i=13 + '[' 13 -le 16 ']' + echo n202 ++ expr 13 + 1 + i=14 + '[' 14 -le 16 ']' + echo n202 ++ expr 14 + 1 + i=15 + '[' 15 -le 16 ']' + echo n202 ++ expr 15 + 1 + i=16 + '[' 16 -le 16 ']' + echo n202 ++ expr 16 + 1 + i=17 + '[' 17 -le 16 ']' + read line + cat /tmp/598.1.ibm/machines n207 n207 n207 n207 n207 n207 n207 n207 n207 n207 n207 n207 n207 n207 n207 n207 n201 n201 n201 n201 n201 n201 n201 n201 n201 n201 n201 n201 n201 n201 n201 n201 n206 n206 n206 n206 n206 n206 n206 n206 n206 n206 n206 n206 n206 n206 n206 n206 n202 n202 n202 n202 n202 n202 n202 n202 n202 n202 n202 n202 n202 n202 n202 n202 + '[' 1 = 1 ']' + rsh_wrapper=/opt/sge6/mpich2_smpd/rsh + '[' '!' -x /opt/sge6/mpich2_smpd/rsh ']' + rshcmd=rsh + case "$ARC" in + ln -s /opt/sge6/mpich2_smpd/rsh /tmp/598.1.ibm/rsh + hash -r + '[' 0 = 1 ']' + port=20598 ++ cat /tmp/598.1.ibm/machines ++ uniq + for host in '`cat $machines | uniq`' + /opt/sge6/mpich2_smpd/bin/lx26-ppc64/start_mpich2 -n n207 /opt/mpich2/gnu//bin/smpd 20598 + for host in '`cat $machines | uniq`' + /opt/sge6/mpich2_smpd/bin/lx26-ppc64/start_mpich2 -n n201 /opt/mpich2/gnu//bin/smpd 20598 + for host in '`cat $machines | uniq`' + /opt/sge6/mpich2_smpd/bin/lx26-ppc64/start_mpich2 -n n206 /opt/mpich2/gnu//bin/smpd 20598 + for host in '`cat $machines | uniq`' + /opt/sge6/mpich2_smpd/bin/lx26-ppc64/start_mpich2 -n n202 /opt/mpich2/gnu//bin/smpd 20598 + actual_retry=1 + '[' 1 -le 10 ']' + echo 'startmpich2.sh: check for smpd daemons (1 of 10)' startmpich2.sh: check for smpd daemons (1 of 10) + numhosts=0 ++ cat /tmp/598.1.ibm/machines ++ uniq + for host in '`cat $machines | uniq`' ++ /opt/mpich2/gnu//bin/smpd -port 20598 -status n207 ++ head -n 1 ++ cut -f 1 -d ' ' rsh n207 /opt/mpich2/gnu//bin/smpd -port 20598 -d 0 rsh n201 /opt/mpich2/gnu//bin/smpd -port 20598 -d 0 rsh n206 /opt/mpich2/gnu//bin/smpd -port 20598 -d 0 Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1 internal ABORT - process 0 rsh n202 /opt/mpich2/gnu//bin/smpd -port 20598 -d 0 + '[' '' == smpd ']' + echo 'startmpich2.sh: missing smpd on n207' startmpich2.sh: missing smpd on n207 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/gnu//bin/smpd -port 20598 -status n201 ++ head -n 1 ++ cut -f 1 -d ' ' Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1 internal ABORT - process 0 + '[' '' == smpd ']' + echo 'startmpich2.sh: missing smpd on n201' startmpich2.sh: missing smpd on n201 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/gnu//bin/smpd -port 20598 -status n206 ++ head -n 1 ++ cut -f 1 -d ' ' Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1 internal ABORT - process 0 + '[' '' == smpd ']' + echo 'startmpich2.sh: missing smpd on n206' startmpich2.sh: missing smpd on n206 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/gnu//bin/smpd -port 20598 -status n202 ++ head -n 1 ++ cut -f 1 -d ' ' Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1 internal ABORT - process 0 + '[' '' == smpd ']' + echo 'startmpich2.sh: missing smpd on n202' startmpich2.sh: missing smpd on n202 + '[' 4 -eq 0 ']' + sleep 5 + (( actual_retry += 1 )) + '[' 2 -le 10 ']' + echo 'startmpich2.sh: check for smpd daemons (2 of 10)' startmpich2.sh: check for smpd daemons (2 of 10) + numhosts=0 ++ cat /tmp/598.1.ibm/machines ++ uniq + for host in '`cat $machines | uniq`' ++ /opt/mpich2/gnu//bin/smpd -port 20598 -status n207 ++ head -n 1 ++ cut -f 1 -d ' ' + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n207' startmpich2.sh: found running smpd on n207 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/gnu//bin/smpd -port 20598 -status n201 ++ head -n 1 ++ cut -f 1 -d ' ' + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n201' startmpich2.sh: found running smpd on n201 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/gnu//bin/smpd -port 20598 -status n206 ++ head -n 1 ++ cut -f 1 -d ' ' + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n206' startmpich2.sh: found running smpd on n206 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/gnu//bin/smpd -port 20598 -status n202 ++ head -n 1 ++ cut -f 1 -d ' ' + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n202' startmpich2.sh: found running smpd on n202 + '[' 4 -eq 4 ']' + echo 'startmpich2.sh: got all 4 of 4 nodes' startmpich2.sh: got all 4 of 4 nodes + exit 0 /tmp/598.1.ibm:/usr/local/bin:/bin:/usr/bin rm: cannot remove `*.phi': No such file or directory rm: cannot remove `*outputkernel*': No such file or directory rm: cannot remove `wnohu*': No such file or directory START END rm: cannot remove `*vector*': No such file or directory -catch_rsh /opt/mpich2/gnu/ Your job finished at Sun May 22 23:21:57 EDT 2011