USER = jtomczak HOMEDIR = /home/users/jtomczak JOB_ID = 836 JOB_NAME = FS-K-20 QUEUE = ibm NSLOTS = 64 NHOSTS = 4 SUB_HOST = rupc06 SUB_DIR = /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta20/nf6/NEW/GO/KERNEL EXE_HOST = n204 NFS_HOST = rupc-caip-02 EXE_DIR = /mnt/p01 OUT_DIR = /tmp TMP_DIR = /tmp/836.1.ibm INFO = Files larger than 100MB will be compressed to reduce the network load. INFO2 = Do NOT remove this file while job is running! PORT = 20836 rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta20/nf6/NEW/GO/KERNEL/0.001: not a plain file rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta20/nf6/NEW/GO/KERNEL/0.002: not a plain file rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta20/nf6/NEW/GO/KERNEL/0.005: not a plain file rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta20/nf6/NEW/GO/KERNEL/0.01: not a plain file rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta20/nf6/NEW/GO/KERNEL/0.02: not a plain file Failure to rcp files! Your job started at Fri Nov 25 15:06:42 EST 2011 on host n204 in /mnt/p01/836jtomczak Submission directory is /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta20/nf6/NEW/GO/KERNEL on host rupc06 ----------------------------------------------------------------------------------------- + echo -catch_rsh /opt/sge6/spool/n204/active_jobs/836.1/pe_hostfile /opt/mpich2/gnu/ -catch_rsh /opt/sge6/spool/n204/active_jobs/836.1/pe_hostfile /opt/mpich2/gnu/ + SLEEPTIME=5 + RETRIES=10 + catch_rsh=0 + catch_hostname=0 + '[' 3 -gt 0 ']' + case "$1" in + catch_rsh=1 + shift + '[' 2 -gt 0 ']' + case "$1" in + break ++ basename /opt/sge6/mpich2_smpd/startmpich2.sh + me=startmpich2.sh + '[' 2 -lt 2 ']' + pe_hostfile=/opt/sge6/spool/n204/active_jobs/836.1/pe_hostfile + MPICH2_ROOT=/opt/mpich2/gnu/ + export MPICH2_ROOT + '[' '!' -x /opt/sge6/mpich2_smpd/bin/lx26-ppc64/start_mpich2 ']' + '[' '!' -x /opt/mpich2/gnu//bin/smpd ']' + '[' '!' -r /opt/sge6/spool/n204/active_jobs/836.1/pe_hostfile ']' + machines=/tmp/836.1.ibm/machines + PeHostfile2MachineFile /opt/sge6/spool/n204/active_jobs/836.1/pe_hostfile + cat /opt/sge6/spool/n204/active_jobs/836.1/pe_hostfile + read line ++ echo n204 16 ibm@n204 UNDEFINED ++ cut -f1 '-d ' ++ cut -f1 -d. + host=n204 ++ echo n204 16 ibm@n204 UNDEFINED ++ cut -f2 '-d ' + nslots=16 + i=1 + '[' 1 -le 16 ']' + echo n204 ++ expr 1 + 1 + i=2 + '[' 2 -le 16 ']' + echo n204 ++ expr 2 + 1 + i=3 + '[' 3 -le 16 ']' + echo n204 ++ expr 3 + 1 + i=4 + '[' 4 -le 16 ']' + echo n204 ++ expr 4 + 1 + i=5 + '[' 5 -le 16 ']' + echo n204 ++ expr 5 + 1 + i=6 + '[' 6 -le 16 ']' + echo n204 ++ expr 6 + 1 + i=7 + '[' 7 -le 16 ']' + echo n204 ++ expr 7 + 1 + i=8 + '[' 8 -le 16 ']' + echo n204 ++ expr 8 + 1 + i=9 + '[' 9 -le 16 ']' + echo n204 ++ expr 9 + 1 + i=10 + '[' 10 -le 16 ']' + echo n204 ++ expr 10 + 1 + i=11 + '[' 11 -le 16 ']' + echo n204 ++ expr 11 + 1 + i=12 + '[' 12 -le 16 ']' + echo n204 ++ expr 12 + 1 + i=13 + '[' 13 -le 16 ']' + echo n204 ++ expr 13 + 1 + i=14 + '[' 14 -le 16 ']' + echo n204 ++ expr 14 + 1 + i=15 + '[' 15 -le 16 ']' + echo n204 ++ expr 15 + 1 + i=16 + '[' 16 -le 16 ']' + echo n204 ++ expr 16 + 1 + i=17 + '[' 17 -le 16 ']' + read line ++ echo n206 16 ibm@n206 UNDEFINED ++ cut -f1 '-d ' ++ cut -f1 -d. + host=n206 ++ echo n206 16 ibm@n206 UNDEFINED ++ cut -f2 '-d ' + nslots=16 + i=1 + '[' 1 -le 16 ']' + echo n206 ++ expr 1 + 1 + i=2 + '[' 2 -le 16 ']' + echo n206 ++ expr 2 + 1 + i=3 + '[' 3 -le 16 ']' + echo n206 ++ expr 3 + 1 + i=4 + '[' 4 -le 16 ']' + echo n206 ++ expr 4 + 1 + i=5 + '[' 5 -le 16 ']' + echo n206 ++ expr 5 + 1 + i=6 + '[' 6 -le 16 ']' + echo n206 ++ expr 6 + 1 + i=7 + '[' 7 -le 16 ']' + echo n206 ++ expr 7 + 1 + i=8 + '[' 8 -le 16 ']' + echo n206 ++ expr 8 + 1 + i=9 + '[' 9 -le 16 ']' + echo n206 ++ expr 9 + 1 + i=10 + '[' 10 -le 16 ']' + echo n206 ++ expr 10 + 1 + i=11 + '[' 11 -le 16 ']' + echo n206 ++ expr 11 + 1 + i=12 + '[' 12 -le 16 ']' + echo n206 ++ expr 12 + 1 + i=13 + '[' 13 -le 16 ']' + echo n206 ++ expr 13 + 1 + i=14 + '[' 14 -le 16 ']' + echo n206 ++ expr 14 + 1 + i=15 + '[' 15 -le 16 ']' + echo n206 ++ expr 15 + 1 + i=16 + '[' 16 -le 16 ']' + echo n206 ++ expr 16 + 1 + i=17 + '[' 17 -le 16 ']' + read line ++ echo n207 16 ibm@n207 UNDEFINED ++ cut -f1 '-d ' ++ cut -f1 -d. + host=n207 ++ echo n207 16 ibm@n207 UNDEFINED ++ cut -f2 '-d ' + nslots=16 + i=1 + '[' 1 -le 16 ']' + echo n207 ++ expr 1 + 1 + i=2 + '[' 2 -le 16 ']' + echo n207 ++ expr 2 + 1 + i=3 + '[' 3 -le 16 ']' + echo n207 ++ expr 3 + 1 + i=4 + '[' 4 -le 16 ']' + echo n207 ++ expr 4 + 1 + i=5 + '[' 5 -le 16 ']' + echo n207 ++ expr 5 + 1 + i=6 + '[' 6 -le 16 ']' + echo n207 ++ expr 6 + 1 + i=7 + '[' 7 -le 16 ']' + echo n207 ++ expr 7 + 1 + i=8 + '[' 8 -le 16 ']' + echo n207 ++ expr 8 + 1 + i=9 + '[' 9 -le 16 ']' + echo n207 ++ expr 9 + 1 + i=10 + '[' 10 -le 16 ']' + echo n207 ++ expr 10 + 1 + i=11 + '[' 11 -le 16 ']' + echo n207 ++ expr 11 + 1 + i=12 + '[' 12 -le 16 ']' + echo n207 ++ expr 12 + 1 + i=13 + '[' 13 -le 16 ']' + echo n207 ++ expr 13 + 1 + i=14 + '[' 14 -le 16 ']' + echo n207 ++ expr 14 + 1 + i=15 + '[' 15 -le 16 ']' + echo n207 ++ expr 15 + 1 + i=16 + '[' 16 -le 16 ']' + echo n207 ++ expr 16 + 1 + i=17 + '[' 17 -le 16 ']' + read line ++ echo n208 16 ibm@n208 UNDEFINED ++ cut -f1 '-d ' ++ cut -f1 -d. + host=n208 ++ echo n208 16 ibm@n208 UNDEFINED ++ cut -f2 '-d ' + nslots=16 + i=1 + '[' 1 -le 16 ']' + echo n208 ++ expr 1 + 1 + i=2 + '[' 2 -le 16 ']' + echo n208 ++ expr 2 + 1 + i=3 + '[' 3 -le 16 ']' + echo n208 ++ expr 3 + 1 + i=4 + '[' 4 -le 16 ']' + echo n208 ++ expr 4 + 1 + i=5 + '[' 5 -le 16 ']' + echo n208 ++ expr 5 + 1 + i=6 + '[' 6 -le 16 ']' + echo n208 ++ expr 6 + 1 + i=7 + '[' 7 -le 16 ']' + echo n208 ++ expr 7 + 1 + i=8 + '[' 8 -le 16 ']' + echo n208 ++ expr 8 + 1 + i=9 + '[' 9 -le 16 ']' + echo n208 ++ expr 9 + 1 + i=10 + '[' 10 -le 16 ']' + echo n208 ++ expr 10 + 1 + i=11 + '[' 11 -le 16 ']' + echo n208 ++ expr 11 + 1 + i=12 + '[' 12 -le 16 ']' + echo n208 ++ expr 12 + 1 + i=13 + '[' 13 -le 16 ']' + echo n208 ++ expr 13 + 1 + i=14 + '[' 14 -le 16 ']' + echo n208 ++ expr 14 + 1 + i=15 + '[' 15 -le 16 ']' + echo n208 ++ expr 15 + 1 + i=16 + '[' 16 -le 16 ']' + echo n208 ++ expr 16 + 1 + i=17 + '[' 17 -le 16 ']' + read line + cat /tmp/836.1.ibm/machines n204 n204 n204 n204 n204 n204 n204 n204 n204 n204 n204 n204 n204 n204 n204 n204 n206 n206 n206 n206 n206 n206 n206 n206 n206 n206 n206 n206 n206 n206 n206 n206 n207 n207 n207 n207 n207 n207 n207 n207 n207 n207 n207 n207 n207 n207 n207 n207 n208 n208 n208 n208 n208 n208 n208 n208 n208 n208 n208 n208 n208 n208 n208 n208 + '[' 1 = 1 ']' + rsh_wrapper=/opt/sge6/mpich2_smpd/rsh + '[' '!' -x /opt/sge6/mpich2_smpd/rsh ']' + rshcmd=rsh + case "$ARC" in + ln -s /opt/sge6/mpich2_smpd/rsh /tmp/836.1.ibm/rsh + hash -r + '[' 0 = 1 ']' + port=20836 ++ cat /tmp/836.1.ibm/machines ++ uniq + for host in '`cat $machines | uniq`' + /opt/sge6/mpich2_smpd/bin/lx26-ppc64/start_mpich2 -n n204 /opt/mpich2/gnu//bin/smpd 20836 + for host in '`cat $machines | uniq`' + /opt/sge6/mpich2_smpd/bin/lx26-ppc64/start_mpich2 -n n206 /opt/mpich2/gnu//bin/smpd 20836 + for host in '`cat $machines | uniq`' + /opt/sge6/mpich2_smpd/bin/lx26-ppc64/start_mpich2 -n n207 /opt/mpich2/gnu//bin/smpd 20836 + for host in '`cat $machines | uniq`' + /opt/sge6/mpich2_smpd/bin/lx26-ppc64/start_mpich2 -n n208 /opt/mpich2/gnu//bin/smpd 20836 + actual_retry=1 + '[' 1 -le 10 ']' + echo 'startmpich2.sh: check for smpd daemons (1 of 10)' startmpich2.sh: check for smpd daemons (1 of 10) + numhosts=0 ++ cat /tmp/836.1.ibm/machines ++ uniq + for host in '`cat $machines | uniq`' ++ /opt/mpich2/gnu//bin/smpd -port 20836 -status n204 rsh n204 /opt/mpich2/gnu//bin/smpd -port 20836 -d 0 ++ head -n 1 ++ cut -f 1 -d ' ' rsh n206 /opt/mpich2/gnu//bin/smpd -port 20836 -d 0 rsh n207 /opt/mpich2/gnu//bin/smpd -port 20836 -d 0 rsh n208 /opt/mpich2/gnu//bin/smpd -port 20836 -d 0 Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1 internal ABORT - process 0 + '[' '' == smpd ']' + echo 'startmpich2.sh: missing smpd on n204' startmpich2.sh: missing smpd on n204 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/gnu//bin/smpd -port 20836 -status n206 ++ head -n 1 ++ cut -f 1 -d ' ' Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1 internal ABORT - process 0 + '[' '' == smpd ']' + echo 'startmpich2.sh: missing smpd on n206' startmpich2.sh: missing smpd on n206 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/gnu//bin/smpd -port 20836 -status n207 ++ head -n 1 ++ cut -f 1 -d ' ' Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1 internal ABORT - process 0 + '[' '' == smpd ']' + echo 'startmpich2.sh: missing smpd on n207' startmpich2.sh: missing smpd on n207 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/gnu//bin/smpd -port 20836 -status n208 ++ head -n 1 ++ cut -f 1 -d ' ' Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1 internal ABORT - process 0 + '[' '' == smpd ']' + echo 'startmpich2.sh: missing smpd on n208' startmpich2.sh: missing smpd on n208 + '[' 4 -eq 0 ']' + sleep 5 + (( actual_retry += 1 )) + '[' 2 -le 10 ']' + echo 'startmpich2.sh: check for smpd daemons (2 of 10)' startmpich2.sh: check for smpd daemons (2 of 10) + numhosts=0 ++ cat /tmp/836.1.ibm/machines ++ uniq + for host in '`cat $machines | uniq`' ++ /opt/mpich2/gnu//bin/smpd -port 20836 -status n204 ++ head -n 1 ++ cut -f 1 -d ' ' + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n204' startmpich2.sh: found running smpd on n204 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/gnu//bin/smpd -port 20836 -status n206 ++ head -n 1 ++ cut -f 1 -d ' ' + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n206' startmpich2.sh: found running smpd on n206 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/gnu//bin/smpd -port 20836 -status n207 ++ head -n 1 ++ cut -f 1 -d ' ' + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n207' startmpich2.sh: found running smpd on n207 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/gnu//bin/smpd -port 20836 -status n208 ++ head -n 1 ++ cut -f 1 -d ' ' + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n208' startmpich2.sh: found running smpd on n208 + '[' 4 -eq 4 ']' + echo 'startmpich2.sh: got all 4 of 4 nodes' startmpich2.sh: got all 4 of 4 nodes + exit 0 /tmp/836.1.ibm:/usr/local/bin:/bin:/usr/bin rm: cannot remove `*.phi': No such file or directory rm: cannot remove `*outputkernel*': No such file or directory rm: cannot remove `wnohu*': No such file or directory START END rm: cannot remove `*vector*': No such file or directory -catch_rsh /opt/mpich2/gnu/ Your job finished at Fri Nov 25 19:08:05 EST 2011