USER = jtomczak HOMEDIR = /home/users/jtomczak JOB_ID = 846 JOB_NAME = FS-MYOP-80-6 QUEUE = ibm NSLOTS = 64 NHOSTS = 4 SUB_HOST = rupc06 SUB_DIR = /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta100/nf6/NEW/GO-small/MYOP EXE_HOST = n207 NFS_HOST = rupc-caip-02 EXE_DIR = /mnt/p01 OUT_DIR = /tmp TMP_DIR = /tmp/846.1.ibm INFO = Files larger than 100MB will be compressed to reduce the network load. INFO2 = Do NOT remove this file while job is running! PORT = 20846 Your job started at Wed Dec 7 15:06:14 EST 2011 on host n207 in /mnt/p01/846jtomczak Submission directory is /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta100/nf6/NEW/GO-small/MYOP on host rupc06 ----------------------------------------------------------------------------------------- + echo -catch_rsh /opt/sge6/spool/n207/active_jobs/846.1/pe_hostfile /opt/mpich2/gnu/ -catch_rsh /opt/sge6/spool/n207/active_jobs/846.1/pe_hostfile /opt/mpich2/gnu/ + SLEEPTIME=5 + RETRIES=10 + catch_rsh=0 + catch_hostname=0 + '[' 3 -gt 0 ']' + case "$1" in + catch_rsh=1 + shift + '[' 2 -gt 0 ']' + case "$1" in + break ++ basename /opt/sge6/mpich2_smpd/startmpich2.sh + me=startmpich2.sh + '[' 2 -lt 2 ']' + pe_hostfile=/opt/sge6/spool/n207/active_jobs/846.1/pe_hostfile + MPICH2_ROOT=/opt/mpich2/gnu/ + export MPICH2_ROOT + '[' '!' -x /opt/sge6/mpich2_smpd/bin/lx26-ppc64/start_mpich2 ']' + '[' '!' -x /opt/mpich2/gnu//bin/smpd ']' + '[' '!' -r /opt/sge6/spool/n207/active_jobs/846.1/pe_hostfile ']' + machines=/tmp/846.1.ibm/machines + PeHostfile2MachineFile /opt/sge6/spool/n207/active_jobs/846.1/pe_hostfile + cat /opt/sge6/spool/n207/active_jobs/846.1/pe_hostfile + read line ++ echo n207 16 ibm@n207 UNDEFINED ++ cut -f1 '-d ' ++ cut -f1 -d. + host=n207 ++ echo n207 16 ibm@n207 UNDEFINED ++ cut -f2 '-d ' + nslots=16 + i=1 + '[' 1 -le 16 ']' + echo n207 ++ expr 1 + 1 + i=2 + '[' 2 -le 16 ']' + echo n207 ++ expr 2 + 1 + i=3 + '[' 3 -le 16 ']' + echo n207 ++ expr 3 + 1 + i=4 + '[' 4 -le 16 ']' + echo n207 ++ expr 4 + 1 + i=5 + '[' 5 -le 16 ']' + echo n207 ++ expr 5 + 1 + i=6 + '[' 6 -le 16 ']' + echo n207 ++ expr 6 + 1 + i=7 + '[' 7 -le 16 ']' + echo n207 ++ expr 7 + 1 + i=8 + '[' 8 -le 16 ']' + echo n207 ++ expr 8 + 1 + i=9 + '[' 9 -le 16 ']' + echo n207 ++ expr 9 + 1 + i=10 + '[' 10 -le 16 ']' + echo n207 ++ expr 10 + 1 + i=11 + '[' 11 -le 16 ']' + echo n207 ++ expr 11 + 1 + i=12 + '[' 12 -le 16 ']' + echo n207 ++ expr 12 + 1 + i=13 + '[' 13 -le 16 ']' + echo n207 ++ expr 13 + 1 + i=14 + '[' 14 -le 16 ']' + echo n207 ++ expr 14 + 1 + i=15 + '[' 15 -le 16 ']' + echo n207 ++ expr 15 + 1 + i=16 + '[' 16 -le 16 ']' + echo n207 ++ expr 16 + 1 + i=17 + '[' 17 -le 16 ']' + read line ++ echo n208 16 ibm@n208 UNDEFINED ++ cut -f1 '-d ' ++ cut -f1 -d. + host=n208 ++ echo n208 16 ibm@n208 UNDEFINED ++ cut -f2 '-d ' + nslots=16 + i=1 + '[' 1 -le 16 ']' + echo n208 ++ expr 1 + 1 + i=2 + '[' 2 -le 16 ']' + echo n208 ++ expr 2 + 1 + i=3 + '[' 3 -le 16 ']' + echo n208 ++ expr 3 + 1 + i=4 + '[' 4 -le 16 ']' + echo n208 ++ expr 4 + 1 + i=5 + '[' 5 -le 16 ']' + echo n208 ++ expr 5 + 1 + i=6 + '[' 6 -le 16 ']' + echo n208 ++ expr 6 + 1 + i=7 + '[' 7 -le 16 ']' + echo n208 ++ expr 7 + 1 + i=8 + '[' 8 -le 16 ']' + echo n208 ++ expr 8 + 1 + i=9 + '[' 9 -le 16 ']' + echo n208 ++ expr 9 + 1 + i=10 + '[' 10 -le 16 ']' + echo n208 ++ expr 10 + 1 + i=11 + '[' 11 -le 16 ']' + echo n208 ++ expr 11 + 1 + i=12 + '[' 12 -le 16 ']' + echo n208 ++ expr 12 + 1 + i=13 + '[' 13 -le 16 ']' + echo n208 ++ expr 13 + 1 + i=14 + '[' 14 -le 16 ']' + echo n208 ++ expr 14 + 1 + i=15 + '[' 15 -le 16 ']' + echo n208 ++ expr 15 + 1 + i=16 + '[' 16 -le 16 ']' + echo n208 ++ expr 16 + 1 + i=17 + '[' 17 -le 16 ']' + read line ++ echo n203 16 ibm@n203 UNDEFINED ++ cut -f1 '-d ' ++ cut -f1 -d. + host=n203 ++ echo n203 16 ibm@n203 UNDEFINED ++ cut -f2 '-d ' + nslots=16 + i=1 + '[' 1 -le 16 ']' + echo n203 ++ expr 1 + 1 + i=2 + '[' 2 -le 16 ']' + echo n203 ++ expr 2 + 1 + i=3 + '[' 3 -le 16 ']' + echo n203 ++ expr 3 + 1 + i=4 + '[' 4 -le 16 ']' + echo n203 ++ expr 4 + 1 + i=5 + '[' 5 -le 16 ']' + echo n203 ++ expr 5 + 1 + i=6 + '[' 6 -le 16 ']' + echo n203 ++ expr 6 + 1 + i=7 + '[' 7 -le 16 ']' + echo n203 ++ expr 7 + 1 + i=8 + '[' 8 -le 16 ']' + echo n203 ++ expr 8 + 1 + i=9 + '[' 9 -le 16 ']' + echo n203 ++ expr 9 + 1 + i=10 + '[' 10 -le 16 ']' + echo n203 ++ expr 10 + 1 + i=11 + '[' 11 -le 16 ']' + echo n203 ++ expr 11 + 1 + i=12 + '[' 12 -le 16 ']' + echo n203 ++ expr 12 + 1 + i=13 + '[' 13 -le 16 ']' + echo n203 ++ expr 13 + 1 + i=14 + '[' 14 -le 16 ']' + echo n203 ++ expr 14 + 1 + i=15 + '[' 15 -le 16 ']' + echo n203 ++ expr 15 + 1 + i=16 + '[' 16 -le 16 ']' + echo n203 ++ expr 16 + 1 + i=17 + '[' 17 -le 16 ']' + read line ++ echo n201 16 ibm@n201 UNDEFINED ++ cut -f1 '-d ' ++ cut -f1 -d. + host=n201 ++ echo n201 16 ibm@n201 UNDEFINED ++ cut -f2 '-d ' + nslots=16 + i=1 + '[' 1 -le 16 ']' + echo n201 ++ expr 1 + 1 + i=2 + '[' 2 -le 16 ']' + echo n201 ++ expr 2 + 1 + i=3 + '[' 3 -le 16 ']' + echo n201 ++ expr 3 + 1 + i=4 + '[' 4 -le 16 ']' + echo n201 ++ expr 4 + 1 + i=5 + '[' 5 -le 16 ']' + echo n201 ++ expr 5 + 1 + i=6 + '[' 6 -le 16 ']' + echo n201 ++ expr 6 + 1 + i=7 + '[' 7 -le 16 ']' + echo n201 ++ expr 7 + 1 + i=8 + '[' 8 -le 16 ']' + echo n201 ++ expr 8 + 1 + i=9 + '[' 9 -le 16 ']' + echo n201 ++ expr 9 + 1 + i=10 + '[' 10 -le 16 ']' + echo n201 ++ expr 10 + 1 + i=11 + '[' 11 -le 16 ']' + echo n201 ++ expr 11 + 1 + i=12 + '[' 12 -le 16 ']' + echo n201 ++ expr 12 + 1 + i=13 + '[' 13 -le 16 ']' + echo n201 ++ expr 13 + 1 + i=14 + '[' 14 -le 16 ']' + echo n201 ++ expr 14 + 1 + i=15 + '[' 15 -le 16 ']' + echo n201 ++ expr 15 + 1 + i=16 + '[' 16 -le 16 ']' + echo n201 ++ expr 16 + 1 + i=17 + '[' 17 -le 16 ']' + read line + cat /tmp/846.1.ibm/machines n207 n207 n207 n207 n207 n207 n207 n207 n207 n207 n207 n207 n207 n207 n207 n207 n208 n208 n208 n208 n208 n208 n208 n208 n208 n208 n208 n208 n208 n208 n208 n208 n203 n203 n203 n203 n203 n203 n203 n203 n203 n203 n203 n203 n203 n203 n203 n203 n201 n201 n201 n201 n201 n201 n201 n201 n201 n201 n201 n201 n201 n201 n201 n201 + '[' 1 = 1 ']' + rsh_wrapper=/opt/sge6/mpich2_smpd/rsh + '[' '!' -x /opt/sge6/mpich2_smpd/rsh ']' + rshcmd=rsh + case "$ARC" in + ln -s /opt/sge6/mpich2_smpd/rsh /tmp/846.1.ibm/rsh + hash -r + '[' 0 = 1 ']' + port=20846 ++ cat /tmp/846.1.ibm/machines ++ uniq + for host in '`cat $machines | uniq`' + /opt/sge6/mpich2_smpd/bin/lx26-ppc64/start_mpich2 -n n207 /opt/mpich2/gnu//bin/smpd 20846 + for host in '`cat $machines | uniq`' + /opt/sge6/mpich2_smpd/bin/lx26-ppc64/start_mpich2 -n n208 /opt/mpich2/gnu//bin/smpd 20846 + for host in '`cat $machines | uniq`' + /opt/sge6/mpich2_smpd/bin/lx26-ppc64/start_mpich2 -n n203 /opt/mpich2/gnu//bin/smpd 20846 + for host in '`cat $machines | uniq`' + /opt/sge6/mpich2_smpd/bin/lx26-ppc64/start_mpich2 -n n201 /opt/mpich2/gnu//bin/smpd 20846 + actual_retry=1 + '[' 1 -le 10 ']' + echo 'startmpich2.sh: check for smpd daemons (1 of 10)' startmpich2.sh: check for smpd daemons (1 of 10) + numhosts=0 ++ cat /tmp/846.1.ibm/machines ++ uniq + for host in '`cat $machines | uniq`' rsh n207 /opt/mpich2/gnu//bin/smpd -port 20846 -d 0 ++ /opt/mpich2/gnu//bin/smpd -port 20846 -status n207 ++ head -n 1 ++ cut -f 1 -d ' ' rsh n208 /opt/mpich2/gnu//bin/smpd -port 20846 -d 0 rsh n203 /opt/mpich2/gnu//bin/smpd -port 20846 -d 0 rsh n201 /opt/mpich2/gnu//bin/smpd -port 20846 -d 0 Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1 internal ABORT - process 0 + '[' '' == smpd ']' + echo 'startmpich2.sh: missing smpd on n207' startmpich2.sh: missing smpd on n207 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/gnu//bin/smpd -port 20846 -status n208 ++ head -n 1 ++ cut -f 1 -d ' ' Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1 internal ABORT - process 0 + '[' '' == smpd ']' + echo 'startmpich2.sh: missing smpd on n208' startmpich2.sh: missing smpd on n208 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/gnu//bin/smpd -port 20846 -status n203 ++ head -n 1 ++ cut -f 1 -d ' ' Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1 internal ABORT - process 0 + '[' '' == smpd ']' + echo 'startmpich2.sh: missing smpd on n203' startmpich2.sh: missing smpd on n203 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/gnu//bin/smpd -port 20846 -status n201 ++ head -n 1 ++ cut -f 1 -d ' ' Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1 internal ABORT - process 0 + '[' '' == smpd ']' + echo 'startmpich2.sh: missing smpd on n201' startmpich2.sh: missing smpd on n201 + '[' 4 -eq 0 ']' + sleep 5 + (( actual_retry += 1 )) + '[' 2 -le 10 ']' + echo 'startmpich2.sh: check for smpd daemons (2 of 10)' startmpich2.sh: check for smpd daemons (2 of 10) + numhosts=0 ++ cat /tmp/846.1.ibm/machines ++ uniq + for host in '`cat $machines | uniq`' ++ /opt/mpich2/gnu//bin/smpd -port 20846 -status n207 ++ head -n 1 ++ cut -f 1 -d ' ' + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n207' startmpich2.sh: found running smpd on n207 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/gnu//bin/smpd -port 20846 -status n208 ++ head -n 1 ++ cut -f 1 -d ' ' + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n208' startmpich2.sh: found running smpd on n208 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/gnu//bin/smpd -port 20846 -status n203 ++ head -n 1 ++ cut -f 1 -d ' ' + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n203' startmpich2.sh: found running smpd on n203 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/gnu//bin/smpd -port 20846 -status n201 ++ head -n 1 ++ cut -f 1 -d ' ' + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n201' startmpich2.sh: found running smpd on n201 + '[' 4 -eq 4 ']' + echo 'startmpich2.sh: got all 4 of 4 nodes' startmpich2.sh: got all 4 of 4 nodes + exit 0 /tmp/846.1.ibm:/usr/local/bin:/bin:/usr/bin gzip: *.gz: No such file or directory rm: cannot remove `*.oc': No such file or directory rm: cannot remove `*.oc?': No such file or directory rm: cannot remove `*outputop**': No such file or directory rm: cannot remove `wnohu*': No such file or directory START END rm: cannot remove `*vector*': No such file or directory rm: cannot remove `*weight*': No such file or directory rm: cannot remove `wienker*': No such file or directory -catch_rsh /opt/mpich2/gnu/ Your job finished at Wed Dec 7 15:10:06 EST 2011