USER = jtomczak HOMEDIR = /home/users/jtomczak JOB_ID = 349 JOB_NAME = FS-K-10 QUEUE = ibm NSLOTS = 48 NHOSTS = 3 SUB_HOST = rupc06 SUB_DIR = /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta10/nf6/NEW/GO/KERNEL EXE_HOST = n205 NFS_HOST = rupc-caip-04 EXE_DIR = /mnt/p06 OUT_DIR = /tmp TMP_DIR = /tmp/349.1.ibm INFO = Files larger than 100MB will be compressed to reduce the network load. INFO2 = Do NOT remove this file while job is running! PORT = 20349 Your job started at Mon Apr 25 10:22:28 EDT 2011 on host n205 in /mnt/p06/349jtomczak Submission directory is /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta10/nf6/NEW/GO/KERNEL on host rupc06 ----------------------------------------------------------------------------------------- + echo -catch_rsh /opt/sge6/spool/n205/active_jobs/349.1/pe_hostfile /opt/mpich2/gnu/ -catch_rsh /opt/sge6/spool/n205/active_jobs/349.1/pe_hostfile /opt/mpich2/gnu/ + SLEEPTIME=5 + RETRIES=10 + catch_rsh=0 + catch_hostname=0 + '[' 3 -gt 0 ']' + case "$1" in + catch_rsh=1 + shift + '[' 2 -gt 0 ']' + case "$1" in + break ++ basename /opt/sge6/mpich2_smpd/startmpich2.sh + me=startmpich2.sh + '[' 2 -lt 2 ']' + pe_hostfile=/opt/sge6/spool/n205/active_jobs/349.1/pe_hostfile + MPICH2_ROOT=/opt/mpich2/gnu/ + export MPICH2_ROOT + '[' '!' -x /opt/sge6/mpich2_smpd/bin/lx26-ppc64/start_mpich2 ']' + '[' '!' -x /opt/mpich2/gnu//bin/smpd ']' + '[' '!' -r /opt/sge6/spool/n205/active_jobs/349.1/pe_hostfile ']' + machines=/tmp/349.1.ibm/machines + PeHostfile2MachineFile /opt/sge6/spool/n205/active_jobs/349.1/pe_hostfile + cat /opt/sge6/spool/n205/active_jobs/349.1/pe_hostfile + read line ++ echo n205 16 ibm@n205 UNDEFINED ++ cut -f1 '-d ' ++ cut -f1 -d. + host=n205 ++ echo n205 16 ibm@n205 UNDEFINED ++ cut -f2 '-d ' + nslots=16 + i=1 + '[' 1 -le 16 ']' + echo n205 ++ expr 1 + 1 + i=2 + '[' 2 -le 16 ']' + echo n205 ++ expr 2 + 1 + i=3 + '[' 3 -le 16 ']' + echo n205 ++ expr 3 + 1 + i=4 + '[' 4 -le 16 ']' + echo n205 ++ expr 4 + 1 + i=5 + '[' 5 -le 16 ']' + echo n205 ++ expr 5 + 1 + i=6 + '[' 6 -le 16 ']' + echo n205 ++ expr 6 + 1 + i=7 + '[' 7 -le 16 ']' + echo n205 ++ expr 7 + 1 + i=8 + '[' 8 -le 16 ']' + echo n205 ++ expr 8 + 1 + i=9 + '[' 9 -le 16 ']' + echo n205 ++ expr 9 + 1 + i=10 + '[' 10 -le 16 ']' + echo n205 ++ expr 10 + 1 + i=11 + '[' 11 -le 16 ']' + echo n205 ++ expr 11 + 1 + i=12 + '[' 12 -le 16 ']' + echo n205 ++ expr 12 + 1 + i=13 + '[' 13 -le 16 ']' + echo n205 ++ expr 13 + 1 + i=14 + '[' 14 -le 16 ']' + echo n205 ++ expr 14 + 1 + i=15 + '[' 15 -le 16 ']' + echo n205 ++ expr 15 + 1 + i=16 + '[' 16 -le 16 ']' + echo n205 ++ expr 16 + 1 + i=17 + '[' 17 -le 16 ']' + read line ++ echo n203 16 ibm@n203 UNDEFINED ++ cut -f1 '-d ' ++ cut -f1 -d. + host=n203 ++ echo n203 16 ibm@n203 UNDEFINED ++ cut -f2 '-d ' + nslots=16 + i=1 + '[' 1 -le 16 ']' + echo n203 ++ expr 1 + 1 + i=2 + '[' 2 -le 16 ']' + echo n203 ++ expr 2 + 1 + i=3 + '[' 3 -le 16 ']' + echo n203 ++ expr 3 + 1 + i=4 + '[' 4 -le 16 ']' + echo n203 ++ expr 4 + 1 + i=5 + '[' 5 -le 16 ']' + echo n203 ++ expr 5 + 1 + i=6 + '[' 6 -le 16 ']' + echo n203 ++ expr 6 + 1 + i=7 + '[' 7 -le 16 ']' + echo n203 ++ expr 7 + 1 + i=8 + '[' 8 -le 16 ']' + echo n203 ++ expr 8 + 1 + i=9 + '[' 9 -le 16 ']' + echo n203 ++ expr 9 + 1 + i=10 + '[' 10 -le 16 ']' + echo n203 ++ expr 10 + 1 + i=11 + '[' 11 -le 16 ']' + echo n203 ++ expr 11 + 1 + i=12 + '[' 12 -le 16 ']' + echo n203 ++ expr 12 + 1 + i=13 + '[' 13 -le 16 ']' + echo n203 ++ expr 13 + 1 + i=14 + '[' 14 -le 16 ']' + echo n203 ++ expr 14 + 1 + i=15 + '[' 15 -le 16 ']' + echo n203 ++ expr 15 + 1 + i=16 + '[' 16 -le 16 ']' + echo n203 ++ expr 16 + 1 + i=17 + '[' 17 -le 16 ']' + read line ++ echo n204 16 ibm@n204 UNDEFINED ++ cut -f1 '-d ' ++ cut -f1 -d. + host=n204 ++ echo n204 16 ibm@n204 UNDEFINED ++ cut -f2 '-d ' + nslots=16 + i=1 + '[' 1 -le 16 ']' + echo n204 ++ expr 1 + 1 + i=2 + '[' 2 -le 16 ']' + echo n204 ++ expr 2 + 1 + i=3 + '[' 3 -le 16 ']' + echo n204 ++ expr 3 + 1 + i=4 + '[' 4 -le 16 ']' + echo n204 ++ expr 4 + 1 + i=5 + '[' 5 -le 16 ']' + echo n204 ++ expr 5 + 1 + i=6 + '[' 6 -le 16 ']' + echo n204 ++ expr 6 + 1 + i=7 + '[' 7 -le 16 ']' + echo n204 ++ expr 7 + 1 + i=8 + '[' 8 -le 16 ']' + echo n204 ++ expr 8 + 1 + i=9 + '[' 9 -le 16 ']' + echo n204 ++ expr 9 + 1 + i=10 + '[' 10 -le 16 ']' + echo n204 ++ expr 10 + 1 + i=11 + '[' 11 -le 16 ']' + echo n204 ++ expr 11 + 1 + i=12 + '[' 12 -le 16 ']' + echo n204 ++ expr 12 + 1 + i=13 + '[' 13 -le 16 ']' + echo n204 ++ expr 13 + 1 + i=14 + '[' 14 -le 16 ']' + echo n204 ++ expr 14 + 1 + i=15 + '[' 15 -le 16 ']' + echo n204 ++ expr 15 + 1 + i=16 + '[' 16 -le 16 ']' + echo n204 ++ expr 16 + 1 + i=17 + '[' 17 -le 16 ']' + read line + cat /tmp/349.1.ibm/machines n205 n205 n205 n205 n205 n205 n205 n205 n205 n205 n205 n205 n205 n205 n205 n205 n203 n203 n203 n203 n203 n203 n203 n203 n203 n203 n203 n203 n203 n203 n203 n203 n204 n204 n204 n204 n204 n204 n204 n204 n204 n204 n204 n204 n204 n204 n204 n204 + '[' 1 = 1 ']' + rsh_wrapper=/opt/sge6/mpich2_smpd/rsh + '[' '!' -x /opt/sge6/mpich2_smpd/rsh ']' + rshcmd=rsh + case "$ARC" in + ln -s /opt/sge6/mpich2_smpd/rsh /tmp/349.1.ibm/rsh + hash -r + '[' 0 = 1 ']' + port=20349 ++ cat /tmp/349.1.ibm/machines ++ uniq + for host in '`cat $machines | uniq`' + /opt/sge6/mpich2_smpd/bin/lx26-ppc64/start_mpich2 -n n205 /opt/mpich2/gnu//bin/smpd 20349 + for host in '`cat $machines | uniq`' + /opt/sge6/mpich2_smpd/bin/lx26-ppc64/start_mpich2 -n n203 /opt/mpich2/gnu//bin/smpd 20349 + for host in '`cat $machines | uniq`' + /opt/sge6/mpich2_smpd/bin/lx26-ppc64/start_mpich2 -n n204 /opt/mpich2/gnu//bin/smpd 20349 + actual_retry=1 + '[' 1 -le 10 ']' + echo 'startmpich2.sh: check for smpd daemons (1 of 10)' startmpich2.sh: check for smpd daemons (1 of 10) + numhosts=0 ++ cat /tmp/349.1.ibm/machines ++ uniq + for host in '`cat $machines | uniq`' ++ /opt/mpich2/gnu//bin/smpd -port 20349 -status n205 ++ head -n 1 ++ cut -f 1 -d ' ' Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1 internal ABORT - process 0 rsh n205 /opt/mpich2/gnu//bin/smpd -port 20349 -d 0 + '[' '' == smpd ']' + echo 'startmpich2.sh: missing smpd on n205' startmpich2.sh: missing smpd on n205 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/gnu//bin/smpd -port 20349 -status n203 rsh n203 /opt/mpich2/gnu//bin/smpd -port 20349 -d 0 ++ head -n 1 rsh n204 /opt/mpich2/gnu//bin/smpd -port 20349 -d 0 ++ cut -f 1 -d ' ' Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1 internal ABORT - process 0 + '[' '' == smpd ']' + echo 'startmpich2.sh: missing smpd on n203' startmpich2.sh: missing smpd on n203 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/gnu//bin/smpd -port 20349 -status n204 ++ head -n 1 ++ cut -f 1 -d ' ' Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1 internal ABORT - process 0 + '[' '' == smpd ']' + echo 'startmpich2.sh: missing smpd on n204' startmpich2.sh: missing smpd on n204 + '[' 3 -eq 0 ']' + sleep 5 + (( actual_retry += 1 )) + '[' 2 -le 10 ']' + echo 'startmpich2.sh: check for smpd daemons (2 of 10)' startmpich2.sh: check for smpd daemons (2 of 10) + numhosts=0 ++ cat /tmp/349.1.ibm/machines ++ uniq + for host in '`cat $machines | uniq`' ++ /opt/mpich2/gnu//bin/smpd -port 20349 -status n205 ++ head -n 1 ++ cut -f 1 -d ' ' + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n205' startmpich2.sh: found running smpd on n205 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/gnu//bin/smpd -port 20349 -status n203 ++ head -n 1 ++ cut -f 1 -d ' ' + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n203' startmpich2.sh: found running smpd on n203 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/gnu//bin/smpd -port 20349 -status n204 ++ head -n 1 ++ cut -f 1 -d ' ' + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n204' startmpich2.sh: found running smpd on n204 + '[' 3 -eq 3 ']' + echo 'startmpich2.sh: got all 3 of 3 nodes' startmpich2.sh: got all 3 of 3 nodes + exit 0 /tmp/349.1.ibm:/usr/local/bin:/bin:/usr/bin rm: cannot remove `*.phi': No such file or directory rm: cannot remove `*outputkernel*': No such file or directory rm: cannot remove `wnohu*': No such file or directory START END rm: cannot remove `*vector*': No such file or directory -catch_rsh /opt/mpich2/gnu/ Your job finished at Mon Apr 25 11:28:54 EDT 2011