USER      =  jtomczak
HOMEDIR   =  /home/users/jtomczak
JOB_ID    =  806
JOB_NAME  =  FS-K-125
QUEUE     =  ibm
NSLOTS    =  127
NHOSTS    =  8

SUB_HOST  =  rupc06
SUB_DIR   =  /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta125/nf6/NEW/GO/KERNEL

EXE_HOST  =  n205
NFS_HOST  =  rupc-caip-02
EXE_DIR   =  /mnt/p01
OUT_DIR   =  /tmp
TMP_DIR   =  /tmp/806.1.ibm

INFO      =  Files larger than 100MB will be compressed to reduce the network load.
INFO2     =  Do NOT remove this file while job is running!

PORT      =  20806

Your job started at Mon Oct 31 15:22:55 EDT 2011 on host n205 in /mnt/p01/806jtomczak
Submission directory is /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta125/nf6/NEW/GO/KERNEL on host rupc06
-----------------------------------------------------------------------------------------
+ echo -catch_rsh /opt/sge6/spool/n205/active_jobs/806.1/pe_hostfile /opt/mpich2/gnu/
-catch_rsh /opt/sge6/spool/n205/active_jobs/806.1/pe_hostfile /opt/mpich2/gnu/
+ SLEEPTIME=5
+ RETRIES=10
+ catch_rsh=0
+ catch_hostname=0
+ '[' 3 -gt 0 ']'
+ case "$1" in
+ catch_rsh=1
+ shift
+ '[' 2 -gt 0 ']'
+ case "$1" in
+ break
++ basename /opt/sge6/mpich2_smpd/startmpich2.sh
+ me=startmpich2.sh
+ '[' 2 -lt 2 ']'
+ pe_hostfile=/opt/sge6/spool/n205/active_jobs/806.1/pe_hostfile
+ MPICH2_ROOT=/opt/mpich2/gnu/
+ export MPICH2_ROOT
+ '[' '!' -x /opt/sge6/mpich2_smpd/bin/lx26-ppc64/start_mpich2 ']'
+ '[' '!' -x /opt/mpich2/gnu//bin/smpd ']'
+ '[' '!' -r /opt/sge6/spool/n205/active_jobs/806.1/pe_hostfile ']'
+ machines=/tmp/806.1.ibm/machines
+ PeHostfile2MachineFile /opt/sge6/spool/n205/active_jobs/806.1/pe_hostfile
+ cat /opt/sge6/spool/n205/active_jobs/806.1/pe_hostfile
+ read line
++ echo n205 16 ibm@n205 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n205
++ echo n205 16 ibm@n205 UNDEFINED
++ cut -f2 '-d '
+ nslots=16
+ i=1
+ '[' 1 -le 16 ']'
+ echo n205
++ expr 1 + 1
+ i=2
+ '[' 2 -le 16 ']'
+ echo n205
++ expr 2 + 1
+ i=3
+ '[' 3 -le 16 ']'
+ echo n205
++ expr 3 + 1
+ i=4
+ '[' 4 -le 16 ']'
+ echo n205
++ expr 4 + 1
+ i=5
+ '[' 5 -le 16 ']'
+ echo n205
++ expr 5 + 1
+ i=6
+ '[' 6 -le 16 ']'
+ echo n205
++ expr 6 + 1
+ i=7
+ '[' 7 -le 16 ']'
+ echo n205
++ expr 7 + 1
+ i=8
+ '[' 8 -le 16 ']'
+ echo n205
++ expr 8 + 1
+ i=9
+ '[' 9 -le 16 ']'
+ echo n205
++ expr 9 + 1
+ i=10
+ '[' 10 -le 16 ']'
+ echo n205
++ expr 10 + 1
+ i=11
+ '[' 11 -le 16 ']'
+ echo n205
++ expr 11 + 1
+ i=12
+ '[' 12 -le 16 ']'
+ echo n205
++ expr 12 + 1
+ i=13
+ '[' 13 -le 16 ']'
+ echo n205
++ expr 13 + 1
+ i=14
+ '[' 14 -le 16 ']'
+ echo n205
++ expr 14 + 1
+ i=15
+ '[' 15 -le 16 ']'
+ echo n205
++ expr 15 + 1
+ i=16
+ '[' 16 -le 16 ']'
+ echo n205
++ expr 16 + 1
+ i=17
+ '[' 17 -le 16 ']'
+ read line
++ echo n202 16 ibm@n202 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n202
++ echo n202 16 ibm@n202 UNDEFINED
++ cut -f2 '-d '
+ nslots=16
+ i=1
+ '[' 1 -le 16 ']'
+ echo n202
++ expr 1 + 1
+ i=2
+ '[' 2 -le 16 ']'
+ echo n202
++ expr 2 + 1
+ i=3
+ '[' 3 -le 16 ']'
+ echo n202
++ expr 3 + 1
+ i=4
+ '[' 4 -le 16 ']'
+ echo n202
++ expr 4 + 1
+ i=5
+ '[' 5 -le 16 ']'
+ echo n202
++ expr 5 + 1
+ i=6
+ '[' 6 -le 16 ']'
+ echo n202
++ expr 6 + 1
+ i=7
+ '[' 7 -le 16 ']'
+ echo n202
++ expr 7 + 1
+ i=8
+ '[' 8 -le 16 ']'
+ echo n202
++ expr 8 + 1
+ i=9
+ '[' 9 -le 16 ']'
+ echo n202
++ expr 9 + 1
+ i=10
+ '[' 10 -le 16 ']'
+ echo n202
++ expr 10 + 1
+ i=11
+ '[' 11 -le 16 ']'
+ echo n202
++ expr 11 + 1
+ i=12
+ '[' 12 -le 16 ']'
+ echo n202
++ expr 12 + 1
+ i=13
+ '[' 13 -le 16 ']'
+ echo n202
++ expr 13 + 1
+ i=14
+ '[' 14 -le 16 ']'
+ echo n202
++ expr 14 + 1
+ i=15
+ '[' 15 -le 16 ']'
+ echo n202
++ expr 15 + 1
+ i=16
+ '[' 16 -le 16 ']'
+ echo n202
++ expr 16 + 1
+ i=17
+ '[' 17 -le 16 ']'
+ read line
++ echo n201 16 ibm@n201 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n201
++ echo n201 16 ibm@n201 UNDEFINED
++ cut -f2 '-d '
+ nslots=16
+ i=1
+ '[' 1 -le 16 ']'
+ echo n201
++ expr 1 + 1
+ i=2
+ '[' 2 -le 16 ']'
+ echo n201
++ expr 2 + 1
+ i=3
+ '[' 3 -le 16 ']'
+ echo n201
++ expr 3 + 1
+ i=4
+ '[' 4 -le 16 ']'
+ echo n201
++ expr 4 + 1
+ i=5
+ '[' 5 -le 16 ']'
+ echo n201
++ expr 5 + 1
+ i=6
+ '[' 6 -le 16 ']'
+ echo n201
++ expr 6 + 1
+ i=7
+ '[' 7 -le 16 ']'
+ echo n201
++ expr 7 + 1
+ i=8
+ '[' 8 -le 16 ']'
+ echo n201
++ expr 8 + 1
+ i=9
+ '[' 9 -le 16 ']'
+ echo n201
++ expr 9 + 1
+ i=10
+ '[' 10 -le 16 ']'
+ echo n201
++ expr 10 + 1
+ i=11
+ '[' 11 -le 16 ']'
+ echo n201
++ expr 11 + 1
+ i=12
+ '[' 12 -le 16 ']'
+ echo n201
++ expr 12 + 1
+ i=13
+ '[' 13 -le 16 ']'
+ echo n201
++ expr 13 + 1
+ i=14
+ '[' 14 -le 16 ']'
+ echo n201
++ expr 14 + 1
+ i=15
+ '[' 15 -le 16 ']'
+ echo n201
++ expr 15 + 1
+ i=16
+ '[' 16 -le 16 ']'
+ echo n201
++ expr 16 + 1
+ i=17
+ '[' 17 -le 16 ']'
+ read line
++ echo n203 16 ibm@n203 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n203
++ echo n203 16 ibm@n203 UNDEFINED
++ cut -f2 '-d '
+ nslots=16
+ i=1
+ '[' 1 -le 16 ']'
+ echo n203
++ expr 1 + 1
+ i=2
+ '[' 2 -le 16 ']'
+ echo n203
++ expr 2 + 1
+ i=3
+ '[' 3 -le 16 ']'
+ echo n203
++ expr 3 + 1
+ i=4
+ '[' 4 -le 16 ']'
+ echo n203
++ expr 4 + 1
+ i=5
+ '[' 5 -le 16 ']'
+ echo n203
++ expr 5 + 1
+ i=6
+ '[' 6 -le 16 ']'
+ echo n203
++ expr 6 + 1
+ i=7
+ '[' 7 -le 16 ']'
+ echo n203
++ expr 7 + 1
+ i=8
+ '[' 8 -le 16 ']'
+ echo n203
++ expr 8 + 1
+ i=9
+ '[' 9 -le 16 ']'
+ echo n203
++ expr 9 + 1
+ i=10
+ '[' 10 -le 16 ']'
+ echo n203
++ expr 10 + 1
+ i=11
+ '[' 11 -le 16 ']'
+ echo n203
++ expr 11 + 1
+ i=12
+ '[' 12 -le 16 ']'
+ echo n203
++ expr 12 + 1
+ i=13
+ '[' 13 -le 16 ']'
+ echo n203
++ expr 13 + 1
+ i=14
+ '[' 14 -le 16 ']'
+ echo n203
++ expr 14 + 1
+ i=15
+ '[' 15 -le 16 ']'
+ echo n203
++ expr 15 + 1
+ i=16
+ '[' 16 -le 16 ']'
+ echo n203
++ expr 16 + 1
+ i=17
+ '[' 17 -le 16 ']'
+ read line
++ echo n207 16 ibm@n207 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n207
++ echo n207 16 ibm@n207 UNDEFINED
++ cut -f2 '-d '
+ nslots=16
+ i=1
+ '[' 1 -le 16 ']'
+ echo n207
++ expr 1 + 1
+ i=2
+ '[' 2 -le 16 ']'
+ echo n207
++ expr 2 + 1
+ i=3
+ '[' 3 -le 16 ']'
+ echo n207
++ expr 3 + 1
+ i=4
+ '[' 4 -le 16 ']'
+ echo n207
++ expr 4 + 1
+ i=5
+ '[' 5 -le 16 ']'
+ echo n207
++ expr 5 + 1
+ i=6
+ '[' 6 -le 16 ']'
+ echo n207
++ expr 6 + 1
+ i=7
+ '[' 7 -le 16 ']'
+ echo n207
++ expr 7 + 1
+ i=8
+ '[' 8 -le 16 ']'
+ echo n207
++ expr 8 + 1
+ i=9
+ '[' 9 -le 16 ']'
+ echo n207
++ expr 9 + 1
+ i=10
+ '[' 10 -le 16 ']'
+ echo n207
++ expr 10 + 1
+ i=11
+ '[' 11 -le 16 ']'
+ echo n207
++ expr 11 + 1
+ i=12
+ '[' 12 -le 16 ']'
+ echo n207
++ expr 12 + 1
+ i=13
+ '[' 13 -le 16 ']'
+ echo n207
++ expr 13 + 1
+ i=14
+ '[' 14 -le 16 ']'
+ echo n207
++ expr 14 + 1
+ i=15
+ '[' 15 -le 16 ']'
+ echo n207
++ expr 15 + 1
+ i=16
+ '[' 16 -le 16 ']'
+ echo n207
++ expr 16 + 1
+ i=17
+ '[' 17 -le 16 ']'
+ read line
++ echo n206 16 ibm@n206 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n206
++ echo n206 16 ibm@n206 UNDEFINED
++ cut -f2 '-d '
+ nslots=16
+ i=1
+ '[' 1 -le 16 ']'
+ echo n206
++ expr 1 + 1
+ i=2
+ '[' 2 -le 16 ']'
+ echo n206
++ expr 2 + 1
+ i=3
+ '[' 3 -le 16 ']'
+ echo n206
++ expr 3 + 1
+ i=4
+ '[' 4 -le 16 ']'
+ echo n206
++ expr 4 + 1
+ i=5
+ '[' 5 -le 16 ']'
+ echo n206
++ expr 5 + 1
+ i=6
+ '[' 6 -le 16 ']'
+ echo n206
++ expr 6 + 1
+ i=7
+ '[' 7 -le 16 ']'
+ echo n206
++ expr 7 + 1
+ i=8
+ '[' 8 -le 16 ']'
+ echo n206
++ expr 8 + 1
+ i=9
+ '[' 9 -le 16 ']'
+ echo n206
++ expr 9 + 1
+ i=10
+ '[' 10 -le 16 ']'
+ echo n206
++ expr 10 + 1
+ i=11
+ '[' 11 -le 16 ']'
+ echo n206
++ expr 11 + 1
+ i=12
+ '[' 12 -le 16 ']'
+ echo n206
++ expr 12 + 1
+ i=13
+ '[' 13 -le 16 ']'
+ echo n206
++ expr 13 + 1
+ i=14
+ '[' 14 -le 16 ']'
+ echo n206
++ expr 14 + 1
+ i=15
+ '[' 15 -le 16 ']'
+ echo n206
++ expr 15 + 1
+ i=16
+ '[' 16 -le 16 ']'
+ echo n206
++ expr 16 + 1
+ i=17
+ '[' 17 -le 16 ']'
+ read line
++ echo n204 16 ibm@n204 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n204
++ echo n204 16 ibm@n204 UNDEFINED
++ cut -f2 '-d '
+ nslots=16
+ i=1
+ '[' 1 -le 16 ']'
+ echo n204
++ expr 1 + 1
+ i=2
+ '[' 2 -le 16 ']'
+ echo n204
++ expr 2 + 1
+ i=3
+ '[' 3 -le 16 ']'
+ echo n204
++ expr 3 + 1
+ i=4
+ '[' 4 -le 16 ']'
+ echo n204
++ expr 4 + 1
+ i=5
+ '[' 5 -le 16 ']'
+ echo n204
++ expr 5 + 1
+ i=6
+ '[' 6 -le 16 ']'
+ echo n204
++ expr 6 + 1
+ i=7
+ '[' 7 -le 16 ']'
+ echo n204
++ expr 7 + 1
+ i=8
+ '[' 8 -le 16 ']'
+ echo n204
++ expr 8 + 1
+ i=9
+ '[' 9 -le 16 ']'
+ echo n204
++ expr 9 + 1
+ i=10
+ '[' 10 -le 16 ']'
+ echo n204
++ expr 10 + 1
+ i=11
+ '[' 11 -le 16 ']'
+ echo n204
++ expr 11 + 1
+ i=12
+ '[' 12 -le 16 ']'
+ echo n204
++ expr 12 + 1
+ i=13
+ '[' 13 -le 16 ']'
+ echo n204
++ expr 13 + 1
+ i=14
+ '[' 14 -le 16 ']'
+ echo n204
++ expr 14 + 1
+ i=15
+ '[' 15 -le 16 ']'
+ echo n204
++ expr 15 + 1
+ i=16
+ '[' 16 -le 16 ']'
+ echo n204
++ expr 16 + 1
+ i=17
+ '[' 17 -le 16 ']'
+ read line
++ echo n208 15 ibm@n208 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n208
++ echo n208 15 ibm@n208 UNDEFINED
++ cut -f2 '-d '
+ nslots=15
+ i=1
+ '[' 1 -le 15 ']'
+ echo n208
++ expr 1 + 1
+ i=2
+ '[' 2 -le 15 ']'
+ echo n208
++ expr 2 + 1
+ i=3
+ '[' 3 -le 15 ']'
+ echo n208
++ expr 3 + 1
+ i=4
+ '[' 4 -le 15 ']'
+ echo n208
++ expr 4 + 1
+ i=5
+ '[' 5 -le 15 ']'
+ echo n208
++ expr 5 + 1
+ i=6
+ '[' 6 -le 15 ']'
+ echo n208
++ expr 6 + 1
+ i=7
+ '[' 7 -le 15 ']'
+ echo n208
++ expr 7 + 1
+ i=8
+ '[' 8 -le 15 ']'
+ echo n208
++ expr 8 + 1
+ i=9
+ '[' 9 -le 15 ']'
+ echo n208
++ expr 9 + 1
+ i=10
+ '[' 10 -le 15 ']'
+ echo n208
++ expr 10 + 1
+ i=11
+ '[' 11 -le 15 ']'
+ echo n208
++ expr 11 + 1
+ i=12
+ '[' 12 -le 15 ']'
+ echo n208
++ expr 12 + 1
+ i=13
+ '[' 13 -le 15 ']'
+ echo n208
++ expr 13 + 1
+ i=14
+ '[' 14 -le 15 ']'
+ echo n208
++ expr 14 + 1
+ i=15
+ '[' 15 -le 15 ']'
+ echo n208
++ expr 15 + 1
+ i=16
+ '[' 16 -le 15 ']'
+ read line
+ cat /tmp/806.1.ibm/machines
n205
n205
n205
n205
n205
n205
n205
n205
n205
n205
n205
n205
n205
n205
n205
n205
n202
n202
n202
n202
n202
n202
n202
n202
n202
n202
n202
n202
n202
n202
n202
n202
n201
n201
n201
n201
n201
n201
n201
n201
n201
n201
n201
n201
n201
n201
n201
n201
n203
n203
n203
n203
n203
n203
n203
n203
n203
n203
n203
n203
n203
n203
n203
n203
n207
n207
n207
n207
n207
n207
n207
n207
n207
n207
n207
n207
n207
n207
n207
n207
n206
n206
n206
n206
n206
n206
n206
n206
n206
n206
n206
n206
n206
n206
n206
n206
n204
n204
n204
n204
n204
n204
n204
n204
n204
n204
n204
n204
n204
n204
n204
n204
n208
n208
n208
n208
n208
n208
n208
n208
n208
n208
n208
n208
n208
n208
n208
+ '[' 1 = 1 ']'
+ rsh_wrapper=/opt/sge6/mpich2_smpd/rsh
+ '[' '!' -x /opt/sge6/mpich2_smpd/rsh ']'
+ rshcmd=rsh
+ case "$ARC" in
+ ln -s /opt/sge6/mpich2_smpd/rsh /tmp/806.1.ibm/rsh
+ hash -r
+ '[' 0 = 1 ']'
+ port=20806
++ cat /tmp/806.1.ibm/machines
++ uniq
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-ppc64/start_mpich2 -n n205 /opt/mpich2/gnu//bin/smpd 20806
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-ppc64/start_mpich2 -n n202 /opt/mpich2/gnu//bin/smpd 20806
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-ppc64/start_mpich2 -n n201 /opt/mpich2/gnu//bin/smpd 20806
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-ppc64/start_mpich2 -n n203 /opt/mpich2/gnu//bin/smpd 20806
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-ppc64/start_mpich2 -n n207 /opt/mpich2/gnu//bin/smpd 20806
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-ppc64/start_mpich2 -n n206 /opt/mpich2/gnu//bin/smpd 20806
rsh n205 /opt/mpich2/gnu//bin/smpd -port 20806 -d 0
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-ppc64/start_mpich2 -n n204 /opt/mpich2/gnu//bin/smpd 20806
rsh n202 /opt/mpich2/gnu//bin/smpd -port 20806 -d 0
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-ppc64/start_mpich2 -n n208 /opt/mpich2/gnu//bin/smpd 20806
rsh n201 /opt/mpich2/gnu//bin/smpd -port 20806 -d 0
+ actual_retry=1
+ '[' 1 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (1 of 10)'
startmpich2.sh: check for smpd daemons (1 of 10)
+ numhosts=0
++ cat /tmp/806.1.ibm/machines
rsh n203 /opt/mpich2/gnu//bin/smpd -port 20806 -d 0
++ uniq
rsh n207 /opt/mpich2/gnu//bin/smpd -port 20806 -d 0
+ for host in '`cat $machines | uniq`'
rsh n206 /opt/mpich2/gnu//bin/smpd -port 20806 -d 0
++ /opt/mpich2/gnu//bin/smpd -port 20806 -status n205
++ head -n 1
++ cut -f 1 -d ' '
rsh n204 /opt/mpich2/gnu//bin/smpd -port 20806 -d 0
rsh n208 /opt/mpich2/gnu//bin/smpd -port 20806 -d 0
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n205'
startmpich2.sh: missing smpd on n205
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/gnu//bin/smpd -port 20806 -status n202
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n202'
startmpich2.sh: missing smpd on n202
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/gnu//bin/smpd -port 20806 -status n201
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n201'
startmpich2.sh: missing smpd on n201
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/gnu//bin/smpd -port 20806 -status n203
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n203'
startmpich2.sh: missing smpd on n203
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/gnu//bin/smpd -port 20806 -status n207
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n207'
startmpich2.sh: missing smpd on n207
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/gnu//bin/smpd -port 20806 -status n206
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n206'
startmpich2.sh: missing smpd on n206
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/gnu//bin/smpd -port 20806 -status n204
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n204'
startmpich2.sh: missing smpd on n204
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/gnu//bin/smpd -port 20806 -status n208
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n208'
startmpich2.sh: missing smpd on n208
+ '[' 8 -eq 0 ']'
+ sleep 5
+ (( actual_retry += 1 ))
+ '[' 2 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (2 of 10)'
startmpich2.sh: check for smpd daemons (2 of 10)
+ numhosts=0
++ cat /tmp/806.1.ibm/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/gnu//bin/smpd -port 20806 -status n205
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n205'
startmpich2.sh: found running smpd on n205
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/gnu//bin/smpd -port 20806 -status n202
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n202'
startmpich2.sh: missing smpd on n202
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/gnu//bin/smpd -port 20806 -status n201
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n201'
startmpich2.sh: missing smpd on n201
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/gnu//bin/smpd -port 20806 -status n203
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n203'
startmpich2.sh: missing smpd on n203
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/gnu//bin/smpd -port 20806 -status n207
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n207'
startmpich2.sh: missing smpd on n207
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/gnu//bin/smpd -port 20806 -status n206
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n206'
startmpich2.sh: missing smpd on n206
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/gnu//bin/smpd -port 20806 -status n204
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n204'
startmpich2.sh: found running smpd on n204
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/gnu//bin/smpd -port 20806 -status n208
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n208'
startmpich2.sh: found running smpd on n208
+ '[' 8 -eq 3 ']'
+ sleep 5
+ (( actual_retry += 1 ))
+ '[' 3 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (3 of 10)'
startmpich2.sh: check for smpd daemons (3 of 10)
+ numhosts=0
++ cat /tmp/806.1.ibm/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/gnu//bin/smpd -port 20806 -status n205
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n205'
startmpich2.sh: found running smpd on n205
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/gnu//bin/smpd -port 20806 -status n202
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n202'
startmpich2.sh: found running smpd on n202
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/gnu//bin/smpd -port 20806 -status n201
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n201'
startmpich2.sh: found running smpd on n201
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/gnu//bin/smpd -port 20806 -status n203
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n203'
startmpich2.sh: missing smpd on n203
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/gnu//bin/smpd -port 20806 -status n207
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n207'
startmpich2.sh: missing smpd on n207
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/gnu//bin/smpd -port 20806 -status n206
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n206'
startmpich2.sh: missing smpd on n206
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/gnu//bin/smpd -port 20806 -status n204
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n204'
startmpich2.sh: found running smpd on n204
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/gnu//bin/smpd -port 20806 -status n208
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n208'
startmpich2.sh: found running smpd on n208
+ '[' 8 -eq 5 ']'
+ sleep 5
+ (( actual_retry += 1 ))
+ '[' 4 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (4 of 10)'
startmpich2.sh: check for smpd daemons (4 of 10)
+ numhosts=0
++ cat /tmp/806.1.ibm/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/gnu//bin/smpd -port 20806 -status n205
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n205'
startmpich2.sh: found running smpd on n205
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/gnu//bin/smpd -port 20806 -status n202
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n202'
startmpich2.sh: found running smpd on n202
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/gnu//bin/smpd -port 20806 -status n201
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n201'
startmpich2.sh: found running smpd on n201
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/gnu//bin/smpd -port 20806 -status n203
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n203'
startmpich2.sh: found running smpd on n203
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/gnu//bin/smpd -port 20806 -status n207
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n207'
startmpich2.sh: found running smpd on n207
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/gnu//bin/smpd -port 20806 -status n206
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n206'
startmpich2.sh: found running smpd on n206
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/gnu//bin/smpd -port 20806 -status n204
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n204'
startmpich2.sh: found running smpd on n204
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/gnu//bin/smpd -port 20806 -status n208
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n208'
startmpich2.sh: found running smpd on n208
+ '[' 8 -eq 8 ']'
+ echo 'startmpich2.sh: got all 8 of 8 nodes'
startmpich2.sh: got all 8 of 8 nodes
+ exit 0
/tmp/806.1.ibm:/usr/local/bin:/bin:/usr/bin
rm: cannot remove `*.phi': No such file or directory
rm: cannot remove `*outputkernel*': No such file or directory
rm: cannot remove `wnohu*': No such file or directory
START
END
rm: cannot remove `*vector*': No such file or directory
-catch_rsh /opt/mpich2/gnu/
Your job finished at Mon Oct 31 17:27:23 EDT 2011