USER      =  haule
HOMEDIR   =  /home/haule
JOB_ID    =  31654
JOB_NAME  =  UO2
QUEUE     =  wp12_hp
NSLOTS    =  100
NHOSTS    =  9

SUB_HOST  =  rupc06
SUB_DIR   =  /mnt/wk06/haule/materials/newDMFT/UO2

EXE_HOST  =  n168
EXE_DIR   =  /mnt/n168
OUT_DIR   =  /scratch
TMP_DIR   =  /tmp/31654.1.wp12_hp

INFO      =  Files larger than 100MB will be compressed to reduce the network load.
INFO2     =  Do NOT remove this file while job is running!

PORT      =  21654
rcp: /mnt/wk06/haule/materials/newDMFT/UO2/31603haule: not a plain file
rcp: /mnt/wk06/haule/materials/newDMFT/UO2/31606haule: not a plain file
rcp: /mnt/wk06/haule/materials/newDMFT/UO2/31607haule: not a plain file
rcp: /mnt/wk06/haule/materials/newDMFT/UO2/31649haule: not a plain file
Failure to rcp files!

Your job started at Mon Jul 16 12:37:09 EDT 2012 on host n168 in /mnt/n168/31654haule
Submission directory is /mnt/wk06/haule/materials/newDMFT/UO2 on host rupc06
-----------------------------------------------------------------------------------------
+ echo -catch_rsh /opt/sge6/spool/n168/active_jobs/31654.1/pe_hostfile /opt/mpich2/intel/11.1/
-catch_rsh /opt/sge6/spool/n168/active_jobs/31654.1/pe_hostfile /opt/mpich2/intel/11.1/
+ SLEEPTIME=5
+ RETRIES=10
+ catch_rsh=0
+ catch_hostname=0
+ '[' 3 -gt 0 ']'
+ case "$1" in
+ catch_rsh=1
+ shift
+ '[' 2 -gt 0 ']'
+ case "$1" in
+ break
++ basename /opt/sge6/mpich2_smpd/startmpich2.sh
+ me=startmpich2.sh
+ '[' 2 -lt 2 ']'
+ pe_hostfile=/opt/sge6/spool/n168/active_jobs/31654.1/pe_hostfile
+ MPICH2_ROOT=/opt/mpich2/intel/11.1/
+ export MPICH2_ROOT
+ '[' '!' -x /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 ']'
+ '[' '!' -x /opt/mpich2/intel/11.1//bin/smpd ']'
+ '[' '!' -r /opt/sge6/spool/n168/active_jobs/31654.1/pe_hostfile ']'
+ machines=/tmp/31654.1.wp12_hp/machines
+ PeHostfile2MachineFile /opt/sge6/spool/n168/active_jobs/31654.1/pe_hostfile
+ cat /opt/sge6/spool/n168/active_jobs/31654.1/pe_hostfile
+ read line
++ echo n168 12 wp12_hp@n168 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n168
++ echo n168 12 wp12_hp@n168 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n168
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n168
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n168
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n168
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n168
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n168
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n168
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n168
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n168
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n168
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n168
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n168
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n169 12 wp12_hp@n169 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n169
++ echo n169 12 wp12_hp@n169 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n169
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n169
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n169
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n169
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n169
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n169
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n169
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n169
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n169
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n169
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n169
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n169
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n172 12 wp12_hp@n172 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n172
++ echo n172 12 wp12_hp@n172 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n172
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n172
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n172
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n172
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n172
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n172
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n172
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n172
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n172
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n172
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n172
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n172
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n173 12 wp12_hp@n173 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n173
++ echo n173 12 wp12_hp@n173 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n173
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n173
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n173
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n173
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n173
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n173
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n173
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n173
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n173
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n173
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n173
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n173
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n174 12 wp12_hp@n174 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n174
++ echo n174 12 wp12_hp@n174 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n174
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n174
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n174
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n174
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n174
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n174
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n174
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n174
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n174
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n174
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n174
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n174
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n175 12 wp12_hp@n175 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n175
++ echo n175 12 wp12_hp@n175 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n175
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n175
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n175
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n175
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n175
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n175
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n175
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n175
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n175
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n175
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n175
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n175
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n176 12 wp12_hp@n176 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n176
++ echo n176 12 wp12_hp@n176 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n176
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n176
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n176
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n176
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n176
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n176
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n176
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n176
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n176
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n176
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n176
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n176
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n177 12 wp12_hp@n177 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n177
++ echo n177 12 wp12_hp@n177 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n177
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n177
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n177
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n177
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n177
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n177
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n177
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n177
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n177
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n177
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n177
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n177
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n178 4 wp12_hp@n178 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n178
++ echo n178 4 wp12_hp@n178 UNDEFINED
++ cut -f2 '-d '
+ nslots=4
+ i=1
+ '[' 1 -le 4 ']'
+ echo n178
++ expr 1 + 1
+ i=2
+ '[' 2 -le 4 ']'
+ echo n178
++ expr 2 + 1
+ i=3
+ '[' 3 -le 4 ']'
+ echo n178
++ expr 3 + 1
+ i=4
+ '[' 4 -le 4 ']'
+ echo n178
++ expr 4 + 1
+ i=5
+ '[' 5 -le 4 ']'
+ read line
+ cat /tmp/31654.1.wp12_hp/machines
n168
n168
n168
n168
n168
n168
n168
n168
n168
n168
n168
n168
n169
n169
n169
n169
n169
n169
n169
n169
n169
n169
n169
n169
n172
n172
n172
n172
n172
n172
n172
n172
n172
n172
n172
n172
n173
n173
n173
n173
n173
n173
n173
n173
n173
n173
n173
n173
n174
n174
n174
n174
n174
n174
n174
n174
n174
n174
n174
n174
n175
n175
n175
n175
n175
n175
n175
n175
n175
n175
n175
n175
n176
n176
n176
n176
n176
n176
n176
n176
n176
n176
n176
n176
n177
n177
n177
n177
n177
n177
n177
n177
n177
n177
n177
n177
n178
n178
n178
n178
+ '[' 1 = 1 ']'
+ rsh_wrapper=/opt/sge6/mpich2_smpd/rsh
+ '[' '!' -x /opt/sge6/mpich2_smpd/rsh ']'
+ rshcmd=rsh
+ case "$ARC" in
+ ln -s /opt/sge6/mpich2_smpd/rsh /tmp/31654.1.wp12_hp/rsh
+ hash -r
+ '[' 0 = 1 ']'
+ port=21654
++ cat /tmp/31654.1.wp12_hp/machines
++ uniq
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n168 /opt/mpich2/intel/11.1//bin/smpd 21654
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n169 /opt/mpich2/intel/11.1//bin/smpd 21654
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n172 /opt/mpich2/intel/11.1//bin/smpd 21654
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n173 /opt/mpich2/intel/11.1//bin/smpd 21654
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n174 /opt/mpich2/intel/11.1//bin/smpd 21654
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n175 /opt/mpich2/intel/11.1//bin/smpd 21654
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n176 /opt/mpich2/intel/11.1//bin/smpd 21654
rsh n168 /opt/mpich2/intel/11.1//bin/smpd -port 21654 -d 0
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n177 /opt/mpich2/intel/11.1//bin/smpd 21654
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n178 /opt/mpich2/intel/11.1//bin/smpd 21654
rsh n169 /opt/mpich2/intel/11.1//bin/smpd -port 21654 -d 0
rsh n172 /opt/mpich2/intel/11.1//bin/smpd -port 21654 -d 0
+ actual_retry=1
+ '[' 1 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (1 of 10)'
rsh n173 /opt/mpich2/intel/11.1//bin/smpd -port 21654 -d 0
startmpich2.sh: check for smpd daemons (1 of 10)
+ numhosts=0
++ cat /tmp/31654.1.wp12_hp/machines
++ uniq
rsh n174 /opt/mpich2/intel/11.1//bin/smpd -port 21654 -d 0
rsh n175 /opt/mpich2/intel/11.1//bin/smpd -port 21654 -d 0
+ for host in '`cat $machines | uniq`'
rsh n176 /opt/mpich2/intel/11.1//bin/smpd -port 21654 -d 0
++ /opt/mpich2/intel/11.1//bin/smpd -port 21654 -status n168
++ head -n 1
++ cut -f 1 -d ' '
rsh n177 /opt/mpich2/intel/11.1//bin/smpd -port 21654 -d 0
rsh n178 /opt/mpich2/intel/11.1//bin/smpd -port 21654 -d 0
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n168'
startmpich2.sh: missing smpd on n168
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21654 -status n169
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n169'
startmpich2.sh: missing smpd on n169
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21654 -status n172
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n172'
startmpich2.sh: missing smpd on n172
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21654 -status n173
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n173'
startmpich2.sh: missing smpd on n173
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21654 -status n174
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n174'
startmpich2.sh: missing smpd on n174
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21654 -status n175
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n175'
startmpich2.sh: missing smpd on n175
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21654 -status n176
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n176'
startmpich2.sh: missing smpd on n176
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21654 -status n177
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n177'
startmpich2.sh: missing smpd on n177
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21654 -status n178
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n178'
startmpich2.sh: missing smpd on n178
+ '[' 9 -eq 0 ']'
+ sleep 5
+ (( actual_retry += 1 ))
+ '[' 2 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (2 of 10)'
startmpich2.sh: check for smpd daemons (2 of 10)
+ numhosts=0
++ cat /tmp/31654.1.wp12_hp/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21654 -status n168
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n168'
startmpich2.sh: found running smpd on n168
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21654 -status n169
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n169'
startmpich2.sh: found running smpd on n169
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21654 -status n172
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n172'
startmpich2.sh: found running smpd on n172
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21654 -status n173
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n173'
startmpich2.sh: found running smpd on n173
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21654 -status n174
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n174'
startmpich2.sh: found running smpd on n174
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21654 -status n175
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n175'
startmpich2.sh: found running smpd on n175
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21654 -status n176
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n176'
startmpich2.sh: found running smpd on n176
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21654 -status n177
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n177'
startmpich2.sh: found running smpd on n177
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21654 -status n178
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n178'
startmpich2.sh: found running smpd on n178
+ '[' 9 -eq 9 ']'
+ echo 'startmpich2.sh: got all 9 of 9 nodes'
startmpich2.sh: got all 9 of 9 nodes
+ exit 0
Ncpu= 74 Nrest= 0
cleaning /mnt/n168/31654haule 
-catch_rsh /opt/mpich2/intel/11.1/
Your job finished at Tue Jul 17 13:17:16 EDT 2012