USER      =  haule
HOMEDIR   =  /home/haule
JOB_ID    =  31670
JOB_NAME  =  UO
QUEUE     =  wp12_hp
NSLOTS    =  100
NHOSTS    =  9

SUB_HOST  =  rupc06
SUB_DIR   =  /mnt/wk06/haule/materials/newDMFT/UO

EXE_HOST  =  n183
EXE_DIR   =  /mnt/n183
OUT_DIR   =  /scratch
TMP_DIR   =  /tmp/31670.1.wp12_hp

INFO      =  Files larger than 100MB will be compressed to reduce the network load.
INFO2     =  Do NOT remove this file while job is running!

PORT      =  21670
rcp: /mnt/wk06/haule/materials/newDMFT/UO/31665haule: not a plain file
rcp: /mnt/wk06/haule/materials/newDMFT/UO/31667haule: not a plain file
rcp: /mnt/wk06/haule/materials/newDMFT/UO/LDA: not a plain file
Failure to rcp files!

Your job started at Mon Jul 16 14:21:48 EDT 2012 on host n183 in /mnt/n183/31670haule
Submission directory is /mnt/wk06/haule/materials/newDMFT/UO on host rupc06
-----------------------------------------------------------------------------------------
+ echo -catch_rsh /opt/sge6/spool/n183/active_jobs/31670.1/pe_hostfile /opt/mpich2/intel/11.1/
-catch_rsh /opt/sge6/spool/n183/active_jobs/31670.1/pe_hostfile /opt/mpich2/intel/11.1/
+ SLEEPTIME=5
+ RETRIES=10
+ catch_rsh=0
+ catch_hostname=0
+ '[' 3 -gt 0 ']'
+ case "$1" in
+ catch_rsh=1
+ shift
+ '[' 2 -gt 0 ']'
+ case "$1" in
+ break
++ basename /opt/sge6/mpich2_smpd/startmpich2.sh
+ me=startmpich2.sh
+ '[' 2 -lt 2 ']'
+ pe_hostfile=/opt/sge6/spool/n183/active_jobs/31670.1/pe_hostfile
+ MPICH2_ROOT=/opt/mpich2/intel/11.1/
+ export MPICH2_ROOT
+ '[' '!' -x /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 ']'
+ '[' '!' -x /opt/mpich2/intel/11.1//bin/smpd ']'
+ '[' '!' -r /opt/sge6/spool/n183/active_jobs/31670.1/pe_hostfile ']'
+ machines=/tmp/31670.1.wp12_hp/machines
+ PeHostfile2MachineFile /opt/sge6/spool/n183/active_jobs/31670.1/pe_hostfile
+ cat /opt/sge6/spool/n183/active_jobs/31670.1/pe_hostfile
+ read line
++ echo n183 12 wp12_hp@n183 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n183
++ echo n183 12 wp12_hp@n183 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n183
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n183
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n183
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n183
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n183
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n183
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n183
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n183
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n183
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n183
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n183
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n183
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n184 12 wp12_hp@n184 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n184
++ echo n184 12 wp12_hp@n184 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n184
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n184
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n184
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n184
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n184
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n184
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n184
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n184
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n184
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n184
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n184
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n184
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n185 12 wp12_hp@n185 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n185
++ echo n185 12 wp12_hp@n185 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n185
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n185
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n185
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n185
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n185
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n185
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n185
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n185
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n185
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n185
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n185
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n185
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n166 12 wp12_hp@n166 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n166
++ echo n166 12 wp12_hp@n166 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n166
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n166
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n166
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n166
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n166
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n166
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n166
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n166
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n166
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n166
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n166
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n166
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n167 12 wp12_hp@n167 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n167
++ echo n167 12 wp12_hp@n167 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n167
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n167
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n167
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n167
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n167
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n167
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n167
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n167
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n167
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n167
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n167
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n167
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n178 8 wp12_hp@n178 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n178
++ echo n178 8 wp12_hp@n178 UNDEFINED
++ cut -f2 '-d '
+ nslots=8
+ i=1
+ '[' 1 -le 8 ']'
+ echo n178
++ expr 1 + 1
+ i=2
+ '[' 2 -le 8 ']'
+ echo n178
++ expr 2 + 1
+ i=3
+ '[' 3 -le 8 ']'
+ echo n178
++ expr 3 + 1
+ i=4
+ '[' 4 -le 8 ']'
+ echo n178
++ expr 4 + 1
+ i=5
+ '[' 5 -le 8 ']'
+ echo n178
++ expr 5 + 1
+ i=6
+ '[' 6 -le 8 ']'
+ echo n178
++ expr 6 + 1
+ i=7
+ '[' 7 -le 8 ']'
+ echo n178
++ expr 7 + 1
+ i=8
+ '[' 8 -le 8 ']'
+ echo n178
++ expr 8 + 1
+ i=9
+ '[' 9 -le 8 ']'
+ read line
++ echo n179 12 wp12_hp@n179 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n179
++ echo n179 12 wp12_hp@n179 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n179
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n179
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n179
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n179
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n179
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n179
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n179
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n179
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n179
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n179
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n179
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n179
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n180 12 wp12_hp@n180 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n180
++ echo n180 12 wp12_hp@n180 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n180
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n180
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n180
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n180
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n180
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n180
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n180
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n180
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n180
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n180
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n180
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n180
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n170 8 wp12_hp@n170 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n170
++ echo n170 8 wp12_hp@n170 UNDEFINED
++ cut -f2 '-d '
+ nslots=8
+ i=1
+ '[' 1 -le 8 ']'
+ echo n170
++ expr 1 + 1
+ i=2
+ '[' 2 -le 8 ']'
+ echo n170
++ expr 2 + 1
+ i=3
+ '[' 3 -le 8 ']'
+ echo n170
++ expr 3 + 1
+ i=4
+ '[' 4 -le 8 ']'
+ echo n170
++ expr 4 + 1
+ i=5
+ '[' 5 -le 8 ']'
+ echo n170
++ expr 5 + 1
+ i=6
+ '[' 6 -le 8 ']'
+ echo n170
++ expr 6 + 1
+ i=7
+ '[' 7 -le 8 ']'
+ echo n170
++ expr 7 + 1
+ i=8
+ '[' 8 -le 8 ']'
+ echo n170
++ expr 8 + 1
+ i=9
+ '[' 9 -le 8 ']'
+ read line
+ cat /tmp/31670.1.wp12_hp/machines
n183
n183
n183
n183
n183
n183
n183
n183
n183
n183
n183
n183
n184
n184
n184
n184
n184
n184
n184
n184
n184
n184
n184
n184
n185
n185
n185
n185
n185
n185
n185
n185
n185
n185
n185
n185
n166
n166
n166
n166
n166
n166
n166
n166
n166
n166
n166
n166
n167
n167
n167
n167
n167
n167
n167
n167
n167
n167
n167
n167
n178
n178
n178
n178
n178
n178
n178
n178
n179
n179
n179
n179
n179
n179
n179
n179
n179
n179
n179
n179
n180
n180
n180
n180
n180
n180
n180
n180
n180
n180
n180
n180
n170
n170
n170
n170
n170
n170
n170
n170
+ '[' 1 = 1 ']'
+ rsh_wrapper=/opt/sge6/mpich2_smpd/rsh
+ '[' '!' -x /opt/sge6/mpich2_smpd/rsh ']'
+ rshcmd=rsh
+ case "$ARC" in
+ ln -s /opt/sge6/mpich2_smpd/rsh /tmp/31670.1.wp12_hp/rsh
+ hash -r
+ '[' 0 = 1 ']'
+ port=21670
++ cat /tmp/31670.1.wp12_hp/machines
++ uniq
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n183 /opt/mpich2/intel/11.1//bin/smpd 21670
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n184 /opt/mpich2/intel/11.1//bin/smpd 21670
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n185 /opt/mpich2/intel/11.1//bin/smpd 21670
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n166 /opt/mpich2/intel/11.1//bin/smpd 21670
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n167 /opt/mpich2/intel/11.1//bin/smpd 21670
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n178 /opt/mpich2/intel/11.1//bin/smpd 21670
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n179 /opt/mpich2/intel/11.1//bin/smpd 21670
rsh n183 /opt/mpich2/intel/11.1//bin/smpd -port 21670 -d 0
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n180 /opt/mpich2/intel/11.1//bin/smpd 21670
rsh n184 /opt/mpich2/intel/11.1//bin/smpd -port 21670 -d 0
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n170 /opt/mpich2/intel/11.1//bin/smpd 21670
rsh n185 /opt/mpich2/intel/11.1//bin/smpd -port 21670 -d 0
rsh n166 /opt/mpich2/intel/11.1//bin/smpd -port 21670 -d 0
+ actual_retry=1
+ '[' 1 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (1 of 10)'
startmpich2.sh: check for smpd daemons (1 of 10)
+ numhosts=0
++ cat /tmp/31670.1.wp12_hp/machines
++ uniq
rsh n167 /opt/mpich2/intel/11.1//bin/smpd -port 21670 -d 0
rsh n178 /opt/mpich2/intel/11.1//bin/smpd -port 21670 -d 0
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21670 -status n183
++ head -n 1
++ cut -f 1 -d ' '
rsh n179 /opt/mpich2/intel/11.1//bin/smpd -port 21670 -d 0
rsh n170 /opt/mpich2/intel/11.1//bin/smpd -port 21670 -d 0
rsh n180 /opt/mpich2/intel/11.1//bin/smpd -port 21670 -d 0
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n183'
startmpich2.sh: missing smpd on n183
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21670 -status n184
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n184'
startmpich2.sh: missing smpd on n184
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21670 -status n185
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n185'
startmpich2.sh: missing smpd on n185
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21670 -status n166
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n166'
startmpich2.sh: missing smpd on n166
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21670 -status n167
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n167'
startmpich2.sh: missing smpd on n167
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21670 -status n178
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n178'
startmpich2.sh: missing smpd on n178
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21670 -status n179
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n179'
startmpich2.sh: missing smpd on n179
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21670 -status n180
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n180'
startmpich2.sh: missing smpd on n180
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21670 -status n170
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n170'
startmpich2.sh: missing smpd on n170
+ '[' 9 -eq 0 ']'
+ sleep 5
+ (( actual_retry += 1 ))
+ '[' 2 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (2 of 10)'
startmpich2.sh: check for smpd daemons (2 of 10)
+ numhosts=0
++ cat /tmp/31670.1.wp12_hp/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21670 -status n183
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n183'
startmpich2.sh: found running smpd on n183
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21670 -status n184
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n184'
startmpich2.sh: found running smpd on n184
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21670 -status n185
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n185'
startmpich2.sh: found running smpd on n185
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21670 -status n166
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n166'
startmpich2.sh: found running smpd on n166
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21670 -status n167
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n167'
startmpich2.sh: found running smpd on n167
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21670 -status n178
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n178'
startmpich2.sh: found running smpd on n178
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21670 -status n179
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n179'
startmpich2.sh: found running smpd on n179
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21670 -status n180
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n180'
startmpich2.sh: found running smpd on n180
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21670 -status n170
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n170'
startmpich2.sh: found running smpd on n170
+ '[' 9 -eq 9 ']'
+ echo 'startmpich2.sh: got all 9 of 9 nodes'
startmpich2.sh: got all 9 of 9 nodes
+ exit 0
Ncpu= 74 Nrest= 0
cleaning /mnt/n183/31670haule 
-catch_rsh /opt/mpich2/intel/11.1/
Your job finished at Tue Jul 17 02:50:29 EDT 2012