USER      =  yinzping
HOMEDIR   =  /home/yinzping
JOB_ID    =  56226
JOB_NAME  =  NFAPM
QUEUE     =  wp10
NSLOTS    =  72
NHOSTS    =  15

SUB_HOST  =  rupc06
SUB_DIR   =  /mnt/wk05/yinzping/WIEN2k/DMFT/FeAs/NaFeAs/P4nmm/U5J8/beta160/Full2/2438yinzping/10607yinzping/22021yinzping

EXE_HOST  =  n148
NFS_HOST  =  caip02s
EXE_DIR   =  /mnt/p06
OUT_DIR   =  /tmp
TMP_DIR   =  /tmp/56226.1.wp10

INFO      =  Files larger than 100MB will be compressed to reduce the network load.
INFO2     =  Do NOT remove this file while job is running!

PORT      =  21226
rcp: /mnt/wk05/yinzping/WIEN2k/DMFT/FeAs/NaFeAs/P4nmm/U5J8/beta160/Full2/2438yinzping/10607yinzping/22021yinzping/Vertex: not a plain file
rcp: /mnt/wk05/yinzping/WIEN2k/DMFT/FeAs/NaFeAs/P4nmm/U5J8/beta160/Full2/2438yinzping/10607yinzping/22021yinzping/imp.0: not a plain file
Failure to rcp files!

Your job started at Thu Sep 12 19:46:23 EDT 2013 on host n148 in /mnt/p06/56226yinzping
Submission directory is /mnt/wk05/yinzping/WIEN2k/DMFT/FeAs/NaFeAs/P4nmm/U5J8/beta160/Full2/2438yinzping/10607yinzping/22021yinzping on host rupc06
-----------------------------------------------------------------------------------------
+ echo -catch_rsh /opt/sge6/spool/n148/active_jobs/56226.1/pe_hostfile /opt/mpich2/intel/11.1/
-catch_rsh /opt/sge6/spool/n148/active_jobs/56226.1/pe_hostfile /opt/mpich2/intel/11.1/
+ SLEEPTIME=5
+ RETRIES=10
+ catch_rsh=0
+ catch_hostname=0
+ '[' 3 -gt 0 ']'
+ case "$1" in
+ catch_rsh=1
+ shift
+ '[' 2 -gt 0 ']'
+ case "$1" in
+ break
++ basename /opt/sge6/mpich2_smpd/startmpich2.sh
+ me=startmpich2.sh
+ '[' 2 -lt 2 ']'
+ pe_hostfile=/opt/sge6/spool/n148/active_jobs/56226.1/pe_hostfile
+ MPICH2_ROOT=/opt/mpich2/intel/11.1/
+ export MPICH2_ROOT
+ '[' '!' -x /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 ']'
+ '[' '!' -x /opt/mpich2/intel/11.1//bin/smpd ']'
+ '[' '!' -r /opt/sge6/spool/n148/active_jobs/56226.1/pe_hostfile ']'
+ machines=/tmp/56226.1.wp10/machines
+ PeHostfile2MachineFile /opt/sge6/spool/n148/active_jobs/56226.1/pe_hostfile
+ cat /opt/sge6/spool/n148/active_jobs/56226.1/pe_hostfile
+ read line
++ echo n148 4 wp10@n148 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n148
++ echo n148 4 wp10@n148 UNDEFINED
++ cut -f2 '-d '
+ nslots=4
+ i=1
+ '[' 1 -le 4 ']'
+ echo n148
++ expr 1 + 1
+ i=2
+ '[' 2 -le 4 ']'
+ echo n148
++ expr 2 + 1
+ i=3
+ '[' 3 -le 4 ']'
+ echo n148
++ expr 3 + 1
+ i=4
+ '[' 4 -le 4 ']'
+ echo n148
++ expr 4 + 1
+ i=5
+ '[' 5 -le 4 ']'
+ read line
++ echo n162 1 wp10@n162 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n162
++ echo n162 1 wp10@n162 UNDEFINED
++ cut -f2 '-d '
+ nslots=1
+ i=1
+ '[' 1 -le 1 ']'
+ echo n162
++ expr 1 + 1
+ i=2
+ '[' 2 -le 1 ']'
+ read line
++ echo n160 1 wp10@n160 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n160
++ echo n160 1 wp10@n160 UNDEFINED
++ cut -f2 '-d '
+ nslots=1
+ i=1
+ '[' 1 -le 1 ']'
+ echo n160
++ expr 1 + 1
+ i=2
+ '[' 2 -le 1 ']'
+ read line
++ echo n149 2 wp10@n149 UNDEFINED
++ cut -f1 -d.
++ cut -f1 '-d '
+ host=n149
++ echo n149 2 wp10@n149 UNDEFINED
++ cut -f2 '-d '
+ nslots=2
+ i=1
+ '[' 1 -le 2 ']'
+ echo n149
++ expr 1 + 1
+ i=2
+ '[' 2 -le 2 ']'
+ echo n149
++ expr 2 + 1
+ i=3
+ '[' 3 -le 2 ']'
+ read line
++ echo n158 1 wp10@n158 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n158
++ echo n158 1 wp10@n158 UNDEFINED
++ cut -f2 '-d '
+ nslots=1
+ i=1
+ '[' 1 -le 1 ']'
+ echo n158
++ expr 1 + 1
+ i=2
+ '[' 2 -le 1 ']'
+ read line
++ echo n147 12 wp10@n147 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n147
++ echo n147 12 wp10@n147 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n147
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n147
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n147
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n147
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n147
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n147
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n147
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n147
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n147
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n147
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n147
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n147
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n163 2 wp10@n163 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n163
++ echo n163 2 wp10@n163 UNDEFINED
++ cut -f2 '-d '
+ nslots=2
+ i=1
+ '[' 1 -le 2 ']'
+ echo n163
++ expr 1 + 1
+ i=2
+ '[' 2 -le 2 ']'
+ echo n163
++ expr 2 + 1
+ i=3
+ '[' 3 -le 2 ']'
+ read line
++ echo n159 3 wp10@n159 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n159
++ echo n159 3 wp10@n159 UNDEFINED
++ cut -f2 '-d '
+ nslots=3
+ i=1
+ '[' 1 -le 3 ']'
+ echo n159
++ expr 1 + 1
+ i=2
+ '[' 2 -le 3 ']'
+ echo n159
++ expr 2 + 1
+ i=3
+ '[' 3 -le 3 ']'
+ echo n159
++ expr 3 + 1
+ i=4
+ '[' 4 -le 3 ']'
+ read line
++ echo n151 7 wp10@n151 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n151
++ echo n151 7 wp10@n151 UNDEFINED
++ cut -f2 '-d '
+ nslots=7
+ i=1
+ '[' 1 -le 7 ']'
+ echo n151
++ expr 1 + 1
+ i=2
+ '[' 2 -le 7 ']'
+ echo n151
++ expr 2 + 1
+ i=3
+ '[' 3 -le 7 ']'
+ echo n151
++ expr 3 + 1
+ i=4
+ '[' 4 -le 7 ']'
+ echo n151
++ expr 4 + 1
+ i=5
+ '[' 5 -le 7 ']'
+ echo n151
++ expr 5 + 1
+ i=6
+ '[' 6 -le 7 ']'
+ echo n151
++ expr 6 + 1
+ i=7
+ '[' 7 -le 7 ']'
+ echo n151
++ expr 7 + 1
+ i=8
+ '[' 8 -le 7 ']'
+ read line
++ echo n150 8 wp10@n150 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n150
++ echo n150 8 wp10@n150 UNDEFINED
++ cut -f2 '-d '
+ nslots=8
+ i=1
+ '[' 1 -le 8 ']'
+ echo n150
++ expr 1 + 1
+ i=2
+ '[' 2 -le 8 ']'
+ echo n150
++ expr 2 + 1
+ i=3
+ '[' 3 -le 8 ']'
+ echo n150
++ expr 3 + 1
+ i=4
+ '[' 4 -le 8 ']'
+ echo n150
++ expr 4 + 1
+ i=5
+ '[' 5 -le 8 ']'
+ echo n150
++ expr 5 + 1
+ i=6
+ '[' 6 -le 8 ']'
+ echo n150
++ expr 6 + 1
+ i=7
+ '[' 7 -le 8 ']'
+ echo n150
++ expr 7 + 1
+ i=8
+ '[' 8 -le 8 ']'
+ echo n150
++ expr 8 + 1
+ i=9
+ '[' 9 -le 8 ']'
+ read line
++ echo n141 7 wp10@n141 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n141
++ echo n141 7 wp10@n141 UNDEFINED
++ cut -f2 '-d '
+ nslots=7
+ i=1
+ '[' 1 -le 7 ']'
+ echo n141
++ expr 1 + 1
+ i=2
+ '[' 2 -le 7 ']'
+ echo n141
++ expr 2 + 1
+ i=3
+ '[' 3 -le 7 ']'
+ echo n141
++ expr 3 + 1
+ i=4
+ '[' 4 -le 7 ']'
+ echo n141
++ expr 4 + 1
+ i=5
+ '[' 5 -le 7 ']'
+ echo n141
++ expr 5 + 1
+ i=6
+ '[' 6 -le 7 ']'
+ echo n141
++ expr 6 + 1
+ i=7
+ '[' 7 -le 7 ']'
+ echo n141
++ expr 7 + 1
+ i=8
+ '[' 8 -le 7 ']'
+ read line
++ echo n154 7 wp10@n154 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n154
++ echo n154 7 wp10@n154 UNDEFINED
++ cut -f2 '-d '
+ nslots=7
+ i=1
+ '[' 1 -le 7 ']'
+ echo n154
++ expr 1 + 1
+ i=2
+ '[' 2 -le 7 ']'
+ echo n154
++ expr 2 + 1
+ i=3
+ '[' 3 -le 7 ']'
+ echo n154
++ expr 3 + 1
+ i=4
+ '[' 4 -le 7 ']'
+ echo n154
++ expr 4 + 1
+ i=5
+ '[' 5 -le 7 ']'
+ echo n154
++ expr 5 + 1
+ i=6
+ '[' 6 -le 7 ']'
+ echo n154
++ expr 6 + 1
+ i=7
+ '[' 7 -le 7 ']'
+ echo n154
++ expr 7 + 1
+ i=8
+ '[' 8 -le 7 ']'
+ read line
++ echo n144 9 wp10@n144 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n144
++ echo n144 9 wp10@n144 UNDEFINED
++ cut -f2 '-d '
+ nslots=9
+ i=1
+ '[' 1 -le 9 ']'
+ echo n144
++ expr 1 + 1
+ i=2
+ '[' 2 -le 9 ']'
+ echo n144
++ expr 2 + 1
+ i=3
+ '[' 3 -le 9 ']'
+ echo n144
++ expr 3 + 1
+ i=4
+ '[' 4 -le 9 ']'
+ echo n144
++ expr 4 + 1
+ i=5
+ '[' 5 -le 9 ']'
+ echo n144
++ expr 5 + 1
+ i=6
+ '[' 6 -le 9 ']'
+ echo n144
++ expr 6 + 1
+ i=7
+ '[' 7 -le 9 ']'
+ echo n144
++ expr 7 + 1
+ i=8
+ '[' 8 -le 9 ']'
+ echo n144
++ expr 8 + 1
+ i=9
+ '[' 9 -le 9 ']'
+ echo n144
++ expr 9 + 1
+ i=10
+ '[' 10 -le 9 ']'
+ read line
++ echo n155 5 wp10@n155 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n155
++ echo n155 5 wp10@n155 UNDEFINED
++ cut -f2 '-d '
+ nslots=5
+ i=1
+ '[' 1 -le 5 ']'
+ echo n155
++ expr 1 + 1
+ i=2
+ '[' 2 -le 5 ']'
+ echo n155
++ expr 2 + 1
+ i=3
+ '[' 3 -le 5 ']'
+ echo n155
++ expr 3 + 1
+ i=4
+ '[' 4 -le 5 ']'
+ echo n155
++ expr 4 + 1
+ i=5
+ '[' 5 -le 5 ']'
+ echo n155
++ expr 5 + 1
+ i=6
+ '[' 6 -le 5 ']'
+ read line
++ echo n146 3 wp10@n146 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n146
++ echo n146 3 wp10@n146 UNDEFINED
++ cut -f2 '-d '
+ nslots=3
+ i=1
+ '[' 1 -le 3 ']'
+ echo n146
++ expr 1 + 1
+ i=2
+ '[' 2 -le 3 ']'
+ echo n146
++ expr 2 + 1
+ i=3
+ '[' 3 -le 3 ']'
+ echo n146
++ expr 3 + 1
+ i=4
+ '[' 4 -le 3 ']'
+ read line
+ cat /tmp/56226.1.wp10/machines
n148
n148
n148
n148
n162
n160
n149
n149
n158
n147
n147
n147
n147
n147
n147
n147
n147
n147
n147
n147
n147
n163
n163
n159
n159
n159
n151
n151
n151
n151
n151
n151
n151
n150
n150
n150
n150
n150
n150
n150
n150
n141
n141
n141
n141
n141
n141
n141
n154
n154
n154
n154
n154
n154
n154
n144
n144
n144
n144
n144
n144
n144
n144
n144
n155
n155
n155
n155
n155
n146
n146
n146
+ '[' 1 = 1 ']'
+ rsh_wrapper=/opt/sge6/mpich2_smpd/rsh
+ '[' '!' -x /opt/sge6/mpich2_smpd/rsh ']'
+ rshcmd=rsh
+ case "$ARC" in
+ ln -s /opt/sge6/mpich2_smpd/rsh /tmp/56226.1.wp10/rsh
+ hash -r
+ '[' 0 = 1 ']'
+ port=21226
++ cat /tmp/56226.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n148 /opt/mpich2/intel/11.1//bin/smpd 21226
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n162 /opt/mpich2/intel/11.1//bin/smpd 21226
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n160 /opt/mpich2/intel/11.1//bin/smpd 21226
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n149 /opt/mpich2/intel/11.1//bin/smpd 21226
rsh n162 /opt/mpich2/intel/11.1//bin/smpd -port 21226 -d 0
rsh n148 /opt/mpich2/intel/11.1//bin/smpd -port 21226 -d 0
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n158 /opt/mpich2/intel/11.1//bin/smpd 21226
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n147 /opt/mpich2/intel/11.1//bin/smpd 21226
rsh n160 /opt/mpich2/intel/11.1//bin/smpd -port 21226 -d 0
rsh n158 /opt/mpich2/intel/11.1//bin/smpd -port 21226 -d 0
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n163 /opt/mpich2/intel/11.1//bin/smpd 21226
rsh n149 /opt/mpich2/intel/11.1//bin/smpd -port 21226 -d 0
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n159 /opt/mpich2/intel/11.1//bin/smpd 21226
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n151 /opt/mpich2/intel/11.1//bin/smpd 21226
rsh n147 /opt/mpich2/intel/11.1//bin/smpd -port 21226 -d 0
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n150 /opt/mpich2/intel/11.1//bin/smpd 21226
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n141 /opt/mpich2/intel/11.1//bin/smpd 21226
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n154 /opt/mpich2/intel/11.1//bin/smpd 21226
rsh n163 /opt/mpich2/intel/11.1//bin/smpd -port 21226 -d 0
rsh n141 /opt/mpich2/intel/11.1//bin/smpd -port 21226 -d 0
rsh n159 /opt/mpich2/intel/11.1//bin/smpd -port 21226 -d 0
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n144 /opt/mpich2/intel/11.1//bin/smpd 21226
+ for host in '`cat $machines | uniq`'
rsh n150 /opt/mpich2/intel/11.1//bin/smpd -port 21226 -d 0
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n155 /opt/mpich2/intel/11.1//bin/smpd 21226
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n146 /opt/mpich2/intel/11.1//bin/smpd 21226
rsh n151 /opt/mpich2/intel/11.1//bin/smpd -port 21226 -d 0
rsh n144 /opt/mpich2/intel/11.1//bin/smpd -port 21226 -d 0
rsh n154 /opt/mpich2/intel/11.1//bin/smpd -port 21226 -d 0
+ actual_retry=1
+ '[' 1 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (1 of 10)'
startmpich2.sh: check for smpd daemons (1 of 10)
+ numhosts=0
++ cat /tmp/56226.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21226 -status n148
++ head -n 1
rsh n155 /opt/mpich2/intel/11.1//bin/smpd -port 21226 -d 0
++ cut -f 1 -d ' '
rsh n146 /opt/mpich2/intel/11.1//bin/smpd -port 21226 -d 0
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n148'
startmpich2.sh: missing smpd on n148
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21226 -status n162
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n162'
startmpich2.sh: missing smpd on n162
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21226 -status n160
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n160'
startmpich2.sh: missing smpd on n160
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21226 -status n149
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n149'
startmpich2.sh: missing smpd on n149
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21226 -status n158
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n158'
startmpich2.sh: missing smpd on n158
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21226 -status n147
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n147'
startmpich2.sh: missing smpd on n147
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21226 -status n163
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n163'
startmpich2.sh: missing smpd on n163
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21226 -status n159
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n159'
startmpich2.sh: missing smpd on n159
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21226 -status n151
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n151'
startmpich2.sh: missing smpd on n151
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21226 -status n150
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n150'
startmpich2.sh: missing smpd on n150
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21226 -status n141
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n141'
startmpich2.sh: missing smpd on n141
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21226 -status n154
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n154'
startmpich2.sh: found running smpd on n154
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21226 -status n144
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n144'
startmpich2.sh: found running smpd on n144
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21226 -status n155
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n155'
startmpich2.sh: found running smpd on n155
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21226 -status n146
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n146'
startmpich2.sh: found running smpd on n146
+ '[' 15 -eq 4 ']'
+ sleep 5
+ (( actual_retry += 1 ))
+ '[' 2 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (2 of 10)'
startmpich2.sh: check for smpd daemons (2 of 10)
+ numhosts=0
++ cat /tmp/56226.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21226 -status n148
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n148'
startmpich2.sh: found running smpd on n148
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21226 -status n162
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n162'
startmpich2.sh: found running smpd on n162
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21226 -status n160
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n160'
startmpich2.sh: found running smpd on n160
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21226 -status n149
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n149'
startmpich2.sh: found running smpd on n149
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21226 -status n158
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n158'
startmpich2.sh: found running smpd on n158
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21226 -status n147
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n147'
startmpich2.sh: found running smpd on n147
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21226 -status n163
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n163'
startmpich2.sh: found running smpd on n163
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21226 -status n159
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n159'
startmpich2.sh: found running smpd on n159
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21226 -status n151
++ cut -f 1 -d ' '
++ head -n 1
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n151'
startmpich2.sh: found running smpd on n151
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21226 -status n150
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n150'
startmpich2.sh: found running smpd on n150
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21226 -status n141
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n141'
startmpich2.sh: found running smpd on n141
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21226 -status n154
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n154'
startmpich2.sh: found running smpd on n154
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21226 -status n144
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n144'
startmpich2.sh: found running smpd on n144
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21226 -status n155
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n155'
startmpich2.sh: found running smpd on n155
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21226 -status n146
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n146'
startmpich2.sh: found running smpd on n146
+ '[' 15 -eq 15 ']'
+ echo 'startmpich2.sh: got all 15 of 15 nodes'
startmpich2.sh: got all 15 of 15 nodes
+ exit 0
rm: cannot remove `./*.?.[12]': No such file or directory
rm: cannot remove `./*.broyd*': No such file or directory
-catch_rsh /opt/mpich2/intel/11.1/
Your job finished at Tue Oct 1 14:25:51 EDT 2013