USER      =  jtomczak
HOMEDIR   =  /home/users/jtomczak
JOB_ID    =  17128
JOB_NAME  =  FS-LOW
QUEUE     =  wp12
NSLOTS    =  110
NHOSTS    =  13

SUB_HOST  =  rupc06
SUB_DIR   =  /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta200/nf6/NEW

EXE_HOST  =  n183
EXE_DIR   =  /mnt/n183
OUT_DIR   =  /scratch
TMP_DIR   =  /tmp/17128.1.wp12

INFO      =  Files larger than 100MB will be compressed to reduce the network load.
INFO2     =  Do NOT remove this file while job is running!

PORT      =  22128
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta200/nf6/NEW/FIX_MU: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta200/nf6/NEW/GO: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta200/nf6/NEW/ancont: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta200/nf6/NEW/imp.0: not a plain file
Failure to rcp files!

Your job started at Fri Oct 28 23:33:57 EDT 2011 on host n183 in /mnt/n183/17128jtomczak
Submission directory is /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta200/nf6/NEW on host rupc06
-----------------------------------------------------------------------------------------
+ echo -catch_rsh /opt/sge6/spool/n183/active_jobs/17128.1/pe_hostfile /opt/mpich2/intel/11.1/
-catch_rsh /opt/sge6/spool/n183/active_jobs/17128.1/pe_hostfile /opt/mpich2/intel/11.1/
+ SLEEPTIME=5
+ RETRIES=10
+ catch_rsh=0
+ catch_hostname=0
+ '[' 3 -gt 0 ']'
+ case "$1" in
+ catch_rsh=1
+ shift
+ '[' 2 -gt 0 ']'
+ case "$1" in
+ break
++ basename /opt/sge6/mpich2_smpd/startmpich2.sh
+ me=startmpich2.sh
+ '[' 2 -lt 2 ']'
+ pe_hostfile=/opt/sge6/spool/n183/active_jobs/17128.1/pe_hostfile
+ MPICH2_ROOT=/opt/mpich2/intel/11.1/
+ export MPICH2_ROOT
+ '[' '!' -x /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 ']'
+ '[' '!' -x /opt/mpich2/intel/11.1//bin/smpd ']'
+ '[' '!' -r /opt/sge6/spool/n183/active_jobs/17128.1/pe_hostfile ']'
+ machines=/tmp/17128.1.wp12/machines
+ PeHostfile2MachineFile /opt/sge6/spool/n183/active_jobs/17128.1/pe_hostfile
+ cat /opt/sge6/spool/n183/active_jobs/17128.1/pe_hostfile
+ read line
++ echo n183 9 wp12@n183 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n183
++ echo n183 9 wp12@n183 UNDEFINED
++ cut -f2 '-d '
+ nslots=9
+ i=1
+ '[' 1 -le 9 ']'
+ echo n183
++ expr 1 + 1
+ i=2
+ '[' 2 -le 9 ']'
+ echo n183
++ expr 2 + 1
+ i=3
+ '[' 3 -le 9 ']'
+ echo n183
++ expr 3 + 1
+ i=4
+ '[' 4 -le 9 ']'
+ echo n183
++ expr 4 + 1
+ i=5
+ '[' 5 -le 9 ']'
+ echo n183
++ expr 5 + 1
+ i=6
+ '[' 6 -le 9 ']'
+ echo n183
++ expr 6 + 1
+ i=7
+ '[' 7 -le 9 ']'
+ echo n183
++ expr 7 + 1
+ i=8
+ '[' 8 -le 9 ']'
+ echo n183
++ expr 8 + 1
+ i=9
+ '[' 9 -le 9 ']'
+ echo n183
++ expr 9 + 1
+ i=10
+ '[' 10 -le 9 ']'
+ read line
++ echo n170 9 wp12@n170 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n170
++ echo n170 9 wp12@n170 UNDEFINED
++ cut -f2 '-d '
+ nslots=9
+ i=1
+ '[' 1 -le 9 ']'
+ echo n170
++ expr 1 + 1
+ i=2
+ '[' 2 -le 9 ']'
+ echo n170
++ expr 2 + 1
+ i=3
+ '[' 3 -le 9 ']'
+ echo n170
++ expr 3 + 1
+ i=4
+ '[' 4 -le 9 ']'
+ echo n170
++ expr 4 + 1
+ i=5
+ '[' 5 -le 9 ']'
+ echo n170
++ expr 5 + 1
+ i=6
+ '[' 6 -le 9 ']'
+ echo n170
++ expr 6 + 1
+ i=7
+ '[' 7 -le 9 ']'
+ echo n170
++ expr 7 + 1
+ i=8
+ '[' 8 -le 9 ']'
+ echo n170
++ expr 8 + 1
+ i=9
+ '[' 9 -le 9 ']'
+ echo n170
++ expr 9 + 1
+ i=10
+ '[' 10 -le 9 ']'
+ read line
++ echo n174 8 wp12@n174 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n174
++ echo n174 8 wp12@n174 UNDEFINED
++ cut -f2 '-d '
+ nslots=8
+ i=1
+ '[' 1 -le 8 ']'
+ echo n174
++ expr 1 + 1
+ i=2
+ '[' 2 -le 8 ']'
+ echo n174
++ expr 2 + 1
+ i=3
+ '[' 3 -le 8 ']'
+ echo n174
++ expr 3 + 1
+ i=4
+ '[' 4 -le 8 ']'
+ echo n174
++ expr 4 + 1
+ i=5
+ '[' 5 -le 8 ']'
+ echo n174
++ expr 5 + 1
+ i=6
+ '[' 6 -le 8 ']'
+ echo n174
++ expr 6 + 1
+ i=7
+ '[' 7 -le 8 ']'
+ echo n174
++ expr 7 + 1
+ i=8
+ '[' 8 -le 8 ']'
+ echo n174
++ expr 8 + 1
+ i=9
+ '[' 9 -le 8 ']'
+ read line
++ echo n185 10 wp12@n185 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n185
++ echo n185 10 wp12@n185 UNDEFINED
++ cut -f2 '-d '
+ nslots=10
+ i=1
+ '[' 1 -le 10 ']'
+ echo n185
++ expr 1 + 1
+ i=2
+ '[' 2 -le 10 ']'
+ echo n185
++ expr 2 + 1
+ i=3
+ '[' 3 -le 10 ']'
+ echo n185
++ expr 3 + 1
+ i=4
+ '[' 4 -le 10 ']'
+ echo n185
++ expr 4 + 1
+ i=5
+ '[' 5 -le 10 ']'
+ echo n185
++ expr 5 + 1
+ i=6
+ '[' 6 -le 10 ']'
+ echo n185
++ expr 6 + 1
+ i=7
+ '[' 7 -le 10 ']'
+ echo n185
++ expr 7 + 1
+ i=8
+ '[' 8 -le 10 ']'
+ echo n185
++ expr 8 + 1
+ i=9
+ '[' 9 -le 10 ']'
+ echo n185
++ expr 9 + 1
+ i=10
+ '[' 10 -le 10 ']'
+ echo n185
++ expr 10 + 1
+ i=11
+ '[' 11 -le 10 ']'
+ read line
++ echo n184 1 wp12@n184 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n184
++ echo n184 1 wp12@n184 UNDEFINED
++ cut -f2 '-d '
+ nslots=1
+ i=1
+ '[' 1 -le 1 ']'
+ echo n184
++ expr 1 + 1
+ i=2
+ '[' 2 -le 1 ']'
+ read line
++ echo n171 11 wp12@n171 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n171
++ echo n171 11 wp12@n171 UNDEFINED
++ cut -f2 '-d '
+ nslots=11
+ i=1
+ '[' 1 -le 11 ']'
+ echo n171
++ expr 1 + 1
+ i=2
+ '[' 2 -le 11 ']'
+ echo n171
++ expr 2 + 1
+ i=3
+ '[' 3 -le 11 ']'
+ echo n171
++ expr 3 + 1
+ i=4
+ '[' 4 -le 11 ']'
+ echo n171
++ expr 4 + 1
+ i=5
+ '[' 5 -le 11 ']'
+ echo n171
++ expr 5 + 1
+ i=6
+ '[' 6 -le 11 ']'
+ echo n171
++ expr 6 + 1
+ i=7
+ '[' 7 -le 11 ']'
+ echo n171
++ expr 7 + 1
+ i=8
+ '[' 8 -le 11 ']'
+ echo n171
++ expr 8 + 1
+ i=9
+ '[' 9 -le 11 ']'
+ echo n171
++ expr 9 + 1
+ i=10
+ '[' 10 -le 11 ']'
+ echo n171
++ expr 10 + 1
+ i=11
+ '[' 11 -le 11 ']'
+ echo n171
++ expr 11 + 1
+ i=12
+ '[' 12 -le 11 ']'
+ read line
++ echo n179 6 wp12@n179 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n179
++ echo n179 6 wp12@n179 UNDEFINED
++ cut -f2 '-d '
+ nslots=6
+ i=1
+ '[' 1 -le 6 ']'
+ echo n179
++ expr 1 + 1
+ i=2
+ '[' 2 -le 6 ']'
+ echo n179
++ expr 2 + 1
+ i=3
+ '[' 3 -le 6 ']'
+ echo n179
++ expr 3 + 1
+ i=4
+ '[' 4 -le 6 ']'
+ echo n179
++ expr 4 + 1
+ i=5
+ '[' 5 -le 6 ']'
+ echo n179
++ expr 5 + 1
+ i=6
+ '[' 6 -le 6 ']'
+ echo n179
++ expr 6 + 1
+ i=7
+ '[' 7 -le 6 ']'
+ read line
++ echo n173 11 wp12@n173 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n173
++ echo n173 11 wp12@n173 UNDEFINED
++ cut -f2 '-d '
+ nslots=11
+ i=1
+ '[' 1 -le 11 ']'
+ echo n173
++ expr 1 + 1
+ i=2
+ '[' 2 -le 11 ']'
+ echo n173
++ expr 2 + 1
+ i=3
+ '[' 3 -le 11 ']'
+ echo n173
++ expr 3 + 1
+ i=4
+ '[' 4 -le 11 ']'
+ echo n173
++ expr 4 + 1
+ i=5
+ '[' 5 -le 11 ']'
+ echo n173
++ expr 5 + 1
+ i=6
+ '[' 6 -le 11 ']'
+ echo n173
++ expr 6 + 1
+ i=7
+ '[' 7 -le 11 ']'
+ echo n173
++ expr 7 + 1
+ i=8
+ '[' 8 -le 11 ']'
+ echo n173
++ expr 8 + 1
+ i=9
+ '[' 9 -le 11 ']'
+ echo n173
++ expr 9 + 1
+ i=10
+ '[' 10 -le 11 ']'
+ echo n173
++ expr 10 + 1
+ i=11
+ '[' 11 -le 11 ']'
+ echo n173
++ expr 11 + 1
+ i=12
+ '[' 12 -le 11 ']'
+ read line
++ echo n176 1 wp12@n176 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n176
++ echo n176 1 wp12@n176 UNDEFINED
++ cut -f2 '-d '
+ nslots=1
+ i=1
+ '[' 1 -le 1 ']'
+ echo n176
++ expr 1 + 1
+ i=2
+ '[' 2 -le 1 ']'
+ read line
++ echo n172 12 wp12@n172 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n172
++ echo n172 12 wp12@n172 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n172
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n172
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n172
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n172
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n172
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n172
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n172
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n172
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n172
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n172
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n172
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n172
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n175 8 wp12@n175 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n175
++ echo n175 8 wp12@n175 UNDEFINED
++ cut -f2 '-d '
+ nslots=8
+ i=1
+ '[' 1 -le 8 ']'
+ echo n175
++ expr 1 + 1
+ i=2
+ '[' 2 -le 8 ']'
+ echo n175
++ expr 2 + 1
+ i=3
+ '[' 3 -le 8 ']'
+ echo n175
++ expr 3 + 1
+ i=4
+ '[' 4 -le 8 ']'
+ echo n175
++ expr 4 + 1
+ i=5
+ '[' 5 -le 8 ']'
+ echo n175
++ expr 5 + 1
+ i=6
+ '[' 6 -le 8 ']'
+ echo n175
++ expr 6 + 1
+ i=7
+ '[' 7 -le 8 ']'
+ echo n175
++ expr 7 + 1
+ i=8
+ '[' 8 -le 8 ']'
+ echo n175
++ expr 8 + 1
+ i=9
+ '[' 9 -le 8 ']'
+ read line
++ echo n181 12 wp12@n181 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n181
++ echo n181 12 wp12@n181 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n181
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n181
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n181
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n181
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n181
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n181
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n181
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n181
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n181
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n181
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n181
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n181
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n182 12 wp12@n182 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n182
++ echo n182 12 wp12@n182 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n182
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n182
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n182
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n182
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n182
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n182
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n182
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n182
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n182
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n182
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n182
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n182
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
+ cat /tmp/17128.1.wp12/machines
n183
n183
n183
n183
n183
n183
n183
n183
n183
n170
n170
n170
n170
n170
n170
n170
n170
n170
n174
n174
n174
n174
n174
n174
n174
n174
n185
n185
n185
n185
n185
n185
n185
n185
n185
n185
n184
n171
n171
n171
n171
n171
n171
n171
n171
n171
n171
n171
n179
n179
n179
n179
n179
n179
n173
n173
n173
n173
n173
n173
n173
n173
n173
n173
n173
n176
n172
n172
n172
n172
n172
n172
n172
n172
n172
n172
n172
n172
n175
n175
n175
n175
n175
n175
n175
n175
n181
n181
n181
n181
n181
n181
n181
n181
n181
n181
n181
n181
n182
n182
n182
n182
n182
n182
n182
n182
n182
n182
n182
n182
+ '[' 1 = 1 ']'
+ rsh_wrapper=/opt/sge6/mpich2_smpd/rsh
+ '[' '!' -x /opt/sge6/mpich2_smpd/rsh ']'
+ rshcmd=rsh
+ case "$ARC" in
+ ln -s /opt/sge6/mpich2_smpd/rsh /tmp/17128.1.wp12/rsh
+ hash -r
+ '[' 0 = 1 ']'
+ port=22128
++ cat /tmp/17128.1.wp12/machines
++ uniq
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n183 /opt/mpich2/intel/11.1//bin/smpd 22128
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n170 /opt/mpich2/intel/11.1//bin/smpd 22128
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n174 /opt/mpich2/intel/11.1//bin/smpd 22128
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n185 /opt/mpich2/intel/11.1//bin/smpd 22128
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n184 /opt/mpich2/intel/11.1//bin/smpd 22128
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n171 /opt/mpich2/intel/11.1//bin/smpd 22128
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n179 /opt/mpich2/intel/11.1//bin/smpd 22128
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n173 /opt/mpich2/intel/11.1//bin/smpd 22128
+ for host in '`cat $machines | uniq`'
rsh n183 /opt/mpich2/intel/11.1//bin/smpd -port 22128 -d 0
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n176 /opt/mpich2/intel/11.1//bin/smpd 22128
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n172 /opt/mpich2/intel/11.1//bin/smpd 22128
rsh n170 /opt/mpich2/intel/11.1//bin/smpd -port 22128 -d 0
rsh n174 /opt/mpich2/intel/11.1//bin/smpd -port 22128 -d 0
+ for host in '`cat $machines | uniq`'
rsh n184 /opt/mpich2/intel/11.1//bin/smpd -port 22128 -d 0
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n175 /opt/mpich2/intel/11.1//bin/smpd 22128
rsh n171 /opt/mpich2/intel/11.1//bin/smpd -port 22128 -d 0
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n181 /opt/mpich2/intel/11.1//bin/smpd 22128
rsh n185 /opt/mpich2/intel/11.1//bin/smpd -port 22128 -d 0
rsh n179 /opt/mpich2/intel/11.1//bin/smpd -port 22128 -d 0
rsh n173 /opt/mpich2/intel/11.1//bin/smpd -port 22128 -d 0
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n182 /opt/mpich2/intel/11.1//bin/smpd 22128
rsh n176 /opt/mpich2/intel/11.1//bin/smpd -port 22128 -d 0
+ actual_retry=1
+ '[' 1 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (1 of 10)'
startmpich2.sh: check for smpd daemons (1 of 10)
+ numhosts=0
++ cat /tmp/17128.1.wp12/machines
rsh n172 /opt/mpich2/intel/11.1//bin/smpd -port 22128 -d 0
++ uniq
rsh n175 /opt/mpich2/intel/11.1//bin/smpd -port 22128 -d 0
+ for host in '`cat $machines | uniq`'
++ head -n 1
++ cut -f 1 -d ' '
++ /opt/mpich2/intel/11.1//bin/smpd -port 22128 -status n183
rsh n182 /opt/mpich2/intel/11.1//bin/smpd -port 22128 -d 0
rsh n181 /opt/mpich2/intel/11.1//bin/smpd -port 22128 -d 0
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n183'
startmpich2.sh: missing smpd on n183
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22128 -status n170
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n170'
startmpich2.sh: missing smpd on n170
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22128 -status n174
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n174'
startmpich2.sh: missing smpd on n174
+ for host in '`cat $machines | uniq`'
++ head -n 1
++ /opt/mpich2/intel/11.1//bin/smpd -port 22128 -status n185
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n185'
startmpich2.sh: missing smpd on n185
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22128 -status n184
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n184'
startmpich2.sh: missing smpd on n184
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22128 -status n171
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n171'
startmpich2.sh: missing smpd on n171
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22128 -status n179
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n179'
startmpich2.sh: missing smpd on n179
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22128 -status n173
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n173'
startmpich2.sh: missing smpd on n173
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22128 -status n176
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n176'
startmpich2.sh: missing smpd on n176
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22128 -status n172
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n172'
startmpich2.sh: missing smpd on n172
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22128 -status n175
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n175'
startmpich2.sh: found running smpd on n175
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22128 -status n181
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n181'
startmpich2.sh: missing smpd on n181
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22128 -status n182
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n182'
startmpich2.sh: missing smpd on n182
+ '[' 13 -eq 1 ']'
+ sleep 5
+ (( actual_retry += 1 ))
+ '[' 2 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (2 of 10)'
startmpich2.sh: check for smpd daemons (2 of 10)
+ numhosts=0
++ cat /tmp/17128.1.wp12/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22128 -status n183
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n183'
startmpich2.sh: found running smpd on n183
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22128 -status n170
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n170'
startmpich2.sh: found running smpd on n170
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22128 -status n174
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n174'
startmpich2.sh: found running smpd on n174
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22128 -status n185
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n185'
startmpich2.sh: found running smpd on n185
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22128 -status n184
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n184'
startmpich2.sh: found running smpd on n184
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22128 -status n171
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n171'
startmpich2.sh: found running smpd on n171
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22128 -status n179
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n179'
startmpich2.sh: found running smpd on n179
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22128 -status n173
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n173'
startmpich2.sh: found running smpd on n173
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22128 -status n176
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n176'
startmpich2.sh: found running smpd on n176
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22128 -status n172
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n172'
startmpich2.sh: found running smpd on n172
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22128 -status n175
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n175'
startmpich2.sh: found running smpd on n175
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22128 -status n181
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n181'
startmpich2.sh: found running smpd on n181
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22128 -status n182
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n182'
startmpich2.sh: found running smpd on n182
+ '[' 13 -eq 13 ']'
+ echo 'startmpich2.sh: got all 13 of 13 nodes'
startmpich2.sh: got all 13 of 13 nodes
+ exit 0
START
END
Mon Nov 14 20:00:09 EST 2011
-catch_rsh /opt/mpich2/intel/11.1/
Your job finished at Mon Nov 14 20:00:14 EST 2011