USER      =  jtomczak
HOMEDIR   =  /home/users/jtomczak
JOB_ID    =  8774
JOB_NAME  =  FS6-80
QUEUE     =  wp10
NSLOTS    =  144
NHOSTS    =  14

SUB_HOST  =  rupc04
SUB_DIR   =  /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta80/nf6/NEW

EXE_HOST  =  n142
NFS_HOST  =  caip02s
EXE_DIR   =  /mnt/p06
OUT_DIR   =  /tmp
TMP_DIR   =  /tmp/8774.1.wp10

INFO      =  Files larger than 100MB will be compressed to reduce the network load.
INFO2     =  Do NOT remove this file while job is running!

PORT      =  23774
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta80/nf6/NEW/FIX_mu: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta80/nf6/NEW/FULL: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta80/nf6/NEW/GO: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta80/nf6/NEW/GO-small: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta80/nf6/NEW/ancont: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta80/nf6/NEW/cdos: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta80/nf6/NEW/imp.0: not a plain file
Failure to rcp files!

Your job started at Fri Jul 8 13:51:06 EDT 2011 on host n142 in /mnt/p06/8774jtomczak
Submission directory is /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta80/nf6/NEW on host rupc04
-----------------------------------------------------------------------------------------
+ echo -catch_rsh /opt/sge6/spool/n142/active_jobs/8774.1/pe_hostfile /opt/mpich2/intel/11.1/
-catch_rsh /opt/sge6/spool/n142/active_jobs/8774.1/pe_hostfile /opt/mpich2/intel/11.1/
+ SLEEPTIME=5
+ RETRIES=10
+ catch_rsh=0
+ catch_hostname=0
+ '[' 3 -gt 0 ']'
+ case "$1" in
+ catch_rsh=1
+ shift
+ '[' 2 -gt 0 ']'
+ case "$1" in
+ break
++ basename /opt/sge6/mpich2_smpd/startmpich2.sh
+ me=startmpich2.sh
+ '[' 2 -lt 2 ']'
+ pe_hostfile=/opt/sge6/spool/n142/active_jobs/8774.1/pe_hostfile
+ MPICH2_ROOT=/opt/mpich2/intel/11.1/
+ export MPICH2_ROOT
+ '[' '!' -x /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 ']'
+ '[' '!' -x /opt/mpich2/intel/11.1//bin/smpd ']'
+ '[' '!' -r /opt/sge6/spool/n142/active_jobs/8774.1/pe_hostfile ']'
+ machines=/tmp/8774.1.wp10/machines
+ PeHostfile2MachineFile /opt/sge6/spool/n142/active_jobs/8774.1/pe_hostfile
+ cat /opt/sge6/spool/n142/active_jobs/8774.1/pe_hostfile
+ read line
++ echo n142 12 wp10@n142 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n142
++ echo n142 12 wp10@n142 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n142
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n142
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n142
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n142
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n142
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n142
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n142
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n142
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n142
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n142
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n142
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n142
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n162 12 wp10@n162 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n162
++ echo n162 12 wp10@n162 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n162
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n162
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n162
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n162
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n162
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n162
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n162
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n162
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n162
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n162
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n162
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n162
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n143 12 wp10@n143 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n143
++ echo n143 12 wp10@n143 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n143
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n143
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n143
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n143
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n143
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n143
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n143
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n143
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n143
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n143
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n143
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n143
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n150 12 wp10@n150 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n150
++ echo n150 12 wp10@n150 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n150
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n150
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n150
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n150
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n150
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n150
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n150
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n150
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n150
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n150
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n150
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n150
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n165 12 wp10@n165 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n165
++ echo n165 12 wp10@n165 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n165
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n165
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n165
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n165
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n165
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n165
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n165
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n165
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n165
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n165
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n165
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n165
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n159 6 wp10@n159 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n159
++ echo n159 6 wp10@n159 UNDEFINED
++ cut -f2 '-d '
+ nslots=6
+ i=1
+ '[' 1 -le 6 ']'
+ echo n159
++ expr 1 + 1
+ i=2
+ '[' 2 -le 6 ']'
+ echo n159
++ expr 2 + 1
+ i=3
+ '[' 3 -le 6 ']'
+ echo n159
++ expr 3 + 1
+ i=4
+ '[' 4 -le 6 ']'
+ echo n159
++ expr 4 + 1
+ i=5
+ '[' 5 -le 6 ']'
+ echo n159
++ expr 5 + 1
+ i=6
+ '[' 6 -le 6 ']'
+ echo n159
++ expr 6 + 1
+ i=7
+ '[' 7 -le 6 ']'
+ read line
++ echo n149 1 wp10@n149 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n149
++ echo n149 1 wp10@n149 UNDEFINED
++ cut -f2 '-d '
+ nslots=1
+ i=1
+ '[' 1 -le 1 ']'
+ echo n149
++ expr 1 + 1
+ i=2
+ '[' 2 -le 1 ']'
+ read line
++ echo n164 12 wp10@n164 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n164
++ echo n164 12 wp10@n164 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n164
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n164
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n164
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n164
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n164
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n164
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n164
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n164
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n164
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n164
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n164
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n164
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n157 12 wp10@n157 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n157
++ echo n157 12 wp10@n157 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n157
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n157
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n157
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n157
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n157
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n157
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n157
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n157
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n157
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n157
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n157
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n157
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n141 12 wp10@n141 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n141
++ echo n141 12 wp10@n141 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n141
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n141
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n141
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n141
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n141
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n141
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n141
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n141
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n141
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n141
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n141
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n141
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n148 5 wp10@n148 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n148
++ echo n148 5 wp10@n148 UNDEFINED
++ cut -f2 '-d '
+ nslots=5
+ i=1
+ '[' 1 -le 5 ']'
+ echo n148
++ expr 1 + 1
+ i=2
+ '[' 2 -le 5 ']'
+ echo n148
++ expr 2 + 1
+ i=3
+ '[' 3 -le 5 ']'
+ echo n148
++ expr 3 + 1
+ i=4
+ '[' 4 -le 5 ']'
+ echo n148
++ expr 4 + 1
+ i=5
+ '[' 5 -le 5 ']'
+ echo n148
++ expr 5 + 1
+ i=6
+ '[' 6 -le 5 ']'
+ read line
++ echo n155 12 wp10@n155 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n155
++ echo n155 12 wp10@n155 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n155
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n155
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n155
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n155
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n155
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n155
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n155
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n155
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n155
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n155
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n155
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n155
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n163 12 wp10@n163 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n163
++ echo n163 12 wp10@n163 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n163
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n163
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n163
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n163
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n163
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n163
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n163
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n163
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n163
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n163
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n163
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n163
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n147 12 wp10@n147 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n147
++ echo n147 12 wp10@n147 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n147
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n147
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n147
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n147
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n147
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n147
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n147
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n147
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n147
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n147
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n147
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n147
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
+ cat /tmp/8774.1.wp10/machines
n142
n142
n142
n142
n142
n142
n142
n142
n142
n142
n142
n142
n162
n162
n162
n162
n162
n162
n162
n162
n162
n162
n162
n162
n143
n143
n143
n143
n143
n143
n143
n143
n143
n143
n143
n143
n150
n150
n150
n150
n150
n150
n150
n150
n150
n150
n150
n150
n165
n165
n165
n165
n165
n165
n165
n165
n165
n165
n165
n165
n159
n159
n159
n159
n159
n159
n149
n164
n164
n164
n164
n164
n164
n164
n164
n164
n164
n164
n164
n157
n157
n157
n157
n157
n157
n157
n157
n157
n157
n157
n157
n141
n141
n141
n141
n141
n141
n141
n141
n141
n141
n141
n141
n148
n148
n148
n148
n148
n155
n155
n155
n155
n155
n155
n155
n155
n155
n155
n155
n155
n163
n163
n163
n163
n163
n163
n163
n163
n163
n163
n163
n163
n147
n147
n147
n147
n147
n147
n147
n147
n147
n147
n147
n147
+ '[' 1 = 1 ']'
+ rsh_wrapper=/opt/sge6/mpich2_smpd/rsh
+ '[' '!' -x /opt/sge6/mpich2_smpd/rsh ']'
+ rshcmd=rsh
+ case "$ARC" in
+ ln -s /opt/sge6/mpich2_smpd/rsh /tmp/8774.1.wp10/rsh
+ hash -r
+ '[' 0 = 1 ']'
+ port=23774
++ cat /tmp/8774.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n142 /opt/mpich2/intel/11.1//bin/smpd 23774
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n162 /opt/mpich2/intel/11.1//bin/smpd 23774
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n143 /opt/mpich2/intel/11.1//bin/smpd 23774
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n150 /opt/mpich2/intel/11.1//bin/smpd 23774
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n165 /opt/mpich2/intel/11.1//bin/smpd 23774
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n159 /opt/mpich2/intel/11.1//bin/smpd 23774
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n149 /opt/mpich2/intel/11.1//bin/smpd 23774
rsh n142 /opt/mpich2/intel/11.1//bin/smpd -port 23774 -d 0
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n164 /opt/mpich2/intel/11.1//bin/smpd 23774
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n157 /opt/mpich2/intel/11.1//bin/smpd 23774
rsh n143 /opt/mpich2/intel/11.1//bin/smpd -port 23774 -d 0
rsh n162 /opt/mpich2/intel/11.1//bin/smpd -port 23774 -d 0
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n141 /opt/mpich2/intel/11.1//bin/smpd 23774
rsh n150 /opt/mpich2/intel/11.1//bin/smpd -port 23774 -d 0
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n148 /opt/mpich2/intel/11.1//bin/smpd 23774
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n155 /opt/mpich2/intel/11.1//bin/smpd 23774
rsh n159 /opt/mpich2/intel/11.1//bin/smpd -port 23774 -d 0
rsh n165 /opt/mpich2/intel/11.1//bin/smpd -port 23774 -d 0
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n163 /opt/mpich2/intel/11.1//bin/smpd 23774
rsh n164 /opt/mpich2/intel/11.1//bin/smpd -port 23774 -d 0
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n147 /opt/mpich2/intel/11.1//bin/smpd 23774
rsh n149 /opt/mpich2/intel/11.1//bin/smpd -port 23774 -d 0
+ actual_retry=1
+ '[' 1 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (1 of 10)'
startmpich2.sh: check for smpd daemons (1 of 10)
+ numhosts=0
rsh n157 /opt/mpich2/intel/11.1//bin/smpd -port 23774 -d 0
rsh n141 /opt/mpich2/intel/11.1//bin/smpd -port 23774 -d 0
++ uniq
++ cat /tmp/8774.1.wp10/machines
+ for host in '`cat $machines | uniq`'
rsh n155 /opt/mpich2/intel/11.1//bin/smpd -port 23774 -d 0
rsh n148 /opt/mpich2/intel/11.1//bin/smpd -port 23774 -d 0
++ /opt/mpich2/intel/11.1//bin/smpd -port 23774 -status n142
++ cut -f 1 -d ' '
++ head -n 1
rsh n163 /opt/mpich2/intel/11.1//bin/smpd -port 23774 -d 0
rsh n147 /opt/mpich2/intel/11.1//bin/smpd -port 23774 -d 0
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n142'
startmpich2.sh: missing smpd on n142
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23774 -status n162
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n162'
startmpich2.sh: missing smpd on n162
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23774 -status n143
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n143'
startmpich2.sh: missing smpd on n143
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23774 -status n150
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n150'
startmpich2.sh: missing smpd on n150
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23774 -status n165
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n165'
startmpich2.sh: missing smpd on n165
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23774 -status n159
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n159'
startmpich2.sh: missing smpd on n159
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23774 -status n149
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n149'
startmpich2.sh: missing smpd on n149
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23774 -status n164
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n164'
startmpich2.sh: missing smpd on n164
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23774 -status n157
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n157'
startmpich2.sh: missing smpd on n157
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23774 -status n141
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n141'
startmpich2.sh: missing smpd on n141
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23774 -status n148
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n148'
startmpich2.sh: missing smpd on n148
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23774 -status n155
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n155'
startmpich2.sh: missing smpd on n155
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23774 -status n163
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n163'
startmpich2.sh: missing smpd on n163
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23774 -status n147
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n147'
startmpich2.sh: missing smpd on n147
+ '[' 14 -eq 0 ']'
+ sleep 5
+ (( actual_retry += 1 ))
+ '[' 2 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (2 of 10)'
startmpich2.sh: check for smpd daemons (2 of 10)
+ numhosts=0
++ cat /tmp/8774.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23774 -status n142
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n142'
startmpich2.sh: found running smpd on n142
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23774 -status n162
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n162'
startmpich2.sh: found running smpd on n162
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23774 -status n143
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n143'
startmpich2.sh: found running smpd on n143
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23774 -status n150
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n150'
startmpich2.sh: found running smpd on n150
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23774 -status n165
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n165'
startmpich2.sh: found running smpd on n165
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23774 -status n159
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n159'
startmpich2.sh: found running smpd on n159
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23774 -status n149
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n149'
startmpich2.sh: found running smpd on n149
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23774 -status n164
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n164'
startmpich2.sh: found running smpd on n164
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23774 -status n157
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n157'
startmpich2.sh: found running smpd on n157
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23774 -status n141
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n141'
startmpich2.sh: found running smpd on n141
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23774 -status n148
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n148'
startmpich2.sh: found running smpd on n148
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23774 -status n155
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n155'
startmpich2.sh: found running smpd on n155
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23774 -status n163
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n163'
startmpich2.sh: found running smpd on n163
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23774 -status n147
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n147'
startmpich2.sh: found running smpd on n147
+ '[' 14 -eq 14 ']'
+ echo 'startmpich2.sh: got all 14 of 14 nodes'
startmpich2.sh: got all 14 of 14 nodes
+ exit 0
START
END
Mon Jul 11 01:21:45 EDT 2011
-catch_rsh /opt/mpich2/intel/11.1/
Your job finished at Mon Jul 11 01:21:50 EDT 2011