USER      =  jtomczak
HOMEDIR   =  /home/users/jtomczak
JOB_ID    =  25847
JOB_NAME  =  FS-LOW
QUEUE     =  wp12
NSLOTS    =  89
NHOSTS    =  10

SUB_HOST  =  rupc04
SUB_DIR   =  /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta200/nf6/NEW

EXE_HOST  =  n174
EXE_DIR   =  /mnt/n174
OUT_DIR   =  /scratch
TMP_DIR   =  /tmp/25847.1.wp12

INFO      =  Files larger than 100MB will be compressed to reduce the network load.
INFO2     =  Do NOT remove this file while job is running!

PORT      =  20847
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta200/nf6/NEW/FIX_MU: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta200/nf6/NEW/GO: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta200/nf6/NEW/GO-old: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta200/nf6/NEW/ancont: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta200/nf6/NEW/cdos: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta200/nf6/NEW/imp.0: not a plain file
Failure to rcp files!

Your job started at Thu Mar 29 07:13:22 EDT 2012 on host n174 in /mnt/n174/25847jtomczak
Submission directory is /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta200/nf6/NEW on host rupc04
-----------------------------------------------------------------------------------------
+ echo -catch_rsh /opt/sge6/spool/n174/active_jobs/25847.1/pe_hostfile /opt/mpich2/intel/11.1/
-catch_rsh /opt/sge6/spool/n174/active_jobs/25847.1/pe_hostfile /opt/mpich2/intel/11.1/
+ SLEEPTIME=5
+ RETRIES=10
+ catch_rsh=0
+ catch_hostname=0
+ '[' 3 -gt 0 ']'
+ case "$1" in
+ catch_rsh=1
+ shift
+ '[' 2 -gt 0 ']'
+ case "$1" in
+ break
++ basename /opt/sge6/mpich2_smpd/startmpich2.sh
+ me=startmpich2.sh
+ '[' 2 -lt 2 ']'
+ pe_hostfile=/opt/sge6/spool/n174/active_jobs/25847.1/pe_hostfile
+ MPICH2_ROOT=/opt/mpich2/intel/11.1/
+ export MPICH2_ROOT
+ '[' '!' -x /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 ']'
+ '[' '!' -x /opt/mpich2/intel/11.1//bin/smpd ']'
+ '[' '!' -r /opt/sge6/spool/n174/active_jobs/25847.1/pe_hostfile ']'
+ machines=/tmp/25847.1.wp12/machines
+ PeHostfile2MachineFile /opt/sge6/spool/n174/active_jobs/25847.1/pe_hostfile
+ cat /opt/sge6/spool/n174/active_jobs/25847.1/pe_hostfile
+ read line
++ echo n174 7 wp12@n174 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n174
++ echo n174 7 wp12@n174 UNDEFINED
++ cut -f2 '-d '
+ nslots=7
+ i=1
+ '[' 1 -le 7 ']'
+ echo n174
++ expr 1 + 1
+ i=2
+ '[' 2 -le 7 ']'
+ echo n174
++ expr 2 + 1
+ i=3
+ '[' 3 -le 7 ']'
+ echo n174
++ expr 3 + 1
+ i=4
+ '[' 4 -le 7 ']'
+ echo n174
++ expr 4 + 1
+ i=5
+ '[' 5 -le 7 ']'
+ echo n174
++ expr 5 + 1
+ i=6
+ '[' 6 -le 7 ']'
+ echo n174
++ expr 6 + 1
+ i=7
+ '[' 7 -le 7 ']'
+ echo n174
++ expr 7 + 1
+ i=8
+ '[' 8 -le 7 ']'
+ read line
++ echo n171 7 wp12@n171 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n171
++ echo n171 7 wp12@n171 UNDEFINED
++ cut -f2 '-d '
+ nslots=7
+ i=1
+ '[' 1 -le 7 ']'
+ echo n171
++ expr 1 + 1
+ i=2
+ '[' 2 -le 7 ']'
+ echo n171
++ expr 2 + 1
+ i=3
+ '[' 3 -le 7 ']'
+ echo n171
++ expr 3 + 1
+ i=4
+ '[' 4 -le 7 ']'
+ echo n171
++ expr 4 + 1
+ i=5
+ '[' 5 -le 7 ']'
+ echo n171
++ expr 5 + 1
+ i=6
+ '[' 6 -le 7 ']'
+ echo n171
++ expr 6 + 1
+ i=7
+ '[' 7 -le 7 ']'
+ echo n171
++ expr 7 + 1
+ i=8
+ '[' 8 -le 7 ']'
+ read line
++ echo n183 12 wp12@n183 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n183
++ echo n183 12 wp12@n183 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n183
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n183
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n183
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n183
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n183
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n183
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n183
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n183
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n183
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n183
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n183
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n183
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n168 2 wp12@n168 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n168
++ echo n168 2 wp12@n168 UNDEFINED
++ cut -f2 '-d '
+ nslots=2
+ i=1
+ '[' 1 -le 2 ']'
+ echo n168
++ expr 1 + 1
+ i=2
+ '[' 2 -le 2 ']'
+ echo n168
++ expr 2 + 1
+ i=3
+ '[' 3 -le 2 ']'
+ read line
++ echo n169 7 wp12@n169 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n169
++ echo n169 7 wp12@n169 UNDEFINED
++ cut -f2 '-d '
+ nslots=7
+ i=1
+ '[' 1 -le 7 ']'
+ echo n169
++ expr 1 + 1
+ i=2
+ '[' 2 -le 7 ']'
+ echo n169
++ expr 2 + 1
+ i=3
+ '[' 3 -le 7 ']'
+ echo n169
++ expr 3 + 1
+ i=4
+ '[' 4 -le 7 ']'
+ echo n169
++ expr 4 + 1
+ i=5
+ '[' 5 -le 7 ']'
+ echo n169
++ expr 5 + 1
+ i=6
+ '[' 6 -le 7 ']'
+ echo n169
++ expr 6 + 1
+ i=7
+ '[' 7 -le 7 ']'
+ echo n169
++ expr 7 + 1
+ i=8
+ '[' 8 -le 7 ']'
+ read line
++ echo n177 9 wp12@n177 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n177
++ echo n177 9 wp12@n177 UNDEFINED
++ cut -f2 '-d '
+ nslots=9
+ i=1
+ '[' 1 -le 9 ']'
+ echo n177
++ expr 1 + 1
+ i=2
+ '[' 2 -le 9 ']'
+ echo n177
++ expr 2 + 1
+ i=3
+ '[' 3 -le 9 ']'
+ echo n177
++ expr 3 + 1
+ i=4
+ '[' 4 -le 9 ']'
+ echo n177
++ expr 4 + 1
+ i=5
+ '[' 5 -le 9 ']'
+ echo n177
++ expr 5 + 1
+ i=6
+ '[' 6 -le 9 ']'
+ echo n177
++ expr 6 + 1
+ i=7
+ '[' 7 -le 9 ']'
+ echo n177
++ expr 7 + 1
+ i=8
+ '[' 8 -le 9 ']'
+ echo n177
++ expr 8 + 1
+ i=9
+ '[' 9 -le 9 ']'
+ echo n177
++ expr 9 + 1
+ i=10
+ '[' 10 -le 9 ']'
+ read line
++ echo n179 9 wp12@n179 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n179
++ echo n179 9 wp12@n179 UNDEFINED
++ cut -f2 '-d '
+ nslots=9
+ i=1
+ '[' 1 -le 9 ']'
+ echo n179
++ expr 1 + 1
+ i=2
+ '[' 2 -le 9 ']'
+ echo n179
++ expr 2 + 1
+ i=3
+ '[' 3 -le 9 ']'
+ echo n179
++ expr 3 + 1
+ i=4
+ '[' 4 -le 9 ']'
+ echo n179
++ expr 4 + 1
+ i=5
+ '[' 5 -le 9 ']'
+ echo n179
++ expr 5 + 1
+ i=6
+ '[' 6 -le 9 ']'
+ echo n179
++ expr 6 + 1
+ i=7
+ '[' 7 -le 9 ']'
+ echo n179
++ expr 7 + 1
+ i=8
+ '[' 8 -le 9 ']'
+ echo n179
++ expr 8 + 1
+ i=9
+ '[' 9 -le 9 ']'
+ echo n179
++ expr 9 + 1
+ i=10
+ '[' 10 -le 9 ']'
+ read line
++ echo n182 12 wp12@n182 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n182
++ echo n182 12 wp12@n182 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n182
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n182
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n182
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n182
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n182
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n182
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n182
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n182
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n182
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n182
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n182
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n182
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n170 12 wp12@n170 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n170
++ echo n170 12 wp12@n170 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n170
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n170
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n170
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n170
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n170
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n170
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n170
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n170
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n170
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n170
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n170
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n170
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n173 12 wp12@n173 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n173
++ echo n173 12 wp12@n173 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n173
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n173
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n173
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n173
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n173
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n173
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n173
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n173
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n173
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n173
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n173
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n173
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
+ cat /tmp/25847.1.wp12/machines
n174
n174
n174
n174
n174
n174
n174
n171
n171
n171
n171
n171
n171
n171
n183
n183
n183
n183
n183
n183
n183
n183
n183
n183
n183
n183
n168
n168
n169
n169
n169
n169
n169
n169
n169
n177
n177
n177
n177
n177
n177
n177
n177
n177
n179
n179
n179
n179
n179
n179
n179
n179
n179
n182
n182
n182
n182
n182
n182
n182
n182
n182
n182
n182
n182
n170
n170
n170
n170
n170
n170
n170
n170
n170
n170
n170
n170
n173
n173
n173
n173
n173
n173
n173
n173
n173
n173
n173
n173
+ '[' 1 = 1 ']'
+ rsh_wrapper=/opt/sge6/mpich2_smpd/rsh
+ '[' '!' -x /opt/sge6/mpich2_smpd/rsh ']'
+ rshcmd=rsh
+ case "$ARC" in
+ ln -s /opt/sge6/mpich2_smpd/rsh /tmp/25847.1.wp12/rsh
+ hash -r
+ '[' 0 = 1 ']'
+ port=20847
++ cat /tmp/25847.1.wp12/machines
++ uniq
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n174 /opt/mpich2/intel/11.1//bin/smpd 20847
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n171 /opt/mpich2/intel/11.1//bin/smpd 20847
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n183 /opt/mpich2/intel/11.1//bin/smpd 20847
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n168 /opt/mpich2/intel/11.1//bin/smpd 20847
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n169 /opt/mpich2/intel/11.1//bin/smpd 20847
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n177 /opt/mpich2/intel/11.1//bin/smpd 20847
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n179 /opt/mpich2/intel/11.1//bin/smpd 20847
rsh n174 /opt/mpich2/intel/11.1//bin/smpd -port 20847 -d 0
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n182 /opt/mpich2/intel/11.1//bin/smpd 20847
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n170 /opt/mpich2/intel/11.1//bin/smpd 20847
rsh n171 /opt/mpich2/intel/11.1//bin/smpd -port 20847 -d 0
rsh n183 /opt/mpich2/intel/11.1//bin/smpd -port 20847 -d 0
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n173 /opt/mpich2/intel/11.1//bin/smpd 20847
+ actual_retry=1
+ '[' 1 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (1 of 10)'
startmpich2.sh: check for smpd daemons (1 of 10)
+ numhosts=0
rsh n168 /opt/mpich2/intel/11.1//bin/smpd -port 20847 -d 0
++ cat /tmp/25847.1.wp12/machines
rsh n169 /opt/mpich2/intel/11.1//bin/smpd -port 20847 -d 0
++ uniq
rsh n177 /opt/mpich2/intel/11.1//bin/smpd -port 20847 -d 0
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20847 -status n174
++ head -n 1
++ cut -f 1 -d ' '
rsh n182 /opt/mpich2/intel/11.1//bin/smpd -port 20847 -d 0
rsh n179 /opt/mpich2/intel/11.1//bin/smpd -port 20847 -d 0
rsh n173 /opt/mpich2/intel/11.1//bin/smpd -port 20847 -d 0
rsh n170 /opt/mpich2/intel/11.1//bin/smpd -port 20847 -d 0
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n174'
startmpich2.sh: missing smpd on n174
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20847 -status n171
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n171'
startmpich2.sh: missing smpd on n171
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20847 -status n183
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n183'
startmpich2.sh: missing smpd on n183
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20847 -status n168
++ cut -f 1 -d ' '
++ head -n 1
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n168'
startmpich2.sh: missing smpd on n168
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20847 -status n169
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n169'
startmpich2.sh: missing smpd on n169
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20847 -status n177
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n177'
startmpich2.sh: missing smpd on n177
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20847 -status n179
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n179'
startmpich2.sh: missing smpd on n179
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20847 -status n182
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n182'
startmpich2.sh: missing smpd on n182
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20847 -status n170
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n170'
startmpich2.sh: missing smpd on n170
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20847 -status n173
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n173'
startmpich2.sh: missing smpd on n173
+ '[' 10 -eq 0 ']'
+ sleep 5
+ (( actual_retry += 1 ))
+ '[' 2 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (2 of 10)'
startmpich2.sh: check for smpd daemons (2 of 10)
+ numhosts=0
++ cat /tmp/25847.1.wp12/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20847 -status n174
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n174'
startmpich2.sh: found running smpd on n174
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20847 -status n171
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n171'
startmpich2.sh: found running smpd on n171
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20847 -status n183
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n183'
startmpich2.sh: found running smpd on n183
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20847 -status n168
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n168'
startmpich2.sh: found running smpd on n168
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20847 -status n169
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n169'
startmpich2.sh: found running smpd on n169
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20847 -status n177
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n177'
startmpich2.sh: found running smpd on n177
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20847 -status n179
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n179'
startmpich2.sh: found running smpd on n179
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20847 -status n182
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n182'
startmpich2.sh: found running smpd on n182
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20847 -status n170
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n170'
startmpich2.sh: found running smpd on n170
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20847 -status n173
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n173'
startmpich2.sh: found running smpd on n173
+ '[' 10 -eq 10 ']'
+ echo 'startmpich2.sh: got all 10 of 10 nodes'
startmpich2.sh: got all 10 of 10 nodes
+ exit 0
rm: cannot remove `*.*.*.*': No such file or directory
rm: cannot remove `imp.0/*.*.*': No such file or directory
rm: cannot remove `imp.0/nohup*': No such file or directory
rm: cannot remove `dmft?.error*': No such file or directory
rm: cannot remove `*.outputdmf?.*': No such file or directory
gzip: *.gz: No such file or directory
START
-catch_rsh /opt/mpich2/intel/11.1/
Your job finished at Mon Apr 30 10:41:04 EDT 2012