USER      =  haule
HOMEDIR   =  /home/haule
JOB_ID    =  21074
JOB_NAME  =  LSCO
QUEUE     =  wp12_hp
NSLOTS    =  120
NHOSTS    =  10

SUB_HOST  =  rupc06
SUB_DIR   =  /mnt/wk06/haule/materials/newDMFT/WinterSchool/La2CuO4

EXE_HOST  =  n170
EXE_DIR   =  /mnt/n170
OUT_DIR   =  /scratch
TMP_DIR   =  /tmp/21074.1.wp12_hp

INFO      =  Files larger than 100MB will be compressed to reduce the network load.
INFO2     =  Do NOT remove this file while job is running!

PORT      =  21074
rcp: /mnt/wk06/haule/materials/newDMFT/WinterSchool/La2CuO4/21068haule: not a plain file
rcp: /mnt/wk06/haule/materials/newDMFT/WinterSchool/La2CuO4/21070haule: not a plain file
rcp: /mnt/wk06/haule/materials/newDMFT/WinterSchool/La2CuO4/21071haule: not a plain file
rcp: /mnt/wk06/haule/materials/newDMFT/WinterSchool/La2CuO4/21072haule: not a plain file
rcp: /mnt/wk06/haule/materials/newDMFT/WinterSchool/La2CuO4/21073haule: not a plain file
rcp: /mnt/wk06/haule/materials/newDMFT/WinterSchool/La2CuO4/LDA: not a plain file
rcp: /mnt/wk06/haule/materials/newDMFT/WinterSchool/La2CuO4/LDA0: not a plain file
Failure to rcp files!

Your job started at Tue Jan 3 03:42:53 EST 2012 on host n170 in /mnt/n170/21074haule
Submission directory is /mnt/wk06/haule/materials/newDMFT/WinterSchool/La2CuO4 on host rupc06
-----------------------------------------------------------------------------------------
+ echo -catch_rsh /opt/sge6/spool/n170/active_jobs/21074.1/pe_hostfile /opt/mpich2/intel/11.1/
-catch_rsh /opt/sge6/spool/n170/active_jobs/21074.1/pe_hostfile /opt/mpich2/intel/11.1/
+ SLEEPTIME=5
+ RETRIES=10
+ catch_rsh=0
+ catch_hostname=0
+ '[' 3 -gt 0 ']'
+ case "$1" in
+ catch_rsh=1
+ shift
+ '[' 2 -gt 0 ']'
+ case "$1" in
+ break
++ basename /opt/sge6/mpich2_smpd/startmpich2.sh
+ me=startmpich2.sh
+ '[' 2 -lt 2 ']'
+ pe_hostfile=/opt/sge6/spool/n170/active_jobs/21074.1/pe_hostfile
+ MPICH2_ROOT=/opt/mpich2/intel/11.1/
+ export MPICH2_ROOT
+ '[' '!' -x /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 ']'
+ '[' '!' -x /opt/mpich2/intel/11.1//bin/smpd ']'
+ '[' '!' -r /opt/sge6/spool/n170/active_jobs/21074.1/pe_hostfile ']'
+ machines=/tmp/21074.1.wp12_hp/machines
+ PeHostfile2MachineFile /opt/sge6/spool/n170/active_jobs/21074.1/pe_hostfile
+ cat /opt/sge6/spool/n170/active_jobs/21074.1/pe_hostfile
+ read line
++ echo n170 12 wp12_hp@n170 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n170
++ echo n170 12 wp12_hp@n170 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n170
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n170
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n170
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n170
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n170
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n170
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n170
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n170
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n170
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n170
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n170
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n170
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n171 12 wp12_hp@n171 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n171
++ echo n171 12 wp12_hp@n171 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n171
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n171
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n171
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n171
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n171
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n171
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n171
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n171
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n171
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n171
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n171
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n171
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n172 12 wp12_hp@n172 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n172
++ echo n172 12 wp12_hp@n172 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n172
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n172
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n172
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n172
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n172
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n172
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n172
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n172
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n172
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n172
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n172
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n172
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n173 12 wp12_hp@n173 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n173
++ echo n173 12 wp12_hp@n173 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n173
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n173
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n173
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n173
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n173
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n173
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n173
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n173
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n173
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n173
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n173
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n173
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n174 12 wp12_hp@n174 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n174
++ echo n174 12 wp12_hp@n174 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n174
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n174
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n174
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n174
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n174
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n174
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n174
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n174
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n174
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n174
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n174
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n174
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n176 12 wp12_hp@n176 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n176
++ echo n176 12 wp12_hp@n176 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n176
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n176
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n176
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n176
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n176
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n176
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n176
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n176
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n176
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n176
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n176
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n176
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n177 12 wp12_hp@n177 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n177
++ echo n177 12 wp12_hp@n177 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n177
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n177
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n177
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n177
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n177
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n177
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n177
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n177
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n177
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n177
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n177
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n177
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n178 12 wp12_hp@n178 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n178
++ echo n178 12 wp12_hp@n178 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n178
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n178
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n178
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n178
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n178
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n178
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n178
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n178
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n178
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n178
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n178
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n178
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n179 12 wp12_hp@n179 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n179
++ echo n179 12 wp12_hp@n179 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n179
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n179
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n179
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n179
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n179
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n179
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n179
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n179
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n179
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n179
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n179
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n179
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n180 12 wp12_hp@n180 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n180
++ echo n180 12 wp12_hp@n180 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n180
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n180
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n180
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n180
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n180
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n180
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n180
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n180
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n180
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n180
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n180
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n180
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
+ cat /tmp/21074.1.wp12_hp/machines
n170
n170
n170
n170
n170
n170
n170
n170
n170
n170
n170
n170
n171
n171
n171
n171
n171
n171
n171
n171
n171
n171
n171
n171
n172
n172
n172
n172
n172
n172
n172
n172
n172
n172
n172
n172
n173
n173
n173
n173
n173
n173
n173
n173
n173
n173
n173
n173
n174
n174
n174
n174
n174
n174
n174
n174
n174
n174
n174
n174
n176
n176
n176
n176
n176
n176
n176
n176
n176
n176
n176
n176
n177
n177
n177
n177
n177
n177
n177
n177
n177
n177
n177
n177
n178
n178
n178
n178
n178
n178
n178
n178
n178
n178
n178
n178
n179
n179
n179
n179
n179
n179
n179
n179
n179
n179
n179
n179
n180
n180
n180
n180
n180
n180
n180
n180
n180
n180
n180
n180
+ '[' 1 = 1 ']'
+ rsh_wrapper=/opt/sge6/mpich2_smpd/rsh
+ '[' '!' -x /opt/sge6/mpich2_smpd/rsh ']'
+ rshcmd=rsh
+ case "$ARC" in
+ ln -s /opt/sge6/mpich2_smpd/rsh /tmp/21074.1.wp12_hp/rsh
+ hash -r
+ '[' 0 = 1 ']'
+ port=21074
++ cat /tmp/21074.1.wp12_hp/machines
++ uniq
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n170 /opt/mpich2/intel/11.1//bin/smpd 21074
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n171 /opt/mpich2/intel/11.1//bin/smpd 21074
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n172 /opt/mpich2/intel/11.1//bin/smpd 21074
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n173 /opt/mpich2/intel/11.1//bin/smpd 21074
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n174 /opt/mpich2/intel/11.1//bin/smpd 21074
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n176 /opt/mpich2/intel/11.1//bin/smpd 21074
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n177 /opt/mpich2/intel/11.1//bin/smpd 21074
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n178 /opt/mpich2/intel/11.1//bin/smpd 21074
rsh n170 /opt/mpich2/intel/11.1//bin/smpd -port 21074 -d 0
rsh n171 /opt/mpich2/intel/11.1//bin/smpd -port 21074 -d 0
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n179 /opt/mpich2/intel/11.1//bin/smpd 21074
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n180 /opt/mpich2/intel/11.1//bin/smpd 21074
rsh n172 /opt/mpich2/intel/11.1//bin/smpd -port 21074 -d 0
rsh n173 /opt/mpich2/intel/11.1//bin/smpd -port 21074 -d 0
+ actual_retry=1
+ '[' 1 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (1 of 10)'
startmpich2.sh: check for smpd daemons (1 of 10)
+ numhosts=0
rsh n174 /opt/mpich2/intel/11.1//bin/smpd -port 21074 -d 0
rsh n177 /opt/mpich2/intel/11.1//bin/smpd -port 21074 -d 0
++ cat /tmp/21074.1.wp12_hp/machines
++ uniq
rsh n176 /opt/mpich2/intel/11.1//bin/smpd -port 21074 -d 0
+ for host in '`cat $machines | uniq`'
rsh n178 /opt/mpich2/intel/11.1//bin/smpd -port 21074 -d 0
++ /opt/mpich2/intel/11.1//bin/smpd -port 21074 -status n170
++ head -n 1
rsh n179 /opt/mpich2/intel/11.1//bin/smpd -port 21074 -d 0
++ cut -f 1 -d ' '
rsh n180 /opt/mpich2/intel/11.1//bin/smpd -port 21074 -d 0
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n170'
startmpich2.sh: missing smpd on n170
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21074 -status n171
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n171'
startmpich2.sh: missing smpd on n171
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21074 -status n172
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n172'
startmpich2.sh: missing smpd on n172
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21074 -status n173
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n173'
startmpich2.sh: missing smpd on n173
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21074 -status n174
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n174'
startmpich2.sh: missing smpd on n174
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21074 -status n176
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n176'
startmpich2.sh: missing smpd on n176
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21074 -status n177
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n177'
startmpich2.sh: found running smpd on n177
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21074 -status n178
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n178'
startmpich2.sh: found running smpd on n178
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21074 -status n179
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n179'
startmpich2.sh: found running smpd on n179
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21074 -status n180
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n180'
startmpich2.sh: found running smpd on n180
+ '[' 10 -eq 4 ']'
+ sleep 5
+ (( actual_retry += 1 ))
+ '[' 2 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (2 of 10)'
startmpich2.sh: check for smpd daemons (2 of 10)
+ numhosts=0
++ cat /tmp/21074.1.wp12_hp/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21074 -status n170
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n170'
startmpich2.sh: found running smpd on n170
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21074 -status n171
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n171'
startmpich2.sh: found running smpd on n171
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21074 -status n172
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n172'
startmpich2.sh: found running smpd on n172
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21074 -status n173
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n173'
startmpich2.sh: found running smpd on n173
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21074 -status n174
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n174'
startmpich2.sh: found running smpd on n174
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21074 -status n176
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n176'
startmpich2.sh: found running smpd on n176
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21074 -status n177
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n177'
startmpich2.sh: found running smpd on n177
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21074 -status n178
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n178'
startmpich2.sh: found running smpd on n178
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21074 -status n179
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n179'
startmpich2.sh: found running smpd on n179
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21074 -status n180
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n180'
startmpich2.sh: found running smpd on n180
+ '[' 10 -eq 10 ']'
+ echo 'startmpich2.sh: got all 10 of 10 nodes'
startmpich2.sh: got all 10 of 10 nodes
+ exit 0
rcp: /mnt/wk06/haule/materials/newDMFT/WinterSchool/La2CuO4/imp.?: No such file or directory
-catch_rsh /opt/mpich2/intel/11.1/
Your job finished at Tue Jan 3 12:00:03 EST 2012