USER      =  yinzping
HOMEDIR   =  /home/yinzping
JOB_ID    =  3827
JOB_NAME  =  Ca122PMfull
QUEUE     =  wp10
NSLOTS    =  36
NHOSTS    =  5

SUB_HOST  =  rupc06
SUB_DIR   =  /mnt/wk05/yinzping/WIEN2k/DMFT/FeAs/Ca122/I4mmm/U5J8/beta160/Full2/2443yinzping

EXE_HOST  =  n163
NFS_HOST  =  caip02s
EXE_DIR   =  /mnt/p06
OUT_DIR   =  /tmp
TMP_DIR   =  /tmp/3827.1.wp10

INFO      =  Files larger than 100MB will be compressed to reduce the network load.
INFO2     =  Do NOT remove this file while job is running!

PORT      =  23827
rcp: /mnt/wk05/yinzping/WIEN2k/DMFT/FeAs/Ca122/I4mmm/U5J8/beta160/Full2/2443yinzping/imp.0: not a plain file
Failure to rcp files!

Your job started at Mon Mar 21 05:45:00 EDT 2011 on host n163 in /mnt/p06/3827yinzping
Submission directory is /mnt/wk05/yinzping/WIEN2k/DMFT/FeAs/Ca122/I4mmm/U5J8/beta160/Full2/2443yinzping on host rupc06
-----------------------------------------------------------------------------------------
+ echo -catch_rsh /opt/sge6/spool/n163/active_jobs/3827.1/pe_hostfile /opt/mpich2/intel/11.1/
-catch_rsh /opt/sge6/spool/n163/active_jobs/3827.1/pe_hostfile /opt/mpich2/intel/11.1/
+ SLEEPTIME=5
+ RETRIES=10
+ catch_rsh=0
+ catch_hostname=0
+ '[' 3 -gt 0 ']'
+ case "$1" in
+ catch_rsh=1
+ shift
+ '[' 2 -gt 0 ']'
+ case "$1" in
+ break
++ basename /opt/sge6/mpich2_smpd/startmpich2.sh
+ me=startmpich2.sh
+ '[' 2 -lt 2 ']'
+ pe_hostfile=/opt/sge6/spool/n163/active_jobs/3827.1/pe_hostfile
+ MPICH2_ROOT=/opt/mpich2/intel/11.1/
+ export MPICH2_ROOT
+ '[' '!' -x /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 ']'
+ '[' '!' -x /opt/mpich2/intel/11.1//bin/smpd ']'
+ '[' '!' -r /opt/sge6/spool/n163/active_jobs/3827.1/pe_hostfile ']'
+ machines=/tmp/3827.1.wp10/machines
+ PeHostfile2MachineFile /opt/sge6/spool/n163/active_jobs/3827.1/pe_hostfile
+ cat /opt/sge6/spool/n163/active_jobs/3827.1/pe_hostfile
+ read line
++ echo n163 2 wp10@n163 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n163
++ echo n163 2 wp10@n163 UNDEFINED
++ cut -f2 '-d '
+ nslots=2
+ i=1
+ '[' 1 -le 2 ']'
+ echo n163
++ expr 1 + 1
+ i=2
+ '[' 2 -le 2 ']'
+ echo n163
++ expr 2 + 1
+ i=3
+ '[' 3 -le 2 ']'
+ read line
++ echo n144 12 wp10@n144 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n144
++ echo n144 12 wp10@n144 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n144
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n144
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n144
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n144
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n144
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n144
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n144
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n144
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n144
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n144
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n144
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n144
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n147 3 wp10@n147 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n147
++ echo n147 3 wp10@n147 UNDEFINED
++ cut -f2 '-d '
+ nslots=3
+ i=1
+ '[' 1 -le 3 ']'
+ echo n147
++ expr 1 + 1
+ i=2
+ '[' 2 -le 3 ']'
+ echo n147
++ expr 2 + 1
+ i=3
+ '[' 3 -le 3 ']'
+ echo n147
++ expr 3 + 1
+ i=4
+ '[' 4 -le 3 ']'
+ read line
++ echo n158 11 wp10@n158 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n158
++ echo n158 11 wp10@n158 UNDEFINED
++ cut -f2 '-d '
+ nslots=11
+ i=1
+ '[' 1 -le 11 ']'
+ echo n158
++ expr 1 + 1
+ i=2
+ '[' 2 -le 11 ']'
+ echo n158
++ expr 2 + 1
+ i=3
+ '[' 3 -le 11 ']'
+ echo n158
++ expr 3 + 1
+ i=4
+ '[' 4 -le 11 ']'
+ echo n158
++ expr 4 + 1
+ i=5
+ '[' 5 -le 11 ']'
+ echo n158
++ expr 5 + 1
+ i=6
+ '[' 6 -le 11 ']'
+ echo n158
++ expr 6 + 1
+ i=7
+ '[' 7 -le 11 ']'
+ echo n158
++ expr 7 + 1
+ i=8
+ '[' 8 -le 11 ']'
+ echo n158
++ expr 8 + 1
+ i=9
+ '[' 9 -le 11 ']'
+ echo n158
++ expr 9 + 1
+ i=10
+ '[' 10 -le 11 ']'
+ echo n158
++ expr 10 + 1
+ i=11
+ '[' 11 -le 11 ']'
+ echo n158
++ expr 11 + 1
+ i=12
+ '[' 12 -le 11 ']'
+ read line
++ echo n141 8 wp10@n141 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n141
++ echo n141 8 wp10@n141 UNDEFINED
++ cut -f2 '-d '
+ nslots=8
+ i=1
+ '[' 1 -le 8 ']'
+ echo n141
++ expr 1 + 1
+ i=2
+ '[' 2 -le 8 ']'
+ echo n141
++ expr 2 + 1
+ i=3
+ '[' 3 -le 8 ']'
+ echo n141
++ expr 3 + 1
+ i=4
+ '[' 4 -le 8 ']'
+ echo n141
++ expr 4 + 1
+ i=5
+ '[' 5 -le 8 ']'
+ echo n141
++ expr 5 + 1
+ i=6
+ '[' 6 -le 8 ']'
+ echo n141
++ expr 6 + 1
+ i=7
+ '[' 7 -le 8 ']'
+ echo n141
++ expr 7 + 1
+ i=8
+ '[' 8 -le 8 ']'
+ echo n141
++ expr 8 + 1
+ i=9
+ '[' 9 -le 8 ']'
+ read line
+ cat /tmp/3827.1.wp10/machines
n163
n163
n144
n144
n144
n144
n144
n144
n144
n144
n144
n144
n144
n144
n147
n147
n147
n158
n158
n158
n158
n158
n158
n158
n158
n158
n158
n158
n141
n141
n141
n141
n141
n141
n141
n141
+ '[' 1 = 1 ']'
+ rsh_wrapper=/opt/sge6/mpich2_smpd/rsh
+ '[' '!' -x /opt/sge6/mpich2_smpd/rsh ']'
+ rshcmd=rsh
+ case "$ARC" in
+ ln -s /opt/sge6/mpich2_smpd/rsh /tmp/3827.1.wp10/rsh
+ hash -r
+ '[' 0 = 1 ']'
+ port=23827
++ cat /tmp/3827.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n163 /opt/mpich2/intel/11.1//bin/smpd 23827
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n144 /opt/mpich2/intel/11.1//bin/smpd 23827
rsh n163 /opt/mpich2/intel/11.1//bin/smpd -port 23827 -d 0
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n147 /opt/mpich2/intel/11.1//bin/smpd 23827
rsh n144 /opt/mpich2/intel/11.1//bin/smpd -port 23827 -d 0
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n158 /opt/mpich2/intel/11.1//bin/smpd 23827
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n141 /opt/mpich2/intel/11.1//bin/smpd 23827
rsh n147 /opt/mpich2/intel/11.1//bin/smpd -port 23827 -d 0
rsh n158 /opt/mpich2/intel/11.1//bin/smpd -port 23827 -d 0
rsh n141 /opt/mpich2/intel/11.1//bin/smpd -port 23827 -d 0
+ actual_retry=1
+ '[' 1 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (1 of 10)'
startmpich2.sh: check for smpd daemons (1 of 10)
+ numhosts=0
++ cat /tmp/3827.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23827 -status n163
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n163'
startmpich2.sh: missing smpd on n163
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23827 -status n144
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n144'
startmpich2.sh: missing smpd on n144
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23827 -status n147
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n147'
startmpich2.sh: missing smpd on n147
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23827 -status n158
++ cut -f 1 -d ' '
++ head -n 1
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n158'
startmpich2.sh: found running smpd on n158
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23827 -status n141
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n141'
startmpich2.sh: found running smpd on n141
+ '[' 5 -eq 2 ']'
+ sleep 5
+ (( actual_retry += 1 ))
+ '[' 2 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (2 of 10)'
startmpich2.sh: check for smpd daemons (2 of 10)
+ numhosts=0
++ cat /tmp/3827.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23827 -status n163
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n163'
startmpich2.sh: found running smpd on n163
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23827 -status n144
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n144'
startmpich2.sh: found running smpd on n144
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23827 -status n147
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n147'
startmpich2.sh: found running smpd on n147
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23827 -status n158
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n158'
startmpich2.sh: found running smpd on n158
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23827 -status n141
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n141'
startmpich2.sh: found running smpd on n141
+ '[' 5 -eq 5 ']'
+ echo 'startmpich2.sh: got all 5 of 5 nodes'
startmpich2.sh: got all 5 of 5 nodes
+ exit 0
rm: cannot remove `./*.broyd*': No such file or directory
-catch_rsh /opt/mpich2/intel/11.1/
Your job finished at Fri Apr 1 17:49:31 EDT 2011