USER      =  yinzping
HOMEDIR   =  /home/yinzping
JOB_ID    =  13406
JOB_NAME  =  LiPMfull2
QUEUE     =  wp10
NSLOTS    =  36
NHOSTS    =  5

SUB_HOST  =  rupc06
SUB_DIR   =  /mnt/wk05/yinzping/WIEN2k/DMFT/FeAs/LiFeAs/P4nmm/U5J8/beta160/Full2/2208yinzping/10606yinzping

EXE_HOST  =  n158
NFS_HOST  =  caip02s
EXE_DIR   =  /mnt/p06
OUT_DIR   =  /tmp
TMP_DIR   =  /tmp/13406.1.wp10

INFO      =  Files larger than 100MB will be compressed to reduce the network load.
INFO2     =  Do NOT remove this file while job is running!

PORT      =  23406
rcp: /mnt/wk05/yinzping/WIEN2k/DMFT/FeAs/LiFeAs/P4nmm/U5J8/beta160/Full2/2208yinzping/10606yinzping/imp.0: not a plain file
Failure to rcp files!

Your job started at Fri Sep 23 22:14:26 EDT 2011 on host n158 in /mnt/p06/13406yinzping
Submission directory is /mnt/wk05/yinzping/WIEN2k/DMFT/FeAs/LiFeAs/P4nmm/U5J8/beta160/Full2/2208yinzping/10606yinzping on host rupc06
-----------------------------------------------------------------------------------------
+ echo -catch_rsh /opt/sge6/spool/n158/active_jobs/13406.1/pe_hostfile /opt/mpich2/intel/11.1/
-catch_rsh /opt/sge6/spool/n158/active_jobs/13406.1/pe_hostfile /opt/mpich2/intel/11.1/
+ SLEEPTIME=5
+ RETRIES=10
+ catch_rsh=0
+ catch_hostname=0
+ '[' 3 -gt 0 ']'
+ case "$1" in
+ catch_rsh=1
+ shift
+ '[' 2 -gt 0 ']'
+ case "$1" in
+ break
++ basename /opt/sge6/mpich2_smpd/startmpich2.sh
+ me=startmpich2.sh
+ '[' 2 -lt 2 ']'
+ pe_hostfile=/opt/sge6/spool/n158/active_jobs/13406.1/pe_hostfile
+ MPICH2_ROOT=/opt/mpich2/intel/11.1/
+ export MPICH2_ROOT
+ '[' '!' -x /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 ']'
+ '[' '!' -x /opt/mpich2/intel/11.1//bin/smpd ']'
+ '[' '!' -r /opt/sge6/spool/n158/active_jobs/13406.1/pe_hostfile ']'
+ machines=/tmp/13406.1.wp10/machines
+ PeHostfile2MachineFile /opt/sge6/spool/n158/active_jobs/13406.1/pe_hostfile
+ cat /opt/sge6/spool/n158/active_jobs/13406.1/pe_hostfile
+ read line
++ echo n158 11 wp10@n158 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n158
++ echo n158 11 wp10@n158 UNDEFINED
++ cut -f2 '-d '
+ nslots=11
+ i=1
+ '[' 1 -le 11 ']'
+ echo n158
++ expr 1 + 1
+ i=2
+ '[' 2 -le 11 ']'
+ echo n158
++ expr 2 + 1
+ i=3
+ '[' 3 -le 11 ']'
+ echo n158
++ expr 3 + 1
+ i=4
+ '[' 4 -le 11 ']'
+ echo n158
++ expr 4 + 1
+ i=5
+ '[' 5 -le 11 ']'
+ echo n158
++ expr 5 + 1
+ i=6
+ '[' 6 -le 11 ']'
+ echo n158
++ expr 6 + 1
+ i=7
+ '[' 7 -le 11 ']'
+ echo n158
++ expr 7 + 1
+ i=8
+ '[' 8 -le 11 ']'
+ echo n158
++ expr 8 + 1
+ i=9
+ '[' 9 -le 11 ']'
+ echo n158
++ expr 9 + 1
+ i=10
+ '[' 10 -le 11 ']'
+ echo n158
++ expr 10 + 1
+ i=11
+ '[' 11 -le 11 ']'
+ echo n158
++ expr 11 + 1
+ i=12
+ '[' 12 -le 11 ']'
+ read line
++ echo n161 11 wp10@n161 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n161
++ echo n161 11 wp10@n161 UNDEFINED
++ cut -f2 '-d '
+ nslots=11
+ i=1
+ '[' 1 -le 11 ']'
+ echo n161
++ expr 1 + 1
+ i=2
+ '[' 2 -le 11 ']'
+ echo n161
++ expr 2 + 1
+ i=3
+ '[' 3 -le 11 ']'
+ echo n161
++ expr 3 + 1
+ i=4
+ '[' 4 -le 11 ']'
+ echo n161
++ expr 4 + 1
+ i=5
+ '[' 5 -le 11 ']'
+ echo n161
++ expr 5 + 1
+ i=6
+ '[' 6 -le 11 ']'
+ echo n161
++ expr 6 + 1
+ i=7
+ '[' 7 -le 11 ']'
+ echo n161
++ expr 7 + 1
+ i=8
+ '[' 8 -le 11 ']'
+ echo n161
++ expr 8 + 1
+ i=9
+ '[' 9 -le 11 ']'
+ echo n161
++ expr 9 + 1
+ i=10
+ '[' 10 -le 11 ']'
+ echo n161
++ expr 10 + 1
+ i=11
+ '[' 11 -le 11 ']'
+ echo n161
++ expr 11 + 1
+ i=12
+ '[' 12 -le 11 ']'
+ read line
++ echo n159 5 wp10@n159 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n159
++ echo n159 5 wp10@n159 UNDEFINED
++ cut -f2 '-d '
+ nslots=5
+ i=1
+ '[' 1 -le 5 ']'
+ echo n159
++ expr 1 + 1
+ i=2
+ '[' 2 -le 5 ']'
+ echo n159
++ expr 2 + 1
+ i=3
+ '[' 3 -le 5 ']'
+ echo n159
++ expr 3 + 1
+ i=4
+ '[' 4 -le 5 ']'
+ echo n159
++ expr 4 + 1
+ i=5
+ '[' 5 -le 5 ']'
+ echo n159
++ expr 5 + 1
+ i=6
+ '[' 6 -le 5 ']'
+ read line
++ echo n164 8 wp10@n164 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n164
++ echo n164 8 wp10@n164 UNDEFINED
++ cut -f2 '-d '
+ nslots=8
+ i=1
+ '[' 1 -le 8 ']'
+ echo n164
++ expr 1 + 1
+ i=2
+ '[' 2 -le 8 ']'
+ echo n164
++ expr 2 + 1
+ i=3
+ '[' 3 -le 8 ']'
+ echo n164
++ expr 3 + 1
+ i=4
+ '[' 4 -le 8 ']'
+ echo n164
++ expr 4 + 1
+ i=5
+ '[' 5 -le 8 ']'
+ echo n164
++ expr 5 + 1
+ i=6
+ '[' 6 -le 8 ']'
+ echo n164
++ expr 6 + 1
+ i=7
+ '[' 7 -le 8 ']'
+ echo n164
++ expr 7 + 1
+ i=8
+ '[' 8 -le 8 ']'
+ echo n164
++ expr 8 + 1
+ i=9
+ '[' 9 -le 8 ']'
+ read line
++ echo n160 1 wp10@n160 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n160
++ echo n160 1 wp10@n160 UNDEFINED
++ cut -f2 '-d '
+ nslots=1
+ i=1
+ '[' 1 -le 1 ']'
+ echo n160
++ expr 1 + 1
+ i=2
+ '[' 2 -le 1 ']'
+ read line
+ cat /tmp/13406.1.wp10/machines
n158
n158
n158
n158
n158
n158
n158
n158
n158
n158
n158
n161
n161
n161
n161
n161
n161
n161
n161
n161
n161
n161
n159
n159
n159
n159
n159
n164
n164
n164
n164
n164
n164
n164
n164
n160
+ '[' 1 = 1 ']'
+ rsh_wrapper=/opt/sge6/mpich2_smpd/rsh
+ '[' '!' -x /opt/sge6/mpich2_smpd/rsh ']'
+ rshcmd=rsh
+ case "$ARC" in
+ ln -s /opt/sge6/mpich2_smpd/rsh /tmp/13406.1.wp10/rsh
+ hash -r
+ '[' 0 = 1 ']'
+ port=23406
++ cat /tmp/13406.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n158 /opt/mpich2/intel/11.1//bin/smpd 23406
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n161 /opt/mpich2/intel/11.1//bin/smpd 23406
rsh n158 /opt/mpich2/intel/11.1//bin/smpd -port 23406 -d 0
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n159 /opt/mpich2/intel/11.1//bin/smpd 23406
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n164 /opt/mpich2/intel/11.1//bin/smpd 23406
rsh n161 /opt/mpich2/intel/11.1//bin/smpd -port 23406 -d 0
rsh n164 /opt/mpich2/intel/11.1//bin/smpd -port 23406 -d 0
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n160 /opt/mpich2/intel/11.1//bin/smpd 23406
+ actual_retry=1
+ '[' 1 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (1 of 10)'
startmpich2.sh: check for smpd daemons (1 of 10)
+ numhosts=0
++ cat /tmp/13406.1.wp10/machines
++ uniq
rsh n160 /opt/mpich2/intel/11.1//bin/smpd -port 23406 -d 0
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23406 -status n158
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n158'
startmpich2.sh: missing smpd on n158
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23406 -status n161
rsh n159 /opt/mpich2/intel/11.1//bin/smpd -port 23406 -d 0
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
++ head -n 1
++ cut -f 1 -d ' '
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n161'
startmpich2.sh: missing smpd on n161
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23406 -status n159
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n159'
startmpich2.sh: missing smpd on n159
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23406 -status n164
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n164'
startmpich2.sh: found running smpd on n164
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23406 -status n160
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n160'
startmpich2.sh: found running smpd on n160
+ '[' 5 -eq 2 ']'
+ sleep 5
+ (( actual_retry += 1 ))
+ '[' 2 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (2 of 10)'
startmpich2.sh: check for smpd daemons (2 of 10)
+ numhosts=0
++ cat /tmp/13406.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23406 -status n158
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n158'
startmpich2.sh: found running smpd on n158
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23406 -status n161
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n161'
startmpich2.sh: found running smpd on n161
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23406 -status n159
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n159'
startmpich2.sh: found running smpd on n159
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23406 -status n164
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n164'
startmpich2.sh: found running smpd on n164
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23406 -status n160
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n160'
startmpich2.sh: found running smpd on n160
+ '[' 5 -eq 5 ']'
+ echo 'startmpich2.sh: got all 5 of 5 nodes'
startmpich2.sh: got all 5 of 5 nodes
+ exit 0
rm: cannot remove `./*.broyd*': No such file or directory
-catch_rsh /opt/mpich2/intel/11.1/
Your job finished at Sat Oct 15 10:43:07 EDT 2011