USER      =  yinzping
HOMEDIR   =  /home/yinzping
JOB_ID    =  29034
JOB_NAME  =  K122PM
QUEUE     =  wp12
NSLOTS    =  36
NHOSTS    =  3

SUB_HOST  =  rupc06
SUB_DIR   =  /mnt/wk05/yinzping/WIEN2k/DMFT/FeAs/K122/I4mmm/U5J8/beta160/Full2/d5.5

EXE_HOST  =  n170
EXE_DIR   =  /mnt/n170
OUT_DIR   =  /scratch
TMP_DIR   =  /tmp/29034.1.wp12

INFO      =  Files larger than 100MB will be compressed to reduce the network load.
INFO2     =  Do NOT remove this file while job is running!

PORT      =  24034
rcp: /mnt/wk05/yinzping/WIEN2k/DMFT/FeAs/K122/I4mmm/U5J8/beta160/Full2/d5.5/28886yinzping: not a plain file
rcp: /mnt/wk05/yinzping/WIEN2k/DMFT/FeAs/K122/I4mmm/U5J8/beta160/Full2/d5.5/28903yinzping: not a plain file
rcp: /mnt/wk05/yinzping/WIEN2k/DMFT/FeAs/K122/I4mmm/U5J8/beta160/Full2/d5.5/28947yinzping: not a plain file
Failure to rcp files!

Your job started at Tue May 29 21:23:39 EDT 2012 on host n170 in /mnt/n170/29034yinzping
Submission directory is /mnt/wk05/yinzping/WIEN2k/DMFT/FeAs/K122/I4mmm/U5J8/beta160/Full2/d5.5 on host rupc06
-----------------------------------------------------------------------------------------
+ echo -catch_rsh /opt/sge6/spool/n170/active_jobs/29034.1/pe_hostfile /opt/mpich2/intel/11.1/
-catch_rsh /opt/sge6/spool/n170/active_jobs/29034.1/pe_hostfile /opt/mpich2/intel/11.1/
+ SLEEPTIME=5
+ RETRIES=10
+ catch_rsh=0
+ catch_hostname=0
+ '[' 3 -gt 0 ']'
+ case "$1" in
+ catch_rsh=1
+ shift
+ '[' 2 -gt 0 ']'
+ case "$1" in
+ break
++ basename /opt/sge6/mpich2_smpd/startmpich2.sh
+ me=startmpich2.sh
+ '[' 2 -lt 2 ']'
+ pe_hostfile=/opt/sge6/spool/n170/active_jobs/29034.1/pe_hostfile
+ MPICH2_ROOT=/opt/mpich2/intel/11.1/
+ export MPICH2_ROOT
+ '[' '!' -x /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 ']'
+ '[' '!' -x /opt/mpich2/intel/11.1//bin/smpd ']'
+ '[' '!' -r /opt/sge6/spool/n170/active_jobs/29034.1/pe_hostfile ']'
+ machines=/tmp/29034.1.wp12/machines
+ PeHostfile2MachineFile /opt/sge6/spool/n170/active_jobs/29034.1/pe_hostfile
+ cat /opt/sge6/spool/n170/active_jobs/29034.1/pe_hostfile
+ read line
++ echo n170 12 wp12@n170 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n170
++ echo n170 12 wp12@n170 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n170
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n170
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n170
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n170
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n170
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n170
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n170
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n170
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n170
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n170
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n170
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n170
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n171 12 wp12@n171 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n171
++ echo n171 12 wp12@n171 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n171
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n171
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n171
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n171
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n171
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n171
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n171
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n171
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n171
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n171
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n171
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n171
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n172 12 wp12@n172 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n172
++ echo n172 12 wp12@n172 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n172
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n172
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n172
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n172
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n172
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n172
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n172
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n172
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n172
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n172
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n172
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n172
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
+ cat /tmp/29034.1.wp12/machines
n170
n170
n170
n170
n170
n170
n170
n170
n170
n170
n170
n170
n171
n171
n171
n171
n171
n171
n171
n171
n171
n171
n171
n171
n172
n172
n172
n172
n172
n172
n172
n172
n172
n172
n172
n172
+ '[' 1 = 1 ']'
+ rsh_wrapper=/opt/sge6/mpich2_smpd/rsh
+ '[' '!' -x /opt/sge6/mpich2_smpd/rsh ']'
+ rshcmd=rsh
+ case "$ARC" in
+ ln -s /opt/sge6/mpich2_smpd/rsh /tmp/29034.1.wp12/rsh
+ hash -r
+ '[' 0 = 1 ']'
+ port=24034
++ cat /tmp/29034.1.wp12/machines
++ uniq
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n170 /opt/mpich2/intel/11.1//bin/smpd 24034
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n171 /opt/mpich2/intel/11.1//bin/smpd 24034
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n172 /opt/mpich2/intel/11.1//bin/smpd 24034
+ actual_retry=1
+ '[' 1 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (1 of 10)'
startmpich2.sh: check for smpd daemons (1 of 10)
+ numhosts=0
++ cat /tmp/29034.1.wp12/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 24034 -status n170
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n170'
startmpich2.sh: missing smpd on n170
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 24034 -status n171
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n171'
startmpich2.sh: missing smpd on n171
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 24034 -status n172
++ head -n 1
++ cut -f 1 -d ' '
rsh n171 /opt/mpich2/intel/11.1//bin/smpd -port 24034 -d 0
rsh n172 /opt/mpich2/intel/11.1//bin/smpd -port 24034 -d 0
rsh n170 /opt/mpich2/intel/11.1//bin/smpd -port 24034 -d 0
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n172'
startmpich2.sh: missing smpd on n172
+ '[' 3 -eq 0 ']'
+ sleep 5
+ (( actual_retry += 1 ))
+ '[' 2 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (2 of 10)'
startmpich2.sh: check for smpd daemons (2 of 10)
+ numhosts=0
++ cat /tmp/29034.1.wp12/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 24034 -status n170
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n170'
startmpich2.sh: found running smpd on n170
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 24034 -status n171
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n171'
startmpich2.sh: found running smpd on n171
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 24034 -status n172
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n172'
startmpich2.sh: found running smpd on n172
+ '[' 3 -eq 3 ']'
+ echo 'startmpich2.sh: got all 3 of 3 nodes'
startmpich2.sh: got all 3 of 3 nodes
+ exit 0
mv: cannot stat `./*.?.[12]': No such file or directory
rm: cannot remove `./*.broyd*': No such file or directory
-catch_rsh /opt/mpich2/intel/11.1/
Your job finished at Sat Jun 9 07:19:13 EDT 2012