USER      =  yinzping
HOMEDIR   =  /home/yinzping
JOB_ID    =  21474
JOB_NAME  =  SePM
QUEUE     =  wp10
NSLOTS    =  36
NHOSTS    =  4

SUB_HOST  =  rupc06
SUB_DIR   =  /mnt/wk05/yinzping/WIEN2k/DMFT/FeSe/P4nmm/PM/U5J8/beta160/Full2/2158yinzping/9953yinzping

EXE_HOST  =  n148
NFS_HOST  =  caip02s
EXE_DIR   =  /mnt/p06
OUT_DIR   =  /tmp
TMP_DIR   =  /tmp/21474.1.wp10

INFO      =  Files larger than 100MB will be compressed to reduce the network load.
INFO2     =  Do NOT remove this file while job is running!

PORT      =  21474
rcp: /mnt/wk05/yinzping/WIEN2k/DMFT/FeSe/P4nmm/PM/U5J8/beta160/Full2/2158yinzping/9953yinzping/21473yinzping: not a plain file
rcp: /mnt/wk05/yinzping/WIEN2k/DMFT/FeSe/P4nmm/PM/U5J8/beta160/Full2/2158yinzping/9953yinzping/imp.0: not a plain file
Failure to rcp files!

Your job started at Wed Jan 11 09:50:39 EST 2012 on host n148 in /mnt/p06/21474yinzping
Submission directory is /mnt/wk05/yinzping/WIEN2k/DMFT/FeSe/P4nmm/PM/U5J8/beta160/Full2/2158yinzping/9953yinzping on host rupc06
-----------------------------------------------------------------------------------------
+ echo -catch_rsh /opt/sge6/spool/n148/active_jobs/21474.1/pe_hostfile /opt/mpich2/intel/11.1/
-catch_rsh /opt/sge6/spool/n148/active_jobs/21474.1/pe_hostfile /opt/mpich2/intel/11.1/
+ SLEEPTIME=5
+ RETRIES=10
+ catch_rsh=0
+ catch_hostname=0
+ '[' 3 -gt 0 ']'
+ case "$1" in
+ catch_rsh=1
+ shift
+ '[' 2 -gt 0 ']'
+ case "$1" in
+ break
++ basename /opt/sge6/mpich2_smpd/startmpich2.sh
+ me=startmpich2.sh
+ '[' 2 -lt 2 ']'
+ pe_hostfile=/opt/sge6/spool/n148/active_jobs/21474.1/pe_hostfile
+ MPICH2_ROOT=/opt/mpich2/intel/11.1/
+ export MPICH2_ROOT
+ '[' '!' -x /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 ']'
+ '[' '!' -x /opt/mpich2/intel/11.1//bin/smpd ']'
+ '[' '!' -r /opt/sge6/spool/n148/active_jobs/21474.1/pe_hostfile ']'
+ machines=/tmp/21474.1.wp10/machines
+ PeHostfile2MachineFile /opt/sge6/spool/n148/active_jobs/21474.1/pe_hostfile
+ cat /opt/sge6/spool/n148/active_jobs/21474.1/pe_hostfile
+ read line
++ echo n148 10 wp10@n148 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n148
++ echo n148 10 wp10@n148 UNDEFINED
++ cut -f2 '-d '
+ nslots=10
+ i=1
+ '[' 1 -le 10 ']'
+ echo n148
++ expr 1 + 1
+ i=2
+ '[' 2 -le 10 ']'
+ echo n148
++ expr 2 + 1
+ i=3
+ '[' 3 -le 10 ']'
+ echo n148
++ expr 3 + 1
+ i=4
+ '[' 4 -le 10 ']'
+ echo n148
++ expr 4 + 1
+ i=5
+ '[' 5 -le 10 ']'
+ echo n148
++ expr 5 + 1
+ i=6
+ '[' 6 -le 10 ']'
+ echo n148
++ expr 6 + 1
+ i=7
+ '[' 7 -le 10 ']'
+ echo n148
++ expr 7 + 1
+ i=8
+ '[' 8 -le 10 ']'
+ echo n148
++ expr 8 + 1
+ i=9
+ '[' 9 -le 10 ']'
+ echo n148
++ expr 9 + 1
+ i=10
+ '[' 10 -le 10 ']'
+ echo n148
++ expr 10 + 1
+ i=11
+ '[' 11 -le 10 ']'
+ read line
++ echo n163 12 wp10@n163 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n163
++ echo n163 12 wp10@n163 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n163
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n163
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n163
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n163
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n163
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n163
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n163
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n163
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n163
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n163
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n163
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n163
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n162 10 wp10@n162 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n162
++ echo n162 10 wp10@n162 UNDEFINED
++ cut -f2 '-d '
+ nslots=10
+ i=1
+ '[' 1 -le 10 ']'
+ echo n162
++ expr 1 + 1
+ i=2
+ '[' 2 -le 10 ']'
+ echo n162
++ expr 2 + 1
+ i=3
+ '[' 3 -le 10 ']'
+ echo n162
++ expr 3 + 1
+ i=4
+ '[' 4 -le 10 ']'
+ echo n162
++ expr 4 + 1
+ i=5
+ '[' 5 -le 10 ']'
+ echo n162
++ expr 5 + 1
+ i=6
+ '[' 6 -le 10 ']'
+ echo n162
++ expr 6 + 1
+ i=7
+ '[' 7 -le 10 ']'
+ echo n162
++ expr 7 + 1
+ i=8
+ '[' 8 -le 10 ']'
+ echo n162
++ expr 8 + 1
+ i=9
+ '[' 9 -le 10 ']'
+ echo n162
++ expr 9 + 1
+ i=10
+ '[' 10 -le 10 ']'
+ echo n162
++ expr 10 + 1
+ i=11
+ '[' 11 -le 10 ']'
+ read line
++ echo n152 4 wp10@n152 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n152
++ echo n152 4 wp10@n152 UNDEFINED
++ cut -f2 '-d '
+ nslots=4
+ i=1
+ '[' 1 -le 4 ']'
+ echo n152
++ expr 1 + 1
+ i=2
+ '[' 2 -le 4 ']'
+ echo n152
++ expr 2 + 1
+ i=3
+ '[' 3 -le 4 ']'
+ echo n152
++ expr 3 + 1
+ i=4
+ '[' 4 -le 4 ']'
+ echo n152
++ expr 4 + 1
+ i=5
+ '[' 5 -le 4 ']'
+ read line
+ cat /tmp/21474.1.wp10/machines
n148
n148
n148
n148
n148
n148
n148
n148
n148
n148
n163
n163
n163
n163
n163
n163
n163
n163
n163
n163
n163
n163
n162
n162
n162
n162
n162
n162
n162
n162
n162
n162
n152
n152
n152
n152
+ '[' 1 = 1 ']'
+ rsh_wrapper=/opt/sge6/mpich2_smpd/rsh
+ '[' '!' -x /opt/sge6/mpich2_smpd/rsh ']'
+ rshcmd=rsh
+ case "$ARC" in
+ ln -s /opt/sge6/mpich2_smpd/rsh /tmp/21474.1.wp10/rsh
+ hash -r
+ '[' 0 = 1 ']'
+ port=21474
++ cat /tmp/21474.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n148 /opt/mpich2/intel/11.1//bin/smpd 21474
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n163 /opt/mpich2/intel/11.1//bin/smpd 21474
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n162 /opt/mpich2/intel/11.1//bin/smpd 21474
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n152 /opt/mpich2/intel/11.1//bin/smpd 21474
+ actual_retry=1
+ '[' 1 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (1 of 10)'
startmpich2.sh: check for smpd daemons (1 of 10)
+ numhosts=0
++ cat /tmp/21474.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21474 -status n148
rsh n148 /opt/mpich2/intel/11.1//bin/smpd -port 21474 -d 0
++ head -n 1
rsh n163 /opt/mpich2/intel/11.1//bin/smpd -port 21474 -d 0
++ cut -f 1 -d ' '
rsh n162 /opt/mpich2/intel/11.1//bin/smpd -port 21474 -d 0
rsh n152 /opt/mpich2/intel/11.1//bin/smpd -port 21474 -d 0
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n148'
startmpich2.sh: missing smpd on n148
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21474 -status n163
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n163'
startmpich2.sh: missing smpd on n163
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21474 -status n162
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n162'
startmpich2.sh: missing smpd on n162
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21474 -status n152
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n152'
startmpich2.sh: missing smpd on n152
+ '[' 4 -eq 0 ']'
+ sleep 5
+ (( actual_retry += 1 ))
+ '[' 2 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (2 of 10)'
startmpich2.sh: check for smpd daemons (2 of 10)
+ numhosts=0
++ cat /tmp/21474.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21474 -status n148
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n148'
startmpich2.sh: found running smpd on n148
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21474 -status n163
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n163'
startmpich2.sh: found running smpd on n163
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21474 -status n162
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n162'
startmpich2.sh: found running smpd on n162
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21474 -status n152
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n152'
startmpich2.sh: found running smpd on n152
+ '[' 4 -eq 4 ']'
+ echo 'startmpich2.sh: got all 4 of 4 nodes'
startmpich2.sh: got all 4 of 4 nodes
+ exit 0
rm: cannot remove `./*.broyd*': No such file or directory
-catch_rsh /opt/mpich2/intel/11.1/
Your job finished at Wed Jan 18 22:28:30 EST 2012