USER      =  yinzping
HOMEDIR   =  /home/yinzping
JOB_ID    =  21970
JOB_NAME  =  SePM
QUEUE     =  wp10
NSLOTS    =  36
NHOSTS    =  4

SUB_HOST  =  rupc06
SUB_DIR   =  /mnt/wk05/yinzping/WIEN2k/DMFT/FeSe/P4nmm/PM/U5J8/beta160/Full2/2158yinzping/9953yinzping/21474yinzping

EXE_HOST  =  n150
NFS_HOST  =  caip02s
EXE_DIR   =  /mnt/p06
OUT_DIR   =  /tmp
TMP_DIR   =  /tmp/21970.1.wp10

INFO      =  Files larger than 100MB will be compressed to reduce the network load.
INFO2     =  Do NOT remove this file while job is running!

PORT      =  21970
rcp: /mnt/wk05/yinzping/WIEN2k/DMFT/FeSe/P4nmm/PM/U5J8/beta160/Full2/2158yinzping/9953yinzping/21474yinzping/Vertex: not a plain file
rcp: /mnt/wk05/yinzping/WIEN2k/DMFT/FeSe/P4nmm/PM/U5J8/beta160/Full2/2158yinzping/9953yinzping/21474yinzping/imp.0: not a plain file
Failure to rcp files!

Your job started at Fri Jan 20 12:33:46 EST 2012 on host n150 in /mnt/p06/21970yinzping
Submission directory is /mnt/wk05/yinzping/WIEN2k/DMFT/FeSe/P4nmm/PM/U5J8/beta160/Full2/2158yinzping/9953yinzping/21474yinzping on host rupc06
-----------------------------------------------------------------------------------------
+ echo -catch_rsh /opt/sge6/spool/n150/active_jobs/21970.1/pe_hostfile /opt/mpich2/intel/11.1/
-catch_rsh /opt/sge6/spool/n150/active_jobs/21970.1/pe_hostfile /opt/mpich2/intel/11.1/
+ SLEEPTIME=5
+ RETRIES=10
+ catch_rsh=0
+ catch_hostname=0
+ '[' 3 -gt 0 ']'
+ case "$1" in
+ catch_rsh=1
+ shift
+ '[' 2 -gt 0 ']'
+ case "$1" in
+ break
++ basename /opt/sge6/mpich2_smpd/startmpich2.sh
+ me=startmpich2.sh
+ '[' 2 -lt 2 ']'
+ pe_hostfile=/opt/sge6/spool/n150/active_jobs/21970.1/pe_hostfile
+ MPICH2_ROOT=/opt/mpich2/intel/11.1/
+ export MPICH2_ROOT
+ '[' '!' -x /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 ']'
+ '[' '!' -x /opt/mpich2/intel/11.1//bin/smpd ']'
+ '[' '!' -r /opt/sge6/spool/n150/active_jobs/21970.1/pe_hostfile ']'
+ machines=/tmp/21970.1.wp10/machines
+ PeHostfile2MachineFile /opt/sge6/spool/n150/active_jobs/21970.1/pe_hostfile
+ cat /opt/sge6/spool/n150/active_jobs/21970.1/pe_hostfile
+ read line
++ echo n150 4 wp10@n150 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n150
++ echo n150 4 wp10@n150 UNDEFINED
++ cut -f2 '-d '
+ nslots=4
+ i=1
+ '[' 1 -le 4 ']'
+ echo n150
++ expr 1 + 1
+ i=2
+ '[' 2 -le 4 ']'
+ echo n150
++ expr 2 + 1
+ i=3
+ '[' 3 -le 4 ']'
+ echo n150
++ expr 3 + 1
+ i=4
+ '[' 4 -le 4 ']'
+ echo n150
++ expr 4 + 1
+ i=5
+ '[' 5 -le 4 ']'
+ read line
++ echo n148 10 wp10@n148 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n148
++ echo n148 10 wp10@n148 UNDEFINED
++ cut -f2 '-d '
+ nslots=10
+ i=1
+ '[' 1 -le 10 ']'
+ echo n148
++ expr 1 + 1
+ i=2
+ '[' 2 -le 10 ']'
+ echo n148
++ expr 2 + 1
+ i=3
+ '[' 3 -le 10 ']'
+ echo n148
++ expr 3 + 1
+ i=4
+ '[' 4 -le 10 ']'
+ echo n148
++ expr 4 + 1
+ i=5
+ '[' 5 -le 10 ']'
+ echo n148
++ expr 5 + 1
+ i=6
+ '[' 6 -le 10 ']'
+ echo n148
++ expr 6 + 1
+ i=7
+ '[' 7 -le 10 ']'
+ echo n148
++ expr 7 + 1
+ i=8
+ '[' 8 -le 10 ']'
+ echo n148
++ expr 8 + 1
+ i=9
+ '[' 9 -le 10 ']'
+ echo n148
++ expr 9 + 1
+ i=10
+ '[' 10 -le 10 ']'
+ echo n148
++ expr 10 + 1
+ i=11
+ '[' 11 -le 10 ']'
+ read line
++ echo n163 12 wp10@n163 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n163
++ echo n163 12 wp10@n163 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n163
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n163
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n163
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n163
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n163
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n163
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n163
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n163
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n163
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n163
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n163
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n163
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n162 10 wp10@n162 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n162
++ echo n162 10 wp10@n162 UNDEFINED
++ cut -f2 '-d '
+ nslots=10
+ i=1
+ '[' 1 -le 10 ']'
+ echo n162
++ expr 1 + 1
+ i=2
+ '[' 2 -le 10 ']'
+ echo n162
++ expr 2 + 1
+ i=3
+ '[' 3 -le 10 ']'
+ echo n162
++ expr 3 + 1
+ i=4
+ '[' 4 -le 10 ']'
+ echo n162
++ expr 4 + 1
+ i=5
+ '[' 5 -le 10 ']'
+ echo n162
++ expr 5 + 1
+ i=6
+ '[' 6 -le 10 ']'
+ echo n162
++ expr 6 + 1
+ i=7
+ '[' 7 -le 10 ']'
+ echo n162
++ expr 7 + 1
+ i=8
+ '[' 8 -le 10 ']'
+ echo n162
++ expr 8 + 1
+ i=9
+ '[' 9 -le 10 ']'
+ echo n162
++ expr 9 + 1
+ i=10
+ '[' 10 -le 10 ']'
+ echo n162
++ expr 10 + 1
+ i=11
+ '[' 11 -le 10 ']'
+ read line
+ cat /tmp/21970.1.wp10/machines
n150
n150
n150
n150
n148
n148
n148
n148
n148
n148
n148
n148
n148
n148
n163
n163
n163
n163
n163
n163
n163
n163
n163
n163
n163
n163
n162
n162
n162
n162
n162
n162
n162
n162
n162
n162
+ '[' 1 = 1 ']'
+ rsh_wrapper=/opt/sge6/mpich2_smpd/rsh
+ '[' '!' -x /opt/sge6/mpich2_smpd/rsh ']'
+ rshcmd=rsh
+ case "$ARC" in
+ ln -s /opt/sge6/mpich2_smpd/rsh /tmp/21970.1.wp10/rsh
+ hash -r
+ '[' 0 = 1 ']'
+ port=21970
++ cat /tmp/21970.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n150 /opt/mpich2/intel/11.1//bin/smpd 21970
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n148 /opt/mpich2/intel/11.1//bin/smpd 21970
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n163 /opt/mpich2/intel/11.1//bin/smpd 21970
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n162 /opt/mpich2/intel/11.1//bin/smpd 21970
+ actual_retry=1
+ '[' 1 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (1 of 10)'
startmpich2.sh: check for smpd daemons (1 of 10)
+ numhosts=0
++ cat /tmp/21970.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
rsh n163 /opt/mpich2/intel/11.1//bin/smpd -port 21970 -d 0
++ /opt/mpich2/intel/11.1//bin/smpd -port 21970 -status n150
++ head -n 1
rsh n150 /opt/mpich2/intel/11.1//bin/smpd -port 21970 -d 0
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
rsh n162 /opt/mpich2/intel/11.1//bin/smpd -port 21970 -d 0
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n150'
startmpich2.sh: missing smpd on n150
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21970 -status n148
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n148'
startmpich2.sh: missing smpd on n148
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21970 -status n163
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n163'
startmpich2.sh: missing smpd on n163
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21970 -status n162
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n162'
startmpich2.sh: missing smpd on n162
+ '[' 4 -eq 0 ']'
+ sleep 5
rsh n148 /opt/mpich2/intel/11.1//bin/smpd -port 21970 -d 0
+ (( actual_retry += 1 ))
+ '[' 2 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (2 of 10)'
startmpich2.sh: check for smpd daemons (2 of 10)
+ numhosts=0
++ cat /tmp/21970.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21970 -status n150
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n150'
startmpich2.sh: found running smpd on n150
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21970 -status n148
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n148'
startmpich2.sh: found running smpd on n148
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21970 -status n163
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n163'
startmpich2.sh: found running smpd on n163
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21970 -status n162
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n162'
startmpich2.sh: found running smpd on n162
+ '[' 4 -eq 4 ']'
+ echo 'startmpich2.sh: got all 4 of 4 nodes'
startmpich2.sh: got all 4 of 4 nodes
+ exit 0
rm: cannot remove `./*.broyd*': No such file or directory
-catch_rsh /opt/mpich2/intel/11.1/
Your job finished at Sun Jan 29 06:45:31 EST 2012