USER      =  yinzping
HOMEDIR   =  /home/yinzping
JOB_ID    =  10606
JOB_NAME  =  LiPMfull2
QUEUE     =  wp10
NSLOTS    =  36
NHOSTS    =  3

SUB_HOST  =  rupc06
SUB_DIR   =  /mnt/wk05/yinzping/WIEN2k/DMFT/FeAs/LiFeAs/P4nmm/U5J8/beta160/Full2/2208yinzping

EXE_HOST  =  n144
NFS_HOST  =  caip03s
EXE_DIR   =  /mnt/p03
OUT_DIR   =  /tmp
TMP_DIR   =  /tmp/10606.1.wp10

INFO      =  Files larger than 100MB will be compressed to reduce the network load.
INFO2     =  Do NOT remove this file while job is running!

PORT      =  20606
rcp: /mnt/wk05/yinzping/WIEN2k/DMFT/FeAs/LiFeAs/P4nmm/U5J8/beta160/Full2/2208yinzping/3DFS: not a plain file
rcp: /mnt/wk05/yinzping/WIEN2k/DMFT/FeAs/LiFeAs/P4nmm/U5J8/beta160/Full2/2208yinzping/FS: not a plain file
rcp: /mnt/wk05/yinzping/WIEN2k/DMFT/FeAs/LiFeAs/P4nmm/U5J8/beta160/Full2/2208yinzping/Gk_k: not a plain file
rcp: /mnt/wk05/yinzping/WIEN2k/DMFT/FeAs/LiFeAs/P4nmm/U5J8/beta160/Full2/2208yinzping/Vertex: not a plain file
rcp: /mnt/wk05/yinzping/WIEN2k/DMFT/FeAs/LiFeAs/P4nmm/U5J8/beta160/Full2/2208yinzping/ancont: not a plain file
rcp: /mnt/wk05/yinzping/WIEN2k/DMFT/FeAs/LiFeAs/P4nmm/U5J8/beta160/Full2/2208yinzping/band: not a plain file
rcp: /mnt/wk05/yinzping/WIEN2k/DMFT/FeAs/LiFeAs/P4nmm/U5J8/beta160/Full2/2208yinzping/dos: not a plain file
rcp: /mnt/wk05/yinzping/WIEN2k/DMFT/FeAs/LiFeAs/P4nmm/U5J8/beta160/Full2/2208yinzping/imp.0: not a plain file
rcp: /mnt/wk05/yinzping/WIEN2k/DMFT/FeAs/LiFeAs/P4nmm/U5J8/beta160/Full2/2208yinzping/optics: not a plain file
Failure to rcp files!

Your job started at Wed Aug 24 12:51:02 EDT 2011 on host n144 in /mnt/p03/10606yinzping
Submission directory is /mnt/wk05/yinzping/WIEN2k/DMFT/FeAs/LiFeAs/P4nmm/U5J8/beta160/Full2/2208yinzping on host rupc06
-----------------------------------------------------------------------------------------
+ echo -catch_rsh /opt/sge6/spool/n144/active_jobs/10606.1/pe_hostfile /opt/mpich2/intel/11.1/
-catch_rsh /opt/sge6/spool/n144/active_jobs/10606.1/pe_hostfile /opt/mpich2/intel/11.1/
+ SLEEPTIME=5
+ RETRIES=10
+ catch_rsh=0
+ catch_hostname=0
+ '[' 3 -gt 0 ']'
+ case "$1" in
+ catch_rsh=1
+ shift
+ '[' 2 -gt 0 ']'
+ case "$1" in
+ break
++ basename /opt/sge6/mpich2_smpd/startmpich2.sh
+ me=startmpich2.sh
+ '[' 2 -lt 2 ']'
+ pe_hostfile=/opt/sge6/spool/n144/active_jobs/10606.1/pe_hostfile
+ MPICH2_ROOT=/opt/mpich2/intel/11.1/
+ export MPICH2_ROOT
+ '[' '!' -x /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 ']'
+ '[' '!' -x /opt/mpich2/intel/11.1//bin/smpd ']'
+ '[' '!' -r /opt/sge6/spool/n144/active_jobs/10606.1/pe_hostfile ']'
+ machines=/tmp/10606.1.wp10/machines
+ PeHostfile2MachineFile /opt/sge6/spool/n144/active_jobs/10606.1/pe_hostfile
+ cat /opt/sge6/spool/n144/active_jobs/10606.1/pe_hostfile
+ read line
++ echo n144 12 wp10@n144 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n144
++ echo n144 12 wp10@n144 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n144
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n144
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n144
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n144
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n144
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n144
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n144
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n144
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n144
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n144
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n144
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n144
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n147 12 wp10@n147 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n147
++ echo n147 12 wp10@n147 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n147
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n147
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n147
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n147
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n147
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n147
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n147
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n147
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n147
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n147
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n147
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n147
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n148 12 wp10@n148 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n148
++ echo n148 12 wp10@n148 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n148
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n148
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n148
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n148
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n148
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n148
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n148
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n148
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n148
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n148
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n148
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n148
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
+ cat /tmp/10606.1.wp10/machines
n144
n144
n144
n144
n144
n144
n144
n144
n144
n144
n144
n144
n147
n147
n147
n147
n147
n147
n147
n147
n147
n147
n147
n147
n148
n148
n148
n148
n148
n148
n148
n148
n148
n148
n148
n148
+ '[' 1 = 1 ']'
+ rsh_wrapper=/opt/sge6/mpich2_smpd/rsh
+ '[' '!' -x /opt/sge6/mpich2_smpd/rsh ']'
+ rshcmd=rsh
+ case "$ARC" in
+ ln -s /opt/sge6/mpich2_smpd/rsh /tmp/10606.1.wp10/rsh
+ hash -r
+ '[' 0 = 1 ']'
+ port=20606
++ cat /tmp/10606.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n144 /opt/mpich2/intel/11.1//bin/smpd 20606
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n147 /opt/mpich2/intel/11.1//bin/smpd 20606
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n148 /opt/mpich2/intel/11.1//bin/smpd 20606
+ actual_retry=1
+ '[' 1 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (1 of 10)'
startmpich2.sh: check for smpd daemons (1 of 10)
+ numhosts=0
++ cat /tmp/10606.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20606 -status n144
++ head -n 1
++ cut -f 1 -d ' '
rsh n144 /opt/mpich2/intel/11.1//bin/smpd -port 20606 -d 0
rsh n147 /opt/mpich2/intel/11.1//bin/smpd -port 20606 -d 0
rsh n148 /opt/mpich2/intel/11.1//bin/smpd -port 20606 -d 0
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n144'
startmpich2.sh: missing smpd on n144
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20606 -status n147
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n147'
startmpich2.sh: missing smpd on n147
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20606 -status n148
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n148'
startmpich2.sh: missing smpd on n148
+ '[' 3 -eq 0 ']'
+ sleep 5
+ (( actual_retry += 1 ))
+ '[' 2 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (2 of 10)'
startmpich2.sh: check for smpd daemons (2 of 10)
+ numhosts=0
++ cat /tmp/10606.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20606 -status n144
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n144'
startmpich2.sh: found running smpd on n144
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20606 -status n147
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n147'
startmpich2.sh: found running smpd on n147
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20606 -status n148
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n148'
startmpich2.sh: found running smpd on n148
+ '[' 3 -eq 3 ']'
+ echo 'startmpich2.sh: got all 3 of 3 nodes'
startmpich2.sh: got all 3 of 3 nodes
+ exit 0
rm: cannot remove `./*.broyd*': No such file or directory
-catch_rsh /opt/mpich2/intel/11.1/
Your job finished at Thu Sep 1 18:58:33 EDT 2011