USER      =  yinzping
HOMEDIR   =  /home/yinzping
JOB_ID    =  11037
JOB_NAME  =  LaPPMfull
QUEUE     =  wp10
NSLOTS    =  36
NHOSTS    =  4

SUB_HOST  =  rupc06
SUB_DIR   =  /mnt/wk05/yinzping/WIEN2k/DMFT/LaFePO/P4nmm/U5J8/beta160/Full2/2446yinzping

EXE_HOST  =  n148
NFS_HOST  =  caip02s
EXE_DIR   =  /mnt/p06
OUT_DIR   =  /tmp
TMP_DIR   =  /tmp/11037.1.wp10

INFO      =  Files larger than 100MB will be compressed to reduce the network load.
INFO2     =  Do NOT remove this file while job is running!

PORT      =  21037
rcp: /mnt/wk05/yinzping/WIEN2k/DMFT/LaFePO/P4nmm/U5J8/beta160/Full2/2446yinzping/Vertex: not a plain file
rcp: /mnt/wk05/yinzping/WIEN2k/DMFT/LaFePO/P4nmm/U5J8/beta160/Full2/2446yinzping/imp.0: not a plain file
Failure to rcp files!

Your job started at Fri Sep 2 09:04:06 EDT 2011 on host n148 in /mnt/p06/11037yinzping
Submission directory is /mnt/wk05/yinzping/WIEN2k/DMFT/LaFePO/P4nmm/U5J8/beta160/Full2/2446yinzping on host rupc06
-----------------------------------------------------------------------------------------
+ echo -catch_rsh /opt/sge6/spool/n148/active_jobs/11037.1/pe_hostfile /opt/mpich2/intel/11.1/
-catch_rsh /opt/sge6/spool/n148/active_jobs/11037.1/pe_hostfile /opt/mpich2/intel/11.1/
+ SLEEPTIME=5
+ RETRIES=10
+ catch_rsh=0
+ catch_hostname=0
+ '[' 3 -gt 0 ']'
+ case "$1" in
+ catch_rsh=1
+ shift
+ '[' 2 -gt 0 ']'
+ case "$1" in
+ break
++ basename /opt/sge6/mpich2_smpd/startmpich2.sh
+ me=startmpich2.sh
+ '[' 2 -lt 2 ']'
+ pe_hostfile=/opt/sge6/spool/n148/active_jobs/11037.1/pe_hostfile
+ MPICH2_ROOT=/opt/mpich2/intel/11.1/
+ export MPICH2_ROOT
+ '[' '!' -x /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 ']'
+ '[' '!' -x /opt/mpich2/intel/11.1//bin/smpd ']'
+ '[' '!' -r /opt/sge6/spool/n148/active_jobs/11037.1/pe_hostfile ']'
+ machines=/tmp/11037.1.wp10/machines
+ PeHostfile2MachineFile /opt/sge6/spool/n148/active_jobs/11037.1/pe_hostfile
+ cat /opt/sge6/spool/n148/active_jobs/11037.1/pe_hostfile
+ read line
++ echo n148 12 wp10@n148 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n148
++ echo n148 12 wp10@n148 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n148
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n148
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n148
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n148
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n148
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n148
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n148
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n148
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n148
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n148
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n148
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n148
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n147 4 wp10@n147 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n147
++ echo n147 4 wp10@n147 UNDEFINED
++ cut -f2 '-d '
+ nslots=4
+ i=1
+ '[' 1 -le 4 ']'
+ echo n147
++ expr 1 + 1
+ i=2
+ '[' 2 -le 4 ']'
+ echo n147
++ expr 2 + 1
+ i=3
+ '[' 3 -le 4 ']'
+ echo n147
++ expr 3 + 1
+ i=4
+ '[' 4 -le 4 ']'
+ echo n147
++ expr 4 + 1
+ i=5
+ '[' 5 -le 4 ']'
+ read line
++ echo n150 12 wp10@n150 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n150
++ echo n150 12 wp10@n150 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n150
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n150
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n150
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n150
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n150
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n150
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n150
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n150
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n150
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n150
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n150
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n150
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n151 8 wp10@n151 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n151
++ echo n151 8 wp10@n151 UNDEFINED
++ cut -f2 '-d '
+ nslots=8
+ i=1
+ '[' 1 -le 8 ']'
+ echo n151
++ expr 1 + 1
+ i=2
+ '[' 2 -le 8 ']'
+ echo n151
++ expr 2 + 1
+ i=3
+ '[' 3 -le 8 ']'
+ echo n151
++ expr 3 + 1
+ i=4
+ '[' 4 -le 8 ']'
+ echo n151
++ expr 4 + 1
+ i=5
+ '[' 5 -le 8 ']'
+ echo n151
++ expr 5 + 1
+ i=6
+ '[' 6 -le 8 ']'
+ echo n151
++ expr 6 + 1
+ i=7
+ '[' 7 -le 8 ']'
+ echo n151
++ expr 7 + 1
+ i=8
+ '[' 8 -le 8 ']'
+ echo n151
++ expr 8 + 1
+ i=9
+ '[' 9 -le 8 ']'
+ read line
+ cat /tmp/11037.1.wp10/machines
n148
n148
n148
n148
n148
n148
n148
n148
n148
n148
n148
n148
n147
n147
n147
n147
n150
n150
n150
n150
n150
n150
n150
n150
n150
n150
n150
n150
n151
n151
n151
n151
n151
n151
n151
n151
+ '[' 1 = 1 ']'
+ rsh_wrapper=/opt/sge6/mpich2_smpd/rsh
+ '[' '!' -x /opt/sge6/mpich2_smpd/rsh ']'
+ rshcmd=rsh
+ case "$ARC" in
+ ln -s /opt/sge6/mpich2_smpd/rsh /tmp/11037.1.wp10/rsh
+ hash -r
+ '[' 0 = 1 ']'
+ port=21037
++ cat /tmp/11037.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n148 /opt/mpich2/intel/11.1//bin/smpd 21037
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n147 /opt/mpich2/intel/11.1//bin/smpd 21037
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n150 /opt/mpich2/intel/11.1//bin/smpd 21037
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n151 /opt/mpich2/intel/11.1//bin/smpd 21037
+ actual_retry=1
+ '[' 1 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (1 of 10)'
startmpich2.sh: check for smpd daemons (1 of 10)
+ numhosts=0
++ cat /tmp/11037.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21037 -status n148
rsh n148 /opt/mpich2/intel/11.1//bin/smpd -port 21037 -d 0
rsh n147 /opt/mpich2/intel/11.1//bin/smpd -port 21037 -d 0
++ head -n 1
++ cut -f 1 -d ' '
rsh n150 /opt/mpich2/intel/11.1//bin/smpd -port 21037 -d 0
rsh n151 /opt/mpich2/intel/11.1//bin/smpd -port 21037 -d 0
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n148'
startmpich2.sh: missing smpd on n148
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21037 -status n147
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n147'
startmpich2.sh: missing smpd on n147
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21037 -status n150
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n150'
startmpich2.sh: missing smpd on n150
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21037 -status n151
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n151'
startmpich2.sh: missing smpd on n151
+ '[' 4 -eq 0 ']'
+ sleep 5
+ (( actual_retry += 1 ))
+ '[' 2 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (2 of 10)'
startmpich2.sh: check for smpd daemons (2 of 10)
+ numhosts=0
++ cat /tmp/11037.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21037 -status n148
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n148'
startmpich2.sh: found running smpd on n148
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21037 -status n147
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n147'
startmpich2.sh: found running smpd on n147
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21037 -status n150
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n150'
startmpich2.sh: found running smpd on n150
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21037 -status n151
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n151'
startmpich2.sh: found running smpd on n151
+ '[' 4 -eq 4 ']'
+ echo 'startmpich2.sh: got all 4 of 4 nodes'
startmpich2.sh: got all 4 of 4 nodes
+ exit 0
rm: cannot remove `./*.broyd*': No such file or directory
-catch_rsh /opt/mpich2/intel/11.1/
Your job finished at Wed Sep 14 04:26:28 EDT 2011