USER      =  yinzping
HOMEDIR   =  /home/yinzping
JOB_ID    =  2082
JOB_NAME  =  K122PMfull2
QUEUE     =  wp10
NSLOTS    =  36
NHOSTS    =  4

SUB_HOST  =  rupc06
SUB_DIR   =  /mnt/wk05/yinzping/WIEN2k/DMFT/FeAs/K122/I4mmm/U5J8/beta160/Full2

EXE_HOST  =  n162
NFS_HOST  =  caip02s
EXE_DIR   =  /mnt/p06
OUT_DIR   =  /tmp
TMP_DIR   =  /tmp/2082.1.wp10

INFO      =  Files larger than 100MB will be compressed to reduce the network load.
INFO2     =  Do NOT remove this file while job is running!

PORT      =  22082

Your job started at Fri Jan 28 10:54:12 EST 2011 on host n162 in /mnt/p06/2082yinzping
Submission directory is /mnt/wk05/yinzping/WIEN2k/DMFT/FeAs/K122/I4mmm/U5J8/beta160/Full2 on host rupc06
-----------------------------------------------------------------------------------------
+ echo -catch_rsh /opt/sge6/spool/n162/active_jobs/2082.1/pe_hostfile /opt/mpich2/intel/11.1/
-catch_rsh /opt/sge6/spool/n162/active_jobs/2082.1/pe_hostfile /opt/mpich2/intel/11.1/
+ SLEEPTIME=5
+ RETRIES=10
+ catch_rsh=0
+ catch_hostname=0
+ '[' 3 -gt 0 ']'
+ case "$1" in
+ catch_rsh=1
+ shift
+ '[' 2 -gt 0 ']'
+ case "$1" in
+ break
++ basename /opt/sge6/mpich2_smpd/startmpich2.sh
+ me=startmpich2.sh
+ '[' 2 -lt 2 ']'
+ pe_hostfile=/opt/sge6/spool/n162/active_jobs/2082.1/pe_hostfile
+ MPICH2_ROOT=/opt/mpich2/intel/11.1/
+ export MPICH2_ROOT
+ '[' '!' -x /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 ']'
+ '[' '!' -x /opt/mpich2/intel/11.1//bin/smpd ']'
+ '[' '!' -r /opt/sge6/spool/n162/active_jobs/2082.1/pe_hostfile ']'
+ machines=/tmp/2082.1.wp10/machines
+ PeHostfile2MachineFile /opt/sge6/spool/n162/active_jobs/2082.1/pe_hostfile
+ cat /opt/sge6/spool/n162/active_jobs/2082.1/pe_hostfile
+ read line
++ echo n162 8 wp10@n162 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n162
++ echo n162 8 wp10@n162 UNDEFINED
++ cut -f2 '-d '
+ nslots=8
+ i=1
+ '[' 1 -le 8 ']'
+ echo n162
++ expr 1 + 1
+ i=2
+ '[' 2 -le 8 ']'
+ echo n162
++ expr 2 + 1
+ i=3
+ '[' 3 -le 8 ']'
+ echo n162
++ expr 3 + 1
+ i=4
+ '[' 4 -le 8 ']'
+ echo n162
++ expr 4 + 1
+ i=5
+ '[' 5 -le 8 ']'
+ echo n162
++ expr 5 + 1
+ i=6
+ '[' 6 -le 8 ']'
+ echo n162
++ expr 6 + 1
+ i=7
+ '[' 7 -le 8 ']'
+ echo n162
++ expr 7 + 1
+ i=8
+ '[' 8 -le 8 ']'
+ echo n162
++ expr 8 + 1
+ i=9
+ '[' 9 -le 8 ']'
+ read line
++ echo n141 12 wp10@n141 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n141
++ echo n141 12 wp10@n141 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n141
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n141
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n141
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n141
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n141
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n141
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n141
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n141
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n141
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n141
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n141
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n141
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n151 12 wp10@n151 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n151
++ echo n151 12 wp10@n151 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n151
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n151
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n151
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n151
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n151
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n151
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n151
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n151
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n151
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n151
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n151
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n151
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n165 4 wp10@n165 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n165
++ echo n165 4 wp10@n165 UNDEFINED
++ cut -f2 '-d '
+ nslots=4
+ i=1
+ '[' 1 -le 4 ']'
+ echo n165
++ expr 1 + 1
+ i=2
+ '[' 2 -le 4 ']'
+ echo n165
++ expr 2 + 1
+ i=3
+ '[' 3 -le 4 ']'
+ echo n165
++ expr 3 + 1
+ i=4
+ '[' 4 -le 4 ']'
+ echo n165
++ expr 4 + 1
+ i=5
+ '[' 5 -le 4 ']'
+ read line
+ cat /tmp/2082.1.wp10/machines
n162
n162
n162
n162
n162
n162
n162
n162
n141
n141
n141
n141
n141
n141
n141
n141
n141
n141
n141
n141
n151
n151
n151
n151
n151
n151
n151
n151
n151
n151
n151
n151
n165
n165
n165
n165
+ '[' 1 = 1 ']'
+ rsh_wrapper=/opt/sge6/mpich2_smpd/rsh
+ '[' '!' -x /opt/sge6/mpich2_smpd/rsh ']'
+ rshcmd=rsh
+ case "$ARC" in
+ ln -s /opt/sge6/mpich2_smpd/rsh /tmp/2082.1.wp10/rsh
+ hash -r
+ '[' 0 = 1 ']'
+ port=22082
++ cat /tmp/2082.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n162 /opt/mpich2/intel/11.1//bin/smpd 22082
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n141 /opt/mpich2/intel/11.1//bin/smpd 22082
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n151 /opt/mpich2/intel/11.1//bin/smpd 22082
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n165 /opt/mpich2/intel/11.1//bin/smpd 22082
+ actual_retry=1
+ '[' 1 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (1 of 10)'
startmpich2.sh: check for smpd daemons (1 of 10)
+ numhosts=0
++ cat /tmp/2082.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22082 -status n162
++ head -n 1
++ cut -f 1 -d ' '
rsh n162 /opt/mpich2/intel/11.1//bin/smpd -port 22082 -d 0
rsh n141 /opt/mpich2/intel/11.1//bin/smpd -port 22082 -d 0
rsh n151 /opt/mpich2/intel/11.1//bin/smpd -port 22082 -d 0
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
rsh n165 /opt/mpich2/intel/11.1//bin/smpd -port 22082 -d 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n162'
startmpich2.sh: missing smpd on n162
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22082 -status n141
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n141'
startmpich2.sh: missing smpd on n141
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22082 -status n151
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n151'
startmpich2.sh: missing smpd on n151
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22082 -status n165
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n165'
startmpich2.sh: missing smpd on n165
+ '[' 4 -eq 0 ']'
+ sleep 5
+ (( actual_retry += 1 ))
+ '[' 2 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (2 of 10)'
startmpich2.sh: check for smpd daemons (2 of 10)
+ numhosts=0
++ cat /tmp/2082.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22082 -status n162
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n162'
startmpich2.sh: found running smpd on n162
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22082 -status n141
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n141'
startmpich2.sh: found running smpd on n141
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22082 -status n151
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n151'
startmpich2.sh: found running smpd on n151
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22082 -status n165
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n165'
startmpich2.sh: found running smpd on n165
+ '[' 4 -eq 4 ']'
+ echo 'startmpich2.sh: got all 4 of 4 nodes'
startmpich2.sh: got all 4 of 4 nodes
+ exit 0
rm: cannot remove `./*.[12345]*.[12]': No such file or directory
rm: cannot remove `./*.?.[12]': No such file or directory
rm: cannot remove `./*.broyd*': No such file or directory
-catch_rsh /opt/mpich2/intel/11.1/
Your job finished at Sun Feb 6 13:16:17 EST 2011