USER      =  yinzping
HOMEDIR   =  /home/yinzping
JOB_ID    =  3826
JOB_NAME  =  LaAsPMfull
QUEUE     =  wp10
NSLOTS    =  36
NHOSTS    =  6

SUB_HOST  =  rupc06
SUB_DIR   =  /mnt/wk05/yinzping/WIEN2k/DMFT/FeAs/LaFeAsO/P4nmm/U5J8/beta160/Full2/2440yinzping

EXE_HOST  =  n145
NFS_HOST  =  caip02s
EXE_DIR   =  /mnt/p06
OUT_DIR   =  /tmp
TMP_DIR   =  /tmp/3826.1.wp10

INFO      =  Files larger than 100MB will be compressed to reduce the network load.
INFO2     =  Do NOT remove this file while job is running!

PORT      =  23826
rcp: /mnt/wk05/yinzping/WIEN2k/DMFT/FeAs/LaFeAsO/P4nmm/U5J8/beta160/Full2/2440yinzping/imp.0: not a plain file
Failure to rcp files!

Your job started at Sun Mar 20 02:40:45 EDT 2011 on host n145 in /mnt/p06/3826yinzping
Submission directory is /mnt/wk05/yinzping/WIEN2k/DMFT/FeAs/LaFeAsO/P4nmm/U5J8/beta160/Full2/2440yinzping on host rupc06
-----------------------------------------------------------------------------------------
+ echo -catch_rsh /opt/sge6/spool/n145/active_jobs/3826.1/pe_hostfile /opt/mpich2/intel/11.1/
-catch_rsh /opt/sge6/spool/n145/active_jobs/3826.1/pe_hostfile /opt/mpich2/intel/11.1/
+ SLEEPTIME=5
+ RETRIES=10
+ catch_rsh=0
+ catch_hostname=0
+ '[' 3 -gt 0 ']'
+ case "$1" in
+ catch_rsh=1
+ shift
+ '[' 2 -gt 0 ']'
+ case "$1" in
+ break
++ basename /opt/sge6/mpich2_smpd/startmpich2.sh
+ me=startmpich2.sh
+ '[' 2 -lt 2 ']'
+ pe_hostfile=/opt/sge6/spool/n145/active_jobs/3826.1/pe_hostfile
+ MPICH2_ROOT=/opt/mpich2/intel/11.1/
+ export MPICH2_ROOT
+ '[' '!' -x /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 ']'
+ '[' '!' -x /opt/mpich2/intel/11.1//bin/smpd ']'
+ '[' '!' -r /opt/sge6/spool/n145/active_jobs/3826.1/pe_hostfile ']'
+ machines=/tmp/3826.1.wp10/machines
+ PeHostfile2MachineFile /opt/sge6/spool/n145/active_jobs/3826.1/pe_hostfile
+ cat /opt/sge6/spool/n145/active_jobs/3826.1/pe_hostfile
+ read line
++ echo n145 1 wp10@n145 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n145
++ echo n145 1 wp10@n145 UNDEFINED
++ cut -f2 '-d '
+ nslots=1
+ i=1
+ '[' 1 -le 1 ']'
+ echo n145
++ expr 1 + 1
+ i=2
+ '[' 2 -le 1 ']'
+ read line
++ echo n161 3 wp10@n161 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n161
++ echo n161 3 wp10@n161 UNDEFINED
++ cut -f2 '-d '
+ nslots=3
+ i=1
+ '[' 1 -le 3 ']'
+ echo n161
++ expr 1 + 1
+ i=2
+ '[' 2 -le 3 ']'
+ echo n161
++ expr 2 + 1
+ i=3
+ '[' 3 -le 3 ']'
+ echo n161
++ expr 3 + 1
+ i=4
+ '[' 4 -le 3 ']'
+ read line
++ echo n162 4 wp10@n162 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n162
++ echo n162 4 wp10@n162 UNDEFINED
++ cut -f2 '-d '
+ nslots=4
+ i=1
+ '[' 1 -le 4 ']'
+ echo n162
++ expr 1 + 1
+ i=2
+ '[' 2 -le 4 ']'
+ echo n162
++ expr 2 + 1
+ i=3
+ '[' 3 -le 4 ']'
+ echo n162
++ expr 3 + 1
+ i=4
+ '[' 4 -le 4 ']'
+ echo n162
++ expr 4 + 1
+ i=5
+ '[' 5 -le 4 ']'
+ read line
++ echo n150 12 wp10@n150 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n150
++ echo n150 12 wp10@n150 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n150
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n150
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n150
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n150
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n150
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n150
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n150
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n150
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n150
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n150
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n150
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n150
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n151 12 wp10@n151 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n151
++ echo n151 12 wp10@n151 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n151
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n151
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n151
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n151
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n151
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n151
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n151
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n151
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n151
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n151
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n151
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n151
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n152 4 wp10@n152 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n152
++ echo n152 4 wp10@n152 UNDEFINED
++ cut -f2 '-d '
+ nslots=4
+ i=1
+ '[' 1 -le 4 ']'
+ echo n152
++ expr 1 + 1
+ i=2
+ '[' 2 -le 4 ']'
+ echo n152
++ expr 2 + 1
+ i=3
+ '[' 3 -le 4 ']'
+ echo n152
++ expr 3 + 1
+ i=4
+ '[' 4 -le 4 ']'
+ echo n152
++ expr 4 + 1
+ i=5
+ '[' 5 -le 4 ']'
+ read line
+ cat /tmp/3826.1.wp10/machines
n145
n161
n161
n161
n162
n162
n162
n162
n150
n150
n150
n150
n150
n150
n150
n150
n150
n150
n150
n150
n151
n151
n151
n151
n151
n151
n151
n151
n151
n151
n151
n151
n152
n152
n152
n152
+ '[' 1 = 1 ']'
+ rsh_wrapper=/opt/sge6/mpich2_smpd/rsh
+ '[' '!' -x /opt/sge6/mpich2_smpd/rsh ']'
+ rshcmd=rsh
+ case "$ARC" in
+ ln -s /opt/sge6/mpich2_smpd/rsh /tmp/3826.1.wp10/rsh
+ hash -r
+ '[' 0 = 1 ']'
+ port=23826
++ cat /tmp/3826.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n145 /opt/mpich2/intel/11.1//bin/smpd 23826
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n161 /opt/mpich2/intel/11.1//bin/smpd 23826
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n162 /opt/mpich2/intel/11.1//bin/smpd 23826
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n150 /opt/mpich2/intel/11.1//bin/smpd 23826
rsh n145 /opt/mpich2/intel/11.1//bin/smpd -port 23826 -d 0
rsh n161 /opt/mpich2/intel/11.1//bin/smpd -port 23826 -d 0
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n151 /opt/mpich2/intel/11.1//bin/smpd 23826
rsh n162 /opt/mpich2/intel/11.1//bin/smpd -port 23826 -d 0
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n152 /opt/mpich2/intel/11.1//bin/smpd 23826
+ actual_retry=1
+ '[' 1 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (1 of 10)'
startmpich2.sh: check for smpd daemons (1 of 10)
+ numhosts=0
rsh n151 /opt/mpich2/intel/11.1//bin/smpd -port 23826 -d 0
++ cat /tmp/3826.1.wp10/machines
rsh n150 /opt/mpich2/intel/11.1//bin/smpd -port 23826 -d 0
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23826 -status n145
++ cut -f 1 -d ' '
++ head -n 1
rsh n152 /opt/mpich2/intel/11.1//bin/smpd -port 23826 -d 0
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n145'
startmpich2.sh: missing smpd on n145
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23826 -status n161
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n161'
startmpich2.sh: missing smpd on n161
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23826 -status n162
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n162'
startmpich2.sh: missing smpd on n162
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23826 -status n150
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n150'
startmpich2.sh: missing smpd on n150
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23826 -status n151
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n151'
startmpich2.sh: missing smpd on n151
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23826 -status n152
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n152'
startmpich2.sh: missing smpd on n152
+ '[' 6 -eq 0 ']'
+ sleep 5
+ (( actual_retry += 1 ))
+ '[' 2 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (2 of 10)'
startmpich2.sh: check for smpd daemons (2 of 10)
+ numhosts=0
++ cat /tmp/3826.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23826 -status n145
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n145'
startmpich2.sh: found running smpd on n145
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23826 -status n161
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n161'
startmpich2.sh: found running smpd on n161
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23826 -status n162
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n162'
startmpich2.sh: found running smpd on n162
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23826 -status n150
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n150'
startmpich2.sh: found running smpd on n150
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23826 -status n151
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n151'
startmpich2.sh: found running smpd on n151
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23826 -status n152
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n152'
startmpich2.sh: found running smpd on n152
+ '[' 6 -eq 6 ']'
+ echo 'startmpich2.sh: got all 6 of 6 nodes'
startmpich2.sh: got all 6 of 6 nodes
+ exit 0
rm: cannot remove `./*.broyd*': No such file or directory
-catch_rsh /opt/mpich2/intel/11.1/
Your job finished at Thu Apr 7 23:35:27 EDT 2011