USER      =  yinzping
HOMEDIR   =  /home/yinzping
JOB_ID    =  17257
JOB_NAME  =  LaAsPMfull
QUEUE     =  wp10
NSLOTS    =  36
NHOSTS    =  4

SUB_HOST  =  rupc06
SUB_DIR   =  /mnt/wk05/yinzping/WIEN2k/DMFT/FeAs/LaFeAsO/P4nmm/U5J8/beta160/Full2/2440yinzping/3826yinzping/11036yinzping

EXE_HOST  =  n143
NFS_HOST  =  caip02s
EXE_DIR   =  /mnt/p06
OUT_DIR   =  /tmp
TMP_DIR   =  /tmp/17257.1.wp10

INFO      =  Files larger than 100MB will be compressed to reduce the network load.
INFO2     =  Do NOT remove this file while job is running!

PORT      =  22257
rcp: /mnt/wk05/yinzping/WIEN2k/DMFT/FeAs/LaFeAsO/P4nmm/U5J8/beta160/Full2/2440yinzping/3826yinzping/11036yinzping/imp.0: not a plain file
Failure to rcp files!

Your job started at Mon Oct 31 09:14:18 EDT 2011 on host n143 in /mnt/p06/17257yinzping
Submission directory is /mnt/wk05/yinzping/WIEN2k/DMFT/FeAs/LaFeAsO/P4nmm/U5J8/beta160/Full2/2440yinzping/3826yinzping/11036yinzping on host rupc06
-----------------------------------------------------------------------------------------
+ echo -catch_rsh /opt/sge6/spool/n143/active_jobs/17257.1/pe_hostfile /opt/mpich2/intel/11.1/
-catch_rsh /opt/sge6/spool/n143/active_jobs/17257.1/pe_hostfile /opt/mpich2/intel/11.1/
+ SLEEPTIME=5
+ RETRIES=10
+ catch_rsh=0
+ catch_hostname=0
+ '[' 3 -gt 0 ']'
+ case "$1" in
+ catch_rsh=1
+ shift
+ '[' 2 -gt 0 ']'
+ case "$1" in
+ break
++ basename /opt/sge6/mpich2_smpd/startmpich2.sh
+ me=startmpich2.sh
+ '[' 2 -lt 2 ']'
+ pe_hostfile=/opt/sge6/spool/n143/active_jobs/17257.1/pe_hostfile
+ MPICH2_ROOT=/opt/mpich2/intel/11.1/
+ export MPICH2_ROOT
+ '[' '!' -x /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 ']'
+ '[' '!' -x /opt/mpich2/intel/11.1//bin/smpd ']'
+ '[' '!' -r /opt/sge6/spool/n143/active_jobs/17257.1/pe_hostfile ']'
+ machines=/tmp/17257.1.wp10/machines
+ PeHostfile2MachineFile /opt/sge6/spool/n143/active_jobs/17257.1/pe_hostfile
+ cat /opt/sge6/spool/n143/active_jobs/17257.1/pe_hostfile
+ read line
++ echo n143 10 wp10@n143 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n143
++ echo n143 10 wp10@n143 UNDEFINED
++ cut -f2 '-d '
+ nslots=10
+ i=1
+ '[' 1 -le 10 ']'
+ echo n143
++ expr 1 + 1
+ i=2
+ '[' 2 -le 10 ']'
+ echo n143
++ expr 2 + 1
+ i=3
+ '[' 3 -le 10 ']'
+ echo n143
++ expr 3 + 1
+ i=4
+ '[' 4 -le 10 ']'
+ echo n143
++ expr 4 + 1
+ i=5
+ '[' 5 -le 10 ']'
+ echo n143
++ expr 5 + 1
+ i=6
+ '[' 6 -le 10 ']'
+ echo n143
++ expr 6 + 1
+ i=7
+ '[' 7 -le 10 ']'
+ echo n143
++ expr 7 + 1
+ i=8
+ '[' 8 -le 10 ']'
+ echo n143
++ expr 8 + 1
+ i=9
+ '[' 9 -le 10 ']'
+ echo n143
++ expr 9 + 1
+ i=10
+ '[' 10 -le 10 ']'
+ echo n143
++ expr 10 + 1
+ i=11
+ '[' 11 -le 10 ']'
+ read line
++ echo n162 12 wp10@n162 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n162
++ echo n162 12 wp10@n162 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n162
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n162
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n162
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n162
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n162
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n162
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n162
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n162
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n162
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n162
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n162
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n162
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n146 12 wp10@n146 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n146
++ echo n146 12 wp10@n146 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n146
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n146
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n146
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n146
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n146
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n146
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n146
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n146
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n146
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n146
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n146
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n146
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n151 2 wp10@n151 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n151
++ echo n151 2 wp10@n151 UNDEFINED
++ cut -f2 '-d '
+ nslots=2
+ i=1
+ '[' 1 -le 2 ']'
+ echo n151
++ expr 1 + 1
+ i=2
+ '[' 2 -le 2 ']'
+ echo n151
++ expr 2 + 1
+ i=3
+ '[' 3 -le 2 ']'
+ read line
+ cat /tmp/17257.1.wp10/machines
n143
n143
n143
n143
n143
n143
n143
n143
n143
n143
n162
n162
n162
n162
n162
n162
n162
n162
n162
n162
n162
n162
n146
n146
n146
n146
n146
n146
n146
n146
n146
n146
n146
n146
n151
n151
+ '[' 1 = 1 ']'
+ rsh_wrapper=/opt/sge6/mpich2_smpd/rsh
+ '[' '!' -x /opt/sge6/mpich2_smpd/rsh ']'
+ rshcmd=rsh
+ case "$ARC" in
+ ln -s /opt/sge6/mpich2_smpd/rsh /tmp/17257.1.wp10/rsh
+ hash -r
+ '[' 0 = 1 ']'
+ port=22257
++ cat /tmp/17257.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n143 /opt/mpich2/intel/11.1//bin/smpd 22257
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n162 /opt/mpich2/intel/11.1//bin/smpd 22257
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n146 /opt/mpich2/intel/11.1//bin/smpd 22257
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n151 /opt/mpich2/intel/11.1//bin/smpd 22257
+ actual_retry=1
+ '[' 1 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (1 of 10)'
startmpich2.sh: check for smpd daemons (1 of 10)
+ numhosts=0
++ cat /tmp/17257.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22257 -status n143
++ head -n 1
rsh n162 /opt/mpich2/intel/11.1//bin/smpd -port 22257 -d 0
rsh n143 /opt/mpich2/intel/11.1//bin/smpd -port 22257 -d 0
++ cut -f 1 -d ' '
rsh n146 /opt/mpich2/intel/11.1//bin/smpd -port 22257 -d 0
rsh n151 /opt/mpich2/intel/11.1//bin/smpd -port 22257 -d 0
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n143'
startmpich2.sh: missing smpd on n143
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22257 -status n162
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n162'
startmpich2.sh: missing smpd on n162
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22257 -status n146
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n146'
startmpich2.sh: missing smpd on n146
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22257 -status n151
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n151'
startmpich2.sh: missing smpd on n151
+ '[' 4 -eq 0 ']'
+ sleep 5
+ (( actual_retry += 1 ))
+ '[' 2 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (2 of 10)'
startmpich2.sh: check for smpd daemons (2 of 10)
+ numhosts=0
++ cat /tmp/17257.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22257 -status n143
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n143'
startmpich2.sh: found running smpd on n143
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22257 -status n162
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n162'
startmpich2.sh: found running smpd on n162
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22257 -status n146
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n146'
startmpich2.sh: found running smpd on n146
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22257 -status n151
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n151'
startmpich2.sh: found running smpd on n151
+ '[' 4 -eq 4 ']'
+ echo 'startmpich2.sh: got all 4 of 4 nodes'
startmpich2.sh: got all 4 of 4 nodes
+ exit 0
rm: cannot remove `./*.broyd*': No such file or directory
-catch_rsh /opt/mpich2/intel/11.1/
Your job finished at Sat Nov 12 01:15:14 EST 2011