USER      =  yinzping
HOMEDIR   =  /home/yinzping
JOB_ID    =  19209
JOB_NAME  =  TePMfull2
QUEUE     =  wp10
NSLOTS    =  36
NHOSTS    =  7

SUB_HOST  =  rupc06
SUB_DIR   =  /mnt/wk05/yinzping/WIEN2k/DMFT/FeTe/P4nmm/PM/U5J8/beta160/Full2/2148yinzping/12168yinzping/18267yinzping

EXE_HOST  =  n145
NFS_HOST  =  caip02s
EXE_DIR   =  /mnt/p06
OUT_DIR   =  /tmp
TMP_DIR   =  /tmp/19209.1.wp10

INFO      =  Files larger than 100MB will be compressed to reduce the network load.
INFO2     =  Do NOT remove this file while job is running!

PORT      =  24209
rcp: /mnt/wk05/yinzping/WIEN2k/DMFT/FeTe/P4nmm/PM/U5J8/beta160/Full2/2148yinzping/12168yinzping/18267yinzping/imp.0: not a plain file
Failure to rcp files!

Your job started at Sat Nov 26 13:21:00 EST 2011 on host n145 in /mnt/p06/19209yinzping
Submission directory is /mnt/wk05/yinzping/WIEN2k/DMFT/FeTe/P4nmm/PM/U5J8/beta160/Full2/2148yinzping/12168yinzping/18267yinzping on host rupc06
-----------------------------------------------------------------------------------------
+ echo -catch_rsh /opt/sge6/spool/n145/active_jobs/19209.1/pe_hostfile /opt/mpich2/intel/11.1/
-catch_rsh /opt/sge6/spool/n145/active_jobs/19209.1/pe_hostfile /opt/mpich2/intel/11.1/
+ SLEEPTIME=5
+ RETRIES=10
+ catch_rsh=0
+ catch_hostname=0
+ '[' 3 -gt 0 ']'
+ case "$1" in
+ catch_rsh=1
+ shift
+ '[' 2 -gt 0 ']'
+ case "$1" in
+ break
++ basename /opt/sge6/mpich2_smpd/startmpich2.sh
+ me=startmpich2.sh
+ '[' 2 -lt 2 ']'
+ pe_hostfile=/opt/sge6/spool/n145/active_jobs/19209.1/pe_hostfile
+ MPICH2_ROOT=/opt/mpich2/intel/11.1/
+ export MPICH2_ROOT
+ '[' '!' -x /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 ']'
+ '[' '!' -x /opt/mpich2/intel/11.1//bin/smpd ']'
+ '[' '!' -r /opt/sge6/spool/n145/active_jobs/19209.1/pe_hostfile ']'
+ machines=/tmp/19209.1.wp10/machines
+ PeHostfile2MachineFile /opt/sge6/spool/n145/active_jobs/19209.1/pe_hostfile
+ cat /opt/sge6/spool/n145/active_jobs/19209.1/pe_hostfile
+ read line
++ echo n145 3 wp10@n145 UNDEFINED
++ cut -f1 -d.
++ cut -f1 '-d '
+ host=n145
++ echo n145 3 wp10@n145 UNDEFINED
++ cut -f2 '-d '
+ nslots=3
+ i=1
+ '[' 1 -le 3 ']'
+ echo n145
++ expr 1 + 1
+ i=2
+ '[' 2 -le 3 ']'
+ echo n145
++ expr 2 + 1
+ i=3
+ '[' 3 -le 3 ']'
+ echo n145
++ expr 3 + 1
+ i=4
+ '[' 4 -le 3 ']'
+ read line
++ echo n156 8 wp10@n156 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n156
++ echo n156 8 wp10@n156 UNDEFINED
++ cut -f2 '-d '
+ nslots=8
+ i=1
+ '[' 1 -le 8 ']'
+ echo n156
++ expr 1 + 1
+ i=2
+ '[' 2 -le 8 ']'
+ echo n156
++ expr 2 + 1
+ i=3
+ '[' 3 -le 8 ']'
+ echo n156
++ expr 3 + 1
+ i=4
+ '[' 4 -le 8 ']'
+ echo n156
++ expr 4 + 1
+ i=5
+ '[' 5 -le 8 ']'
+ echo n156
++ expr 5 + 1
+ i=6
+ '[' 6 -le 8 ']'
+ echo n156
++ expr 6 + 1
+ i=7
+ '[' 7 -le 8 ']'
+ echo n156
++ expr 7 + 1
+ i=8
+ '[' 8 -le 8 ']'
+ echo n156
++ expr 8 + 1
+ i=9
+ '[' 9 -le 8 ']'
+ read line
++ echo n143 2 wp10@n143 UNDEFINED
++ cut -f1 -d.
++ cut -f1 '-d '
+ host=n143
++ echo n143 2 wp10@n143 UNDEFINED
++ cut -f2 '-d '
+ nslots=2
+ i=1
+ '[' 1 -le 2 ']'
+ echo n143
++ expr 1 + 1
+ i=2
+ '[' 2 -le 2 ']'
+ echo n143
++ expr 2 + 1
+ i=3
+ '[' 3 -le 2 ']'
+ read line
++ cut -f1 '-d '
++ cut -f1 -d.
++ echo n147 8 wp10@n147 UNDEFINED
+ host=n147
++ echo n147 8 wp10@n147 UNDEFINED
++ cut -f2 '-d '
+ nslots=8
+ i=1
+ '[' 1 -le 8 ']'
+ echo n147
++ expr 1 + 1
+ i=2
+ '[' 2 -le 8 ']'
+ echo n147
++ expr 2 + 1
+ i=3
+ '[' 3 -le 8 ']'
+ echo n147
++ expr 3 + 1
+ i=4
+ '[' 4 -le 8 ']'
+ echo n147
++ expr 4 + 1
+ i=5
+ '[' 5 -le 8 ']'
+ echo n147
++ expr 5 + 1
+ i=6
+ '[' 6 -le 8 ']'
+ echo n147
++ expr 6 + 1
+ i=7
+ '[' 7 -le 8 ']'
+ echo n147
++ expr 7 + 1
+ i=8
+ '[' 8 -le 8 ']'
+ echo n147
++ expr 8 + 1
+ i=9
+ '[' 9 -le 8 ']'
+ read line
++ cut -f1 '-d '
++ cut -f1 -d.
++ echo n159 4 wp10@n159 UNDEFINED
+ host=n159
++ echo n159 4 wp10@n159 UNDEFINED
++ cut -f2 '-d '
+ nslots=4
+ i=1
+ '[' 1 -le 4 ']'
+ echo n159
++ expr 1 + 1
+ i=2
+ '[' 2 -le 4 ']'
+ echo n159
++ expr 2 + 1
+ i=3
+ '[' 3 -le 4 ']'
+ echo n159
++ expr 3 + 1
+ i=4
+ '[' 4 -le 4 ']'
+ echo n159
++ expr 4 + 1
+ i=5
+ '[' 5 -le 4 ']'
+ read line
++ cut -f1 '-d '
++ cut -f1 -d.
++ echo n152 8 wp10@n152 UNDEFINED
+ host=n152
++ echo n152 8 wp10@n152 UNDEFINED
++ cut -f2 '-d '
+ nslots=8
+ i=1
+ '[' 1 -le 8 ']'
+ echo n152
++ expr 1 + 1
+ i=2
+ '[' 2 -le 8 ']'
+ echo n152
++ expr 2 + 1
+ i=3
+ '[' 3 -le 8 ']'
+ echo n152
++ expr 3 + 1
+ i=4
+ '[' 4 -le 8 ']'
+ echo n152
++ expr 4 + 1
+ i=5
+ '[' 5 -le 8 ']'
+ echo n152
++ expr 5 + 1
+ i=6
+ '[' 6 -le 8 ']'
+ echo n152
++ expr 6 + 1
+ i=7
+ '[' 7 -le 8 ']'
+ echo n152
++ expr 7 + 1
+ i=8
+ '[' 8 -le 8 ']'
+ echo n152
++ expr 8 + 1
+ i=9
+ '[' 9 -le 8 ']'
+ read line
++ echo n148 3 wp10@n148 UNDEFINED
++ cut -f1 -d.
++ cut -f1 '-d '
+ host=n148
++ cut -f2 '-d '
++ echo n148 3 wp10@n148 UNDEFINED
+ nslots=3
+ i=1
+ '[' 1 -le 3 ']'
+ echo n148
++ expr 1 + 1
+ i=2
+ '[' 2 -le 3 ']'
+ echo n148
++ expr 2 + 1
+ i=3
+ '[' 3 -le 3 ']'
+ echo n148
++ expr 3 + 1
+ i=4
+ '[' 4 -le 3 ']'
+ read line
+ cat /tmp/19209.1.wp10/machines
n145
n145
n145
n156
n156
n156
n156
n156
n156
n156
n156
n143
n143
n147
n147
n147
n147
n147
n147
n147
n147
n159
n159
n159
n159
n152
n152
n152
n152
n152
n152
n152
n152
n148
n148
n148
+ '[' 1 = 1 ']'
+ rsh_wrapper=/opt/sge6/mpich2_smpd/rsh
+ '[' '!' -x /opt/sge6/mpich2_smpd/rsh ']'
+ rshcmd=rsh
+ case "$ARC" in
+ ln -s /opt/sge6/mpich2_smpd/rsh /tmp/19209.1.wp10/rsh
+ hash -r
+ '[' 0 = 1 ']'
+ port=24209
++ uniq
++ cat /tmp/19209.1.wp10/machines
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n145 /opt/mpich2/intel/11.1//bin/smpd 24209
rsh n145 /opt/mpich2/intel/11.1//bin/smpd -port 24209 -d 0
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n156 /opt/mpich2/intel/11.1//bin/smpd 24209
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n143 /opt/mpich2/intel/11.1//bin/smpd 24209
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n147 /opt/mpich2/intel/11.1//bin/smpd 24209
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n159 /opt/mpich2/intel/11.1//bin/smpd 24209
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n152 /opt/mpich2/intel/11.1//bin/smpd 24209
rsh n147 /opt/mpich2/intel/11.1//bin/smpd -port 24209 -d 0
rsh n143 /opt/mpich2/intel/11.1//bin/smpd -port 24209 -d 0
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n148 /opt/mpich2/intel/11.1//bin/smpd 24209
+ actual_retry=1
+ '[' 1 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (1 of 10)'
startmpich2.sh: check for smpd daemons (1 of 10)
+ numhosts=0
rsh n152 /opt/mpich2/intel/11.1//bin/smpd -port 24209 -d 0
rsh n159 /opt/mpich2/intel/11.1//bin/smpd -port 24209 -d 0
++ cat /tmp/19209.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 24209 -status n145
rsh n148 /opt/mpich2/intel/11.1//bin/smpd -port 24209 -d 0
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n145'
startmpich2.sh: missing smpd on n145
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 24209 -status n156
++ cut -f 1 -d ' '
++ head -n 1
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n156'
startmpich2.sh: missing smpd on n156
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 24209 -status n143
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n143'
startmpich2.sh: missing smpd on n143
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 24209 -status n147
++ head -n 1
++ cut -f 1 -d ' '
rsh n156 /opt/mpich2/intel/11.1//bin/smpd -port 24209 -d 0
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n147'
startmpich2.sh: missing smpd on n147
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 24209 -status n159
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n159'
startmpich2.sh: missing smpd on n159
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 24209 -status n152
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n152'
startmpich2.sh: found running smpd on n152
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 24209 -status n148
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n148'
startmpich2.sh: missing smpd on n148
+ '[' 7 -eq 1 ']'
+ sleep 5
+ (( actual_retry += 1 ))
+ '[' 2 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (2 of 10)'
startmpich2.sh: check for smpd daemons (2 of 10)
+ numhosts=0
++ cat /tmp/19209.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 24209 -status n145
++ cut -f 1 -d ' '
++ head -n 1
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n145'
startmpich2.sh: found running smpd on n145
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 24209 -status n156
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n156'
startmpich2.sh: found running smpd on n156
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 24209 -status n143
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n143'
startmpich2.sh: found running smpd on n143
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 24209 -status n147
++ cut -f 1 -d ' '
++ head -n 1
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n147'
startmpich2.sh: found running smpd on n147
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 24209 -status n159
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n159'
startmpich2.sh: found running smpd on n159
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 24209 -status n152
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n152'
startmpich2.sh: found running smpd on n152
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 24209 -status n148
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n148'
startmpich2.sh: found running smpd on n148
+ '[' 7 -eq 7 ']'
+ echo 'startmpich2.sh: got all 7 of 7 nodes'
startmpich2.sh: got all 7 of 7 nodes
+ exit 0
rm: cannot remove `./*.broyd*': No such file or directory
-catch_rsh /opt/mpich2/intel/11.1/
Your job finished at Thu Dec 8 04:29:24 EST 2011