USER      =  yinzping
HOMEDIR   =  /home/yinzping
JOB_ID    =  33178
JOB_NAME  =  K122PM
QUEUE     =  wp12
NSLOTS    =  48
NHOSTS    =  6

SUB_HOST  =  rupc06
SUB_DIR   =  /mnt/wk05/yinzping/WIEN2k/DMFT/FeAs/K122/I4mmm/U5J8/beta160/Full2/d5.5/29034yinzping

EXE_HOST  =  n171
EXE_DIR   =  /mnt/n171
OUT_DIR   =  /scratch
TMP_DIR   =  /tmp/33178.1.wp12

INFO      =  Files larger than 100MB will be compressed to reduce the network load.
INFO2     =  Do NOT remove this file while job is running!

PORT      =  23178
rcp: /mnt/wk05/yinzping/WIEN2k/DMFT/FeAs/K122/I4mmm/U5J8/beta160/Full2/d5.5/29034yinzping/backup: not a plain file
rcp: /mnt/wk05/yinzping/WIEN2k/DMFT/FeAs/K122/I4mmm/U5J8/beta160/Full2/d5.5/29034yinzping/imp.0: not a plain file
Failure to rcp files!

Your job started at Sat Aug 11 19:57:56 EDT 2012 on host n171 in /mnt/n171/33178yinzping
Submission directory is /mnt/wk05/yinzping/WIEN2k/DMFT/FeAs/K122/I4mmm/U5J8/beta160/Full2/d5.5/29034yinzping on host rupc06
-----------------------------------------------------------------------------------------
+ echo -catch_rsh /opt/sge6/spool/n171/active_jobs/33178.1/pe_hostfile /opt/mpich2/intel/11.1/
-catch_rsh /opt/sge6/spool/n171/active_jobs/33178.1/pe_hostfile /opt/mpich2/intel/11.1/
+ SLEEPTIME=5
+ RETRIES=10
+ catch_rsh=0
+ catch_hostname=0
+ '[' 3 -gt 0 ']'
+ case "$1" in
+ catch_rsh=1
+ shift
+ '[' 2 -gt 0 ']'
+ case "$1" in
+ break
++ basename /opt/sge6/mpich2_smpd/startmpich2.sh
+ me=startmpich2.sh
+ '[' 2 -lt 2 ']'
+ pe_hostfile=/opt/sge6/spool/n171/active_jobs/33178.1/pe_hostfile
+ MPICH2_ROOT=/opt/mpich2/intel/11.1/
+ export MPICH2_ROOT
+ '[' '!' -x /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 ']'
+ '[' '!' -x /opt/mpich2/intel/11.1//bin/smpd ']'
+ '[' '!' -r /opt/sge6/spool/n171/active_jobs/33178.1/pe_hostfile ']'
+ machines=/tmp/33178.1.wp12/machines
+ PeHostfile2MachineFile /opt/sge6/spool/n171/active_jobs/33178.1/pe_hostfile
+ cat /opt/sge6/spool/n171/active_jobs/33178.1/pe_hostfile
+ read line
++ echo n171 4 wp12@n171 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n171
++ echo n171 4 wp12@n171 UNDEFINED
++ cut -f2 '-d '
+ nslots=4
+ i=1
+ '[' 1 -le 4 ']'
+ echo n171
++ expr 1 + 1
+ i=2
+ '[' 2 -le 4 ']'
+ echo n171
++ expr 2 + 1
+ i=3
+ '[' 3 -le 4 ']'
+ echo n171
++ expr 3 + 1
+ i=4
+ '[' 4 -le 4 ']'
+ echo n171
++ expr 4 + 1
+ i=5
+ '[' 5 -le 4 ']'
+ read line
++ echo n176 4 wp12@n176 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n176
++ echo n176 4 wp12@n176 UNDEFINED
++ cut -f2 '-d '
+ nslots=4
+ i=1
+ '[' 1 -le 4 ']'
+ echo n176
++ expr 1 + 1
+ i=2
+ '[' 2 -le 4 ']'
+ echo n176
++ expr 2 + 1
+ i=3
+ '[' 3 -le 4 ']'
+ echo n176
++ expr 3 + 1
+ i=4
+ '[' 4 -le 4 ']'
+ echo n176
++ expr 4 + 1
+ i=5
+ '[' 5 -le 4 ']'
+ read line
++ echo n177 12 wp12@n177 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n177
++ echo n177 12 wp12@n177 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n177
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n177
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n177
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n177
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n177
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n177
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n177
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n177
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n177
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n177
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n177
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n177
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n172 12 wp12@n172 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n172
++ echo n172 12 wp12@n172 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n172
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n172
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n172
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n172
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n172
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n172
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n172
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n172
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n172
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n172
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n172
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n172
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n173 12 wp12@n173 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n173
++ echo n173 12 wp12@n173 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n173
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n173
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n173
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n173
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n173
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n173
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n173
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n173
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n173
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n173
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n173
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n173
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n178 4 wp12@n178 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n178
++ echo n178 4 wp12@n178 UNDEFINED
++ cut -f2 '-d '
+ nslots=4
+ i=1
+ '[' 1 -le 4 ']'
+ echo n178
++ expr 1 + 1
+ i=2
+ '[' 2 -le 4 ']'
+ echo n178
++ expr 2 + 1
+ i=3
+ '[' 3 -le 4 ']'
+ echo n178
++ expr 3 + 1
+ i=4
+ '[' 4 -le 4 ']'
+ echo n178
++ expr 4 + 1
+ i=5
+ '[' 5 -le 4 ']'
+ read line
+ cat /tmp/33178.1.wp12/machines
n171
n171
n171
n171
n176
n176
n176
n176
n177
n177
n177
n177
n177
n177
n177
n177
n177
n177
n177
n177
n172
n172
n172
n172
n172
n172
n172
n172
n172
n172
n172
n172
n173
n173
n173
n173
n173
n173
n173
n173
n173
n173
n173
n173
n178
n178
n178
n178
+ '[' 1 = 1 ']'
+ rsh_wrapper=/opt/sge6/mpich2_smpd/rsh
+ '[' '!' -x /opt/sge6/mpich2_smpd/rsh ']'
+ rshcmd=rsh
+ case "$ARC" in
+ ln -s /opt/sge6/mpich2_smpd/rsh /tmp/33178.1.wp12/rsh
+ hash -r
+ '[' 0 = 1 ']'
+ port=23178
++ cat /tmp/33178.1.wp12/machines
++ uniq
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n171 /opt/mpich2/intel/11.1//bin/smpd 23178
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n176 /opt/mpich2/intel/11.1//bin/smpd 23178
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n177 /opt/mpich2/intel/11.1//bin/smpd 23178
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n172 /opt/mpich2/intel/11.1//bin/smpd 23178
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n173 /opt/mpich2/intel/11.1//bin/smpd 23178
+ for host in '`cat $machines | uniq`'
rsh n171 /opt/mpich2/intel/11.1//bin/smpd -port 23178 -d 0
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n178 /opt/mpich2/intel/11.1//bin/smpd 23178
+ actual_retry=1
+ '[' 1 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (1 of 10)'
startmpich2.sh: check for smpd daemons (1 of 10)
+ numhosts=0
++ cat /tmp/33178.1.wp12/machines
++ uniq
rsh n177 /opt/mpich2/intel/11.1//bin/smpd -port 23178 -d 0
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23178 -status n171
++ head -n 1
rsh n176 /opt/mpich2/intel/11.1//bin/smpd -port 23178 -d 0
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
rsh n172 /opt/mpich2/intel/11.1//bin/smpd -port 23178 -d 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n171'
startmpich2.sh: missing smpd on n171
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23178 -status n176
rsh n173 /opt/mpich2/intel/11.1//bin/smpd -port 23178 -d 0
++ head -n 1
++ cut -f 1 -d ' '
rsh n178 /opt/mpich2/intel/11.1//bin/smpd -port 23178 -d 0
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n176'
startmpich2.sh: missing smpd on n176
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23178 -status n177
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n177'
startmpich2.sh: missing smpd on n177
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23178 -status n172
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n172'
startmpich2.sh: missing smpd on n172
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23178 -status n173
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n173'
startmpich2.sh: missing smpd on n173
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23178 -status n178
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n178'
startmpich2.sh: missing smpd on n178
+ '[' 6 -eq 0 ']'
+ sleep 5
+ (( actual_retry += 1 ))
+ '[' 2 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (2 of 10)'
startmpich2.sh: check for smpd daemons (2 of 10)
+ numhosts=0
++ cat /tmp/33178.1.wp12/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23178 -status n171
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n171'
startmpich2.sh: found running smpd on n171
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23178 -status n176
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n176'
startmpich2.sh: found running smpd on n176
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23178 -status n177
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n177'
startmpich2.sh: found running smpd on n177
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23178 -status n172
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n172'
startmpich2.sh: found running smpd on n172
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23178 -status n173
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n173'
startmpich2.sh: found running smpd on n173
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23178 -status n178
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n178'
startmpich2.sh: found running smpd on n178
+ '[' 6 -eq 6 ']'
+ echo 'startmpich2.sh: got all 6 of 6 nodes'
startmpich2.sh: got all 6 of 6 nodes
+ exit 0
rm: cannot remove `./*.broyd*': No such file or directory
-catch_rsh /opt/mpich2/intel/11.1/
Your job finished at Thu Sep 6 20:42:16 EDT 2012