USER      =  yinzping
HOMEDIR   =  /home/yinzping
JOB_ID    =  17256
JOB_NAME  =  LaPPMfull
QUEUE     =  wp10
NSLOTS    =  36
NHOSTS    =  5

SUB_HOST  =  rupc06
SUB_DIR   =  /mnt/wk05/yinzping/WIEN2k/DMFT/LaFePO/P4nmm/U5J8/beta160/Full2/2446yinzping/11037yinzping

EXE_HOST  =  n153
NFS_HOST  =  caip02s
EXE_DIR   =  /mnt/p05
OUT_DIR   =  /tmp
TMP_DIR   =  /tmp/17256.1.wp10

INFO      =  Files larger than 100MB will be compressed to reduce the network load.
INFO2     =  Do NOT remove this file while job is running!

PORT      =  22256
rcp: /mnt/wk05/yinzping/WIEN2k/DMFT/LaFePO/P4nmm/U5J8/beta160/Full2/2446yinzping/11037yinzping/imp.0: not a plain file
Failure to rcp files!

Your job started at Mon Oct 31 09:10:40 EDT 2011 on host n153 in /mnt/p05/17256yinzping
Submission directory is /mnt/wk05/yinzping/WIEN2k/DMFT/LaFePO/P4nmm/U5J8/beta160/Full2/2446yinzping/11037yinzping on host rupc06
-----------------------------------------------------------------------------------------
+ echo -catch_rsh /opt/sge6/spool/n153/active_jobs/17256.1/pe_hostfile /opt/mpich2/intel/11.1/
-catch_rsh /opt/sge6/spool/n153/active_jobs/17256.1/pe_hostfile /opt/mpich2/intel/11.1/
+ SLEEPTIME=5
+ RETRIES=10
+ catch_rsh=0
+ catch_hostname=0
+ '[' 3 -gt 0 ']'
+ case "$1" in
+ catch_rsh=1
+ shift
+ '[' 2 -gt 0 ']'
+ case "$1" in
+ break
++ basename /opt/sge6/mpich2_smpd/startmpich2.sh
+ me=startmpich2.sh
+ '[' 2 -lt 2 ']'
+ pe_hostfile=/opt/sge6/spool/n153/active_jobs/17256.1/pe_hostfile
+ MPICH2_ROOT=/opt/mpich2/intel/11.1/
+ export MPICH2_ROOT
+ '[' '!' -x /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 ']'
+ '[' '!' -x /opt/mpich2/intel/11.1//bin/smpd ']'
+ '[' '!' -r /opt/sge6/spool/n153/active_jobs/17256.1/pe_hostfile ']'
+ machines=/tmp/17256.1.wp10/machines
+ PeHostfile2MachineFile /opt/sge6/spool/n153/active_jobs/17256.1/pe_hostfile
+ cat /opt/sge6/spool/n153/active_jobs/17256.1/pe_hostfile
+ read line
++ echo n153 4 wp10@n153 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n153
++ echo n153 4 wp10@n153 UNDEFINED
++ cut -f2 '-d '
+ nslots=4
+ i=1
+ '[' 1 -le 4 ']'
+ echo n153
++ expr 1 + 1
+ i=2
+ '[' 2 -le 4 ']'
+ echo n153
++ expr 2 + 1
+ i=3
+ '[' 3 -le 4 ']'
+ echo n153
++ expr 3 + 1
+ i=4
+ '[' 4 -le 4 ']'
+ echo n153
++ expr 4 + 1
+ i=5
+ '[' 5 -le 4 ']'
+ read line
++ echo n156 12 wp10@n156 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n156
++ echo n156 12 wp10@n156 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n156
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n156
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n156
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n156
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n156
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n156
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n156
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n156
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n156
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n156
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n156
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n156
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n148 6 wp10@n148 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n148
++ echo n148 6 wp10@n148 UNDEFINED
++ cut -f2 '-d '
+ nslots=6
+ i=1
+ '[' 1 -le 6 ']'
+ echo n148
++ expr 1 + 1
+ i=2
+ '[' 2 -le 6 ']'
+ echo n148
++ expr 2 + 1
+ i=3
+ '[' 3 -le 6 ']'
+ echo n148
++ expr 3 + 1
+ i=4
+ '[' 4 -le 6 ']'
+ echo n148
++ expr 4 + 1
+ i=5
+ '[' 5 -le 6 ']'
+ echo n148
++ expr 5 + 1
+ i=6
+ '[' 6 -le 6 ']'
+ echo n148
++ expr 6 + 1
+ i=7
+ '[' 7 -le 6 ']'
+ read line
++ echo n158 12 wp10@n158 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n158
++ echo n158 12 wp10@n158 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n158
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n158
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n158
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n158
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n158
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n158
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n158
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n158
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n158
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n158
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n158
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n158
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n165 2 wp10@n165 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n165
++ echo n165 2 wp10@n165 UNDEFINED
++ cut -f2 '-d '
+ nslots=2
+ i=1
+ '[' 1 -le 2 ']'
+ echo n165
++ expr 1 + 1
+ i=2
+ '[' 2 -le 2 ']'
+ echo n165
++ expr 2 + 1
+ i=3
+ '[' 3 -le 2 ']'
+ read line
+ cat /tmp/17256.1.wp10/machines
n153
n153
n153
n153
n156
n156
n156
n156
n156
n156
n156
n156
n156
n156
n156
n156
n148
n148
n148
n148
n148
n148
n158
n158
n158
n158
n158
n158
n158
n158
n158
n158
n158
n158
n165
n165
+ '[' 1 = 1 ']'
+ rsh_wrapper=/opt/sge6/mpich2_smpd/rsh
+ '[' '!' -x /opt/sge6/mpich2_smpd/rsh ']'
+ rshcmd=rsh
+ case "$ARC" in
+ ln -s /opt/sge6/mpich2_smpd/rsh /tmp/17256.1.wp10/rsh
+ hash -r
+ '[' 0 = 1 ']'
+ port=22256
++ cat /tmp/17256.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n153 /opt/mpich2/intel/11.1//bin/smpd 22256
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n156 /opt/mpich2/intel/11.1//bin/smpd 22256
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n148 /opt/mpich2/intel/11.1//bin/smpd 22256
rsh n156 /opt/mpich2/intel/11.1//bin/smpd -port 22256 -d 0
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n158 /opt/mpich2/intel/11.1//bin/smpd 22256
+ for host in '`cat $machines | uniq`'
rsh n153 /opt/mpich2/intel/11.1//bin/smpd -port 22256 -d 0
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n165 /opt/mpich2/intel/11.1//bin/smpd 22256
+ actual_retry=1
+ '[' 1 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (1 of 10)'
startmpich2.sh: check for smpd daemons (1 of 10)
+ numhosts=0
++ cat /tmp/17256.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22256 -status n153
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
rsh n148 /opt/mpich2/intel/11.1//bin/smpd -port 22256 -d 0
rsh n165 /opt/mpich2/intel/11.1//bin/smpd -port 22256 -d 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n153'
startmpich2.sh: missing smpd on n153
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22256 -status n156
rsh n158 /opt/mpich2/intel/11.1//bin/smpd -port 22256 -d 0
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n156'
startmpich2.sh: missing smpd on n156
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22256 -status n148
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n148'
startmpich2.sh: found running smpd on n148
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22256 -status n158
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n158'
startmpich2.sh: missing smpd on n158
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22256 -status n165
++ cut -f 1 -d ' '
++ head -n 1
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n165'
startmpich2.sh: missing smpd on n165
+ '[' 5 -eq 1 ']'
+ sleep 5
+ (( actual_retry += 1 ))
+ '[' 2 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (2 of 10)'
startmpich2.sh: check for smpd daemons (2 of 10)
+ numhosts=0
++ cat /tmp/17256.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22256 -status n153
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n153'
startmpich2.sh: found running smpd on n153
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22256 -status n156
++ cut -f 1 -d ' '
++ head -n 1
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n156'
startmpich2.sh: found running smpd on n156
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22256 -status n148
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n148'
startmpich2.sh: found running smpd on n148
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22256 -status n158
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n158'
startmpich2.sh: found running smpd on n158
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22256 -status n165
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n165'
startmpich2.sh: found running smpd on n165
+ '[' 5 -eq 5 ']'
+ echo 'startmpich2.sh: got all 5 of 5 nodes'
startmpich2.sh: got all 5 of 5 nodes
+ exit 0
rm: cannot remove `./*.broyd*': No such file or directory
-catch_rsh /opt/mpich2/intel/11.1/
Your job finished at Sat Nov 12 14:38:27 EST 2011