USER      =  yinzping
HOMEDIR   =  /home/yinzping
JOB_ID    =  22021
JOB_NAME  =  NFAPM
QUEUE     =  wp12
NSLOTS    =  36
NHOSTS    =  5

SUB_HOST  =  rupc06
SUB_DIR   =  /mnt/wk05/yinzping/WIEN2k/DMFT/FeAs/NaFeAs/P4nmm/U5J8/beta160/Full2/2438yinzping/10607yinzping

EXE_HOST  =  n180
EXE_DIR   =  /mnt/n180
OUT_DIR   =  /scratch
TMP_DIR   =  /tmp/22021.1.wp12

INFO      =  Files larger than 100MB will be compressed to reduce the network load.
INFO2     =  Do NOT remove this file while job is running!

PORT      =  22021
rcp: /mnt/wk05/yinzping/WIEN2k/DMFT/FeAs/NaFeAs/P4nmm/U5J8/beta160/Full2/2438yinzping/10607yinzping/Vertex: not a plain file
rcp: /mnt/wk05/yinzping/WIEN2k/DMFT/FeAs/NaFeAs/P4nmm/U5J8/beta160/Full2/2438yinzping/10607yinzping/imp.0: not a plain file
Failure to rcp files!

Your job started at Sat Jan 21 01:06:18 EST 2012 on host n180 in /mnt/n180/22021yinzping
Submission directory is /mnt/wk05/yinzping/WIEN2k/DMFT/FeAs/NaFeAs/P4nmm/U5J8/beta160/Full2/2438yinzping/10607yinzping on host rupc06
-----------------------------------------------------------------------------------------
+ echo -catch_rsh /opt/sge6/spool/n180/active_jobs/22021.1/pe_hostfile /opt/mpich2/intel/11.1/
-catch_rsh /opt/sge6/spool/n180/active_jobs/22021.1/pe_hostfile /opt/mpich2/intel/11.1/
+ SLEEPTIME=5
+ RETRIES=10
+ catch_rsh=0
+ catch_hostname=0
+ '[' 3 -gt 0 ']'
+ case "$1" in
+ catch_rsh=1
+ shift
+ '[' 2 -gt 0 ']'
+ case "$1" in
+ break
++ basename /opt/sge6/mpich2_smpd/startmpich2.sh
+ me=startmpich2.sh
+ '[' 2 -lt 2 ']'
+ pe_hostfile=/opt/sge6/spool/n180/active_jobs/22021.1/pe_hostfile
+ MPICH2_ROOT=/opt/mpich2/intel/11.1/
+ export MPICH2_ROOT
+ '[' '!' -x /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 ']'
+ '[' '!' -x /opt/mpich2/intel/11.1//bin/smpd ']'
+ '[' '!' -r /opt/sge6/spool/n180/active_jobs/22021.1/pe_hostfile ']'
+ machines=/tmp/22021.1.wp12/machines
+ PeHostfile2MachineFile /opt/sge6/spool/n180/active_jobs/22021.1/pe_hostfile
+ cat /opt/sge6/spool/n180/active_jobs/22021.1/pe_hostfile
+ read line
++ echo n180 12 wp12@n180 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n180
++ echo n180 12 wp12@n180 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n180
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n180
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n180
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n180
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n180
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n180
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n180
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n180
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n180
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n180
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n180
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n180
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n168 1 wp12@n168 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n168
++ echo n168 1 wp12@n168 UNDEFINED
++ cut -f2 '-d '
+ nslots=1
+ i=1
+ '[' 1 -le 1 ']'
+ echo n168
++ expr 1 + 1
+ i=2
+ '[' 2 -le 1 ']'
+ read line
++ echo n169 8 wp12@n169 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n169
++ echo n169 8 wp12@n169 UNDEFINED
++ cut -f2 '-d '
+ nslots=8
+ i=1
+ '[' 1 -le 8 ']'
+ echo n169
++ expr 1 + 1
+ i=2
+ '[' 2 -le 8 ']'
+ echo n169
++ expr 2 + 1
+ i=3
+ '[' 3 -le 8 ']'
+ echo n169
++ expr 3 + 1
+ i=4
+ '[' 4 -le 8 ']'
+ echo n169
++ expr 4 + 1
+ i=5
+ '[' 5 -le 8 ']'
+ echo n169
++ expr 5 + 1
+ i=6
+ '[' 6 -le 8 ']'
+ echo n169
++ expr 6 + 1
+ i=7
+ '[' 7 -le 8 ']'
+ echo n169
++ expr 7 + 1
+ i=8
+ '[' 8 -le 8 ']'
+ echo n169
++ expr 8 + 1
+ i=9
+ '[' 9 -le 8 ']'
+ read line
++ echo n185 8 wp12@n185 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n185
++ echo n185 8 wp12@n185 UNDEFINED
++ cut -f2 '-d '
+ nslots=8
+ i=1
+ '[' 1 -le 8 ']'
+ echo n185
++ expr 1 + 1
+ i=2
+ '[' 2 -le 8 ']'
+ echo n185
++ expr 2 + 1
+ i=3
+ '[' 3 -le 8 ']'
+ echo n185
++ expr 3 + 1
+ i=4
+ '[' 4 -le 8 ']'
+ echo n185
++ expr 4 + 1
+ i=5
+ '[' 5 -le 8 ']'
+ echo n185
++ expr 5 + 1
+ i=6
+ '[' 6 -le 8 ']'
+ echo n185
++ expr 6 + 1
+ i=7
+ '[' 7 -le 8 ']'
+ echo n185
++ expr 7 + 1
+ i=8
+ '[' 8 -le 8 ']'
+ echo n185
++ expr 8 + 1
+ i=9
+ '[' 9 -le 8 ']'
+ read line
++ echo n182 7 wp12@n182 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n182
++ echo n182 7 wp12@n182 UNDEFINED
++ cut -f2 '-d '
+ nslots=7
+ i=1
+ '[' 1 -le 7 ']'
+ echo n182
++ expr 1 + 1
+ i=2
+ '[' 2 -le 7 ']'
+ echo n182
++ expr 2 + 1
+ i=3
+ '[' 3 -le 7 ']'
+ echo n182
++ expr 3 + 1
+ i=4
+ '[' 4 -le 7 ']'
+ echo n182
++ expr 4 + 1
+ i=5
+ '[' 5 -le 7 ']'
+ echo n182
++ expr 5 + 1
+ i=6
+ '[' 6 -le 7 ']'
+ echo n182
++ expr 6 + 1
+ i=7
+ '[' 7 -le 7 ']'
+ echo n182
++ expr 7 + 1
+ i=8
+ '[' 8 -le 7 ']'
+ read line
+ cat /tmp/22021.1.wp12/machines
n180
n180
n180
n180
n180
n180
n180
n180
n180
n180
n180
n180
n168
n169
n169
n169
n169
n169
n169
n169
n169
n185
n185
n185
n185
n185
n185
n185
n185
n182
n182
n182
n182
n182
n182
n182
+ '[' 1 = 1 ']'
+ rsh_wrapper=/opt/sge6/mpich2_smpd/rsh
+ '[' '!' -x /opt/sge6/mpich2_smpd/rsh ']'
+ rshcmd=rsh
+ case "$ARC" in
+ ln -s /opt/sge6/mpich2_smpd/rsh /tmp/22021.1.wp12/rsh
+ hash -r
+ '[' 0 = 1 ']'
+ port=22021
++ cat /tmp/22021.1.wp12/machines
++ uniq
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n180 /opt/mpich2/intel/11.1//bin/smpd 22021
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n168 /opt/mpich2/intel/11.1//bin/smpd 22021
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n169 /opt/mpich2/intel/11.1//bin/smpd 22021
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n185 /opt/mpich2/intel/11.1//bin/smpd 22021
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n182 /opt/mpich2/intel/11.1//bin/smpd 22021
+ actual_retry=1
+ '[' 1 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (1 of 10)'
startmpich2.sh: check for smpd daemons (1 of 10)
+ numhosts=0
++ uniq
++ cat /tmp/22021.1.wp12/machines
rsh n180 /opt/mpich2/intel/11.1//bin/smpd -port 22021 -d 0
+ for host in '`cat $machines | uniq`'
rsh n168 /opt/mpich2/intel/11.1//bin/smpd -port 22021 -d 0
++ /opt/mpich2/intel/11.1//bin/smpd -port 22021 -status n180
++ head -n 1
++ cut -f 1 -d ' '
rsh n169 /opt/mpich2/intel/11.1//bin/smpd -port 22021 -d 0
rsh n185 /opt/mpich2/intel/11.1//bin/smpd -port 22021 -d 0
rsh n182 /opt/mpich2/intel/11.1//bin/smpd -port 22021 -d 0
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n180'
startmpich2.sh: missing smpd on n180
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22021 -status n168
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n168'
startmpich2.sh: missing smpd on n168
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22021 -status n169
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n169'
startmpich2.sh: missing smpd on n169
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22021 -status n185
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n185'
startmpich2.sh: missing smpd on n185
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22021 -status n182
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n182'
startmpich2.sh: missing smpd on n182
+ '[' 5 -eq 0 ']'
+ sleep 5
+ (( actual_retry += 1 ))
+ '[' 2 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (2 of 10)'
startmpich2.sh: check for smpd daemons (2 of 10)
+ numhosts=0
++ cat /tmp/22021.1.wp12/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22021 -status n180
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n180'
startmpich2.sh: found running smpd on n180
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22021 -status n168
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n168'
startmpich2.sh: found running smpd on n168
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22021 -status n169
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n169'
startmpich2.sh: found running smpd on n169
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22021 -status n185
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n185'
startmpich2.sh: found running smpd on n185
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22021 -status n182
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n182'
startmpich2.sh: found running smpd on n182
+ '[' 5 -eq 5 ']'
+ echo 'startmpich2.sh: got all 5 of 5 nodes'
startmpich2.sh: got all 5 of 5 nodes
+ exit 0
rm: cannot remove `./*.broyd*': No such file or directory
-catch_rsh /opt/mpich2/intel/11.1/
Your job finished at Wed Feb 1 03:00:17 EST 2012