USER      =  yinzping
HOMEDIR   =  /home/yinzping
JOB_ID    =  10607
JOB_NAME  =  NaPMfull2
QUEUE     =  wp10
NSLOTS    =  36
NHOSTS    =  3

SUB_HOST  =  rupc06
SUB_DIR   =  /mnt/wk05/yinzping/WIEN2k/DMFT/FeAs/NaFeAs/P4nmm/U5J8/beta160/Full2/2438yinzping

EXE_HOST  =  n150
NFS_HOST  =  caip03s
EXE_DIR   =  /mnt/p03
OUT_DIR   =  /tmp
TMP_DIR   =  /tmp/10607.1.wp10

INFO      =  Files larger than 100MB will be compressed to reduce the network load.
INFO2     =  Do NOT remove this file while job is running!

PORT      =  20607
rcp: /mnt/wk05/yinzping/WIEN2k/DMFT/FeAs/NaFeAs/P4nmm/U5J8/beta160/Full2/2438yinzping/Gk_k: not a plain file
rcp: /mnt/wk05/yinzping/WIEN2k/DMFT/FeAs/NaFeAs/P4nmm/U5J8/beta160/Full2/2438yinzping/Vertex: not a plain file
rcp: /mnt/wk05/yinzping/WIEN2k/DMFT/FeAs/NaFeAs/P4nmm/U5J8/beta160/Full2/2438yinzping/ancont: not a plain file
rcp: /mnt/wk05/yinzping/WIEN2k/DMFT/FeAs/NaFeAs/P4nmm/U5J8/beta160/Full2/2438yinzping/dos: not a plain file
rcp: /mnt/wk05/yinzping/WIEN2k/DMFT/FeAs/NaFeAs/P4nmm/U5J8/beta160/Full2/2438yinzping/imp.0: not a plain file
Failure to rcp files!

Your job started at Wed Aug 24 12:59:07 EDT 2011 on host n150 in /mnt/p03/10607yinzping
Submission directory is /mnt/wk05/yinzping/WIEN2k/DMFT/FeAs/NaFeAs/P4nmm/U5J8/beta160/Full2/2438yinzping on host rupc06
-----------------------------------------------------------------------------------------
+ echo -catch_rsh /opt/sge6/spool/n150/active_jobs/10607.1/pe_hostfile /opt/mpich2/intel/11.1/
-catch_rsh /opt/sge6/spool/n150/active_jobs/10607.1/pe_hostfile /opt/mpich2/intel/11.1/
+ SLEEPTIME=5
+ RETRIES=10
+ catch_rsh=0
+ catch_hostname=0
+ '[' 3 -gt 0 ']'
+ case "$1" in
+ catch_rsh=1
+ shift
+ '[' 2 -gt 0 ']'
+ case "$1" in
+ break
++ basename /opt/sge6/mpich2_smpd/startmpich2.sh
+ me=startmpich2.sh
+ '[' 2 -lt 2 ']'
+ pe_hostfile=/opt/sge6/spool/n150/active_jobs/10607.1/pe_hostfile
+ MPICH2_ROOT=/opt/mpich2/intel/11.1/
+ export MPICH2_ROOT
+ '[' '!' -x /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 ']'
+ '[' '!' -x /opt/mpich2/intel/11.1//bin/smpd ']'
+ '[' '!' -r /opt/sge6/spool/n150/active_jobs/10607.1/pe_hostfile ']'
+ machines=/tmp/10607.1.wp10/machines
+ PeHostfile2MachineFile /opt/sge6/spool/n150/active_jobs/10607.1/pe_hostfile
+ cat /opt/sge6/spool/n150/active_jobs/10607.1/pe_hostfile
+ read line
++ echo n150 12 wp10@n150 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n150
++ echo n150 12 wp10@n150 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n150
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n150
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n150
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n150
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n150
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n150
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n150
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n150
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n150
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n150
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n150
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n150
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n151 12 wp10@n151 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n151
++ echo n151 12 wp10@n151 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n151
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n151
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n151
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n151
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n151
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n151
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n151
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n151
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n151
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n151
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n151
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n151
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n152 12 wp10@n152 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n152
++ echo n152 12 wp10@n152 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n152
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n152
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n152
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n152
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n152
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n152
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n152
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n152
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n152
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n152
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n152
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n152
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
+ cat /tmp/10607.1.wp10/machines
n150
n150
n150
n150
n150
n150
n150
n150
n150
n150
n150
n150
n151
n151
n151
n151
n151
n151
n151
n151
n151
n151
n151
n151
n152
n152
n152
n152
n152
n152
n152
n152
n152
n152
n152
n152
+ '[' 1 = 1 ']'
+ rsh_wrapper=/opt/sge6/mpich2_smpd/rsh
+ '[' '!' -x /opt/sge6/mpich2_smpd/rsh ']'
+ rshcmd=rsh
+ case "$ARC" in
+ ln -s /opt/sge6/mpich2_smpd/rsh /tmp/10607.1.wp10/rsh
+ hash -r
+ '[' 0 = 1 ']'
+ port=20607
++ cat /tmp/10607.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n150 /opt/mpich2/intel/11.1//bin/smpd 20607
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n151 /opt/mpich2/intel/11.1//bin/smpd 20607
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n152 /opt/mpich2/intel/11.1//bin/smpd 20607
+ actual_retry=1
+ '[' 1 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (1 of 10)'
startmpich2.sh: check for smpd daemons (1 of 10)
+ numhosts=0
++ cat /tmp/10607.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20607 -status n150
++ head -n 1
rsh n150 /opt/mpich2/intel/11.1//bin/smpd -port 20607 -d 0
++ cut -f 1 -d ' '
rsh n151 /opt/mpich2/intel/11.1//bin/smpd -port 20607 -d 0
rsh n152 /opt/mpich2/intel/11.1//bin/smpd -port 20607 -d 0
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n150'
startmpich2.sh: missing smpd on n150
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20607 -status n151
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n151'
startmpich2.sh: missing smpd on n151
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20607 -status n152
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n152'
startmpich2.sh: missing smpd on n152
+ '[' 3 -eq 0 ']'
+ sleep 5
+ (( actual_retry += 1 ))
+ '[' 2 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (2 of 10)'
startmpich2.sh: check for smpd daemons (2 of 10)
+ numhosts=0
++ cat /tmp/10607.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20607 -status n150
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n150'
startmpich2.sh: found running smpd on n150
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20607 -status n151
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n151'
startmpich2.sh: found running smpd on n151
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20607 -status n152
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n152'
startmpich2.sh: found running smpd on n152
+ '[' 3 -eq 3 ']'
+ echo 'startmpich2.sh: got all 3 of 3 nodes'
startmpich2.sh: got all 3 of 3 nodes
+ exit 0
rm: cannot remove `./*.?.[12]': No such file or directory
rm: cannot remove `./*.broyd*': No such file or directory
-catch_rsh /opt/mpich2/intel/11.1/
Your job finished at Fri Sep 2 08:58:32 EDT 2011