USER      =  yinzping
HOMEDIR   =  /home/yinzping
JOB_ID    =  18267
JOB_NAME  =  TePMfull2
QUEUE     =  wp10
NSLOTS    =  36
NHOSTS    =  6

SUB_HOST  =  rupc06
SUB_DIR   =  /mnt/wk05/yinzping/WIEN2k/DMFT/FeTe/P4nmm/PM/U5J8/beta160/Full2/2148yinzping/12168yinzping

EXE_HOST  =  n143
NFS_HOST  =  caip02s
EXE_DIR   =  /mnt/p06
OUT_DIR   =  /tmp
TMP_DIR   =  /tmp/18267.1.wp10

INFO      =  Files larger than 100MB will be compressed to reduce the network load.
INFO2     =  Do NOT remove this file while job is running!

PORT      =  23267
rcp: /mnt/wk05/yinzping/WIEN2k/DMFT/FeTe/P4nmm/PM/U5J8/beta160/Full2/2148yinzping/12168yinzping/ancont: not a plain file
rcp: /mnt/wk05/yinzping/WIEN2k/DMFT/FeTe/P4nmm/PM/U5J8/beta160/Full2/2148yinzping/12168yinzping/dos: not a plain file
rcp: /mnt/wk05/yinzping/WIEN2k/DMFT/FeTe/P4nmm/PM/U5J8/beta160/Full2/2148yinzping/12168yinzping/imp.0: not a plain file
rcp: /mnt/wk05/yinzping/WIEN2k/DMFT/FeTe/P4nmm/PM/U5J8/beta160/Full2/2148yinzping/12168yinzping/optics: not a plain file
Failure to rcp files!

Your job started at Mon Nov 14 16:00:32 EST 2011 on host n143 in /mnt/p06/18267yinzping
Submission directory is /mnt/wk05/yinzping/WIEN2k/DMFT/FeTe/P4nmm/PM/U5J8/beta160/Full2/2148yinzping/12168yinzping on host rupc06
-----------------------------------------------------------------------------------------
+ echo -catch_rsh /opt/sge6/spool/n143/active_jobs/18267.1/pe_hostfile /opt/mpich2/intel/11.1/
-catch_rsh /opt/sge6/spool/n143/active_jobs/18267.1/pe_hostfile /opt/mpich2/intel/11.1/
+ SLEEPTIME=5
+ RETRIES=10
+ catch_rsh=0
+ catch_hostname=0
+ '[' 3 -gt 0 ']'
+ case "$1" in
+ catch_rsh=1
+ shift
+ '[' 2 -gt 0 ']'
+ case "$1" in
+ break
++ basename /opt/sge6/mpich2_smpd/startmpich2.sh
+ me=startmpich2.sh
+ '[' 2 -lt 2 ']'
+ pe_hostfile=/opt/sge6/spool/n143/active_jobs/18267.1/pe_hostfile
+ MPICH2_ROOT=/opt/mpich2/intel/11.1/
+ export MPICH2_ROOT
+ '[' '!' -x /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 ']'
+ '[' '!' -x /opt/mpich2/intel/11.1//bin/smpd ']'
+ '[' '!' -r /opt/sge6/spool/n143/active_jobs/18267.1/pe_hostfile ']'
+ machines=/tmp/18267.1.wp10/machines
+ PeHostfile2MachineFile /opt/sge6/spool/n143/active_jobs/18267.1/pe_hostfile
+ cat /opt/sge6/spool/n143/active_jobs/18267.1/pe_hostfile
+ read line
++ echo n143 2 wp10@n143 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n143
++ echo n143 2 wp10@n143 UNDEFINED
++ cut -f2 '-d '
+ nslots=2
+ i=1
+ '[' 1 -le 2 ']'
+ echo n143
++ expr 1 + 1
+ i=2
+ '[' 2 -le 2 ']'
+ echo n143
++ expr 2 + 1
+ i=3
+ '[' 3 -le 2 ']'
+ read line
++ echo n147 8 wp10@n147 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n147
++ echo n147 8 wp10@n147 UNDEFINED
++ cut -f2 '-d '
+ nslots=8
+ i=1
+ '[' 1 -le 8 ']'
+ echo n147
++ expr 1 + 1
+ i=2
+ '[' 2 -le 8 ']'
+ echo n147
++ expr 2 + 1
+ i=3
+ '[' 3 -le 8 ']'
+ echo n147
++ expr 3 + 1
+ i=4
+ '[' 4 -le 8 ']'
+ echo n147
++ expr 4 + 1
+ i=5
+ '[' 5 -le 8 ']'
+ echo n147
++ expr 5 + 1
+ i=6
+ '[' 6 -le 8 ']'
+ echo n147
++ expr 6 + 1
+ i=7
+ '[' 7 -le 8 ']'
+ echo n147
++ expr 7 + 1
+ i=8
+ '[' 8 -le 8 ']'
+ echo n147
++ expr 8 + 1
+ i=9
+ '[' 9 -le 8 ']'
+ read line
++ echo n159 4 wp10@n159 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n159
++ echo n159 4 wp10@n159 UNDEFINED
++ cut -f2 '-d '
+ nslots=4
+ i=1
+ '[' 1 -le 4 ']'
+ echo n159
++ expr 1 + 1
+ i=2
+ '[' 2 -le 4 ']'
+ echo n159
++ expr 2 + 1
+ i=3
+ '[' 3 -le 4 ']'
+ echo n159
++ expr 3 + 1
+ i=4
+ '[' 4 -le 4 ']'
+ echo n159
++ expr 4 + 1
+ i=5
+ '[' 5 -le 4 ']'
+ read line
++ echo n152 8 wp10@n152 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n152
++ echo n152 8 wp10@n152 UNDEFINED
++ cut -f2 '-d '
+ nslots=8
+ i=1
+ '[' 1 -le 8 ']'
+ echo n152
++ expr 1 + 1
+ i=2
+ '[' 2 -le 8 ']'
+ echo n152
++ expr 2 + 1
+ i=3
+ '[' 3 -le 8 ']'
+ echo n152
++ expr 3 + 1
+ i=4
+ '[' 4 -le 8 ']'
+ echo n152
++ expr 4 + 1
+ i=5
+ '[' 5 -le 8 ']'
+ echo n152
++ expr 5 + 1
+ i=6
+ '[' 6 -le 8 ']'
+ echo n152
++ expr 6 + 1
+ i=7
+ '[' 7 -le 8 ']'
+ echo n152
++ expr 7 + 1
+ i=8
+ '[' 8 -le 8 ']'
+ echo n152
++ expr 8 + 1
+ i=9
+ '[' 9 -le 8 ']'
+ read line
++ echo n148 6 wp10@n148 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n148
++ echo n148 6 wp10@n148 UNDEFINED
++ cut -f2 '-d '
+ nslots=6
+ i=1
+ '[' 1 -le 6 ']'
+ echo n148
++ expr 1 + 1
+ i=2
+ '[' 2 -le 6 ']'
+ echo n148
++ expr 2 + 1
+ i=3
+ '[' 3 -le 6 ']'
+ echo n148
++ expr 3 + 1
+ i=4
+ '[' 4 -le 6 ']'
+ echo n148
++ expr 4 + 1
+ i=5
+ '[' 5 -le 6 ']'
+ echo n148
++ expr 5 + 1
+ i=6
+ '[' 6 -le 6 ']'
+ echo n148
++ expr 6 + 1
+ i=7
+ '[' 7 -le 6 ']'
+ read line
++ echo n149 8 wp10@n149 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n149
++ echo n149 8 wp10@n149 UNDEFINED
++ cut -f2 '-d '
+ nslots=8
+ i=1
+ '[' 1 -le 8 ']'
+ echo n149
++ expr 1 + 1
+ i=2
+ '[' 2 -le 8 ']'
+ echo n149
++ expr 2 + 1
+ i=3
+ '[' 3 -le 8 ']'
+ echo n149
++ expr 3 + 1
+ i=4
+ '[' 4 -le 8 ']'
+ echo n149
++ expr 4 + 1
+ i=5
+ '[' 5 -le 8 ']'
+ echo n149
++ expr 5 + 1
+ i=6
+ '[' 6 -le 8 ']'
+ echo n149
++ expr 6 + 1
+ i=7
+ '[' 7 -le 8 ']'
+ echo n149
++ expr 7 + 1
+ i=8
+ '[' 8 -le 8 ']'
+ echo n149
++ expr 8 + 1
+ i=9
+ '[' 9 -le 8 ']'
+ read line
+ cat /tmp/18267.1.wp10/machines
n143
n143
n147
n147
n147
n147
n147
n147
n147
n147
n159
n159
n159
n159
n152
n152
n152
n152
n152
n152
n152
n152
n148
n148
n148
n148
n148
n148
n149
n149
n149
n149
n149
n149
n149
n149
+ '[' 1 = 1 ']'
+ rsh_wrapper=/opt/sge6/mpich2_smpd/rsh
+ '[' '!' -x /opt/sge6/mpich2_smpd/rsh ']'
+ rshcmd=rsh
+ case "$ARC" in
+ ln -s /opt/sge6/mpich2_smpd/rsh /tmp/18267.1.wp10/rsh
+ hash -r
+ '[' 0 = 1 ']'
+ port=23267
++ cat /tmp/18267.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n143 /opt/mpich2/intel/11.1//bin/smpd 23267
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n147 /opt/mpich2/intel/11.1//bin/smpd 23267
rsh n143 /opt/mpich2/intel/11.1//bin/smpd -port 23267 -d 0
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n159 /opt/mpich2/intel/11.1//bin/smpd 23267
rsh n147 /opt/mpich2/intel/11.1//bin/smpd -port 23267 -d 0
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n152 /opt/mpich2/intel/11.1//bin/smpd 23267
rsh n159 /opt/mpich2/intel/11.1//bin/smpd -port 23267 -d 0
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n148 /opt/mpich2/intel/11.1//bin/smpd 23267
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n149 /opt/mpich2/intel/11.1//bin/smpd 23267
rsh n152 /opt/mpich2/intel/11.1//bin/smpd -port 23267 -d 0
rsh n148 /opt/mpich2/intel/11.1//bin/smpd -port 23267 -d 0
+ actual_retry=1
+ '[' 1 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (1 of 10)'
startmpich2.sh: check for smpd daemons (1 of 10)
+ numhosts=0
++ cat /tmp/18267.1.wp10/machines
++ uniq
rsh n149 /opt/mpich2/intel/11.1//bin/smpd -port 23267 -d 0
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23267 -status n143
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n143'
startmpich2.sh: missing smpd on n143
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23267 -status n147
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n147'
startmpich2.sh: missing smpd on n147
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23267 -status n159
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n159'
startmpich2.sh: missing smpd on n159
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23267 -status n152
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n152'
startmpich2.sh: missing smpd on n152
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23267 -status n148
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n148'
startmpich2.sh: found running smpd on n148
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23267 -status n149
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n149'
startmpich2.sh: found running smpd on n149
+ '[' 6 -eq 2 ']'
+ sleep 5
+ (( actual_retry += 1 ))
+ '[' 2 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (2 of 10)'
startmpich2.sh: check for smpd daemons (2 of 10)
+ numhosts=0
++ cat /tmp/18267.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23267 -status n143
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n143'
startmpich2.sh: found running smpd on n143
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23267 -status n147
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n147'
startmpich2.sh: found running smpd on n147
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23267 -status n159
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n159'
startmpich2.sh: found running smpd on n159
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23267 -status n152
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n152'
startmpich2.sh: found running smpd on n152
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23267 -status n148
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n148'
startmpich2.sh: found running smpd on n148
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23267 -status n149
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n149'
startmpich2.sh: found running smpd on n149
+ '[' 6 -eq 6 ']'
+ echo 'startmpich2.sh: got all 6 of 6 nodes'
startmpich2.sh: got all 6 of 6 nodes
+ exit 0
rm: cannot remove `./*.broyd*': No such file or directory
-catch_rsh /opt/mpich2/intel/11.1/
Your job finished at Sat Nov 26 12:22:34 EST 2011