USER      =  yinzping
HOMEDIR   =  /home/yinzping
JOB_ID    =  12168
JOB_NAME  =  TePMfull2
QUEUE     =  wp10
NSLOTS    =  36
NHOSTS    =  5

SUB_HOST  =  rupc06
SUB_DIR   =  /mnt/wk05/yinzping/WIEN2k/DMFT/FeTe/P4nmm/PM/U5J8/beta160/Full2/2148yinzping

EXE_HOST  =  n155
NFS_HOST  =  caip02s
EXE_DIR   =  /mnt/p05
OUT_DIR   =  /tmp
TMP_DIR   =  /tmp/12168.1.wp10

INFO      =  Files larger than 100MB will be compressed to reduce the network load.
INFO2     =  Do NOT remove this file while job is running!

PORT      =  22168
rcp: /mnt/wk05/yinzping/WIEN2k/DMFT/FeTe/P4nmm/PM/U5J8/beta160/Full2/2148yinzping/3DFS: not a plain file
rcp: /mnt/wk05/yinzping/WIEN2k/DMFT/FeTe/P4nmm/PM/U5J8/beta160/Full2/2148yinzping/FS: not a plain file
rcp: /mnt/wk05/yinzping/WIEN2k/DMFT/FeTe/P4nmm/PM/U5J8/beta160/Full2/2148yinzping/Gk_k: not a plain file
rcp: /mnt/wk05/yinzping/WIEN2k/DMFT/FeTe/P4nmm/PM/U5J8/beta160/Full2/2148yinzping/Vertex: not a plain file
rcp: /mnt/wk05/yinzping/WIEN2k/DMFT/FeTe/P4nmm/PM/U5J8/beta160/Full2/2148yinzping/ancont: not a plain file
rcp: /mnt/wk05/yinzping/WIEN2k/DMFT/FeTe/P4nmm/PM/U5J8/beta160/Full2/2148yinzping/band: not a plain file
rcp: /mnt/wk05/yinzping/WIEN2k/DMFT/FeTe/P4nmm/PM/U5J8/beta160/Full2/2148yinzping/dos: not a plain file
rcp: /mnt/wk05/yinzping/WIEN2k/DMFT/FeTe/P4nmm/PM/U5J8/beta160/Full2/2148yinzping/imp.0: not a plain file
rcp: /mnt/wk05/yinzping/WIEN2k/DMFT/FeTe/P4nmm/PM/U5J8/beta160/Full2/2148yinzping/optics: not a plain file
Failure to rcp files!

Your job started at Wed Sep 14 04:03:39 EDT 2011 on host n155 in /mnt/p05/12168yinzping
Submission directory is /mnt/wk05/yinzping/WIEN2k/DMFT/FeTe/P4nmm/PM/U5J8/beta160/Full2/2148yinzping on host rupc06
-----------------------------------------------------------------------------------------
+ echo -catch_rsh /opt/sge6/spool/n155/active_jobs/12168.1/pe_hostfile /opt/mpich2/intel/11.1/
-catch_rsh /opt/sge6/spool/n155/active_jobs/12168.1/pe_hostfile /opt/mpich2/intel/11.1/
+ SLEEPTIME=5
+ RETRIES=10
+ catch_rsh=0
+ catch_hostname=0
+ '[' 3 -gt 0 ']'
+ case "$1" in
+ catch_rsh=1
+ shift
+ '[' 2 -gt 0 ']'
+ case "$1" in
+ break
++ basename /opt/sge6/mpich2_smpd/startmpich2.sh
+ me=startmpich2.sh
+ '[' 2 -lt 2 ']'
+ pe_hostfile=/opt/sge6/spool/n155/active_jobs/12168.1/pe_hostfile
+ MPICH2_ROOT=/opt/mpich2/intel/11.1/
+ export MPICH2_ROOT
+ '[' '!' -x /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 ']'
+ '[' '!' -x /opt/mpich2/intel/11.1//bin/smpd ']'
+ '[' '!' -r /opt/sge6/spool/n155/active_jobs/12168.1/pe_hostfile ']'
+ machines=/tmp/12168.1.wp10/machines
+ PeHostfile2MachineFile /opt/sge6/spool/n155/active_jobs/12168.1/pe_hostfile
+ cat /opt/sge6/spool/n155/active_jobs/12168.1/pe_hostfile
+ read line
++ echo n155 1 wp10@n155 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n155
++ echo n155 1 wp10@n155 UNDEFINED
++ cut -f2 '-d '
+ nslots=1
+ i=1
+ '[' 1 -le 1 ']'
+ echo n155
++ expr 1 + 1
+ i=2
+ '[' 2 -le 1 ']'
+ read line
++ echo n165 12 wp10@n165 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n165
++ echo n165 12 wp10@n165 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n165
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n165
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n165
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n165
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n165
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n165
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n165
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n165
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n165
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n165
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n165
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n165
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n152 12 wp10@n152 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n152
++ echo n152 12 wp10@n152 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n152
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n152
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n152
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n152
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n152
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n152
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n152
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n152
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n152
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n152
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n152
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n152
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n163 5 wp10@n163 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n163
++ echo n163 5 wp10@n163 UNDEFINED
++ cut -f2 '-d '
+ nslots=5
+ i=1
+ '[' 1 -le 5 ']'
+ echo n163
++ expr 1 + 1
+ i=2
+ '[' 2 -le 5 ']'
+ echo n163
++ expr 2 + 1
+ i=3
+ '[' 3 -le 5 ']'
+ echo n163
++ expr 3 + 1
+ i=4
+ '[' 4 -le 5 ']'
+ echo n163
++ expr 4 + 1
+ i=5
+ '[' 5 -le 5 ']'
+ echo n163
++ expr 5 + 1
+ i=6
+ '[' 6 -le 5 ']'
+ read line
++ echo n160 6 wp10@n160 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n160
++ echo n160 6 wp10@n160 UNDEFINED
++ cut -f2 '-d '
+ nslots=6
+ i=1
+ '[' 1 -le 6 ']'
+ echo n160
++ expr 1 + 1
+ i=2
+ '[' 2 -le 6 ']'
+ echo n160
++ expr 2 + 1
+ i=3
+ '[' 3 -le 6 ']'
+ echo n160
++ expr 3 + 1
+ i=4
+ '[' 4 -le 6 ']'
+ echo n160
++ expr 4 + 1
+ i=5
+ '[' 5 -le 6 ']'
+ echo n160
++ expr 5 + 1
+ i=6
+ '[' 6 -le 6 ']'
+ echo n160
++ expr 6 + 1
+ i=7
+ '[' 7 -le 6 ']'
+ read line
+ cat /tmp/12168.1.wp10/machines
n155
n165
n165
n165
n165
n165
n165
n165
n165
n165
n165
n165
n165
n152
n152
n152
n152
n152
n152
n152
n152
n152
n152
n152
n152
n163
n163
n163
n163
n163
n160
n160
n160
n160
n160
n160
+ '[' 1 = 1 ']'
+ rsh_wrapper=/opt/sge6/mpich2_smpd/rsh
+ '[' '!' -x /opt/sge6/mpich2_smpd/rsh ']'
+ rshcmd=rsh
+ case "$ARC" in
+ ln -s /opt/sge6/mpich2_smpd/rsh /tmp/12168.1.wp10/rsh
+ hash -r
+ '[' 0 = 1 ']'
+ port=22168
++ cat /tmp/12168.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n155 /opt/mpich2/intel/11.1//bin/smpd 22168
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n165 /opt/mpich2/intel/11.1//bin/smpd 22168
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n152 /opt/mpich2/intel/11.1//bin/smpd 22168
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n163 /opt/mpich2/intel/11.1//bin/smpd 22168
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n160 /opt/mpich2/intel/11.1//bin/smpd 22168
+ actual_retry=1
+ '[' 1 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (1 of 10)'
startmpich2.sh: check for smpd daemons (1 of 10)
+ numhosts=0
++ uniq
++ cat /tmp/12168.1.wp10/machines
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22168 -status n155
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n155'
startmpich2.sh: missing smpd on n155
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22168 -status n165
++ head -n 1
++ cut -f 1 -d ' '
rsh n160 /opt/mpich2/intel/11.1//bin/smpd -port 22168 -d 0
rsh n152 /opt/mpich2/intel/11.1//bin/smpd -port 22168 -d 0
rsh n163 /opt/mpich2/intel/11.1//bin/smpd -port 22168 -d 0
rsh n155 /opt/mpich2/intel/11.1//bin/smpd -port 22168 -d 0
rsh n165 /opt/mpich2/intel/11.1//bin/smpd -port 22168 -d 0
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n165'
startmpich2.sh: missing smpd on n165
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22168 -status n152
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n152'
startmpich2.sh: missing smpd on n152
+ for host in '`cat $machines | uniq`'
++ head -n 1
++ cut -f 1 -d ' '
++ /opt/mpich2/intel/11.1//bin/smpd -port 22168 -status n163
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n163'
startmpich2.sh: missing smpd on n163
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22168 -status n160
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n160'
startmpich2.sh: missing smpd on n160
+ '[' 5 -eq 0 ']'
+ sleep 5
+ (( actual_retry += 1 ))
+ '[' 2 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (2 of 10)'
startmpich2.sh: check for smpd daemons (2 of 10)
+ numhosts=0
++ cat /tmp/12168.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22168 -status n155
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n155'
startmpich2.sh: found running smpd on n155
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22168 -status n165
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n165'
startmpich2.sh: found running smpd on n165
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22168 -status n152
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n152'
startmpich2.sh: found running smpd on n152
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22168 -status n163
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n163'
startmpich2.sh: found running smpd on n163
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22168 -status n160
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n160'
startmpich2.sh: found running smpd on n160
+ '[' 5 -eq 5 ']'
+ echo 'startmpich2.sh: got all 5 of 5 nodes'
startmpich2.sh: got all 5 of 5 nodes
+ exit 0
rm: cannot remove `./*.?.[12]': No such file or directory
rm: cannot remove `./*.broyd*': No such file or directory
-catch_rsh /opt/mpich2/intel/11.1/
Your job finished at Sat Sep 24 14:04:39 EDT 2011