USER      =  jtomczak
HOMEDIR   =  /home/users/jtomczak
JOB_ID    =  6703
JOB_NAME  =  FS-b150-6
QUEUE     =  wp10
NSLOTS    =  64
NHOSTS    =  6

SUB_HOST  =  rupc06
SUB_DIR   =  /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta150/nf6/NEW

EXE_HOST  =  n141
NFS_HOST  =  caip02s
EXE_DIR   =  /mnt/p06
OUT_DIR   =  /tmp
TMP_DIR   =  /tmp/6703.1.wp10

INFO      =  Files larger than 100MB will be compressed to reduce the network load.
INFO2     =  Do NOT remove this file while job is running!

PORT      =  21703
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta150/nf6/NEW/FIX_mu: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta150/nf6/NEW/GO: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta150/nf6/NEW/GO-small: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta150/nf6/NEW/ancont: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta150/nf6/NEW/cdos: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta150/nf6/NEW/cdos-old: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta150/nf6/NEW/imp.0: not a plain file
Failure to rcp files!

Your job started at Fri May 20 17:49:35 EDT 2011 on host n141 in /mnt/p06/6703jtomczak
Submission directory is /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta150/nf6/NEW on host rupc06
-----------------------------------------------------------------------------------------
+ echo -catch_rsh /opt/sge6/spool/n141/active_jobs/6703.1/pe_hostfile /opt/mpich2/intel/11.1/
-catch_rsh /opt/sge6/spool/n141/active_jobs/6703.1/pe_hostfile /opt/mpich2/intel/11.1/
+ SLEEPTIME=5
+ RETRIES=10
+ catch_rsh=0
+ catch_hostname=0
+ '[' 3 -gt 0 ']'
+ case "$1" in
+ catch_rsh=1
+ shift
+ '[' 2 -gt 0 ']'
+ case "$1" in
+ break
++ basename /opt/sge6/mpich2_smpd/startmpich2.sh
+ me=startmpich2.sh
+ '[' 2 -lt 2 ']'
+ pe_hostfile=/opt/sge6/spool/n141/active_jobs/6703.1/pe_hostfile
+ MPICH2_ROOT=/opt/mpich2/intel/11.1/
+ export MPICH2_ROOT
+ '[' '!' -x /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 ']'
+ '[' '!' -x /opt/mpich2/intel/11.1//bin/smpd ']'
+ '[' '!' -r /opt/sge6/spool/n141/active_jobs/6703.1/pe_hostfile ']'
+ machines=/tmp/6703.1.wp10/machines
+ PeHostfile2MachineFile /opt/sge6/spool/n141/active_jobs/6703.1/pe_hostfile
+ cat /opt/sge6/spool/n141/active_jobs/6703.1/pe_hostfile
+ read line
++ echo n141 12 wp10@n141 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n141
++ echo n141 12 wp10@n141 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n141
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n141
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n141
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n141
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n141
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n141
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n141
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n141
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n141
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n141
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n141
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n141
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n145 12 wp10@n145 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n145
++ echo n145 12 wp10@n145 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n145
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n145
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n145
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n145
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n145
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n145
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n145
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n145
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n145
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n145
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n145
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n145
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n146 12 wp10@n146 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n146
++ echo n146 12 wp10@n146 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n146
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n146
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n146
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n146
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n146
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n146
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n146
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n146
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n146
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n146
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n146
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n146
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n147 12 wp10@n147 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n147
++ echo n147 12 wp10@n147 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n147
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n147
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n147
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n147
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n147
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n147
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n147
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n147
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n147
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n147
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n147
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n147
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n148 12 wp10@n148 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n148
++ echo n148 12 wp10@n148 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n148
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n148
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n148
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n148
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n148
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n148
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n148
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n148
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n148
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n148
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n148
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n148
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n149 4 wp10@n149 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n149
++ echo n149 4 wp10@n149 UNDEFINED
++ cut -f2 '-d '
+ nslots=4
+ i=1
+ '[' 1 -le 4 ']'
+ echo n149
++ expr 1 + 1
+ i=2
+ '[' 2 -le 4 ']'
+ echo n149
++ expr 2 + 1
+ i=3
+ '[' 3 -le 4 ']'
+ echo n149
++ expr 3 + 1
+ i=4
+ '[' 4 -le 4 ']'
+ echo n149
++ expr 4 + 1
+ i=5
+ '[' 5 -le 4 ']'
+ read line
+ cat /tmp/6703.1.wp10/machines
n141
n141
n141
n141
n141
n141
n141
n141
n141
n141
n141
n141
n145
n145
n145
n145
n145
n145
n145
n145
n145
n145
n145
n145
n146
n146
n146
n146
n146
n146
n146
n146
n146
n146
n146
n146
n147
n147
n147
n147
n147
n147
n147
n147
n147
n147
n147
n147
n148
n148
n148
n148
n148
n148
n148
n148
n148
n148
n148
n148
n149
n149
n149
n149
+ '[' 1 = 1 ']'
+ rsh_wrapper=/opt/sge6/mpich2_smpd/rsh
+ '[' '!' -x /opt/sge6/mpich2_smpd/rsh ']'
+ rshcmd=rsh
+ case "$ARC" in
+ ln -s /opt/sge6/mpich2_smpd/rsh /tmp/6703.1.wp10/rsh
+ hash -r
+ '[' 0 = 1 ']'
+ port=21703
++ cat /tmp/6703.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n141 /opt/mpich2/intel/11.1//bin/smpd 21703
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n145 /opt/mpich2/intel/11.1//bin/smpd 21703
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n146 /opt/mpich2/intel/11.1//bin/smpd 21703
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n147 /opt/mpich2/intel/11.1//bin/smpd 21703
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n148 /opt/mpich2/intel/11.1//bin/smpd 21703
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n149 /opt/mpich2/intel/11.1//bin/smpd 21703
rsh n141 /opt/mpich2/intel/11.1//bin/smpd -port 21703 -d 0
rsh n145 /opt/mpich2/intel/11.1//bin/smpd -port 21703 -d 0
rsh n146 /opt/mpich2/intel/11.1//bin/smpd -port 21703 -d 0
+ actual_retry=1
+ '[' 1 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (1 of 10)'
startmpich2.sh: check for smpd daemons (1 of 10)
+ numhosts=0
rsh n147 /opt/mpich2/intel/11.1//bin/smpd -port 21703 -d 0
++ cat /tmp/6703.1.wp10/machines
++ uniq
rsh n148 /opt/mpich2/intel/11.1//bin/smpd -port 21703 -d 0
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21703 -status n141
++ head -n 1
rsh n149 /opt/mpich2/intel/11.1//bin/smpd -port 21703 -d 0
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n141'
startmpich2.sh: missing smpd on n141
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21703 -status n145
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n145'
startmpich2.sh: missing smpd on n145
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21703 -status n146
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n146'
startmpich2.sh: missing smpd on n146
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21703 -status n147
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n147'
startmpich2.sh: missing smpd on n147
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21703 -status n148
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n148'
startmpich2.sh: missing smpd on n148
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21703 -status n149
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n149'
startmpich2.sh: missing smpd on n149
+ '[' 6 -eq 0 ']'
+ sleep 5
+ (( actual_retry += 1 ))
+ '[' 2 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (2 of 10)'
startmpich2.sh: check for smpd daemons (2 of 10)
+ numhosts=0
++ cat /tmp/6703.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21703 -status n141
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n141'
startmpich2.sh: found running smpd on n141
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21703 -status n145
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n145'
startmpich2.sh: found running smpd on n145
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21703 -status n146
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n146'
startmpich2.sh: found running smpd on n146
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21703 -status n147
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n147'
startmpich2.sh: found running smpd on n147
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21703 -status n148
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n148'
startmpich2.sh: found running smpd on n148
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21703 -status n149
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n149'
startmpich2.sh: found running smpd on n149
+ '[' 6 -eq 6 ']'
+ echo 'startmpich2.sh: got all 6 of 6 nodes'
startmpich2.sh: got all 6 of 6 nodes
+ exit 0
gzip: eigvals.dat already exists;	not overwritten
START
END
Wed May 25 08:34:10 EDT 2011
-catch_rsh /opt/mpich2/intel/11.1/
Your job finished at Wed May 25 08:34:15 EDT 2011