USER      =  jtomczak
HOMEDIR   =  /home/users/jtomczak
JOB_ID    =  16490
JOB_NAME  =  F125
QUEUE     =  wp10
NSLOTS    =  76
NHOSTS    =  7

SUB_HOST  =  rupc04
SUB_DIR   =  /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta125/nf6/NEW

EXE_HOST  =  n165
NFS_HOST  =  caip02s
EXE_DIR   =  /mnt/p06
OUT_DIR   =  /tmp
TMP_DIR   =  /tmp/16490.1.wp10

INFO      =  Files larger than 100MB will be compressed to reduce the network load.
INFO2     =  Do NOT remove this file while job is running!

PORT      =  21490
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta125/nf6/NEW/15782jtomczakT: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta125/nf6/NEW/16415jtomczak: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta125/nf6/NEW/ancont: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta125/nf6/NEW/cdos: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta125/nf6/NEW/imp.0: not a plain file
Failure to rcp files!

Your job started at Tue Oct 25 10:24:41 EDT 2011 on host n165 in /mnt/p06/16490jtomczak
Submission directory is /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta125/nf6/NEW on host rupc04
-----------------------------------------------------------------------------------------
+ echo -catch_rsh /opt/sge6/spool/n165/active_jobs/16490.1/pe_hostfile /opt/mpich2/intel/11.1/
-catch_rsh /opt/sge6/spool/n165/active_jobs/16490.1/pe_hostfile /opt/mpich2/intel/11.1/
+ SLEEPTIME=5
+ RETRIES=10
+ catch_rsh=0
+ catch_hostname=0
+ '[' 3 -gt 0 ']'
+ case "$1" in
+ catch_rsh=1
+ shift
+ '[' 2 -gt 0 ']'
+ case "$1" in
+ break
++ basename /opt/sge6/mpich2_smpd/startmpich2.sh
+ me=startmpich2.sh
+ '[' 2 -lt 2 ']'
+ pe_hostfile=/opt/sge6/spool/n165/active_jobs/16490.1/pe_hostfile
+ MPICH2_ROOT=/opt/mpich2/intel/11.1/
+ export MPICH2_ROOT
+ '[' '!' -x /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 ']'
+ '[' '!' -x /opt/mpich2/intel/11.1//bin/smpd ']'
+ '[' '!' -r /opt/sge6/spool/n165/active_jobs/16490.1/pe_hostfile ']'
+ machines=/tmp/16490.1.wp10/machines
+ PeHostfile2MachineFile /opt/sge6/spool/n165/active_jobs/16490.1/pe_hostfile
+ cat /opt/sge6/spool/n165/active_jobs/16490.1/pe_hostfile
+ read line
++ echo n165 12 wp10@n165 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n165
++ echo n165 12 wp10@n165 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n165
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n165
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n165
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n165
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n165
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n165
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n165
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n165
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n165
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n165
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n165
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n165
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n143 8 wp10@n143 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n143
++ echo n143 8 wp10@n143 UNDEFINED
++ cut -f2 '-d '
+ nslots=8
+ i=1
+ '[' 1 -le 8 ']'
+ echo n143
++ expr 1 + 1
+ i=2
+ '[' 2 -le 8 ']'
+ echo n143
++ expr 2 + 1
+ i=3
+ '[' 3 -le 8 ']'
+ echo n143
++ expr 3 + 1
+ i=4
+ '[' 4 -le 8 ']'
+ echo n143
++ expr 4 + 1
+ i=5
+ '[' 5 -le 8 ']'
+ echo n143
++ expr 5 + 1
+ i=6
+ '[' 6 -le 8 ']'
+ echo n143
++ expr 6 + 1
+ i=7
+ '[' 7 -le 8 ']'
+ echo n143
++ expr 7 + 1
+ i=8
+ '[' 8 -le 8 ']'
+ echo n143
++ expr 8 + 1
+ i=9
+ '[' 9 -le 8 ']'
+ read line
++ echo n162 12 wp10@n162 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n162
++ echo n162 12 wp10@n162 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n162
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n162
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n162
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n162
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n162
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n162
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n162
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n162
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n162
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n162
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n162
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n162
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n146 12 wp10@n146 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n146
++ echo n146 12 wp10@n146 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n146
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n146
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n146
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n146
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n146
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n146
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n146
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n146
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n146
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n146
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n146
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n146
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n151 12 wp10@n151 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n151
++ echo n151 12 wp10@n151 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n151
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n151
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n151
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n151
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n151
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n151
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n151
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n151
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n151
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n151
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n151
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n151
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n141 12 wp10@n141 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n141
++ echo n141 12 wp10@n141 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n141
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n141
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n141
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n141
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n141
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n141
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n141
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n141
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n141
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n141
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n141
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n141
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n159 8 wp10@n159 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n159
++ echo n159 8 wp10@n159 UNDEFINED
++ cut -f2 '-d '
+ nslots=8
+ i=1
+ '[' 1 -le 8 ']'
+ echo n159
++ expr 1 + 1
+ i=2
+ '[' 2 -le 8 ']'
+ echo n159
++ expr 2 + 1
+ i=3
+ '[' 3 -le 8 ']'
+ echo n159
++ expr 3 + 1
+ i=4
+ '[' 4 -le 8 ']'
+ echo n159
++ expr 4 + 1
+ i=5
+ '[' 5 -le 8 ']'
+ echo n159
++ expr 5 + 1
+ i=6
+ '[' 6 -le 8 ']'
+ echo n159
++ expr 6 + 1
+ i=7
+ '[' 7 -le 8 ']'
+ echo n159
++ expr 7 + 1
+ i=8
+ '[' 8 -le 8 ']'
+ echo n159
++ expr 8 + 1
+ i=9
+ '[' 9 -le 8 ']'
+ read line
+ cat /tmp/16490.1.wp10/machines
n165
n165
n165
n165
n165
n165
n165
n165
n165
n165
n165
n165
n143
n143
n143
n143
n143
n143
n143
n143
n162
n162
n162
n162
n162
n162
n162
n162
n162
n162
n162
n162
n146
n146
n146
n146
n146
n146
n146
n146
n146
n146
n146
n146
n151
n151
n151
n151
n151
n151
n151
n151
n151
n151
n151
n151
n141
n141
n141
n141
n141
n141
n141
n141
n141
n141
n141
n141
n159
n159
n159
n159
n159
n159
n159
n159
+ '[' 1 = 1 ']'
+ rsh_wrapper=/opt/sge6/mpich2_smpd/rsh
+ '[' '!' -x /opt/sge6/mpich2_smpd/rsh ']'
+ rshcmd=rsh
+ case "$ARC" in
+ ln -s /opt/sge6/mpich2_smpd/rsh /tmp/16490.1.wp10/rsh
+ hash -r
+ '[' 0 = 1 ']'
+ port=21490
++ cat /tmp/16490.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n165 /opt/mpich2/intel/11.1//bin/smpd 21490
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n143 /opt/mpich2/intel/11.1//bin/smpd 21490
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n162 /opt/mpich2/intel/11.1//bin/smpd 21490
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n146 /opt/mpich2/intel/11.1//bin/smpd 21490
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n151 /opt/mpich2/intel/11.1//bin/smpd 21490
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n141 /opt/mpich2/intel/11.1//bin/smpd 21490
rsh n143 /opt/mpich2/intel/11.1//bin/smpd -port 21490 -d 0
+ for host in '`cat $machines | uniq`'
rsh n165 /opt/mpich2/intel/11.1//bin/smpd -port 21490 -d 0
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n159 /opt/mpich2/intel/11.1//bin/smpd 21490
+ actual_retry=1
+ '[' 1 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (1 of 10)'
startmpich2.sh: check for smpd daemons (1 of 10)
+ numhosts=0
rsh n162 /opt/mpich2/intel/11.1//bin/smpd -port 21490 -d 0
++ uniq
++ cat /tmp/16490.1.wp10/machines
+ for host in '`cat $machines | uniq`'
rsh n151 /opt/mpich2/intel/11.1//bin/smpd -port 21490 -d 0
++ /opt/mpich2/intel/11.1//bin/smpd -port 21490 -status n165
++ head -n 1
++ cut -f 1 -d ' '
rsh n141 /opt/mpich2/intel/11.1//bin/smpd -port 21490 -d 0
rsh n146 /opt/mpich2/intel/11.1//bin/smpd -port 21490 -d 0
rsh n159 /opt/mpich2/intel/11.1//bin/smpd -port 21490 -d 0
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n165'
startmpich2.sh: missing smpd on n165
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21490 -status n143
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n143'
startmpich2.sh: found running smpd on n143
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21490 -status n162
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n162'
startmpich2.sh: missing smpd on n162
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21490 -status n146
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n146'
startmpich2.sh: missing smpd on n146
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21490 -status n151
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n151'
startmpich2.sh: missing smpd on n151
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21490 -status n141
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n141'
startmpich2.sh: missing smpd on n141
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21490 -status n159
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n159'
startmpich2.sh: found running smpd on n159
+ '[' 7 -eq 2 ']'
+ sleep 5
+ (( actual_retry += 1 ))
+ '[' 2 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (2 of 10)'
startmpich2.sh: check for smpd daemons (2 of 10)
+ numhosts=0
++ cat /tmp/16490.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21490 -status n165
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n165'
startmpich2.sh: found running smpd on n165
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21490 -status n143
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n143'
startmpich2.sh: found running smpd on n143
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21490 -status n162
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n162'
startmpich2.sh: found running smpd on n162
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21490 -status n146
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n146'
startmpich2.sh: found running smpd on n146
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21490 -status n151
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n151'
startmpich2.sh: found running smpd on n151
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21490 -status n141
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n141'
startmpich2.sh: found running smpd on n141
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21490 -status n159
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n159'
startmpich2.sh: found running smpd on n159
+ '[' 7 -eq 7 ']'
+ echo 'startmpich2.sh: got all 7 of 7 nodes'
startmpich2.sh: got all 7 of 7 nodes
+ exit 0
gzip: eigvals.dat already exists;	not overwritten
START
END
Mon Oct 31 07:20:03 EDT 2011
-catch_rsh /opt/mpich2/intel/11.1/
Your job finished at Mon Oct 31 07:20:08 EDT 2011