USER      =  jtomczak
HOMEDIR   =  /home/users/jtomczak
JOB_ID    =  7792
JOB_NAME  =  FS-LOW
QUEUE     =  wp10
NSLOTS    =  38
NHOSTS    =  9

SUB_HOST  =  rupc04
SUB_DIR   =  /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta250

EXE_HOST  =  n160
NFS_HOST  =  caip02s
EXE_DIR   =  /mnt/p05
OUT_DIR   =  /tmp
TMP_DIR   =  /tmp/7792.1.wp10

INFO      =  Files larger than 100MB will be compressed to reduce the network load.
INFO2     =  Do NOT remove this file while job is running!

PORT      =  22792
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta250/imp.0: not a plain file
Failure to rcp files!

Your job started at Sat Jun 11 18:27:22 EDT 2011 on host n160 in /mnt/p05/7792jtomczak
Submission directory is /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta250 on host rupc04
-----------------------------------------------------------------------------------------
+ echo -catch_rsh /opt/sge6/spool/n160/active_jobs/7792.1/pe_hostfile /opt/mpich2/intel/11.1/
-catch_rsh /opt/sge6/spool/n160/active_jobs/7792.1/pe_hostfile /opt/mpich2/intel/11.1/
+ SLEEPTIME=5
+ RETRIES=10
+ catch_rsh=0
+ catch_hostname=0
+ '[' 3 -gt 0 ']'
+ case "$1" in
+ catch_rsh=1
+ shift
+ '[' 2 -gt 0 ']'
+ case "$1" in
+ break
++ basename /opt/sge6/mpich2_smpd/startmpich2.sh
+ me=startmpich2.sh
+ '[' 2 -lt 2 ']'
+ pe_hostfile=/opt/sge6/spool/n160/active_jobs/7792.1/pe_hostfile
+ MPICH2_ROOT=/opt/mpich2/intel/11.1/
+ export MPICH2_ROOT
+ '[' '!' -x /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 ']'
+ '[' '!' -x /opt/mpich2/intel/11.1//bin/smpd ']'
+ '[' '!' -r /opt/sge6/spool/n160/active_jobs/7792.1/pe_hostfile ']'
+ machines=/tmp/7792.1.wp10/machines
+ PeHostfile2MachineFile /opt/sge6/spool/n160/active_jobs/7792.1/pe_hostfile
+ cat /opt/sge6/spool/n160/active_jobs/7792.1/pe_hostfile
+ read line
++ echo n160 1 wp10@n160 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n160
++ echo n160 1 wp10@n160 UNDEFINED
++ cut -f2 '-d '
+ nslots=1
+ i=1
+ '[' 1 -le 1 ']'
+ echo n160
++ expr 1 + 1
+ i=2
+ '[' 2 -le 1 ']'
+ read line
++ echo n154 1 wp10@n154 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n154
++ echo n154 1 wp10@n154 UNDEFINED
++ cut -f2 '-d '
+ nslots=1
+ i=1
+ '[' 1 -le 1 ']'
+ echo n154
++ expr 1 + 1
+ i=2
+ '[' 2 -le 1 ']'
+ read line
++ echo n149 8 wp10@n149 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n149
++ echo n149 8 wp10@n149 UNDEFINED
++ cut -f2 '-d '
+ nslots=8
+ i=1
+ '[' 1 -le 8 ']'
+ echo n149
++ expr 1 + 1
+ i=2
+ '[' 2 -le 8 ']'
+ echo n149
++ expr 2 + 1
+ i=3
+ '[' 3 -le 8 ']'
+ echo n149
++ expr 3 + 1
+ i=4
+ '[' 4 -le 8 ']'
+ echo n149
++ expr 4 + 1
+ i=5
+ '[' 5 -le 8 ']'
+ echo n149
++ expr 5 + 1
+ i=6
+ '[' 6 -le 8 ']'
+ echo n149
++ expr 6 + 1
+ i=7
+ '[' 7 -le 8 ']'
+ echo n149
++ expr 7 + 1
+ i=8
+ '[' 8 -le 8 ']'
+ echo n149
++ expr 8 + 1
+ i=9
+ '[' 9 -le 8 ']'
+ read line
++ echo n150 1 wp10@n150 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n150
++ echo n150 1 wp10@n150 UNDEFINED
++ cut -f2 '-d '
+ nslots=1
+ i=1
+ '[' 1 -le 1 ']'
+ echo n150
++ expr 1 + 1
+ i=2
+ '[' 2 -le 1 ']'
+ read line
++ echo n161 3 wp10@n161 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n161
++ echo n161 3 wp10@n161 UNDEFINED
++ cut -f2 '-d '
+ nslots=3
+ i=1
+ '[' 1 -le 3 ']'
+ echo n161
++ expr 1 + 1
+ i=2
+ '[' 2 -le 3 ']'
+ echo n161
++ expr 2 + 1
+ i=3
+ '[' 3 -le 3 ']'
+ echo n161
++ expr 3 + 1
+ i=4
+ '[' 4 -le 3 ']'
+ read line
++ echo n141 12 wp10@n141 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n141
++ echo n141 12 wp10@n141 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n141
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n141
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n141
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n141
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n141
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n141
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n141
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n141
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n141
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n141
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n141
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n141
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n143 6 wp10@n143 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n143
++ echo n143 6 wp10@n143 UNDEFINED
++ cut -f2 '-d '
+ nslots=6
+ i=1
+ '[' 1 -le 6 ']'
+ echo n143
++ expr 1 + 1
+ i=2
+ '[' 2 -le 6 ']'
+ echo n143
++ expr 2 + 1
+ i=3
+ '[' 3 -le 6 ']'
+ echo n143
++ expr 3 + 1
+ i=4
+ '[' 4 -le 6 ']'
+ echo n143
++ expr 4 + 1
+ i=5
+ '[' 5 -le 6 ']'
+ echo n143
++ expr 5 + 1
+ i=6
+ '[' 6 -le 6 ']'
+ echo n143
++ expr 6 + 1
+ i=7
+ '[' 7 -le 6 ']'
+ read line
++ echo n163 4 wp10@n163 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n163
++ echo n163 4 wp10@n163 UNDEFINED
++ cut -f2 '-d '
+ nslots=4
+ i=1
+ '[' 1 -le 4 ']'
+ echo n163
++ expr 1 + 1
+ i=2
+ '[' 2 -le 4 ']'
+ echo n163
++ expr 2 + 1
+ i=3
+ '[' 3 -le 4 ']'
+ echo n163
++ expr 3 + 1
+ i=4
+ '[' 4 -le 4 ']'
+ echo n163
++ expr 4 + 1
+ i=5
+ '[' 5 -le 4 ']'
+ read line
++ echo n148 2 wp10@n148 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n148
++ echo n148 2 wp10@n148 UNDEFINED
++ cut -f2 '-d '
+ nslots=2
+ i=1
+ '[' 1 -le 2 ']'
+ echo n148
++ expr 1 + 1
+ i=2
+ '[' 2 -le 2 ']'
+ echo n148
++ expr 2 + 1
+ i=3
+ '[' 3 -le 2 ']'
+ read line
+ cat /tmp/7792.1.wp10/machines
n160
n154
n149
n149
n149
n149
n149
n149
n149
n149
n150
n161
n161
n161
n141
n141
n141
n141
n141
n141
n141
n141
n141
n141
n141
n141
n143
n143
n143
n143
n143
n143
n163
n163
n163
n163
n148
n148
+ '[' 1 = 1 ']'
+ rsh_wrapper=/opt/sge6/mpich2_smpd/rsh
+ '[' '!' -x /opt/sge6/mpich2_smpd/rsh ']'
+ rshcmd=rsh
+ case "$ARC" in
+ ln -s /opt/sge6/mpich2_smpd/rsh /tmp/7792.1.wp10/rsh
+ hash -r
+ '[' 0 = 1 ']'
+ port=22792
++ cat /tmp/7792.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n160 /opt/mpich2/intel/11.1//bin/smpd 22792
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n154 /opt/mpich2/intel/11.1//bin/smpd 22792
rsh n160 /opt/mpich2/intel/11.1//bin/smpd -port 22792 -d 0
rsh n154 /opt/mpich2/intel/11.1//bin/smpd -port 22792 -d 0
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n149 /opt/mpich2/intel/11.1//bin/smpd 22792
rsh n149 /opt/mpich2/intel/11.1//bin/smpd -port 22792 -d 0
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n150 /opt/mpich2/intel/11.1//bin/smpd 22792
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n161 /opt/mpich2/intel/11.1//bin/smpd 22792
rsh n150 /opt/mpich2/intel/11.1//bin/smpd -port 22792 -d 0
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n141 /opt/mpich2/intel/11.1//bin/smpd 22792
rsh n141 /opt/mpich2/intel/11.1//bin/smpd -port 22792 -d 0
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n143 /opt/mpich2/intel/11.1//bin/smpd 22792
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n163 /opt/mpich2/intel/11.1//bin/smpd 22792
rsh n143 /opt/mpich2/intel/11.1//bin/smpd -port 22792 -d 0
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n148 /opt/mpich2/intel/11.1//bin/smpd 22792
rsh n148 /opt/mpich2/intel/11.1//bin/smpd -port 22792 -d 0
+ actual_retry=1
+ '[' 1 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (1 of 10)'
startmpich2.sh: check for smpd daemons (1 of 10)
+ numhosts=0
++ cat /tmp/7792.1.wp10/machines
rsh n163 /opt/mpich2/intel/11.1//bin/smpd -port 22792 -d 0
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22792 -status n160
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n160'
startmpich2.sh: missing smpd on n160
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22792 -status n154
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n154'
startmpich2.sh: missing smpd on n154
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22792 -status n149
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n149'
startmpich2.sh: found running smpd on n149
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22792 -status n150
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n150'
startmpich2.sh: missing smpd on n150
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22792 -status n161
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n161'
startmpich2.sh: missing smpd on n161
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22792 -status n141
++ head -n 1
++ cut -f 1 -d ' '
rsh n161 /opt/mpich2/intel/11.1//bin/smpd -port 22792 -d 0
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n141'
startmpich2.sh: found running smpd on n141
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22792 -status n143
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n143'
startmpich2.sh: found running smpd on n143
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22792 -status n163
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n163'
startmpich2.sh: found running smpd on n163
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22792 -status n148
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n148'
startmpich2.sh: found running smpd on n148
+ '[' 9 -eq 5 ']'
+ sleep 5
+ (( actual_retry += 1 ))
+ '[' 2 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (2 of 10)'
startmpich2.sh: check for smpd daemons (2 of 10)
+ numhosts=0
++ cat /tmp/7792.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22792 -status n160
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n160'
startmpich2.sh: found running smpd on n160
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22792 -status n154
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n154'
startmpich2.sh: found running smpd on n154
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22792 -status n149
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n149'
startmpich2.sh: found running smpd on n149
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22792 -status n150
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n150'
startmpich2.sh: found running smpd on n150
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22792 -status n161
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n161'
startmpich2.sh: found running smpd on n161
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22792 -status n141
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n141'
startmpich2.sh: found running smpd on n141
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22792 -status n143
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n143'
startmpich2.sh: found running smpd on n143
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22792 -status n163
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n163'
startmpich2.sh: found running smpd on n163
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22792 -status n148
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n148'
startmpich2.sh: found running smpd on n148
+ '[' 9 -eq 9 ']'
+ echo 'startmpich2.sh: got all 9 of 9 nodes'
startmpich2.sh: got all 9 of 9 nodes
+ exit 0
rm: cannot remove `*.*.*.*': No such file or directory
rm: cannot remove `imp.0/*.*.*': No such file or directory
rm: cannot remove `imp.0/nohup*': No such file or directory
rm: cannot remove `dmft?.error*': No such file or directory
rm: cannot remove `*.outputdmf?.*': No such file or directory
START
END
Sat Jun 11 20:54:09 EDT 2011
-catch_rsh /opt/mpich2/intel/11.1/
Your job finished at Sat Jun 11 20:54:14 EDT 2011