USER      =  jtomczak
HOMEDIR   =  /home/users/jtomczak
JOB_ID    =  5281
JOB_NAME  =  FS6-80
QUEUE     =  wp10
NSLOTS    =  60
NHOSTS    =  6

SUB_HOST  =  rupc04
SUB_DIR   =  /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta80/nf6

EXE_HOST  =  n159
NFS_HOST  =  caip02s
EXE_DIR   =  /mnt/p06
OUT_DIR   =  /tmp
TMP_DIR   =  /tmp/5281.1.wp10

INFO      =  Files larger than 100MB will be compressed to reduce the network load.
INFO2     =  Do NOT remove this file while job is running!

PORT      =  20281
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta80/nf6/GO: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta80/nf6/GO-small: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta80/nf6/UPDATE: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta80/nf6/ancont: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta80/nf6/cdos: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta80/nf6/cdos_fine: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta80/nf6/imp.0: not a plain file
Failure to rcp files!

Your job started at Fri Apr 22 23:49:55 EDT 2011 on host n159 in /mnt/p06/5281jtomczak
Submission directory is /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta80/nf6 on host rupc04
-----------------------------------------------------------------------------------------
+ echo -catch_rsh /opt/sge6/spool/n159/active_jobs/5281.1/pe_hostfile /opt/mpich2/intel/11.1/
-catch_rsh /opt/sge6/spool/n159/active_jobs/5281.1/pe_hostfile /opt/mpich2/intel/11.1/
+ SLEEPTIME=5
+ RETRIES=10
+ catch_rsh=0
+ catch_hostname=0
+ '[' 3 -gt 0 ']'
+ case "$1" in
+ catch_rsh=1
+ shift
+ '[' 2 -gt 0 ']'
+ case "$1" in
+ break
++ basename /opt/sge6/mpich2_smpd/startmpich2.sh
+ me=startmpich2.sh
+ '[' 2 -lt 2 ']'
+ pe_hostfile=/opt/sge6/spool/n159/active_jobs/5281.1/pe_hostfile
+ MPICH2_ROOT=/opt/mpich2/intel/11.1/
+ export MPICH2_ROOT
+ '[' '!' -x /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 ']'
+ '[' '!' -x /opt/mpich2/intel/11.1//bin/smpd ']'
+ '[' '!' -r /opt/sge6/spool/n159/active_jobs/5281.1/pe_hostfile ']'
+ machines=/tmp/5281.1.wp10/machines
+ PeHostfile2MachineFile /opt/sge6/spool/n159/active_jobs/5281.1/pe_hostfile
+ cat /opt/sge6/spool/n159/active_jobs/5281.1/pe_hostfile
+ read line
++ echo n159 8 wp10@n159 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n159
++ echo n159 8 wp10@n159 UNDEFINED
++ cut -f2 '-d '
+ nslots=8
+ i=1
+ '[' 1 -le 8 ']'
+ echo n159
++ expr 1 + 1
+ i=2
+ '[' 2 -le 8 ']'
+ echo n159
++ expr 2 + 1
+ i=3
+ '[' 3 -le 8 ']'
+ echo n159
++ expr 3 + 1
+ i=4
+ '[' 4 -le 8 ']'
+ echo n159
++ expr 4 + 1
+ i=5
+ '[' 5 -le 8 ']'
+ echo n159
++ expr 5 + 1
+ i=6
+ '[' 6 -le 8 ']'
+ echo n159
++ expr 6 + 1
+ i=7
+ '[' 7 -le 8 ']'
+ echo n159
++ expr 7 + 1
+ i=8
+ '[' 8 -le 8 ']'
+ echo n159
++ expr 8 + 1
+ i=9
+ '[' 9 -le 8 ']'
+ read line
++ echo n160 12 wp10@n160 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n160
++ echo n160 12 wp10@n160 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n160
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n160
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n160
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n160
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n160
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n160
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n160
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n160
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n160
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n160
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n160
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n160
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n161 12 wp10@n161 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n161
++ echo n161 12 wp10@n161 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n161
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n161
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n161
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n161
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n161
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n161
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n161
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n161
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n161
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n161
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n161
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n161
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n162 12 wp10@n162 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n162
++ echo n162 12 wp10@n162 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n162
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n162
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n162
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n162
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n162
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n162
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n162
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n162
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n162
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n162
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n162
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n162
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n163 12 wp10@n163 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n163
++ echo n163 12 wp10@n163 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n163
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n163
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n163
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n163
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n163
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n163
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n163
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n163
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n163
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n163
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n163
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n163
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n164 4 wp10@n164 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n164
++ echo n164 4 wp10@n164 UNDEFINED
++ cut -f2 '-d '
+ nslots=4
+ i=1
+ '[' 1 -le 4 ']'
+ echo n164
++ expr 1 + 1
+ i=2
+ '[' 2 -le 4 ']'
+ echo n164
++ expr 2 + 1
+ i=3
+ '[' 3 -le 4 ']'
+ echo n164
++ expr 3 + 1
+ i=4
+ '[' 4 -le 4 ']'
+ echo n164
++ expr 4 + 1
+ i=5
+ '[' 5 -le 4 ']'
+ read line
+ cat /tmp/5281.1.wp10/machines
n159
n159
n159
n159
n159
n159
n159
n159
n160
n160
n160
n160
n160
n160
n160
n160
n160
n160
n160
n160
n161
n161
n161
n161
n161
n161
n161
n161
n161
n161
n161
n161
n162
n162
n162
n162
n162
n162
n162
n162
n162
n162
n162
n162
n163
n163
n163
n163
n163
n163
n163
n163
n163
n163
n163
n163
n164
n164
n164
n164
+ '[' 1 = 1 ']'
+ rsh_wrapper=/opt/sge6/mpich2_smpd/rsh
+ '[' '!' -x /opt/sge6/mpich2_smpd/rsh ']'
+ rshcmd=rsh
+ case "$ARC" in
+ ln -s /opt/sge6/mpich2_smpd/rsh /tmp/5281.1.wp10/rsh
+ hash -r
+ '[' 0 = 1 ']'
+ port=20281
++ cat /tmp/5281.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n159 /opt/mpich2/intel/11.1//bin/smpd 20281
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n160 /opt/mpich2/intel/11.1//bin/smpd 20281
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n161 /opt/mpich2/intel/11.1//bin/smpd 20281
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n162 /opt/mpich2/intel/11.1//bin/smpd 20281
rsh n159 /opt/mpich2/intel/11.1//bin/smpd -port 20281 -d 0
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n163 /opt/mpich2/intel/11.1//bin/smpd 20281
rsh n161 /opt/mpich2/intel/11.1//bin/smpd -port 20281 -d 0
rsh n160 /opt/mpich2/intel/11.1//bin/smpd -port 20281 -d 0
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n164 /opt/mpich2/intel/11.1//bin/smpd 20281
rsh n162 /opt/mpich2/intel/11.1//bin/smpd -port 20281 -d 0
+ actual_retry=1
+ '[' 1 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (1 of 10)'
startmpich2.sh: check for smpd daemons (1 of 10)
+ numhosts=0
++ cat /tmp/5281.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20281 -status n159
rsh n163 /opt/mpich2/intel/11.1//bin/smpd -port 20281 -d 0
++ head -n 1
++ cut -f 1 -d ' '
rsh n164 /opt/mpich2/intel/11.1//bin/smpd -port 20281 -d 0
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n159'
startmpich2.sh: missing smpd on n159
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20281 -status n160
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n160'
startmpich2.sh: missing smpd on n160
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20281 -status n161
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n161'
startmpich2.sh: missing smpd on n161
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20281 -status n162
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n162'
startmpich2.sh: missing smpd on n162
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20281 -status n163
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n163'
startmpich2.sh: missing smpd on n163
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20281 -status n164
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n164'
startmpich2.sh: missing smpd on n164
+ '[' 6 -eq 0 ']'
+ sleep 5
+ (( actual_retry += 1 ))
+ '[' 2 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (2 of 10)'
startmpich2.sh: check for smpd daemons (2 of 10)
+ numhosts=0
++ cat /tmp/5281.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20281 -status n159
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n159'
startmpich2.sh: found running smpd on n159
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20281 -status n160
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n160'
startmpich2.sh: found running smpd on n160
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20281 -status n161
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n161'
startmpich2.sh: found running smpd on n161
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20281 -status n162
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n162'
startmpich2.sh: found running smpd on n162
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20281 -status n163
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n163'
startmpich2.sh: found running smpd on n163
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20281 -status n164
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n164'
startmpich2.sh: found running smpd on n164
+ '[' 6 -eq 6 ']'
+ echo 'startmpich2.sh: got all 6 of 6 nodes'
startmpich2.sh: got all 6 of 6 nodes
+ exit 0
START
END
Sat Apr 23 01:18:17 EDT 2011
-catch_rsh /opt/mpich2/intel/11.1/
Your job finished at Sat Apr 23 01:18:22 EDT 2011