USER      =  jtomczak
HOMEDIR   =  /home/users/jtomczak
JOB_ID    =  5802
JOB_NAME  =  F100
QUEUE     =  wp10
NSLOTS    =  32
NHOSTS    =  4

SUB_HOST  =  rupc06
SUB_DIR   =  /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta100/nf6/NEW

EXE_HOST  =  n164
NFS_HOST  =  caip02s
EXE_DIR   =  /mnt/p06
OUT_DIR   =  /tmp
TMP_DIR   =  /tmp/5802.1.wp10

INFO      =  Files larger than 100MB will be compressed to reduce the network load.
INFO2     =  Do NOT remove this file while job is running!

PORT      =  20802
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta100/nf6/NEW/GO: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta100/nf6/NEW/GO-small: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta100/nf6/NEW/SPEC: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta100/nf6/NEW/ancont: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta100/nf6/NEW/cdos: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta100/nf6/NEW/imp.0: not a plain file
Failure to rcp files!

Your job started at Wed May 4 19:38:28 EDT 2011 on host n164 in /mnt/p06/5802jtomczak
Submission directory is /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta100/nf6/NEW on host rupc06
-----------------------------------------------------------------------------------------
+ echo -catch_rsh /opt/sge6/spool/n164/active_jobs/5802.1/pe_hostfile /opt/mpich2/intel/11.1/
-catch_rsh /opt/sge6/spool/n164/active_jobs/5802.1/pe_hostfile /opt/mpich2/intel/11.1/
+ SLEEPTIME=5
+ RETRIES=10
+ catch_rsh=0
+ catch_hostname=0
+ '[' 3 -gt 0 ']'
+ case "$1" in
+ catch_rsh=1
+ shift
+ '[' 2 -gt 0 ']'
+ case "$1" in
+ break
++ basename /opt/sge6/mpich2_smpd/startmpich2.sh
+ me=startmpich2.sh
+ '[' 2 -lt 2 ']'
+ pe_hostfile=/opt/sge6/spool/n164/active_jobs/5802.1/pe_hostfile
+ MPICH2_ROOT=/opt/mpich2/intel/11.1/
+ export MPICH2_ROOT
+ '[' '!' -x /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 ']'
+ '[' '!' -x /opt/mpich2/intel/11.1//bin/smpd ']'
+ '[' '!' -r /opt/sge6/spool/n164/active_jobs/5802.1/pe_hostfile ']'
+ machines=/tmp/5802.1.wp10/machines
+ PeHostfile2MachineFile /opt/sge6/spool/n164/active_jobs/5802.1/pe_hostfile
+ cat /opt/sge6/spool/n164/active_jobs/5802.1/pe_hostfile
+ read line
++ echo n164 4 wp10@n164 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n164
++ echo n164 4 wp10@n164 UNDEFINED
++ cut -f2 '-d '
+ nslots=4
+ i=1
+ '[' 1 -le 4 ']'
+ echo n164
++ expr 1 + 1
+ i=2
+ '[' 2 -le 4 ']'
+ echo n164
++ expr 2 + 1
+ i=3
+ '[' 3 -le 4 ']'
+ echo n164
++ expr 3 + 1
+ i=4
+ '[' 4 -le 4 ']'
+ echo n164
++ expr 4 + 1
+ i=5
+ '[' 5 -le 4 ']'
+ read line
++ echo n162 12 wp10@n162 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n162
++ echo n162 12 wp10@n162 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n162
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n162
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n162
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n162
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n162
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n162
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n162
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n162
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n162
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n162
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n162
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n162
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n163 12 wp10@n163 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n163
++ echo n163 12 wp10@n163 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n163
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n163
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n163
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n163
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n163
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n163
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n163
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n163
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n163
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n163
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n163
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n163
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n161 4 wp10@n161 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n161
++ echo n161 4 wp10@n161 UNDEFINED
++ cut -f2 '-d '
+ nslots=4
+ i=1
+ '[' 1 -le 4 ']'
+ echo n161
++ expr 1 + 1
+ i=2
+ '[' 2 -le 4 ']'
+ echo n161
++ expr 2 + 1
+ i=3
+ '[' 3 -le 4 ']'
+ echo n161
++ expr 3 + 1
+ i=4
+ '[' 4 -le 4 ']'
+ echo n161
++ expr 4 + 1
+ i=5
+ '[' 5 -le 4 ']'
+ read line
+ cat /tmp/5802.1.wp10/machines
n164
n164
n164
n164
n162
n162
n162
n162
n162
n162
n162
n162
n162
n162
n162
n162
n163
n163
n163
n163
n163
n163
n163
n163
n163
n163
n163
n163
n161
n161
n161
n161
+ '[' 1 = 1 ']'
+ rsh_wrapper=/opt/sge6/mpich2_smpd/rsh
+ '[' '!' -x /opt/sge6/mpich2_smpd/rsh ']'
+ rshcmd=rsh
+ case "$ARC" in
+ ln -s /opt/sge6/mpich2_smpd/rsh /tmp/5802.1.wp10/rsh
+ hash -r
+ '[' 0 = 1 ']'
+ port=20802
++ cat /tmp/5802.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n164 /opt/mpich2/intel/11.1//bin/smpd 20802
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n162 /opt/mpich2/intel/11.1//bin/smpd 20802
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n163 /opt/mpich2/intel/11.1//bin/smpd 20802
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n161 /opt/mpich2/intel/11.1//bin/smpd 20802
rsh n162 /opt/mpich2/intel/11.1//bin/smpd -port 20802 -d 0
+ actual_retry=1
+ '[' 1 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (1 of 10)'
startmpich2.sh: check for smpd daemons (1 of 10)
+ numhosts=0
++ cat /tmp/5802.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20802 -status n164
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n164'
startmpich2.sh: missing smpd on n164
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20802 -status n162
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n162'
startmpich2.sh: missing smpd on n162
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20802 -status n163
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n163'
startmpich2.sh: missing smpd on n163
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20802 -status n161
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n161'
startmpich2.sh: missing smpd on n161
+ '[' 4 -eq 0 ']'
+ sleep 5
rsh n163 /opt/mpich2/intel/11.1//bin/smpd -port 20802 -d 0
rsh n164 /opt/mpich2/intel/11.1//bin/smpd -port 20802 -d 0
rsh n161 /opt/mpich2/intel/11.1//bin/smpd -port 20802 -d 0
+ (( actual_retry += 1 ))
+ '[' 2 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (2 of 10)'
startmpich2.sh: check for smpd daemons (2 of 10)
+ numhosts=0
++ cat /tmp/5802.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20802 -status n164
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n164'
startmpich2.sh: found running smpd on n164
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20802 -status n162
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n162'
startmpich2.sh: found running smpd on n162
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20802 -status n163
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n163'
startmpich2.sh: found running smpd on n163
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20802 -status n161
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n161'
startmpich2.sh: found running smpd on n161
+ '[' 4 -eq 4 ']'
+ echo 'startmpich2.sh: got all 4 of 4 nodes'
startmpich2.sh: got all 4 of 4 nodes
+ exit 0
START
END
Sat May  7 07:05:20 EDT 2011
-catch_rsh /opt/mpich2/intel/11.1/
Your job finished at Sat May 7 07:05:25 EDT 2011