USER      =  jtomczak
HOMEDIR   =  /home/users/jtomczak
JOB_ID    =  18571
JOB_NAME  =  FS-b150-6
QUEUE     =  wp10
NSLOTS    =  61
NHOSTS    =  7

SUB_HOST  =  rupc04
SUB_DIR   =  /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta150/nf6/NEW

EXE_HOST  =  n149
NFS_HOST  =  caip02s
EXE_DIR   =  /mnt/p06
OUT_DIR   =  /tmp
TMP_DIR   =  /tmp/18571.1.wp10

INFO      =  Files larger than 100MB will be compressed to reduce the network load.
INFO2     =  Do NOT remove this file while job is running!

PORT      =  23571
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta150/nf6/NEW/FIX_mu: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta150/nf6/NEW/FIX_mu_2: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta150/nf6/NEW/FIX_mu_3: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta150/nf6/NEW/GO: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta150/nf6/NEW/GO-small: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta150/nf6/NEW/ancont: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta150/nf6/NEW/cdos: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta150/nf6/NEW/imp.0: not a plain file
Failure to rcp files!

Your job started at Fri Nov 18 20:05:56 EST 2011 on host n149 in /mnt/p06/18571jtomczak
Submission directory is /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta150/nf6/NEW on host rupc04
-----------------------------------------------------------------------------------------
+ echo -catch_rsh /opt/sge6/spool/n149/active_jobs/18571.1/pe_hostfile /opt/mpich2/intel/11.1/
-catch_rsh /opt/sge6/spool/n149/active_jobs/18571.1/pe_hostfile /opt/mpich2/intel/11.1/
+ SLEEPTIME=5
+ RETRIES=10
+ catch_rsh=0
+ catch_hostname=0
+ '[' 3 -gt 0 ']'
+ case "$1" in
+ catch_rsh=1
+ shift
+ '[' 2 -gt 0 ']'
+ case "$1" in
+ break
++ basename /opt/sge6/mpich2_smpd/startmpich2.sh
+ me=startmpich2.sh
+ '[' 2 -lt 2 ']'
+ pe_hostfile=/opt/sge6/spool/n149/active_jobs/18571.1/pe_hostfile
+ MPICH2_ROOT=/opt/mpich2/intel/11.1/
+ export MPICH2_ROOT
+ '[' '!' -x /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 ']'
+ '[' '!' -x /opt/mpich2/intel/11.1//bin/smpd ']'
+ '[' '!' -r /opt/sge6/spool/n149/active_jobs/18571.1/pe_hostfile ']'
+ machines=/tmp/18571.1.wp10/machines
+ PeHostfile2MachineFile /opt/sge6/spool/n149/active_jobs/18571.1/pe_hostfile
+ cat /opt/sge6/spool/n149/active_jobs/18571.1/pe_hostfile
+ read line
++ echo n149 4 wp10@n149 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n149
++ echo n149 4 wp10@n149 UNDEFINED
++ cut -f2 '-d '
+ nslots=4
+ i=1
+ '[' 1 -le 4 ']'
+ echo n149
++ expr 1 + 1
+ i=2
+ '[' 2 -le 4 ']'
+ echo n149
++ expr 2 + 1
+ i=3
+ '[' 3 -le 4 ']'
+ echo n149
++ expr 3 + 1
+ i=4
+ '[' 4 -le 4 ']'
+ echo n149
++ expr 4 + 1
+ i=5
+ '[' 5 -le 4 ']'
+ read line
++ echo n153 8 wp10@n153 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n153
++ echo n153 8 wp10@n153 UNDEFINED
++ cut -f2 '-d '
+ nslots=8
+ i=1
+ '[' 1 -le 8 ']'
+ echo n153
++ expr 1 + 1
+ i=2
+ '[' 2 -le 8 ']'
+ echo n153
++ expr 2 + 1
+ i=3
+ '[' 3 -le 8 ']'
+ echo n153
++ expr 3 + 1
+ i=4
+ '[' 4 -le 8 ']'
+ echo n153
++ expr 4 + 1
+ i=5
+ '[' 5 -le 8 ']'
+ echo n153
++ expr 5 + 1
+ i=6
+ '[' 6 -le 8 ']'
+ echo n153
++ expr 6 + 1
+ i=7
+ '[' 7 -le 8 ']'
+ echo n153
++ expr 7 + 1
+ i=8
+ '[' 8 -le 8 ']'
+ echo n153
++ expr 8 + 1
+ i=9
+ '[' 9 -le 8 ']'
+ read line
++ echo n151 6 wp10@n151 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n151
++ echo n151 6 wp10@n151 UNDEFINED
++ cut -f2 '-d '
+ nslots=6
+ i=1
+ '[' 1 -le 6 ']'
+ echo n151
++ expr 1 + 1
+ i=2
+ '[' 2 -le 6 ']'
+ echo n151
++ expr 2 + 1
+ i=3
+ '[' 3 -le 6 ']'
+ echo n151
++ expr 3 + 1
+ i=4
+ '[' 4 -le 6 ']'
+ echo n151
++ expr 4 + 1
+ i=5
+ '[' 5 -le 6 ']'
+ echo n151
++ expr 5 + 1
+ i=6
+ '[' 6 -le 6 ']'
+ echo n151
++ expr 6 + 1
+ i=7
+ '[' 7 -le 6 ']'
+ read line
++ echo n150 12 wp10@n150 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n150
++ echo n150 12 wp10@n150 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n150
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n150
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n150
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n150
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n150
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n150
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n150
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n150
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n150
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n150
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n150
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n150
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n164 11 wp10@n164 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n164
++ echo n164 11 wp10@n164 UNDEFINED
++ cut -f2 '-d '
+ nslots=11
+ i=1
+ '[' 1 -le 11 ']'
+ echo n164
++ expr 1 + 1
+ i=2
+ '[' 2 -le 11 ']'
+ echo n164
++ expr 2 + 1
+ i=3
+ '[' 3 -le 11 ']'
+ echo n164
++ expr 3 + 1
+ i=4
+ '[' 4 -le 11 ']'
+ echo n164
++ expr 4 + 1
+ i=5
+ '[' 5 -le 11 ']'
+ echo n164
++ expr 5 + 1
+ i=6
+ '[' 6 -le 11 ']'
+ echo n164
++ expr 6 + 1
+ i=7
+ '[' 7 -le 11 ']'
+ echo n164
++ expr 7 + 1
+ i=8
+ '[' 8 -le 11 ']'
+ echo n164
++ expr 8 + 1
+ i=9
+ '[' 9 -le 11 ']'
+ echo n164
++ expr 9 + 1
+ i=10
+ '[' 10 -le 11 ']'
+ echo n164
++ expr 10 + 1
+ i=11
+ '[' 11 -le 11 ']'
+ echo n164
++ expr 11 + 1
+ i=12
+ '[' 12 -le 11 ']'
+ read line
++ echo n155 12 wp10@n155 UNDEFINED
++ cut -f1 -d.
++ cut -f1 '-d '
+ host=n155
++ echo n155 12 wp10@n155 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n155
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n155
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n155
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n155
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n155
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n155
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n155
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n155
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n155
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n155
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n155
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n155
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n157 8 wp10@n157 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n157
++ echo n157 8 wp10@n157 UNDEFINED
++ cut -f2 '-d '
+ nslots=8
+ i=1
+ '[' 1 -le 8 ']'
+ echo n157
++ expr 1 + 1
+ i=2
+ '[' 2 -le 8 ']'
+ echo n157
++ expr 2 + 1
+ i=3
+ '[' 3 -le 8 ']'
+ echo n157
++ expr 3 + 1
+ i=4
+ '[' 4 -le 8 ']'
+ echo n157
++ expr 4 + 1
+ i=5
+ '[' 5 -le 8 ']'
+ echo n157
++ expr 5 + 1
+ i=6
+ '[' 6 -le 8 ']'
+ echo n157
++ expr 6 + 1
+ i=7
+ '[' 7 -le 8 ']'
+ echo n157
++ expr 7 + 1
+ i=8
+ '[' 8 -le 8 ']'
+ echo n157
++ expr 8 + 1
+ i=9
+ '[' 9 -le 8 ']'
+ read line
+ cat /tmp/18571.1.wp10/machines
n149
n149
n149
n149
n153
n153
n153
n153
n153
n153
n153
n153
n151
n151
n151
n151
n151
n151
n150
n150
n150
n150
n150
n150
n150
n150
n150
n150
n150
n150
n164
n164
n164
n164
n164
n164
n164
n164
n164
n164
n164
n155
n155
n155
n155
n155
n155
n155
n155
n155
n155
n155
n155
n157
n157
n157
n157
n157
n157
n157
n157
+ '[' 1 = 1 ']'
+ rsh_wrapper=/opt/sge6/mpich2_smpd/rsh
+ '[' '!' -x /opt/sge6/mpich2_smpd/rsh ']'
+ rshcmd=rsh
+ case "$ARC" in
+ ln -s /opt/sge6/mpich2_smpd/rsh /tmp/18571.1.wp10/rsh
+ hash -r
+ '[' 0 = 1 ']'
+ port=23571
++ cat /tmp/18571.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n149 /opt/mpich2/intel/11.1//bin/smpd 23571
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n153 /opt/mpich2/intel/11.1//bin/smpd 23571
rsh n149 /opt/mpich2/intel/11.1//bin/smpd -port 23571 -d 0
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n151 /opt/mpich2/intel/11.1//bin/smpd 23571
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n150 /opt/mpich2/intel/11.1//bin/smpd 23571
rsh n153 /opt/mpich2/intel/11.1//bin/smpd -port 23571 -d 0
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n164 /opt/mpich2/intel/11.1//bin/smpd 23571
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n155 /opt/mpich2/intel/11.1//bin/smpd 23571
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n157 /opt/mpich2/intel/11.1//bin/smpd 23571
rsh n151 /opt/mpich2/intel/11.1//bin/smpd -port 23571 -d 0
+ actual_retry=1
+ '[' 1 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (1 of 10)'
startmpich2.sh: check for smpd daemons (1 of 10)
+ numhosts=0
++ cat /tmp/18571.1.wp10/machines
rsh n150 /opt/mpich2/intel/11.1//bin/smpd -port 23571 -d 0
++ uniq
rsh n155 /opt/mpich2/intel/11.1//bin/smpd -port 23571 -d 0
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23571 -status n149
++ head -n 1
++ cut -f 1 -d ' '
rsh n157 /opt/mpich2/intel/11.1//bin/smpd -port 23571 -d 0
rsh n164 /opt/mpich2/intel/11.1//bin/smpd -port 23571 -d 0
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n149'
startmpich2.sh: missing smpd on n149
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23571 -status n153
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n153'
startmpich2.sh: missing smpd on n153
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23571 -status n151
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n151'
startmpich2.sh: missing smpd on n151
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23571 -status n150
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n150'
startmpich2.sh: missing smpd on n150
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23571 -status n164
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n164'
startmpich2.sh: missing smpd on n164
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23571 -status n155
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n155'
startmpich2.sh: missing smpd on n155
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23571 -status n157
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n157'
startmpich2.sh: missing smpd on n157
+ '[' 7 -eq 0 ']'
+ sleep 5
+ (( actual_retry += 1 ))
+ '[' 2 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (2 of 10)'
startmpich2.sh: check for smpd daemons (2 of 10)
+ numhosts=0
++ cat /tmp/18571.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23571 -status n149
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n149'
startmpich2.sh: found running smpd on n149
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23571 -status n153
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n153'
startmpich2.sh: found running smpd on n153
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23571 -status n151
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n151'
startmpich2.sh: found running smpd on n151
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23571 -status n150
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n150'
startmpich2.sh: found running smpd on n150
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23571 -status n164
++ cut -f 1 -d ' '
++ head -n 1
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n164'
startmpich2.sh: found running smpd on n164
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23571 -status n155
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n155'
startmpich2.sh: found running smpd on n155
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23571 -status n157
++ cut -f 1 -d ' '
++ head -n 1
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n157'
startmpich2.sh: found running smpd on n157
+ '[' 7 -eq 7 ']'
+ echo 'startmpich2.sh: got all 7 of 7 nodes'
startmpich2.sh: got all 7 of 7 nodes
+ exit 0
START
-catch_rsh /opt/mpich2/intel/11.1/
Your job finished at Fri Dec 2 19:07:48 EST 2011