#!/bin/sh 
. $(dirname $0)/common.sh 

#
# make sure a file exists, and return it's full path in $verified_file
#
function abort()
{
	unlock_install
	die $*
}

function abort_log
{
	unlock_install
	die_log $*
}
function verify_file()
{
        local file=$1
        [ -e $file ] || abort "no such file $file"
        [ ${file:0:1} != "/" ] && verified_file=$(pwd)/$file && return
        verified_file=$file
}

function get_inet_eth(){
       all_eth=$(ip a s|egrep "^[0-9]+: eth"|cut -f2 -d:)

       for eth in $all_eth ; do
               addr=$(ip -f inet a s dev $eth|tail -1|tr -s " "|cut -f3 -d" "|cut -f1 -d/)
               if [ ! -z $addr ] ; then
                       echo $eth:$addr
                       break
               fi
       done
}

#
# validate ipoib_conf
#
function check_ipoib_conf()
{
	n_total_lines=$(cat $ipoib_conf |grep -v "^$"|grep -v "^#"|wc -l)
	n_ib_lines=$(cat $ipoib_conf |grep -v "^$"|grep -v "^#"|cut -f1 -d:|grep  ib|wc -l)
	[ $n_ib_lines -lt $n_total_lines ] && \
		error "$ipoib_conf has wrong syntax. not all lines start with ib"
        
	n_ib_lines_uniq=$(cat $ipoib_conf |grep -v "^$"|grep -v "^#"|cut -f1 -d:|grep  ib|sort -u |wc -l)
	if [ $n_ib_lines_uniq -ne $n_ib_lines ]
	then
		abort "$ipoib_conf has wrong syntax. not all lines are unique"
	fi
	# TODO still need to check if each line is valid
}

#
# parse ipoib.conf file, and set up network scripts accordingly
# 
function parse_ipoib_conf()
{
	for line in `cat $ipoib_conf |grep -v "^$"|grep -v "^#"`
	do
		ib_if=$(echo $line|cut -f1 -d:)
		network=$(echo $line|cut -f2 -d:)
		if [ $network != "dhcp" ]
		then
			eth_if=$(echo $line|cut -f3 -d:)
			ib-config-as-eth -ib ${ib_if} -eth ${eth_if} -network $network -nocolor -overwrite
		else
			ib-config-as-eth -ib ${ib_if} -dhcp -nocolor -overwrite
		fi
	done
}


#
# make sure we are not ahead of time to prevent make errors
#
function validate_time_of_day()
{
	machine_date=$(date +%s)
	package_date=$(date +%s -r $mydir/$myname)
	if [ $package_date -gt $machine_date ]
	then
		abort "Package date is in the future. Current time should be later than $(date -r $mydir/$myname) or run with --set-timeofday" 
	fi
}
	
#
# get the kver, arch, and os type [redhat/sles/unknown]
#
function get_os_vars()
{
	K_VER=$(uname -r)
	ARCH=$(uname -m)
	OS=$linux_dist
	[ $OS = unknown ] && warning Running on unknownm OS 
}

#
# fix Module.symvers - update the symbol versions there
# TODO this has issues - makes 0x crc's
#
function fix_ib_symvers()
{
#	if [ ! -f $mydir/${OFED_DIR}/RPMS/*/kernel-ib-[0-9]* ]
#	then
#		echo no need to update file Module.symvers
#		return 0
#	fi
	rm -rf /tmp/gs-syms.$$
	mkdir /tmp/gs-syms.$$
	pushd /tmp/gs-syms.$$
	for f in $(rpm -qlp $mydir/$OFED_DIR/RPMS/*/kernel-ib-[0-9]*)
	do 
		mkdir -p ./$(dirname $f)
	done
	rpm2cpio $mydir/${OFED_DIR}/RPMS/*/kernel-ib-[0-9]* |cpio -i
	rm -f /tmp/syms
	n_mods=0
	for mod in $(find -name *.ko) ; do
		nm -o $mod |grep __crc >> /tmp/syms
		n_mods=$((n_mods+1))
	done

	n_syms=$(wc -l /tmp/syms |cut -f1 -d" ")
	echo "Found $n_syms InfiniBand symbols in $n_mods InfiniBand modules"
	n=1

	MOD_SYMVERS_IB=./Module.symvers.ib
	MOD_SYMVERS_PATCH=./Module.symvers.patch
	for f in /lib/modules/${K_VER}/source/Module.symvers /lib/modules/${K_VER}/build/Module.symvers
	do
		[ -f $f ] && MOD_SYMVERS_KERNEL=$f && break
	done
	[ -f "$f" ] || echo file Module.symvers not found

	if [ ! -z $MOD_SYMVERS_KERNEL ]
	then 
		rm -f $MOD_SYMVERS_IB
		while [ $n -le $n_syms ]
		do
			line=$(head -$n /tmp/syms|tail -1)
			line1=$(echo $line|cut -f1 -d:)
			line2=$(echo $line|cut -f2 -d:)
			file=$(echo $line1|cut -f6- -d/)
			file=$(echo $file|cut -f1 -d.)
			crc=$(echo $line2|cut -f1 -d" "|tail --bytes=9)
			sym=$(echo $line2|cut -f3 -d" ")
			sym=${sym:6}
			
			echo -e  "0x$crc\t$sym\t$file" >>${MOD_SYMVERS_IB}
			if [ -z $allsyms ]
			then
				allsyms=$sym
			else
				allsyms="$allsyms|$sym"
			fi
			n=$((n+1))
		done
		egrep -v "$allsyms" ${MOD_SYMVERS_KERNEL} >>${MOD_SYMVERS_IB}
		diff -u $MOD_SYMVERS_KERNEL ${MOD_SYMVERS_IB} >${MOD_SYMVERS_PATCH}
		patch -d $(dirname ${MOD_SYMVERS_KERNEL}) <${MOD_SYMVERS_PATCH}
		mkdir -p ${prefix}/backup
		cp ${MOD_SYMVERS_PATCH} ${prefix}/backup
	fi
}

#
#
#
function fix_ofed_conf_file()
{
	case $OS in 
	suse)
		if rpm -q kernel-source >/dev/null
		then
			echo rpm kernel-source is installed. Leaving OFED configuration as is.
			fix_for_ib_devel=0
		else
			fix_for_ib_devel=1
		fi
		;;
	redhat)
		if rpm -q kernel-devel >/dev/null
		then
			echo rpm kernel-devel is installed. Leaving OFED configuration as is.
			fix_for_ib_devel=0
		else
			fix_for_ib_devel=1
		fi
		;;
	*)
		echo Unknown Linux distribution. Leaving OFED configuration as is.
		fix_for_ib_devel=0
		;;
	esac
	
	if [ $fix_for_ib_devel -ne 0 ]
	then
		sed -i -e 's/kernel_ib_devel=y/kernel_ib_devel=n/g' ${ofed_conf_file}
		echo kernel_ib_devel was dropped from OFED configuraion
	fi

	if rpm -q tcl-devel >/dev/null
	then
		echo tcl-devel is installed. Leaving OFED configuration as is.
	else
		sed -i -e 's/ibutils=y/ibutils=n/g' ${ofed_conf_file}
		echo ibutils was dropped from OFED configuraion
	fi
	
}

#
# save OS parameters in a file
# 
function save_os_vars()
{
	cat <<EOF >$1
OS=$OS
K_VER=${K_VER}
ARCH=$ARCH
EOF
}

#
# create a binary package
#
function make_bin_package()
{
	make_bin_type=$1
	pushd $mydir/.. > /dev/null

	GS_VER=$GSVersion

	GS_DIR=$GS_VER
	BIN_NAME=$GS_VER-$OS-k$K_VER-$ARCH-$make_bin_type
	rm -rf $BIN_NAME $BIN_NAME.tar.bz2 $BIN_NAME.md5sum
	cp -a $GS_DIR $BIN_NAME
	rm -f $BIN_NAME/version-info
	for d in $(ls -d $BIN_NAME/RPMS/$rpm_target_cpu/*)
	do
		[ $(basename $d) != $OS ] && rm -rf $d
	done
	for d in $(ls -d $BIN_NAME/OFED-*/RPMS/*)
	do 
		[ $(basename $d) != $K_VER ] && rm -rf $d
	done
	rm -rf $BIN_NAME/SOURCES $BIN_NAME/SPECS $BIN_NAME/SRPMS $BIN_NAME/BUILD
	echo $install_args_to_keep > $BIN_NAME/install-args
	OFED_DIR_BIN=$(basename $(ls -d $BIN_NAME/OFED*))
	save_os_vars $BIN_NAME/bin-info
	mv $BIN_NAME/$OFED_DIR_BIN/SOURCES $BIN_NAME/$OFED_DIR_BIN/SOURCES.0
	mkdir $BIN_NAME/$OFED_DIR_BIN/SOURCES
	cp  $BIN_NAME/$OFED_DIR_BIN/SOURCES.0/propel.sh $BIN_NAME/$OFED_DIR_BIN/SOURCES

	OFED_SRC_RPMS=$(ls $BIN_NAME/$OFED_DIR_BIN/SRPMS/* |egrep -v "ofed-docs|ofed-scripts|openmpi|ibutils|ib-bonding|rds-tools")
	rm -rf  $BIN_NAME/SRPMS/*  $BIN_NAME/SOURCES/*  $BIN_NAME/SPECS/* $OFED_SRC_RPMS
	patch -s -p1 -d $BIN_NAME/$OFED_DIR_BIN/ < $BIN_NAME/install/ofed_install_bin.patch
	
	rm -rf `find $BIN_NAME -name \.svn`
	if (test $make_bin_type = "slim" || test  $make_bin_type = "mini")
	then
		rm -rf $BIN_NAME/$OFED_DIR_BIN
	fi
	if ( test  $make_bin_type = "mini")
	then
		rm -rf $BIN_NAME/patches
		rm -f -rf $BIN_NAME/RPMS/$rpm_target_cpu/$OS/ib-enhanced-services*
		rm -f -rf $BIN_NAME/RPMS/$rpm_target_cpu/$OS/vltmpi-OPENIB*
		rm -f -rf $BIN_NAME/RPMS/$rpm_target_cpu/$OS/openmpi*
	fi
	echo GSCustomizations=\"$CUSTOM_LIST\" >> $BIN_NAME/version-info
	echo GSPackageType=$make_bin_type  >> $BIN_NAME/version-info
	ex tar jcvf $BIN_NAME.tar.bz2 $BIN_NAME || abort_log Failed making binary package


	md5sum $BIN_NAME.tar.bz2 > $BIN_NAME.md5sum
	info Binary package: $(pwd)/$BIN_NAME.tar.bz2
	info Binary package checksum: $(pwd)/$BIN_NAME.md5sum
	popd > /dev/null
}

#
# customize the SRPMS with our patches
#
function copy_voltaire_patches()
{
	cd $mydir/patches
	ex ./customize.sh --ofed-dir $mydir/$OFED_DIR/ $CUSTOM_LIST || \
		abort_log "Customization of OFED failed"
}

#
# build a binary package and install it
#
function build_pkg()
{
	PKG=$1
	shift
	NAME=$1
	NAME=${NAME:-$PKG}
	shift
	RPMBUILD_OPTS=$@
	PKG_SRC=$(test -f $mydir/SRPMS/$PKG-[0-9]*.src.rpm && basename $(ls $mydir/SRPMS/$PKG-[0-9]*.src.rpm))

	rpm -q $NAME >/dev/null && abort "$NAME is already installed"
	
	PKG_BIN=$mydir/RPMS/$rpm_target_cpu/$OS/$NAME-[^debuginfo]*.rpm
	if [ ! -f ${PKG_BIN} ]
	then
		[ -z ${PKG_SRC} ] && abort_log "Source RPM of $PKG not found"

		echo Building $NAME
		if ! ex rpmbuild --rebuild --define \"_topdir $mydir\" ${RPMBUILD_OPTS} $mydir/SRPMS/${PKG_SRC}
		then
			abort_log "Building of $NAME failed"
		else
			mv $mydir/RPMS/${rpm_target_cpu}/$NAME-[^debuginfo]*.rpm $mydir/RPMS/${rpm_target_cpu}/$OS/
			rm -f $mydir/RPMS/${rpm_target_cpu}/*debuginfo*.rpm 
		fi
	fi
	
	PKG_BIN=$mydir/RPMS/$rpm_target_cpu/$OS/$NAME-[^debuginfo]*.rpm
	[ -z $PKG_BIN ] && abort "RPM of $NAME not found"

	# install
	ex rpm -ivh ${PKG_BIN} || abort_log "Installation of ${PKG_BIN} failed"
}

#
# remove old stacks
#
function remove_old_stacks()
{
	# Check if old host stack is installed
	for stack_name in ibhost-hpc ibhost-biz
	do
		rpm -q ${stack_name}>/dev/null && abort "Please remove first old InfiniBand stack ${stack_name}"
	done

	[ -f ${prefix}/version ] || return

	abort "Please remove previous installation: "$(cat ${prefix}/version | sed -e s/GridStack=//)
}

#
# Open and install OFED
#
function install_ofed()	
{
	if [ $dont_patch -eq 1 ]
	then
		echo installation will continue without applying patches
	else
		copy_voltaire_patches
	fi
	cd $mydir/$OFED_DIR

	info "Installing OFED. This may take some time. Please wait."
	export OFA_KERNEL_PARAMS="--without-modprobe --without-ipoibconf --with-madeye"
	ex ./install.sh -c ${ofed_conf_file} || abort_log "Installaion of OFED failed"
	info "Installation of OFED finished"
	if [ $fix_symvers -ne 0 ]
	then
		fix_ib_symvers
	else
		rm -f ${prefix}/backup/Module.symvers.patch
	fi
	
	# disable openibd
	chkconfig openibd off
}

#
# fix /etc/infiniband/openib.conf
#
function fix_openib_conf()
{
	if [ -f /etc/infiniband/openib.conf ]
	then
		sed -e {s/IPATH_LOAD=yes/IPATH_LOAD=no/g} /etc/infiniband/openib.conf > /tmp/openib.conf
		cp /tmp/openib.conf /etc/infiniband/openib.conf
		sed -e {s/UCM_LOAD=no/UCM_LOAD=yes/g} /etc/infiniband/openib.conf > /tmp/openib.conf
		cp /tmp/openib.conf /etc/infiniband/openib.conf
		sed -e {s/RDS_LOAD=no/RDS_LOAD=yes/g} /etc/infiniband/openib.conf > /tmp/openib.conf
		cp /tmp/openib.conf /etc/infiniband/openib.conf
	else
		abort /etc/infiniband/openib.conf is missing after OFED installation
	fi
}

#
# fix /etc/modprobe.conf
#
function fix_modprobe_conf()
{
	sdp_line="alias net-pf-27           ib_sdp"
	grep -q "${sdp_line}" /etc/modprobe.conf || echo "${sdp_line}" >> /etc/modprobe.conf

	local_sa_line="options ib_local_sa cache_timeout"
	if grep -q "${local_sa_line}" /etc/modprobe.conf
	then
		grep -v "${local_sa_line}" /etc/modprobe.conf >/tmp/modprobe.conf
		cp /tmp/modprobe.conf /etc/modprobe.conf
	fi
	echo "${local_sa_line}=0" >>/etc/modprobe.conf
}

#
# fix udev rules
#
function fix_udev_rules(){
	if [ -f /etc/sysconfig/network-scripts/net.hotplug ] ; then
		if (patch -p0 --dry-run --forward < $mydir/install/net_hotplug.diff &> /dev/null) ; then
			ex patch -p0 -s < $mydir/install/net_hotplug.diff
		fi
	fi
	if [ -f /etc/udev/rules.d/31-network.rules ] ; then
		if (patch -p0 --dry-run --forward < $mydir/install/31-network_rules-sles10.diff &> /dev/null) ; then
			ex patch -p0 < $mydir/install/31-network_rules-sles10.diff
		fi
	fi
	if [ -f /etc/udev/rules.d/31-network.rules ] ; then
		if (patch -p0 --dry-run --forward < $mydir/install/31-network_rules-sles10sp1.diff &> /dev/null) ; then
			ex patch -p0 < $mydir/install/31-network_rules-sles10sp1.diff
		fi
	fi
}
 
#
# write network configuration scripts
#
function config_network()
{
	if [ $GSPackageType != "mini" ] ;  then
		[ $linux_dist = suse ] && \
			patch  -p1 -d /etc/sysconfig/network/scripts/ < ${prefix}/functions-ib.diff

		if [ $linux_dist = redhat  ] ; then
			if [ -f /sbin/ifup-pre-local ] ; then
				line1=$(head -1 /sbin/ifup-pre-local)
				[ -z $line1 ] && line1=empty
				[ ${line1:0:2} != '#!' ]  && echo "#!/bin/bash" > /sbin/ifup-pre-local
			else
				echo "#!/bin/bash" > /sbin/ifup-pre-local
			fi
			echo "/etc/sysconfig/network-scripts/ifup-pre-ib \$1" >> /sbin/ifup-pre-local
			chmod +x /sbin/ifup-pre-local

			if [  -f /sbin/ifdown-pre-local ] ; then
				line1=$(head -1 /sbin/ifdown-pre-local)
				[ -z $line1 ] && line1=empty
				[ ${line1:0:2} != '#!' ] && echo "#!/bin/bash" > /sbin/ifdown-pre-local
			else
				echo "#!/bin/bash" > /sbin/ifdown-pre-local
			fi
			echo "/etc/sysconfig/network-scripts/ifdown-post-ib \$1" >> /sbin/ifdown-pre-local
			chmod +x /sbin/ifdown-pre-local
		fi
	fi

	# copy dhclient file, if not existed before
	[ -f /etc/dhclient.conf ] || cp ${prefix}/dhclient.conf /etc/dhclient.conf

	# set up ipoib address
	if [ ! -z ${ipoib_conf} ]
	then
		[ $GSPackageType != mini ] && parse_ipoib_conf
	else
		if [ -x /usr/local/bin/ib-config-as-eth ] ; then
			if [ ! -f $netscript_dir/ifcfg-ib0 ] ; then
				res=$(get_inet_eth)
				if [ ! -z $res ] ; then
					eth=$(echo $res|cut -f1 -d:)
					add=$(echo $res|cut -f2 -d:)
					if [ $(echo $add|cut -f1 -d.) != 192 ] ; then
						/usr/local/bin/ib-config-as-eth  -network 192.168.0.0 -eth $eth -netmask 255.255.255.0
					else
						/usr/local/bin/ib-config-as-eth  -network 193.168.0.0 -eth $eth -netmask 255.255.255.0
					fi

				else
					info No reference Ethernet interface was found to configure ib0
				fi
			fi
		fi
	fi
}

#
# Build and install Voltaire MPI
#
function install_vmpi()
{
	vltmpi_pkgs=$(rpm -qa|grep vltmpi-OPENIB)
	[ -z "$vltmpi_pkgs" ] || rpm -e $vltmpi_pkgs
	build_pkg vltmpi-OPENIB
}

#
# Build and install OpenMPI
#
function install_ompi()
{
	# list of compilers
	MPI_COMPILER_openmpi="gcc"

###########


	OPENMPI_SRC_RPM=$(/bin/ls -1 ${mydir}/SRPMS/openmpi*src.rpm 2>/dev/null)
	if [[ -n "${OPENMPI_SRC_RPM}" && -s ${OPENMPI_SRC_RPM} ]]
	then
		OPENMPI_NAME=$(rpm --queryformat "[%{NAME}]" -qp $OPENMPI_SRC_RPM)
		OPENMPI_VER=$(rpm --queryformat "[%{VERSION}]" -qp $OPENMPI_SRC_RPM)
		OPENMPI_REL=$(rpm --queryformat "[%{RELEASE}]" -qp $OPENMPI_SRC_RPM)
		OPENMPI_VERSION=${OPENMPI_VER}-${OPENMPI_REL}
	fi

	[ -z $OPENMPI_NAME ] && OPENMPI_NAME=openmpi

	for mpi_comp in ${MPI_COMPILER_openmpi}
	do
		OMPI_PREFIX="/opt/${OPENMPI_NAME}/${OPENMPI_VERSION}/${mpi_comp}"
		case $(uname -m) in
		i?86|ia64)
			OMPI_LIBDIR="lib"
			;;
		*)
			OMPI_LIBDIR="lib64"
			;;
		esac

		OMPI_LDFLAGS="LDFLAGS=-L/usr/${OMPI_LIBDIR}"
		# We have just the CXX FLAGS in a separate variable because
		# the PGI C++ compiler needs a special flag.
		OMPI_WRAPPER_CXX_FLAGS=
		# Not all compilers like the default $RPM_OPT_FLAGS
		USE_DEFAULT_RPM_OPT_FLAGS=1

		case ${mpi_comp} in
		pathscale)
			COMP_OPTIONS="CC=pathcc"
			if which pathCC >/dev/null 2>&1; then
				COMP_OPTIONS="$COMP_OPTIONS CXX=pathCC"
			else
				COMP_OPTIONS="$COMP_OPTIONS --disable-mpi-cxx"
			fi
			if which pathf90 >/dev/null 2>&1; then
				COMP_OPTIONS="$COMP_OPTIONS F77=pathf90 FC=pathf90"
			else
				COMP_OPTIONS="$COMP_OPTIONS --disable-mpi-f77 --disable-mpi-f90"
			fi
			# On fedora6 and redhat5 the pathscale compiler fails with default $RPM_OPT_FLAGS
			#if [ $DISTRIBUTION == "fedora6" -o $DISTRIBUTION == "redhat5" ]; then
				USE_DEFAULT_RPM_OPT_FLAGS=0
 			#fi
            ;;
		intel)
			COMP_OPTIONS="CC=icc"
			if which icpc >/dev/null 2>&1; then
				COMP_OPTIONS="$COMP_OPTIONS CXX=icpc"
			else
				COMP_OPTIONS="$COMP_OPTIONS --disable-mpi-cxx"
			fi
			if which ifort >/dev/null 2>&1; then
				COMP_OPTIONS="$COMP_OPTIONS F77=ifort FC=ifort"
			else
				COMP_OPTIONS="$COMP_OPTIONS --disable-mpi-f77 --disable-mpi-f90"
			fi
            ;;
		pgi)
			COMP_OPTIONS="CC=pgcc"
			USE_DEFAULT_RPM_OPT_FLAGS=0
			if which pgCC >/dev/null 2>&1; then
				COMP_OPTIONS="$COMP_OPTIONS CXX=pgCC"
				# See http://www.pgroup.com/userforum/viewtopic.php?p=2371
				OMPI_WRAPPER_CXX_FLAGS="-fpic"
			else
				COMP_OPTIONS="$COMP_OPTIONS --disable-mpi-cxx"
			fi
			if which pgf77 >/dev/null 2>&1; then
				COMP_OPTIONS="$COMP_OPTIONS F77=pgf77"
			else
				COMP_OPTIONS="$COMP_OPTIONS --disable-mpi-f77"
			fi
			if which pgf90 >/dev/null 2>&1; then
				# *Must* put in FCFLAGS=-O2 so that -g doesn't get
				# snuck in there (pgi 6.2-5 has a problem with
				# modules and -g).
				COMP_OPTIONS="$COMP_OPTIONS FC=pgf90 FCFLAGS=-O2"
			else
				COMP_OPTIONS="$COMP_OPTIONS --disable-mpi-f90"
			fi
            ;;
		gcc)
			COMP_OPTIONS="CC=gcc"
			if which g++ >/dev/null 2>&1; then
				COMP_OPTIONS="$COMP_OPTIONS CXX=g++"
			else
				COMP_OPTIONS="$COMP_OPTIONS --disable-mpi-cxx"
			fi
			if which gfortran >/dev/null 2>&1; then
				COMP_OPTIONS="$COMP_OPTIONS F77=gfortran FC=gfortran"
			elif which g77 >/dev/null 2>&1; then
				COMP_OPTIONS="$COMP_OPTIONS F77=g77 --disable-mpi-f90"
			else
				COMP_OPTIONS="$COMP_OPTIONS --disable-mpi-f77 --disable-mpi-f90"
			fi
            ;;
		*)
			COMP_OPTIONS="CC=${mpi_comp} --disable-mpi-f77 --disable-mpi-f90"
        esac
        
		case $(uname -m) in
		ppc64)
			# In the ppc64 case, add -m64 to all the relevant
			# flags because it's not the default.
			COMP_OPTIONS="${COMP_OPTIONS} CFLAGS='-m64 -O2' CXXFLAGS='-m64 -O2' FCFLAGS='-m64 -O2' FFLAGS='-m64 -O2' --with-wrapper-ldflags='-g -O2 -m64 -L/usr/lib64' --with-wrapper-cflags=-m64 --with-wrapper-cxxflags=-m64 --with-wrapper-fflags=-m64 --with-wrapper-fcflags=-m64"
			OMPI_WRAPPER_CXX_FLAGS="-m64 $OMPI_WRAPPER_CXX_FLAGS"
            ;;
		*)
            ;;
        esac

		# add C++ flags
        COMP_OPTIONS="$COMP_OPTIONS --enable-mpirun-prefix-by-default"
        test "$OMPI_WRAPPER_CXX_FLAGS" != "" && \
        	 COMP_OPTIONS="$COMP_OPTIONS --with-wrapper-cxxflags='$OMPI_WRAPPER_CXX_FLAGS'"

		NAME=${OPENMPI_NAME}_${mpi_comp}
		
		# build the package
        build_pkg openmpi ${NAME} \
                --define \"_name ${NAME}\" \
				--define \"_prefix ${OMPI_PREFIX}\" \
				--define \"_mandir %{_prefix}/share/man\" \
                --define \"configure_options --with-openib=${OFED_PREFIX} ${COMP_OPTIONS}\" \
                --define \"mflags -j 4\" \
                --define \"use_default_rpm_opt_flags ${USE_DEFAULT_RPM_OPT_FLAGS}\"
#				--define "_defaultdocdir ${STACK_PREFIX}/mpi/${mpi_comp}/${OPENMPI_NAME}-${OPENMPI_VERSION}" \
#               --define "buildroot ${BUILD_ROOT}" \
#               --define "mpi_selector ${STACK_PREFIX}/bin/mpi-selector" \
#               --define "ofed 1" \
            
    done
}

#
# Unload IB modules
#
function stop_stack()
{
	if [ -x /etc/init.d/gridstack ]
	then
		info "Stopping GridStack"
		/etc/init.d/gridstack stop >$LOG
	elif [ -x /etc/init.d/openibd ]
	then
		info "Stopping OFED"
		/etc/init.d/openibd stop >$LOG
	else
		lsmod |egrep "^ib_|^rdma_|rds"|cut -f1 -d" "|xargs modprobe -r
	fi
}

#
# print usage
#
function usage()
{
	cat <<EOF
  Usage: $(basename $0) [OPTIONS]
	
  --gs-config <config-name>     Use alternative configuration for GridStack
  --ofed-config <config-name>   Use alternative configuration for OFED
  --ipoib-config <conf-file>    Specify IPoIB interface configurfation file
  --no-v-mpi                    Don't install VoltaireMPI
  --no-o-mpi                    Don't install OpenMPI
  --v-mpi-compiler-ifc <path>   Use IFC compiler for VoltaireMPI
  --v-mpi-compiler-icc <path>   Use IFC compiler for VoltaireMPI
  --v-mpi-compiler-path <path>  Use PathScale compiler for VoltaireMPI
  --v-mpi-compiler-pgi <path>   Use PGI compiler for VoltaireMPI
  --make-bin-package <full|slim|mini>  
                                Write a binary package
  --custom <cust-name>          Build a customized version   
  --dont-patch                  Do not apply patches to source code
  --unlock                      Remove install lock
  --fix-symvers                 Fix IB kernel symbol versions
  --nocolor                     Do not print colored output
  --set-timeofday               Set system date to the time of this script
  --no-stop                     Do not stop running stack 
  
EOF
}

#
# read options and set variables accordingly
#
function get_install_options()
{
	while [[ ! -z "$1" ]]
	do
		case $1 in
		--set-timeofday)
			date -s "$(date -r $mydir/$myname)" >/dev/null
			shift
			;;
		--no-v-mpi)
			v_mpi=0;
			install_args_to_keep="$install_args_to_keep $1"
			shift
			;;
		--no-o-mpi)
			o_mpi=0;
			install_args_to_keep="$install_args_to_keep $1"
			shift
			;;
		--v-mpi-compiler-ifc)
			export IFC=$2
			export PATH=$IFC/bin:$PATH
			shift 2
			;;
		--v-mpi-compiler-icc)
			export ICC=$2
			export PATH=$ICC/bin:$PATH
			shift 2
			;;
		--v-mpi-compiler-path)
			export PATHSCALE=$2
			export PATH=$PATHSCALE/bin:$PATH
			export PATHSCALE_SUBSCRIPTION_DAEMON=lynx.voltaire.com:7143
			shift 2
			;;
		--v-mpi-compiler-pgi)
			export PGI=$2
			export PATH=$PGI/linux86-64/7.0-5/bin:$PATH
			shift 2
			;;
		--unlock)
			unlock_install
			shift
			;;
		--dont-patch)
			dont_patch=1
			shift
			;;
		--make-bin-package)
			bin_type=$2
			case $bin_type in
			full|slim|mini)
				;;
			-*)
				abort "--make-bin-package option expects an argument: full, slim or mini"
				;;
			*) 
				abort "binary type $bin_type is invalid" 
				;;
			esac
			make_bin_packages="$make_bin_packages $bin_type"
			shift 2
			;;
		--ofed-config)
			ofed_conf=$2
			install_args_to_keep="$install_args_to_keep $1 $2"
			shift 2
			;;
		--gs-config)
			gs_conf=$2
			install_args_to_keep="$install_args_to_keep $1 $2"
			shift 2
			;;
		--ipoib-config)
			verify_file $2
			ipoib_conf=${verified_file}
			check_ipoib_conf
			install_args_to_keep="${install_args_to_keep} $1 $2"
			shift 2
			;;
		--custom)
			[ -d $mydir/patches/$2 ] || \
				abort "Available Customizations: $(cd $mydir/patches ; ls -d *|xargs)"
			CUSTOM_LIST="${CUSTOM_LIST} $2"
			shift 2
			;;
		--fix-symvers)
			fix_symvers=1
			shift
			;;
		--nocolor)
			disable_color_print
			shift
			;;
		--no-stop)
			stop=0
			shift
			;;
		-h|--help)
			usage
			unlock_install
			exit
			;;
		*)
			abort "Invalid option $1"
		esac
	done
}

function lock_install(){
	[ -f $lock ] && abort "Installation is locked by another process. Try again later or use --unlock"
	touch $lock
	echo hostname:$$ >> $lock
	date >> $lock
}

function unlock_install(){
	rm -f $lock
}

#####################################
#                                   #
#            Entry Point            #
#                                   #
#####################################

# build settings
mydir=$(cd $(dirname $0) && pwd)
lock=$mydir/lock
myname=$(basename $0)
prefix=/usr/voltaire
LOG=/tmp/gs-install.$$.log
rpm_target_cpu=$(rpm --eval '%{_target_cpu}')
rpm_dir=$(rpm --eval '%{_topdir}')
OFED_DIR=$(ls -d $mydir/OFED* 2>/dev/null|sort|tail -1)
gs_conf_file=${prefix}/gs.conf
ofed_conf_file=${prefix}/ofed.conf
OFED_PREFIX=/usr
get_os_vars

# default values for arguments
v_mpi=1
o_mpi=1
stop=1
dont_patch=0
fix_symvers=0
make_bin_packages=""
CUSTOM_LIST="Common"
ofed_conf=default
gs_conf=default
install_args_to_keep=""


trap_handler()
{
	unlock_install	
	die "Installation aborted by user"
}

# Get user's trap (Ctrl-C or kill signals)
trap 'trap_handler' 2 9 15

while [[ ! -z "$1" ]] ;  do
        case $1 in
        --profile)
                [ -f $mydir/bin-info ] && abort "--profile is not allowed for binary package"
                verify_file $mydir/install/profile.$2
                install_args_to_use="$install_args_to_use $(cat ${verified_file})"
                shift 2
                ;;
        *)
                install_args_to_use="$install_args_to_use $1"
                shift
                ;;
        esac
done

install_args_to_use="$install_args_to_use $([ -f $mydir/install-args ] && cat $mydir/install-args)"
get_install_options $install_args_to_use
info "Using install options: ${install_args_to_use}"
lock_install

# create log file
touch $LOG
info "Log file is $LOG"

# make sure requested conf files exist
[ -f $mydir/install/vlt-gs.conf.${gs_conf} ] || \
	abort "GridStack configuration '${gs_conf}' does not exist"
[ -f $mydir/install/ofed.conf.${ofed_conf} ] || \
	abort "$mydir/install/ofed.conf.${ofed_conf} does not exist"

# create version ID
if [ ! -f $mydir/version-info ]
then
	echo GSCustomizations=\"${CUSTOM_LIST}\" >> $mydir/version-info
	echo GSPackageType=full >> $mydir/version-info
fi
source $mydir/version-info

# make sure OpenMPI will find has sysfs
if [ "$GSPackageType" = "full" ] && [ ${o_mpi} -ne 0 ]
then
	if [ ! -f /usr/include/sysfs/libsysfs.h ]
	then
		error "File /usr/include/sysfs/libsysfs.h is missing. It seems that OpenMPI cannot be installed."
		error "Disable installation of OpenMPI by passing --no-o-mpi to $myname"
		abort   "or install RPM libsysfs-devel, sysfsutils or sysfsutils-devel \(depending on your OS\)"
	fi
fi


remove_old_stacks
validate_time_of_day


# create directory for packages
mkdir -p ${prefix}


# prepare for building RPM's
mkdir -p $mydir/BUILD
mkdir -p $mydir/RPMS/${rpm_target_cpu}/$OS

# Install gridstack-scripts before all other packages.
# we do this to be able to uninstall correctly, if
# install fails at some point
info "Installing gridstack-scripts"
build_pkg gridstack-scripts




# add package type to the version file
cat $mydir/version-info >> ${prefix}/version
source ${prefix}/version


# local OFED installation directory
[ -z ${OFED_DIR} ] || OFED_DIR=$(basename $OFED_DIR)


# copy configuration files
info "Using GS configuration: ${gs_conf}"
cp $mydir/install/vlt-gs.conf.${gs_conf} ${gs_conf_file}
info "Using OFED configuration: ${ofed_conf}"
cp $mydir/install/ofed.conf.${ofed_conf} ${ofed_conf_file}

fix_ofed_conf_file

if [ -f $mydir/bin-info ]
then
	dont_patch=1
	[ -z "${make_bin_packages}" ] || \
		abort "can't make binary package from binary package"

	[ ${GSPackageType} != "full" ] && [ ! -f /etc/infiniband/info ] && \
		abort "You are trying to install a non full GridStack package but OFED is not installed.\
Please install OFED first and than try again."

	save_os_vars /tmp/bin-info
	diff $mydir/bin-info /tmp/bin-info >/dev/null || \
		abort "Binary package doesnt seem to match this system."
fi


# stop old stack
[ ${stop} -eq 1 ] && stop_stack



# install OFED
[ -z ${OFED_DIR} ] || install_ofed

# post-install steps
fix_openib_conf
fix_modprobe_conf
fix_udev_rules


# remove opensm
rpm -q opensm >/dev/null &&	rpm -e --nodeps opensm opensm-devel

	
#Install Voltaire MPI
if [ "$GSPackageType" != "mini" ] && [ ${v_mpi} -ne 0 ]
then
	info "Installing vltmpi-OPENIB"
	install_vmpi
fi

#Install Open MPI
if [ "$GSPackageType" != "mini" ] && [ ${o_mpi} -ne 0 ]
then
	info "Installing openmpi"
	install_ompi
fi

# Install Voltaire Packages
if [ $GSPackageType != "mini" ]
then
	info "Installing ib-enhanced-services"
	build_pkg ib-enhanced-services
fi

info "Installing ib-his"
build_pkg ib-his

info "Configuring network"
config_network


unlock_install
for bin_type in $make_bin_packages ; do
	info Making binary package: $bin_type
	make_bin_package $bin_type
done

echo "Installation finished"
if [ $GSPackageType != "full" ]
then
	ofed_buildid_file=""
	if rpm -q ofed-scripts >/dev/null
	then
		ofed_buildid_file=$(rpm -ql ofed-docs|grep BUILD_ID)
	fi
    [ -z ${ofed_buildid_file} ] && abort "OFED BUILD_ID is missing"
	
	if ! grep -q VLT_PATCH ${ofed_buildid_file}
	then
		info You installed GridStack without OFED.
		info It seems that you installed OFED without Voltaire customizations.
		info Please make sure that your GridStack doesnt require any Customization before you proceed.
		info You can also go to the patches directory and customize OFED now.
	fi
fi

echo "Please log out from the shell and login again in order to update your PATH environment variable"


## SUCCESS ##
