#!/usr/local/bin/cbsd
#v12.1.8
CIXARG=""
CIXOPTARG="cleanup delay environment jname inter quiet"
MYDESC="Start jail"
ADDHELP="
${H3_COLOR}Description${N0_COLOR}:

 Start the jail container. When used in a directory with a CBSDfile,
the jstart command only processes the environments described in the CBSDfile.

This script supports the reserved word WHERE, which allows you to write SQL condition,
please see 'schema jails' in ~cbsd/var/db/local SQLite3 DB for flexible queries.

${H3_COLOR}Options${N0_COLOR}:

 ${N2_COLOR}cleanup=${N0_COLOR}    - 1 (by default) - run jcleanup before start, '0' - skip jcleanup
 ${N2_COLOR}delay=${N0_COLOR}      - <sec>, delay N secbefore start, mainly to smooth the astart,
               default is: '0', no delay.
 ${N2_COLOR}environment${N0_COLOR} - pass environment, e.g.: 'environment=\"FOO=bar\" environment=\"VAR1=boo\"'
               or path to 'env' file;
 ${N2_COLOR}inter=${N0_COLOR}      - set 1 to prevent any questions and to accept answers by default.
 ${N2_COLOR}jname=${N0_COLOR}      - target jail. If jail='*' or jail='pri*' then start all jails or
               jails whose names begin with 'pri', e.g. 'prison1', 'prisonX'...
 ${N2_COLOR}quiet=${N0_COLOR}      - 0,1: be quiet, dont output verbose message.

${H3_COLOR}Examples${N0_COLOR}:

 # cbsd jstart
 # cbsd jstart jname='memcach*'
 # cbsd jstart jname=\"test\" environment=\"VAR1=var1\" environment=\"VAR2=var2\"
 # cbsd jstart WHERE astart=1 AND vnet=1 AND ip4_addr LIKE \'1%\'

${H3_COLOR}See also${N0_COLOR}:

 cbsd jstop --help

"

CBSDMODULE="jail"
EXTHELP="wf_jstop_jstart"

. ${subrdir}/nc.subr
. ${subrdir}/tools.subr		# for select_jail_by_list

readconf buildworld.conf
jname=

# check for cloud function when CBSDfile exist
Makefile="${CIX_PWD}/CBSDfile"
if [ ! -r "${Makefile}" ]; then
		[ -z "${1}" ] && select_jail_by_list -s "List of offline jail" -a "Off" -r ${sqlreplica}
fi

delay=0
odelay=
oenvironment=
environment=
cixinit
[ -z "${cleanup}" ] && cleanup=1

if [ -z "${ci_gw4}" -a "${ci_gw4}" != "0" ]; then
	if [ -n "${default_ci_gw4}" -a "${default_ci_gw4}" != "0" ]; then
		oci_gw4="${default_ci_gw4}"
	fi
fi

readconf jail-freebsd-default.conf

. ${subrdir}/cbsdinit.subr
. ${subrdir}/system.subr
. ${subrdir}/universe.subr
. ${subrdir}/ipfw.subr	# fwcounter

[ -z "${quiet}" ] && quiet=0

xenvironment=
# adjust jail_list by CBSDfile
if [ -r "${Makefile}" ]; then
	[ -z "${CBSDFILE_RECURSIVE}" ] && ${ECHO} "${N1_COLOR}found CBSDfile: ${N2_COLOR}${Makefile}${N0_COLOR}" 1>&2
	. ${Makefile}
	all_jail_list=$( ${GREP_CMD} -E '^jail_[a-zA-Z0-9_@%:][-a-zA-Z0-9_@%:]*\(\)$' ${Makefile} | ${XARGS_CMD} | ${TR_CMD} -d "()" | ${SED_CMD} s#jail_##g )

	# cbsd jstart <env1> <env2> route
	jname_in_args=0

	if [ -n "${ojname}" ]; then
		jail_list="${ojname}"
	else
		# cbsd jstart <env1> <env2> route
		for i in $*; do
			strpos --str="${i}" --search="="
			if [ $? -eq 0 ]; then
				jname_in_args=1
				# not param=value - jail?
				jname_in_cbsdfile=0
				for j in ${all_jail_list}; do
					[ "${i}" != "${j}" ] && continue
					jname_in_cbsdfile=1
				done
				if [ ${jname_in_cbsdfile} -eq 1 ]; then
					if [ -z "${ojname}" ]; then
						ojname="${i}"
					else
						ojname="${ojname} ${i}"
					fi
				else
					${ECHO} "${N1_COLOR}${CIX_APP}: env absent in CBSDfile: ${N2_COLOR}${i}${N0_COLOR}" 1>&2
				fi
			fi
		done
		if [ ${jname_in_args} -eq 1 ]; then
			jail_list="${ojname}"
		else
			jail_list="${all_jail_list}"
		fi
	fi

	unset ojname jname

	[ -z "${jail_list}" ] && err 1 "${N1_COLOR}${CIX_APP}: give me jname${N0_COLOR}"

	# multiple?
	strpos --str="${jail_list}" --search=" "
	if [ $? -eq 0 ]; then
		# alone
		jname="${jail_list}"
		unset jail_list
	fi

	if [ -n "${CLOUD_URL}" -a -n "${CLOUD_KEY}" ]; then
		cbsd_api=1
	else
		cbsd_api=0
	fi
else
	cbsd_api=0

	# trim args from "$*"
	for i in ${CIX_OTHER_ARGS}; do
		strpos --str="${i}" --search="="
		_pos=$?

		if [ ${_pos} -ne 0 ]; then
			_arg_len=$( strlen ${i} )
			_pref=$(( _arg_len - _pos ))
			ARG=$( substr --pos=0 --len=${_pos} --str="${i}" )

			case "${ARG}" in
				environment)
					VAL=$( substr --pos=$(( _pos + 2 )) --len=${_pref} --str="${i}" | ${TR_CMD} -d '"' )
					if [ -z "${xenvironment}" ]; then
						xenvironment="${VAL}"
					else
						xenvironment="${xenvironment} ${VAL}"
					fi
					;;
			esac
			shift
			continue
		fi

		if [ -z "${jail_list}" ]; then
			jail_list="${i}"
		else
			jail_list="${jail_list} ${i}"
		fi
	done

	if [ -n "${CIXINIT_SQL_CONDITION}" ]; then
		jail_list=$( cbsdsqlro local "SELECT jname FROM jails WHERE emulator='jail' AND ${CIXINIT_SQL_CONDITION}" 2>/dev/null | ${XARGS_CMD} )
	fi

	[ -n "${ojname}" ] && jail_list="${ojname}"

	# multiple?
	strpos --str="${jail_list}" --search=" "
	if [ $? -eq 0 ]; then
		# alone
		jname="${jail_list}"
		unset jail_list
	fi
fi

[ -z "${jname}" -a -z "${jail_list}" ] && err 1 "${N1_COLOR}No jail specified${N0_COLOR}"

# for external_exec-related command
. ${subrdir}/jcreate.subr

. ${subrdir}/jstart.subr
. ${subrdir}/time.subr

# MAIN for multiple jails
if [ "${mod_cbsd_queue_enabled}" = "YES" -a -z "${MOD_CBSD_QUEUE_DISABLED}" ]; then
	readconf cbsd_queue.conf
	[ -z "${cbsd_queue_backend}" ] && MOD_CBSD_QUEUE_DISABLED="1"
fi

TRAP=""
emulator="jail"	# for jname_is_multiple
# jail_list can be init in CBSDfile case
[ -z "${jail_list}" ] && jname_is_multiple

if [ -n "${jail_list}" ]; then
	TMP_JLIST="${jail_list}"
else
	TMP_JLIST="${jname}"
fi

JLIST=

if [ -z "${jname}" ]; then
	# check for actual vm list in arg list
	jail_num=0
	jname=
	for i in ${TMP_JLIST}; do
		cbsdsqlro_vars local "SELECT jname FROM jails WHERE jname='${i}' AND emulator='${emulator}' LIMIT 1" exist
		if [ -n "${exist}" ]; then
			if [ -n "${JLIST}" ]; then
				JLIST="${JLIST} ${i}"
			else
				JLIST="${i}"
			fi
			if [ ${jail_num} -eq 0 ]; then
				jname="${i}"
			else
				# when > 1, jname unused below
				unset jname
			fi
			jail_num=$(( jail_num + 1 ))
		fi
	done

	# this is multiple list, split it by parallel jstart execution
	if [ ${jail_num} -gt 1 ]; then
		cbsdlogger NOTICE ${CIX_APP}: executing for multiple starting: ${JLIST}
		# multiple astart always non interactive
		export inter=0

		gettime st_time

		if [ -z "${parallel}" -o "${parallel}" = "0" ]; then
			for jname in ${JLIST}; do
				/usr/local/bin/cbsd jstart inter=0 jname=${jname}
			done
		else
			_seq=0
			wait_pid_files=

			for jname in ${JLIST}; do
				cbsdsqlro_vars local "SELECT boot_delay FROM jails WHERE jname='${jname}' LIMIT 1" _boot_delay
				if [ -z "${_boot_delay}" ]; then
					_boot_delay=0
				fi

				if [ ${_boot_delay} -eq 0 ]; then
					_boot_delay="${_seq}"
				fi

				env NOINTER=1 ${miscdir}/daemonize -E NOINTER=1 -p ${ftmpdir}/jstart.${jname}.$$ /usr/local/bin/cbsd jstart inter=0 jname=${jname} delay=${_boot_delay}
				if [ -z "${wait_pid_files}" ]; then
					wait_pid_files="${ftmpdir}/jstart.${jname}.$$"
				else
					wait_pid_files="${wait_pid_files} ${ftmpdir}/jstart.${jname}.$$"
				fi

				_seq=$(( _seq + 1 ))
			done

			#lets save .pid file
			sleep 1

			# wait loop
			for i in ${wait_pid_files}; do
				[ ! -r ${i} ] && continue
				read -r _pid < ${i}
				[ -z "${_pid}" ] && continue
				cbsd_pwait --pid=${_pid} --timeout=${parallel}
			done

			wait_for_fpid -a start -t ${parallel}
		fi

		gettime end_time
		diff_time=$(( end_time - st_time ))
		diff_time=$( displaytime ${diff_time} )
		${ECHO} "${N1_COLOR}${CIX_APP} done ${N2_COLOR}in ${diff_time}${N0_COLOR}"
		cbsdlogger NOTICE ${CIX_APP}: executing for multiple done in ${diff_time}: ${JLIST}
		err 0 "${N1_COLOR}Multiple start: ${N2_COLOR}done${N0_COLOR}"
	fi
fi

# MAIN
# single jname
[ -z "${jname}" ] && jname="${1}"
[ -z "${jname}" ] && err 1 "${N1_COLOR}Give me jail name${N0_COLOR}"

. ${subrdir}/rcconf.subr

if [ $? -eq 1 ]; then
	# remote start
	[ ${sqlreplica} -eq 0 ] && err 1 "${N1_COLOR}No such jail: ${N2_COLOR}${jname}${N0_COLOR}"
	capture remotenode jwhereis ${jname}
	[ -z "${remotenode}" ] && err 1 "${N1_COLOR}No such jail: ${N2_COLOR}${jname}${N0_COLOR}"
	for i in ${remotenode}; do
		if [ "${i}" = "${nodename}" ]; then
			[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Remote jstart: found on nodename ${N2_COLOR}${nodename}${N1_COLOR}. Skipped${N0_COLOR}"
			continue
		fi
		[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Remote jstart: ${N2_COLOR}${jname} ${N1_COLOR}on${N2_COLOR} ${i}${N0_COLOR}"
		rexe node=${i} cbsd jstart jname=${jname}
		_ret=$?
		if [ ${_ret} -eq 0 ]; then
			# updating state and put task for retrinv inventory
			[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Updating inventory...${N0_COLOR}"
			task autoflush=2 mode=new retrinv node=${i} tryoffline=1 data=db > /dev/null 2>&1
		fi
	done
	exit 0
fi

# mainly to smooth mass start in astart.
delay="${odelay:-$delay}"

if [ ${delay} -ne 0 ]; then
	cbsdlogger NOTICE ${CIX_APP}: delayed boot for ${jname} via delay= params: ${delay} sec.
	sleep ${delay}
fi

# start jail
gettime st_time

if [  "${ver}" = "native" ]; then
	capture tmpver ${UNAME_CMD} -r
	ver=${tmpver%%-*}
	unset tmpver
	[ "${stable}" = "1" ] && ver=${ver%%.*}
	cbsdsqlrw local "UPDATE jails SET ver='${ver}' WHERE jname='${jname}'"
	[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}adjust version: ${N2_COLOR}native -> ${ver}${N0_COLOR}"
fi

if [ "${arch}" = "native" ]; then
	capture arch ${UNAME_CMD} -m
	[ "${arch}" = "x86_64" ] && arch="amd64"

	init_target_arch

	cbsdsqlrw local "UPDATE jails SET arch='${arch}' WHERE jname='${jname}'"
	[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}adjust version: ${N2_COLOR}native -> ${ver}${N0_COLOR}"
fi

over="${ver}"

# Determine stable value. Must be after buildconf
strpos --str="${over}" --search="."

# auto-detect for stable/release
pos=$?
if [ ${pos} -eq 0 ]; then
	stable=1
	ostable=1
else
	stable=0
	ostable=0
fi

[ ${status} -eq 2 ] && err 1 "${N1_COLOR}Jail in slave mode. Please ${N2_COLOR}cbsd jswmode mode=master${N1_COLOR} first${N0_COLOR}"
[ ${status} -eq 3 ] && err 1 "${N1_COLOR}Jail in maintenance mode${N0_COLOR}"
[ ${jid} -ne 0 ] && err 1 "${N1_COLOR}Jail ${jname} already running, jid: ${N2_COLOR}${jid}${N0_COLOR}"

if [ "${vnet}" = "1" -a "${vimage_feature}" = "0" ]; then
	[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Jail ${N2_COLOR}${jname}${N1_COLOR} have vnet=1 flags but your kernel is not support VIMAGE${N0_COLOR}"
	[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Please recompile kernel with: ${N2_COLOR}options VIMAGE${N0_COLOR}"
	sleep 3
	vnet=0
fi

[ "${emulator}" = "bhyve" -a -z "${mdsize}" ] && err 1 "${N1_COLOR}Bhyve required for file image${N0_COLOR}"

TRAP=""

#Check for shared lock
jaillock="${jailsysdir}/${jname}/locked"
if [ -f "${jaillock}" ]; then
	read -r masterhost < ${jaillock}
	if [ "${masterhost}" = "${nodename}" -o -z "${masterhost}" ]; then
		cbsdlogger NOTICE ${CIX_APP}: jail ${jname}: remove my stale lock file: ${jaillock}
		[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Remove my stale lock file: ${N2_COLOR}${jaillock}${N0_COLOR}"
		${RM_CMD} -f ${jaillock}
	else
		gettime cur_time
		eval $( ${STAT_CMD} -s ${jaillock} )
		difftime=$(( ( cur_time - st_mtime ) / 60 ))

		# 30 minutes outdated lock
		if [ ${difftime} -gt 30 ]; then
			cbsdlogger NOTICE ${CIX_APP}: jail ${jname}: locked by ${masterhost} node via ${jaillock} file but lock file age is too old: ${difftime}. Removing!
			[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Jail ${N2_COLOR}${jname}${N1_COLOR} locked by ${N2_COLOR}${masterhost}${N1_COLOR} node${N0_COLOR}"
			[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}But lock age is too old: ${difftime} min. Removing!!!${N0_COLOR}"
			${RM_CMD} -f ${jaillock}
		else
			# still fresh
			cbsdlogger NOTICE ${CIX_APP}: jail ${jname}: locked by ${masterhost} node via ${jaillock} file and lock file age time is fresh: ${difftime}.
			[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Jail ${N2_COLOR}${jname}${N1_COLOR} locked by ${N2_COLOR}${masterhost}${N1_COLOR} node, lock age: ${difftime} min.${N0_COLOR}"
			log_err 1 "${N1_COLOR}You may remove the lockfile if you believe that jail is not running on this node: ${N2_COLOR}rm -f ${jaillock}${N0_COLOR}"
		fi
	fi
fi

TRAP="${TRAP} ${RM_CMD} -f ${jaillock};"
trap "${TRAP}" HUP INT ABRT BUS TERM EXIT

#Check for md vnode backend
if [ -n "${mdsize}" -a "${mdsize}" != "0" ]; then
	MDFILE="${jailsysdir}/${jname}/image.dat"
	if [ ! -f "${MDFILE}" -a ! -h "${MDFILE}" ]; then
		cbsdlogger NOTICE ${CIX_APP}: jail: ${jname}: no such ${MDFILE} but mdsize flags is not null, skip
		[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}No such ${MDFILE} but mdsize flags is not null. Skip${N0_COLOR}" && continue
	fi
fi

# Update Redis
if [ "${mod_cbsd_redis_enabled}" = "YES" -a -z "${MOD_CBSD_REDIS_DISABLED}" ]; then
	cbsdredis hset "jail:${jname}" node "${nodename}" status 1 || echo "WARNING: failed to update Redis!"
	cbsdredis publish cbsd_events '{"cmd":"jstart", "node":"'${nodename}'", "jail":"'${jname}'", "status":1}'
fi

# CBSD QUEUE
if [ "${mod_cbsd_queue_enabled}" = "YES" -a -z "${MOD_CBSD_QUEUE_DISABLED}" ]; then
	readconf cbsd_queue.conf
	if [ -z "${cbsd_queue_backend}" ]; then
		MOD_CBSD_QUEUE_DISABLED="1"
	else
		[ -n "${cbsd_jail_queue_name}" ] && ${cbsd_queue_backend} cbsd_queue_name=${cbsd_jail_queue_name} id=${jname} cmd=jstart status=1 workdir="${workdir}"
	fi
fi

[ "${emulator}" = "bhyve" ] && start_bhyve

init_target_arch
init_basedir

if [ ${baserw} -eq 0 ]; then
	if [ -r "${BASE_DIR_LOCKFILE}" ]; then
		read -r locked_by < ${BASE_DIR_LOCKFILE}
		log_err 1 "${N1_COLOR}basedir locked: ${N2_COLOR}${BASE_DIR}${N1_COLOR}, by pid: ${N2_COLOR}${locked_by}${N1_COLOR}. Please try later or remove ${BASE_DIR_LOCKFILE}${N0_COLOR}"
	fi
fi

[ ${cleanup} -eq 1 ] && jcleanup jname=${jname}

[ ! -d "${path}" ] && ${MKDIR_CMD} ${path}
[ -z "${mnt_start}" ] && mnt_start="0"

[ -n "${ointer}" ] && inter="${ointer}"
[ -n "${inter}" ] && export NOINTER=1

if [ "${mnt_start}" != "0" ]; then
	if [ ! -r "${mnt_start}" -o ! -x "${mnt_start}" ]; then
		err 1 "mnt_start script not exist or not executable: ${mnt_start}"
	fi
	[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Execute mnt_start script: ${N2_COLOR}${mnt_start}${N0_COLOR}..."
	# external mount, reset zfsfeat
	zfsfeat=0
	[ ! -d ${data} ] && ${MKDIR_CMD} -p ${data}
	[ ! -d ${jailfstabdir}/${jname} ] && ${MKDIR_CMD} -p ${jailfstabdir}/${jname}
	[ ! -d ${jailsysdir}/${jname} ] && ${MKDIR_CMD} -p ${jailsysdir}/${jname}
	${mnt_start} -d ${data} -f ${jailfstabdir}/${jname} -j ${jname} -r ${jailrcconfdir} -s ${jailsysdir}/${jname}
	_ret=$?
	if [ ${_ret} -ne 0 ]; then
		err 1 "${W1_COLOR}error: ${N1_COLOR}mnt_start script failed: ${N2_COLOR}${mnt_start} ${jname}${N0_COLOR}"
	fi
fi

check_environment_script "tests"

# check for DHCP in ip4_addr string and replace them by ip
OIFS="${IFS}"
IFS=","
_tmp_ip4_addr=
_mod=0

for _tmp in ${ip4_addr}; do
	case ${_tmp} in
		[Dd][Hh][Cc][PP])
			capture _tmp dhcpd
			[ $? -eq 2 ] && log_err 1 "${N1_COLOR}No free IP address for DHCP in nodeippool${N0_COLOR}"
			cbsdlogger NOTICE ${CIX_APP}: set ip4_addr for ${jname} via DHCP: ${_tmp}
			_mod=1
			;;
		[Dd][Hh][Cc][PP][vV]6)
			capture _tmp dhcpdv6
			[ $? -eq 2 ] && log_err 1 "${N1_COLOR}No free IP6 address for DHCP in nodeip6pool${N0_COLOR}"
			cbsdlogger NOTICE ${CIX_APP}: set ip4_addr for ${jname} via DHCPv6: ${_tmp}
			_mod=1
			;;
		*)
			;;
	esac

	if [ -z "${_tmp_ip4_addr}" ]; then
		_tmp_ip4_addr="${_tmp}"
	else
		_tmp_ip4_addr="${_tmp_ip4_addr},${_tmp}"
	fi
done

IFS="${OIFS}"

if [ ${_mod} -eq 1 ]; then
	jset jname=${jname} ip4_addr="${_tmp_ip4_addr}"
	ip4_addr="${_tmp_ip4_addr}"
fi

geniplist ${ip4_addr}

if [ -n "${interface}" -a "${interface}" != "0" -a "${vnet}" -eq "0" ]; then

	if [ "${interface}" != "auto" ]; then
		# check for interface existance
		case "${platform}" in
			Linux)
				${IP_CMD} link show ${interface} >/dev/null 2>&1
				_ret=$?
				;;
			*)
				${IFCONFIG_CMD} ${interface} >/dev/null 2>&1
				_ret=$?
				;;
		esac

		if [ ${_ret} -ne 0 ]; then
			${ECHO} "${W1_COLOR}Warning: ${N1_COLOR}interface not exist for jail ${jname}: ${N2_COLOR}${interface}${N0_COLOR}"
			cbsdlogger NOTICE ${CIX_APP}: Warning: interface not exist for jail ${jname}: ${interface}
		fi
	fi

	### CHECK FOR IP ####
	for ips in ${IPS}; do
		iptype "${ips}" || true
		[ -z "${IWM}" ] && continue
		[ -n "${VHID}" ] && continue
		#prevent to use nodeip
		[ "${IWM}" = "${nodeip}" ] && err 1 "${N1_COLOR}Error: Jail can not take nodeip when interface is not equal 0: ${N2_COLOR}${nodeip}${N0_COLOR}"
		checkip ip=${IWM} check=1
		IPUSE=$?
		case ${IPUSE} in
			0)
				[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}ip ${IWM} not in pool range${N0_COLOR}"
				continue
			;;
			2)
				[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Ip ${IWM} already exists in LAN${N0_COLOR}"
				continue
			;;
		esac
	done
fi

# args for makejconf
epairb_list=
# for traffic count
epaira_list=
# NIC names from DB (for interface renaming inside jail)
nic_name_list=

if [ "${vnet}" = "1" ]; then
	# for vnet we can make another action
	. ${subrdir}/vnet.subr
	prepare_jail_vnet	# sets epaira_list, epairb_list, nic_name_list
fi

#test for zfs mounted & mount if not
case ${zfsfeat} in
	1)
		. ${subrdir}/zfs.subr
		zfsmnt ${data}
		if [ $? -eq 2 ]; then
			# ZFS encryption?
			capture zfs_encryption_val ${ZFS_CMD} get -Ho value encryption ${ZPOOL}
			if [ -n "${zfs_encryption_val}" -a "${zfs_encryption_val}" != "off" ]; then
				${ECHO} "${N1_COLOR}${CIX_APP}: encrypted dataset: ${N2_COLOR}${zfs_encryption_val}${N0_COLOR}"
				zfs_encryption_keyformat=$( ${ZFS_CMD} get -Ho value keyformat ${ZPOOL} 2>/dev/null )
				${ECHO} "${N1_COLOR}${CIX_APP}: encryption keyformat: ${N2_COLOR}${zfs_encryption_keyformat}${N0_COLOR}"
				if [ "${zfs_encryption_keyformat}" = "passphrase" ]; then
					if [ -n "${NOINTER}" -o "${inter}" = "1" ]; then
						err 0 "${N1_COLOR}${CIX_APP} keyformat is passphrase, skipp for non-interactive mode${N0_COLOR}"
					else
						${ZFS_CMD} load-key ${ZPOOL}
						# 255 code valid here, already loaded
					fi
				fi
			fi
			${ZFS_CMD} mount "${ZPOOL}"
			is_mounted ${data} || err 1 "${N1_COLOR}${CIX_APP}: unable to mount zfs pool: ${N2_COLOR}${ZPOOL} -> ${data}${N0_COLOR}"
		fi
	;;
esac

if [ "${ver}" != "empty" ]; then
	if [ ${baserw} -eq 1 ]; then
		path="${data}"
		[ ! -f "${path}/bin/sh" ] && switch_baserw ${path} 1
		[ -f ${mount_fstab} ] && ${RM_CMD} -f ${mount_fstab}
	fi

	if [ ${baserw} -eq 0 -a ! -f "${mount_fstab}" ]; then
		switch_baserw ${path} 2
	fi
fi

# MD area
if [ -n "${mdsize}" -a "${mdsize}" != "0" ]; then
	cbsd mountmd mdfile="${MDFILE}" jroot="${data}"
fi

if [ "${ver}" != "empty" ]; then
	[  -d "${data}/etc" -o -d "${data}/bin" ] || err 1 "${N1_COLOR}No such data structure: ${N2_COLOR}${data}${N0_COLOR}"
fi

# cp local default resolv.conf skel
if [ ${floatresolv} -eq 1 ]; then
	[ -d ${data}/etc ] && makeresolv jname=${jname}
fi

MOUNTOPT=""

if [ "${mount_src}" = "1" ]; then
	SRCDIR="${srcdir}/src_${ver}/src"
	MOUNTOPT="${MOUNTOPT} -s ${SRCDIR}"
fi

if [ "${mount_obj}" = "1" ]; then
	SRCDIR="${srcdir}/obj_${arch}_${ver}/obj"
	MOUNTOPT="${MOUNTOPT} -o ${SRCDIR}"
fi

if [ "${mount_kernel}" = "1" ]; then
	if [ -d ${basejaildir}/kernel_GENERIC_${arch}_${ver}/boot/kernel ]; then
		SRCDIR="${basejaildir}/kernel_GENERIC_${arch}_${ver}/boot/kernel"
		MOUNTOPT="${MOUNTOPT} -k ${SRCDIR}"
	fi
fi

if [ "${mount_ports}" = "1" ]; then
	case "${platform}" in
		DragonFly)
			MOUNTOPT="${MOUNTOPT} -p /usr/dports"
			;;
		*)
			MOUNTOPT="${MOUNTOPT} -p /usr/ports -d ${data}/var/cache/distfiles"
			;;
	esac
fi

[ "${ver}" != "empty" ] && mountbase -v ${ver} -a ${arch} ${MOUNTOPT}

# sign of zfs attach inside jail: we need special route for this case
# remove orphaned sign if exist: then touched it by mountfstab script
with_zfs_attach="${jailsysdir}/${jname}/with_zfs_attach"
[ -r ${with_zfs_attach} ] && ${RM_CMD} -f ${with_zfs_attach}

mount_jail_fstab

# hook
if cix_read_dir ${jailsysdir}/${jname}/master_prestart.d/ >/dev/null 2>&1; then
	export_jail_data_for_external_hook
	external_exec_master_script "master_prestart.d"
fi

if [ "${ver}" = "empty" ]; then
	#path="/"
	[ -z "${exec_start}" ] && exec_start="${jailsysdir}/${jname}/run.sh"
else
	[ ! -d "${data}/var/cache/pkg" ] && ${MKDIR_CMD} -p "${data}/var/cache/pkg"
fi

#determine that jail is FreeBSD. Useful for vnet operation in makejconf and
is_freebsd=0

if [ ${baserw} -eq 1 ]; then
	elftest=${data}/bin/sh
else
	elftest="${BASE_DIR}/bin/sh"
fi
[ -f "${elftest}" ] && capture osname ${miscdir}/elf_tables --osname ${elftest}

[ "${osname}" = "freebsd" ] && is_freebsd=1

# check for freshed version of the base via elf from /bin/sh
if [ ${is_freebsd} -eq 1 -a ${baserw} -eq 1 ]; then
	if [ -n ${BASE_DIR} -a -f "${BASE_DIR}/bin/sh" ]; then
		capture baseelf ${miscdir}/elf_tables --ver ${BASE_DIR}/bin/sh
		capture jailelf ${miscdir}/elf_tables --ver ${elftest}
		if [ ${baseelf} -gt ${jailelf} ]; then
			[ ${quiet} -ne 1 ] && ${ECHO} "${BOLD}Notice: ${N1_COLOR}You have a more recent version of the base in ${BASE_DIR} (${N2_COLOR}${baseelf}${N1_COLOR}/${N2_COLOR}${jailelf}${N1_COLOR}).${N0_COLOR}"
			[ ${quiet} -ne 1 ] && ${ECHO} "${BOLD}Notice: ${N1_COLOR}Please consider upgrading jail base via ${N2_COLOR}cbsd jupgrade${N0_COLOR}"
		fi
	fi
fi

if [ "${emulator_flags}" = "ocijail" ]; then
	ocijail_create
	exit 0
fi

set -e
makejconf jname=${jname} out=${ftmpdir}/${jname}.conf ip6wa=${HAVE_IPV6} epair=${epairb_list} fbsd=${is_freebsd} quiet=${quiet}
set +e

#rctl/limits area
jrctl jname=${jname} mode=set quiet=${quiet}
if [ -r "${jailsysdir}/${jname}/helpers/jrctl.sqlite" ]; then
	cbsdsqlro_vars ${jailsysdir}/${jname}/helpers/jrctl.sqlite "SELECT cur FROM forms WHERE param='nice'" nice
fi

setup_jail_nice
etcupdate_check

[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Starting jail: ${N2_COLOR}${jname}, parallel timeout=${parallel}${N0_COLOR}"
TRAP="${TRAP} ${RM_CMD} -f ${ftmpdir}/jstart.${jname}.$$;"
trap "${TRAP}" HUP INT ABRT BUS TERM EXIT

# check that the emulator is able to execute commands
capture res emulator_exec_check
ret=$?

if [ ${ret} -ne 0 ]; then
	[ ${quiet} -ne 1 ] && $ECHO "${N1_COLOR}emulator error: ${N2_COLOR}${res}${N0_COLOR}"
	jcleanup jname="${jname}"
	exit 0
fi

# TODO/NOTES:
# when path have no /var and/or /tmp (e.g empty fstab file), after jail start we have
# undesirable/undesirable tmpfs! bug/issue ?
# tmpfs on /usr/jails/jails/jail2/var (tmpfs, local)
# tmpfs on /usr/jails/jails/jail2/tmp (tmpfs, local)

capture CPUSET get_cpuset_cmd

[ -r ${ftmpdir}/jstart.${jname}.err ] && ${RM_CMD} -f ${ftmpdir}/jstart.${jname}.err

case "${platform}" in
	DragonFly)
		# legacy jail tools only
		#cat ${ftmpdir}/${jname}.conf
		${miscdir}/mountpoint ${path}/dev || ${MOUNT_CMD} -t devfs devfs ${path}/dev
		echo "[debug] ${JAIL_CMD} ${path} ${host_hostname} ${ip4_addr} ${exec_start}"
		${miscdir}/daemonize -e ${ftmpdir}/jstart.${jname}.err -p ${ftmpdir}/jstart.${jname}.$$ ${NICE_CMD} -n ${nice} ${JAIL_CMD} ${path} ${host_hostname} ${ip4_addr} ${exec_start}
		ret=$?
		;;
	*)
		#cbsdlogger NOTICE ${CIX_APP}: ${CPUSET} ${miscdir}/daemonize -e ${ftmpdir}/jstart.${jname}.err -p ${ftmpdir}/jstart.${jname}.$$ ${NICE_CMD} -n ${nice} ${JAIL_CMD} -f ${ftmpdir}/${jname}.conf -c ${jname}
		cbsdlogger NOTICE ${CIX_APP}: ${CPUSET} ${miscdir}/daemonize -e ${ftmpdir}/jstart.${jname}.err -p ${ftmpdir}/jstart.${jname}.$$ ${NICE_CMD} -n ${nice} ${miscdir}/jail_env ${jname} ${ftmpdir}/${jname}.conf
		#echo "${JAIL_CMD} -f ${ftmpdir}/${jname}.conf -c ${jname}"

		case "${quiet}" in
			0)
				#_cmd="${CPUSET} ${miscdir}/daemonize -e ${ftmpdir}/jstart.${jname}.err -p ${ftmpdir}/jstart.${jname}.$$ ${NICE_CMD} -n ${nice} ${JAIL_CMD} -f ${ftmpdir}/${jname}.conf -c ${jname}"
				_cmd="${CPUSET} ${miscdir}/daemonize -e ${ftmpdir}/jstart.${jname}.err -p ${ftmpdir}/jstart.${jname}.$$ ${NICE_CMD} -n ${nice} ${miscdir}/jail_env ${jname} ${ftmpdir}/${jname}.conf"
				;;
			1)
				#_cmd="${CPUSET} ${miscdir}/daemonize -e ${ftmpdir}/jstart.${jname}.err -p ${ftmpdir}/jstart.${jname}.$$ ${NICE_CMD} -n ${nice} ${JAIL_CMD} -f ${ftmpdir}/${jname}.conf -c ${jname} > /dev/null"
				_cmd="${CPUSET} ${miscdir}/daemonize -e ${ftmpdir}/jstart.${jname}.err -p ${ftmpdir}/jstart.${jname}.$$ ${NICE_CMD} -n ${nice} ${miscdir}/jail_env ${jname} ${ftmpdir}/${jname}.conf > /dev/null"
				;;
		esac

		${_cmd}
		ret=$?
		;;
	esac

if [ ${ret} -ne 0 ]; then
	if [ -r ${ftmpdir}/jstart.${jname}.err ]; then
		${CAT_CMD} ${ftmpdir}/jstart.${jname}.err
		${RM_CMD} ${ftmpdir}/jstart.${jname}.err
	fi
	jcleanup jname=${jname}
	log_err 1 "${N1_COLOR}${CIX_APP}: daemonize error${N0_COLOR}"
fi

trap "" HUP INT ABRT BUS TERM EXIT

if [ -r "${ftmpdir}/jstart.${jname}.$$" ]; then
	read -r _pid < ${ftmpdir}/jstart.${jname}.$$
	cbsd_pwait --pid=${_pid} --timeout=${parallel} > /dev/null 2>&1
fi
sleep 1

jstart_slow=0

# wait for jid
for i in 1 2 3 4 5 6; do
	get_jid
	ret=$?
	[ ${ret} -eq 1 -a -n "${myjid}" ] && break
	if [ "${quiet}" != "1" ]; then
		printf "${CLRLINE}"
		printf "${CURSORRST}"
		printf "${N1_COLOR}${CIX_APP}: waiting for ${jname} JID: ${N2_COLOR}${i}/6 ... ${N0_COLOR}"
		jstart_slow=1
	fi
	sleep 1
done

if [ ${ret} -eq 0 ]; then
	[ "${quiet}" != "1" ] && ${ECHO} "${W1_COLOR}ERROR${N0_COLOR}"
	if [ -r ${ftmpdir}/jstart.${jname}.err ]; then
		${CAT_CMD} ${ftmpdir}/jstart.${jname}.err
		${RM_CMD} ${ftmpdir}/jstart.${jname}.err
	fi
	jcleanup jname=${jname}
	log_err 1 "${N1_COLOR}${CIX_APP}: jail failed${N0_COLOR}"
fi

[ ${jstart_slow} -eq 1 -a "${quiet}" != "1" ] && ${ECHO} "${H1_COLOR}${myjid}${N0_COLOR}"
unset jstart_slow

### late start ###
# late exec_start: for zfsattached and vnet-based jails
# this is necessary for any operations that must be performed after
# creating the jail but before running real exec_start
late_start=

# sign of zfs attach inside jail: we need special route for this case
# remove orphaned sign if exist: then touched it by mountfstab script
if [ -r ${with_zfs_attach} -a ${is_freebsd} -eq 1 ]; then
	[ -r ${mount_fstab} ] && attachzfs fstab=${mount_fstab} jname=${jname}
	[ -r ${mount_fstab}.local ] && attachzfs fstab=${mount_fstab}.local jname=${jname}
	# real exec_start for zfs attached jail
	# todo: FIB? NICE?
	${CPUSET} ${JEXEC_CMD} ${jname} ${ZFS_CMD} mount -a
	late_start="${exec_start}"
fi

# for VNET-based jail we also need special route
# due to jail.conf doesn't support for multiple NICs
if [ ${vnet} -eq 1 ]; then
	setup_jail_vnet
	late_start="${exec_start}"
fi

if [ -n "${late_start}" ]; then
	[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}late_start in progress...${N2_COLOR}"
	# todo: FIB? NICE?
	${CPUSET} ${JEXEC_CMD} ${jname} ${late_start}
fi
### /late start ###

# update state_time
cbsdsqlrw local "UPDATE jails SET state_time=(strftime('%s','now')) WHERE jname='${jname}'"

# make id file
UNDHOSTNAME=$( echo ${hostname} | ${TR_CMD}  "." "_" )
FID="/var/run/jail_${UNDHOSTNAME}.id"
echo ${ST} > ${FID}

# todo: CPUSET, NICE, FIB?

exec_cbsdjail_first_boot

# hook
if cix_read_dir ${jailsysdir}/${jname}/start.d/ >/dev/null 2>&1; then
	external_exec_script -s start.d
fi

get_jid
jid=${myjid}

if [ "${platform}" = "DragonFly" ]; then
	[ ${allow_raw_sockets} -eq 1 ] && ${SYSCTL_CMD} -w jail.${jid}.net_raw_sockets=1
fi

# hook
if cix_read_dir ${jailsysdir}/${jname}/master_poststart.d/ >/dev/null 2>&1; then
	external_exec_master_script "master_poststart.d"
fi

if [ ${myjid} -gt 0 ]; then
	status="1"
	# make shared lock
	echo "${nodename}" > ${jaillock}
	${CHOWN_CMD} ${cbsduser}:${cbsduser} ${jaillock}
else
	status="0"
	# looks like jail start is failed, execute post script
	exec_master_poststop

	# hook
	if cix_read_dir ${jailsysdir}/${jname}/master_poststop.d/ >/dev/null 2>&1; then
		external_exec_master_script "master_poststop.d"
	fi

	exec_poststop
	jcleanup jname=${jname}

	# redis
	if [ "${mod_cbsd_redis_enabled}" = "YES" -a -z "${MOD_CBSD_REDIS_DISABLED}" ]; then
		cbasredis hset "jail:${jname}" status -1 || echo "WARNING: Failed to update Redis"
		cbasredis hdel "jail:${jname}" jid || echo "WARNING: Failed to update Redis"
		cbsdredis publish cbsd_events '{"cmd":"jstart", "node":"'${nodename}'", "jail":"'${jname}'", "status":-1}'
	fi
fi

cbsdsqlrw local "UPDATE jails SET jid=${myjid},status=${status} where jname='${jname}'"
${RM_CMD} -f ${ftmpdir}/${jname}.conf
fwcounter
expose mode=apply jname=${jname}

if [ ${myjid} -gt 0 ]; then
	if [ -r "${jailsysdir}/${jname}/cpu" ]; then
		. ${jailsysdir}/${jname}/cpu
		echo "${CPUSET_CMD} ${cpuset} -j ${myjid}" 1>&2
		${CPUSET_CMD} ${cpuset} -j ${myjid}
	else
		if [ "${cpuset}" != "0" ]; then
			echo "${CPUSET_CMD} ${cpuset} -j ${myjid}" 1>&2
			${CPUSET_CMD} ${cpuset} -j ${myjid}
		fi
	fi
fi

# VNC auto start
if [ -x "${distmoduledir}/vncterm.d/cbsdvnc" ]; then
	[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}vncterm is installed, launch vncterm session...${N0_COLOR}"
	${miscdir}/daemonize /usr/local/bin/cbsd vncterm jname=${jname} mode=run addr=127.0.0.1 >/dev/null 2>&1
fi

# CBSD QUEUE
if [ "${mod_cbsd_queue_enabled}" = "YES" -a -z "${MOD_CBSD_QUEUE_DISABLED}" ]; then
	[ -n "${cbsd_jail_queue_name}" ] && ${cbsd_queue_backend} cbsd_queue_name=${cbsd_jail_queue_name} id=${jname} cmd=jstart status=2 data_status=1 workdir="${workdir}"
fi

# API compat: ~cbsd/jails-system/info*.*
${miscdir}/daemonize ${toolsdir}/save-jail-info jname=${jname}

gettime end_time
diff_time=$(( end_time - st_time ))
# redis
if [ ${myjid} -gt 0 -a "${mod_cbsd_redis_enabled}" = "YES" -a -z "${MOD_CBSD_REDIS_DISABLED}" ]; then
	cbsdredis hset "jail:${jname}" status 2 jid ${myjid} started ${st_time} echo "WARNING: Failed to update Redis"
	cbsdredis publish cbsd_events '{"cmd":"jstart", "node":"'${nodename}'", "jail":"'${jname}'", "status":0, "duration":'${diff_time}'}'
fi
capture diff_time displaytime ${diff_time}

if [ -f "${jailsysdir}/${jname}/jail-message" -a ${quiet} -ne 1 ]; then
	${ECHO} "${N1_COLOR}---- ${N2_COLOR}jail messages ${N1_COLOR}----${N0_COLOR}" 1>&2
	# todo: trim special/carp/interface value?
	${SED_CMD} -e "s|\${ip4_addr}|${ip4_addr}|g" "${jailsysdir}/${jname}/jail-message" 1>&2
	${ECHO} "${N1_COLOR}---- ${N2_COLOR}jail messages ${N1_COLOR}----${N0_COLOR}" 1>&2
fi

[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}${CIX_APP} done ${N2_COLOR}in ${diff_time}${N0_COLOR}"
cbsdlogger NOTICE ${CIX_APP}: jail ${jname} started in ${diff_time}

[ -r ${ftmpdir}/jstart.${jname}.$$ ] && ${RM_CMD} -f ${ftmpdir}/jstart.${jname}.$$
[ -r ${ftmpdir}/jstart.${jname}.err ] && ${RM_CMD} -f ${ftmpdir}/jstart.${jname}.err

exit 0
