以下是对该脚本的详细分析:
一、基础检查
- DORIS_HOME检查
if [[ -z "${DORIS_HOME}" ]]; then
echo "Error: DORIS_HOME is not set"
exit 1
fi
• 作用:确保环境变量DORIS_HOME
已设置,否则报错退出
- 操作系统检查
if [[ -n "${OSTYPE}" ]]; then
if [[ "${OSTYPE}" != "linux-gnu" ]] && [[ "${OSTYPE:0:6}" != "darwin" ]]; then
echo "Error: Unsupported OS type: ${OSTYPE}"
exit 1
fi
fi
• 作用:仅支持Linux和macOS系统
二、macOS特殊处理
- Homebrew检查
if ! command -v brew &>/dev/null; then
echo "Error: Homebrew is missing..."
exit 1
fi
• 作用:macOS必须安装Homebrew包管理器
- 生成环境配置文件
cat >"${DORIS_HOME}/custom_env_mac.sh" <<EOF
...
EOF
• 作用:生成包含Homebrew工具路径的配置文件
- 路径配置
EXPORT_CELLARS="\${HOMEBREW_REPO_PREFIX}/opt/\${cellar}/bin:\${EXPORT_CELLARS}"
export PATH="\${EXPORT_CELLARS}:/usr/bin:\${PATH}"
• 作用:将Homebrew安装的关键工具(automake/cmake等)加入PATH
三、工具链配置
- Python检查
if ! ${PYTHON} --version; then
echo "Error: ${PYTHON} not found..."
exit 1
fi
• 作用:验证Python环境可用性
- 工具链选择
if [[ "${DORIS_TOOLCHAIN}" == "gcc" ]]; then
# GCC配置
elif [[ "${DORIS_TOOLCHAIN}" == "clang" ]]; then
# Clang配置,C/C++/Objective-C编译器前端,将源代码转换为LLVM中间表示(IR),比GCC更快的编译速度
else
echo "Error: unknown toolchain..."
fi
• 特点:
• 支持GCC和Clang两种工具链
• 自动检测编译器路径
• 配置llvm-cov/llvm-profdata等工具
四、编译优化配置
- ccache配置
export CCACHE_COMPILERCHECK=content
if [[ "${ENABLE_PCH}" == "ON" ]]; then
export CCACHE_PCH_EXTSUM=true
export CCACHE_SLOPPINESS="pch_defines,time_macros"
else
export CCACHE_NOPCH_EXTSUM=true
export CCACHE_SLOPPINESS="default"
fi
• 作用:通过ccache加速编译过程
- 兼容性配置
CLANG_COMPATIBLE_FLAGS="$(...)"
export CLANG_COMPATIBLE_FLAGS
• 作用:生成Clang兼容的编译标志
五、依赖工具检查
- Flex版本检查
flex_ver="$(flex --version | awk '{print $2}')"
required_ver="2.6.0"
if [[ ! "$(...)" = "${required_ver}" ]]; then
echo "Error: flex version too old..."
fi
• 要求:flex >= 2.6.0(因C++17兼容性问题)
- Java环境检查
if [[ "${JAVA_VER}" -lt 52 ]]; then
echo "Error: require JDK 1.8+..."
fi
• 要求:Java 8及以上版本
- 构建工具检查
# Maven检查
if ! "${MVN_CMD}" --version; then exit 1; fi
# CMake检查
if ! "${CMAKE_CMD}" --version; then exit 1; fi
# Ninja检测
if NINJA_VERSION="$(ninja --version 2>/dev/null)"; then
GENERATOR="Ninja"
BUILD_SYSTEM="ninja"
fi
• 关键工具:Maven、CMake、Ninja
六、环境变量配置
export PKG_CONFIG_PATH="${DORIS_HOME}/thirdparty/installed/lib64/pkgconfig:${PKG_CONFIG_PATH}"
• 作用:添加第三方库的pkg-config路径
七、设计特点总结
跨平台支持
• 区分Linux/macOS处理
• macOS自动配置Homebrew环境
• 兼容不同工具链(GCC/Clang)环境隔离
• 生成custom_env_mac.sh
隔离macOS配置
• 支持custom_env.sh
自定义配置严格检查
• 工具版本检查(flex/java)
• 依赖工具存在性检查
• 路径有效性验证性能优化
• ccache配置
• Ninja检测
• 预编译头支持可扩展性
• 通过环境变量定制工具路径
• 支持自定义Maven/CMake路径
八、典型执行流程
- 设置DORIS_HOME
- 检查操作系统
- macOS环境初始化
- 加载自定义配置
- 设置工具链
- 检查编译依赖
- 配置构建系统
- 设置第三方库路径
九、工具链协作流程
工具链协作流程
十、代码
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# check DORIS_HOME
export LC_ALL=C
# 确保环境变量DORIS_HOME已设置,否则报错退出
if [[ -z "${DORIS_HOME}" ]]; then
echo "Error: DORIS_HOME is not set"
exit 1
fi
# 仅支持Linux和macOS系统
if [[ -n "${OSTYPE}" ]]; then
if [[ "${OSTYPE}" != "linux-gnu" ]] && [[ "${OSTYPE:0:6}" != "darwin" ]]; then
echo "Error: Unsupported OS type: ${OSTYPE}"
exit 1
fi
fi
if [[ "$(uname -s)" == 'Darwin' ]]; then
# macOS必须安装Homebrew包管理器
if ! command -v brew &>/dev/null; then
echo "Error: Homebrew is missing. Please install it first due to we use Homebrew to manage the tools which are needed to build the project."
exit 1
fi
# 生成包含Homebrew工具路径的配置文件
cat >"${DORIS_HOME}/custom_env_mac.sh" <<EOF
# This file is generated automatically. PLEASE DO NOT MODIFY IT.
HOMEBREW_REPO_PREFIX="$(brew --prefix)"
CELLARS=(
automake
autoconf
libtool
pkg-config
texinfo
coreutils
gnu-getopt
python@3
cmake
ninja
ccache
bison
byacc
gettext
wget
pcre
maven
llvm@16
)
# 将Homebrew安装的关键工具(automake/cmake等)加入PATH
for cellar in "\${CELLARS[@]}"; do
EXPORT_CELLARS="\${HOMEBREW_REPO_PREFIX}/opt/\${cellar}/bin:\${EXPORT_CELLARS}"
done
export PATH="\${EXPORT_CELLARS}:/usr/bin:\${PATH}"
export DORIS_BUILD_PYTHON_VERSION='python3'
export NODE_OPTIONS='--openssl-legacy-provider'
EOF
DORIS_HOME_ABSOLUATE_PATH="$(
set -e
cd "${DORIS_HOME}"
pwd
)"
SOURCE_MAC_ENV_CONTENT="source '${DORIS_HOME_ABSOLUATE_PATH}/custom_env_mac.sh'"
if [[ ! -f "${DORIS_HOME}/custom_env.sh" ]] ||
! grep "${SOURCE_MAC_ENV_CONTENT}" "${DORIS_HOME}/custom_env.sh" &>/dev/null; then
echo "${SOURCE_MAC_ENV_CONTENT}" >>"${DORIS_HOME}/custom_env.sh"
fi
fi
# include custom environment variables
if [[ -f "${DORIS_HOME}/custom_env.sh" ]]; then
# shellcheck disable=1091
. "${DORIS_HOME}/custom_env.sh"
fi
# 设置DORIS三方依赖包目录
if [[ -z "${DORIS_THIRDPARTY}" ]]; then
export DORIS_THIRDPARTY="${DORIS_HOME}/thirdparty"
fi
# 验证Python环境可用性
if [[ -z "${DORIS_BUILD_PYTHON_VERSION}" ]]; then
DORIS_BUILD_PYTHON_VERSION="python"
fi
export PYTHON="${DORIS_BUILD_PYTHON_VERSION}"
if ! ${PYTHON} --version; then
echo "Error: ${PYTHON} is not found, maybe you should set DORIS_BUILD_PYTHON_VERSION."
exit 1
fi
if [[ -z "${DORIS_TOOLCHAIN}" ]]; then
if [[ "$(uname -s)" == 'Darwin' ]]; then
DORIS_TOOLCHAIN=clang
else
DORIS_TOOLCHAIN=clang
fi
fi
# 工具链选择,支持GCC和Clang两种工具链
if [[ "${DORIS_TOOLCHAIN}" == "gcc" ]]; then
# set GCC HOME
if [[ -z "${DORIS_GCC_HOME}" ]]; then
DORIS_GCC_HOME="$(dirname "$(command -v gcc)")"/..
export DORIS_GCC_HOME
fi
export CC="${DORIS_GCC_HOME}/bin/gcc"
export CXX="${DORIS_GCC_HOME}/bin/g++"
if test -x "${DORIS_GCC_HOME}/bin/ld"; then
export DORIS_BIN_UTILS="${DORIS_GCC_HOME}/bin/"
fi
ENABLE_PCH='OFF'
# 工具链选择,支持GCC和Clang两种工具链
elif [[ "${DORIS_TOOLCHAIN}" == "clang" ]]; then
# set CLANG HOME
if [[ -z "${DORIS_CLANG_HOME}" ]]; then
DORIS_CLANG_HOME="$(dirname "$(command -v clang)")"/..
export DORIS_CLANG_HOME
fi
export CC="${DORIS_CLANG_HOME}/bin/clang"
export CXX="${DORIS_CLANG_HOME}/bin/clang++"
if test -x "${DORIS_CLANG_HOME}/bin/ld.lld"; then
export DORIS_BIN_UTILS="${DORIS_CLANG_HOME}/bin/"
fi
if [[ -f "${DORIS_CLANG_HOME}/bin/llvm-symbolizer" ]]; then
export ASAN_SYMBOLIZER_PATH="${DORIS_CLANG_HOME}/bin/llvm-symbolizer"
fi
covs=()
while IFS='' read -r line; do covs+=("${line}"); done <<<"$(find "${DORIS_CLANG_HOME}" -name "llvm-cov*")"
if [[ ${#covs[@]} -ge 1 ]]; then
LLVM_COV="${covs[0]}"
else
LLVM_COV="$(command -v llvm-cov)"
fi
# 配置llvm-cov/llvm-profdata等工具,编译器基础设施框架,对中间代码进行多层级优化,将优化后的IR转换为目标机器码
export LLVM_COV
profdatas=()
while IFS='' read -r line; do profdatas+=("${line}"); done <<<"$(find "${DORIS_CLANG_HOME}" -name "llvm-profdata*")"
if [[ ${#profdatas[@]} -ge 1 ]]; then
LLVM_PROFDATA="${profdatas[0]}"
else
LLVM_PROFDATA="$(command -v llvm-profdata)"
fi
export LLVM_PROFDATA
if [[ -z "${ENABLE_PCH}" ]]; then
ENABLE_PCH='ON'
fi
else
echo "Error: unknown DORIS_TOOLCHAIN=${DORIS_TOOLCHAIN}, currently only 'gcc' and 'clang' are supported"
exit 1
fi
# 通过ccache加速编译过程,编译缓存加速工具,减少重复编译时间(增量构建加速50%-90%)
export CCACHE_COMPILERCHECK=content
if [[ "${ENABLE_PCH}" == "ON" ]]; then
export CCACHE_PCH_EXTSUM=true
export CCACHE_SLOPPINESS="pch_defines,time_macros"
else
export CCACHE_NOPCH_EXTSUM=true
export CCACHE_SLOPPINESS="default"
fi
if [[ -z "${DORIS_BIN_UTILS}" ]]; then
export DORIS_BIN_UTILS='/usr/bin/'
fi
if [[ -z "${DORIS_GCC_HOME}" ]]; then
DORIS_GCC_HOME="$(dirname "$(command -v gcc)")/.."
export DORIS_GCC_HOME
fi
if [[ ! -f "${DORIS_GCC_HOME}/bin/gcc" ]]; then
echo "Error: wrong directory DORIS_GCC_HOME=${DORIS_GCC_HOME}"
exit 1
fi
# 生成Clang兼容的编译标志
# export CLANG COMPATIBLE FLAGS
CLANG_COMPATIBLE_FLAGS="$(echo | "${DORIS_GCC_HOME}/bin/gcc" -Wp,-v -xc++ - -fsyntax-only 2>&1 |
grep -E '^\s+/' | awk '{print "-I" $1}' | tr '\n' ' ')"
export CLANG_COMPATIBLE_FLAGS
# if is called from build-thirdparty.sh, no need to check these tools
if test -z "${BUILD_THIRDPARTY_WIP:-}"; then
# 在c++ 17中禁止使用register关键字
# flex生成的c++代码在2.6.0版本之后删除了register关键字
# 所以我们需要检查flex版本在这里,以避免编译失败
# 词法分析器生成器,工作流程:
# 定义词法规则(正则表达式)
# 生成C语言词法分析器
# 与Bison(语法分析器)配合使用
# 典型应用:
# 配置文件解析(如JSON/YAML)
# 领域特定语言(DSL)开发
# 编译器前端实现
flex_ver="$(flex --version | awk '{print $2}')"
required_ver="2.6.0"
if [[ ! "$(printf '%s\n' "${required_ver}" "${flex_ver}" | sort -V | head -n1)" = "${required_ver}" ]]; then
echo "Error: flex version (${flex_ver}) must be greater than or equal to ${required_ver}"
exit 1
fi
# check java home
if [[ -z "${JAVA_HOME}" ]]; then
JAVA="$(command -v java)"
JAVAP="$(command -v javap)"
else
JAVA="${JAVA_HOME}/bin/java"
JAVAP="${JAVA_HOME}/bin/javap"
fi
export JAVA
if [[ ! -x "${JAVA}" ]]; then
echo "The JAVA_HOME environment variable is not defined correctly"
echo "This environment variable is needed to run this program"
echo "NB: JAVA_HOME should point to a JDK not a JRE"
exit 1
fi
JAVA_VER="$("${JAVAP}" -verbose java.lang.String | grep "major version" | cut -d " " -f5)"
# Java 8及以上版本
if [[ "${JAVA_VER}" -lt 52 ]]; then
echo "Error: require JAVA with JDK version at least 1.8"
exit 1
fi
# Maven检查
MVN_CMD='mvn'
if [[ -n "${CUSTOM_MVN}" ]]; then
MVN_CMD="${CUSTOM_MVN}"
fi
if ! "${MVN_CMD}" --version; then
echo "Error: mvn is not found"
exit 1
fi
export MVN_CMD
fi
if [[ "$(uname -s)" == 'Darwin' ]]; then
if ! command -v libtoolize &>/dev/null && command -v glibtoolize &>/dev/null; then
shopt -s expand_aliases
alias libtoolize='glibtoolize'
fi
fi
# 跨平台构建系统生成器,生成多种构建系统文件:Makefile(Unix)、Ninja(高速构建)
CMAKE_CMD='cmake'
if [[ -n "${CUSTOM_CMAKE}" ]]; then
CMAKE_CMD="${CUSTOM_CMAKE}"
fi
# CMake检查
if ! "${CMAKE_CMD}" --version; then
echo "Error: cmake is not found"
exit 1
fi
export CMAKE_CMD
GENERATOR="Unix Makefiles"
BUILD_SYSTEM="make"
# Ninja检测,高性能构建系统,与CMake配合
if NINJA_VERSION="$(ninja --version 2>/dev/null)"; then
echo "ninja ${NINJA_VERSION}"
GENERATOR="Ninja"
BUILD_SYSTEM="ninja"
fi
if CCACHE_VERSION="$(ccache --version 2>/dev/null)"; then
echo "${CCACHE_VERSION}" | head -n 1
# shellcheck disable=2034
CMAKE_USE_CCACHE="-DCMAKE_CXX_COMPILER_LAUNCHER=ccache"
fi
export GENERATOR
export BUILD_SYSTEM
# 添加第三方库的pkg-config路径
export PKG_CONFIG_PATH="${DORIS_HOME}/thirdparty/installed/lib64/pkgconfig:${PKG_CONFIG_PATH}"
该脚本是Doris项目构建系统的核心环境配置脚本,通过严格的检查和灵活的配置,确保在不同平台上都能获得一致的构建环境。