全棧交叉編譯X86完成過程經驗分享

周榮華發表於2022-04-23

1 CMAKE的交叉編譯配置

主要是C和C++編譯器的配置和SYSROOT的配置。

set (CMAKE_SYSTEM_NAME "Linux")
set (CMAKE_SYSTEM_PROCESSOR "x86_64")

set (default_toolchain_folder /home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux)
if (EXISTS "${default_toolchain_folder}/bin")
    set (toolchain_path "${default_toolchain_folder}/bin")
else ()
    message (FATAL_ERROR "No clang toolchain  ${default_toolchain_folder} support on this platform")
endif ()

set (CMAKE_C_COMPILER   "${toolchain_path}/x86_64-linux-gnu-gcc")
set (CMAKE_CXX_COMPILER "${toolchain_path}/x86_64-linux-gnu-g++")

# Where is the target environment.
set (CMAKE_FIND_ROOT_PATH "${default_toolchain_folder}")
set (CMAKE_SYSROOT "${default_toolchain_folder}")
#set (CMAKE_SYSROOT "${default_toolchain_folder}")
#set (CMAKE_SYSROOT "${default_toolchain_folder}/x86_64-linux-gnu")
set (ONLY_CMAKE_FIND_ROOT_PATH ON)
# Search for programs in the build host directories.
set (CMAKE_FIND_ROOT_PATH_MODE_PROGRAM BOTH)

# Search for libraries and headers in both the target and host directories.
set (CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
set (CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)

# for opencv
set (CV_DISABLE_OPTIMIZATION ON)
set (CPU_BASELINE "")
set (CPU_DISPATCH "")

add_compile_options (--sysroot=/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux)

add_link_options (--sysroot=/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux)

set (CPP_STDLIB_FILE
    "${default_toolchain_folder}/x86_64-linux-gnu/lib64/libstdc++.so"
    "${default_toolchain_folder}/x86_64-linux-gnu/lib64/libstdc++.so.6"
    "${default_toolchain_folder}/x86_64-linux-gnu/lib64/libstdc++.so.6.0.21"
    )

# Use thin archives to save disk space
if (NOT DISABLE_THIN)
    set (CMAKE_C_ARCHIVE_CREATE   "<CMAKE_AR> Tcr <TARGET> <LINK_FLAGS> <OBJECTS>")
    set (CMAKE_C_ARCHIVE_APPEND   "<CMAKE_AR> Tr  <TARGET> <LINK_FLAGS> <OBJECTS>")
    set (CMAKE_CXX_ARCHIVE_CREATE "<CMAKE_AR> Tcr <TARGET> <LINK_FLAGS> <OBJECTS>")
    set (CMAKE_CXX_ARCHIVE_APPEND "<CMAKE_AR> Tr  <TARGET> <LINK_FLAGS> <OBJECTS>")
endif()

set (MPI_mpi_LIBRARY "${default_toolchain_folder}/lib/libmpi.so")
set (MPI_C_HEADER_DIR "${default_toolchain_folder}/x86_64-linux-gnu/include/")
set (MPI_CXX_HEADER_DIR "${default_toolchain_folder}/x86_64-linux-gnu/include/")
set (MPI_mpi_cxx_LIBRARY "${default_toolchain_folder}/lib/libmpi_cxx.so")
set (MPI_LIBRARIES ${MPI_mpi_LIBRARY} ${MPI_mpi_cxx_LIBRARY})

set (CMAKE_TOOLCHAIN_FILE_FULL_PATH ${CMAKE_SOURCE_DIR}/infra/cmake/module/toolchain/linux_cross_gcc5_glibc2.17_x86_64.cmake)
#message("CMAKE_TOOLCHAIN_FILE_FULL_PATH is ${CMAKE_TOOLCHAIN_FILE_FULL_PATH}")


option (CROSS_COMPILE "if is cross compile"   TRUE)

 

還有一些是第三方庫的cmake巨集定義,例如openmpi,zlib:

set (MPI_mpi_LIBRARY "${default_toolchain_folder}/lib/libmpi.so")
set (MPI_C_HEADER_DIR "${default_toolchain_folder}/x86_64-linux-gnu/include/")
set (MPI_CXX_HEADER_DIR "${default_toolchain_folder}/x86_64-linux-gnu/include/")
set (MPI_mpi_cxx_LIBRARY "${default_toolchain_folder}/lib/libmpi_cxx.so")
set (MPI_LIBRARIES ${MPI_mpi_LIBRARY} ${MPI_mpi_cxx_LIBRARY})

 

2 工具鏈本身的修改

2.1 sdk庫和標頭檔案的依賴

tops程式碼棧之前編譯過程中依賴sdk的安裝,所以建立了交叉編譯工具鏈到sdk的lib庫和dtu標頭檔案路徑的軟連結,後面改成cmake_build下面的子目錄傳進來的做法之後,很多庫在改變sysroot之後找不到了,所以在交叉編譯工具鏈的根目錄下面建立了到/home的軟連結,確保改變sysroot之後也能找到相關的庫和標頭檔案。

2.2 x86_64-linux-gnu/lib和lib/x86_64-linux-gnu有什麼差別

不同編譯軟體的搜尋路徑不一樣,所以這些編譯軟體編譯出來的庫會存在搜尋路徑不一致的情況,這時為了統一,在某個目錄放好原始檔之後,另外一個目錄需要建好軟連結。

同樣的差異還體現在include和x86_64-linux-gnu/include。

當前編譯器的搜尋路徑可以用下面的命令查詢。

庫檔案的搜尋路徑的查詢命令“/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/bin/x86_64-linux-gnu-gcc -print-search-dirs”,路徑用“:”分隔:

/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/bin/x86_64-linux-gnu-gcc -print-search-dirs
install: /home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/lib/gcc/x86_64-linux-gnu/5.5.0/
programs: =/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/libexec/gcc/x86_64-linux-gnu/5.5.0/:/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/libexec/gcc/x86_64-linux-gnu/5.5.0/:/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/libexec/gcc/x86_64-linux-gnu/:/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/lib/gcc/x86_64-linux-gnu/5.5.0/:/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/lib/gcc/x86_64-linux-gnu/:/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/lib/gcc/x86_64-linux-gnu/5.5.0/../../../../x86_64-linux-gnu/bin/x86_64-linux-gnu/5.5.0/:/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/lib/gcc/x86_64-linux-gnu/5.5.0/../../../../x86_64-linux-gnu/bin/
libraries: =/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/lib/gcc/x86_64-linux-gnu/5.5.0/:/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/lib/gcc/x86_64-linux-gnu/5.5.0/../../../../x86_64-linux-gnu/lib/x86_64-linux-gnu/5.5.0/:/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/lib/gcc/x86_64-linux-gnu/5.5.0/../../../../x86_64-linux-gnu/lib/../lib64/:/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/lib/gcc/x86_64-linux-gnu/5.5.0/../../../../x86_64-linux-gnu/lib/

 

 

標頭檔案搜尋路徑的查詢命令“/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/bin/x86_64-linux-gnu-gcc -E -v -”:

/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/bin/x86_64-linux-gnu-gcc -E -v -
Using built-in specs.
COLLECT_GCC=/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/bin/x86_64-linux-gnu-gcc
Target: x86_64-linux-gnu
Configured with: ../configure --prefix=/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux --target=x86_64-linux-gnu --with-glibc-version=2.17 --enable-languages=c,c++ --disable-multilib
Thread model: posix
gcc version 5.5.0 (GCC)
COLLECT_GCC_OPTIONS='-E' '-v' '-mtune=generic' '-march=x86-64'
 /home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/libexec/gcc/x86_64-linux-gnu/5.5.0/cc1 -E -quiet -v - -mtune=generic -march=x86-64
ignoring nonexistent directory "/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/lib/gcc/x86_64-linux-gnu/5.5.0/../../../../x86_64-linux-gnu/sys-include"
#include "..." search starts here:
#include <...> search starts here:
 /home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/lib/gcc/x86_64-linux-gnu/5.5.0/include
 /home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/lib/gcc/x86_64-linux-gnu/5.5.0/include-fixed
 /home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/lib/gcc/x86_64-linux-gnu/5.5.0/../../../../x86_64-linux-gnu/include
End of search list.

 

3 第三方元件找不到

一般需要新增到交叉編譯工具鏈中。

3.1 大的第三方元件找不到

以python找不到為例。

需要在交叉編譯工具鏈中增加python版本,當前增加了python3.5和python3.6,後續還需要增加python3.7和3.8,注意編譯的時候需要增加sysroot的CFLAGS。

這類比較大的第三方元件需要用現有的交叉編譯工具鏈進行編譯。

3.2 小的第三方元件找不到

以sqlit找不到為例。

查了一下ubuntu1604上的sqlite包,發現裡面不含glibc2.17以上的符號,直接下載增加到交叉編譯工具鏈中。

查詢:

apt search sqlite

下載:

apt download libsqlite0   //庫檔案

apt download libsqlite0-dev  //標頭檔案

 

解壓:

deb包解壓需要2步,先解壓出data.tar.xz,再解壓出資料檔案:

ar -xf libsqlite0-dev_2.8.17-12fakesync1_amd64.deb

tar -xf data.tar.xz

ar -xf libsqlite0_2.8.17-12fakesync1_amd64.deb
tar -xf data.tar.xz

 

解壓完之後的目錄結構如下,其中lib和include是我們需要的庫和標頭檔案,share部分文件不需要。將lib和include拷貝到交叉編譯工具鏈的目錄。

tree usr/
usr/
|-- include
|   `-- sqlite.h
|-- lib
|   |-- libsqlite.a
|   |-- libsqlite.la
|   |-- libsqlite.so -> libsqlite.so.0.8.6
|   |-- libsqlite.so.0 -> libsqlite.so.0.8.6
|   |-- libsqlite.so.0.8.6
|   `-- pkgconfig
|       `-- sqlite.pc
`-- share
    `-- doc
        |-- libsqlite0
        |   |-- README.Debian
        |   |-- changelog.Debian.gz
        |   `-- copyright
        `-- libsqlite0-dev
            |-- changelog.Debian.gz -> ../libsqlite0/changelog.Debian.gz
            `-- copyright

 

3.3 判斷庫是否使用了高版本GLIBC介面的方法

很多對外發布的庫裡面是不含symbol的,所以用nm命令查不到任何內容,這時可以使用strings命令來查,下面的命令結果表面libsqlite.so.0.8.6中需要使用GLIBC_2.2.5 / GLIBC_2.3 / GLIBC_2.4 / GLIBC_2.15 這幾個版本的介面,當前環境只要GLIBC高於2.14就可以正常執行,我們當前的交叉編譯工具鏈的版本是2.17,這種情況下就不需要額外自己編譯這個庫了。

strings usr/lib/libsqlite.so.0.8.6 | grep "^GLIBC"
GLIBC_2.3
GLIBC_2.14
GLIBC_2.4
GLIBC_2.2.5

 

4 非頂級tops的獨立的cmake專案編譯

主要是交叉編譯工具鏈的傳遞,考慮到獨立的cmake專案對應的infra的相對路徑可能不一致,所以不能傳相對路徑的交叉編譯工具鏈配置檔案給獨立的cmake專案,每個交叉編譯工具鏈增加

CMAKE_TOOLCHAIN_FILE_FULL_PATH 變數類定義自己的全路徑交叉工具鏈配置檔名。
set (CMAKE_TOOLCHAIN_FILE_FULL_PATH ${CMAKE_SOURCE_DIR}/xxx/linux_cross_gcc5_glibc2.17_x86_64.cmake)

 

5 bazel專案的適配

先在sdk/toolchain/下面配置好,其他bazel專案建立到這個目錄的軟連結,確保全域性只有一套bazel的交叉編譯工具鏈配置。

--crosstool_top=//toolchain/linux_cross_gcc5_glibc2.17_x86_64:cross_suite

 

對TensorFlow,如果配置了交叉編譯工具鏈,還要配置host工具鏈,方便部分臨時編譯的host程式能通過執行來生成新的原始碼,host工具鏈配置是bazel自帶的。

--host_crosstool_top=@bazel_tools//tools/cpp:toolchain

 

bazel跨版本間的相容性很差,一般網上別人給的toolchain配置,如果版本和當前不一樣,一般都不能直接用,建議參考bazel編譯過程中bazel自己生成的,一般在bazel編譯路徑的下面目錄中可以找到:

external/bazel_tools/tools/cpp/BUILD

external/bazel_tools/tools/cpp/cc_toolchain_config.bzl

bazel的配置項很多,大多數可以不改,但需要裁減掉不是相關host或者target的編譯配置。

BUILD檔案:

package(default_visibility = ["//visibility:public"])

licenses(["notice"])  # Apache 2.0

load(":cc_toolchain_config.bzl", "cc_toolchain_config")

# The toolchain type used to distinguish cc toolchains.
toolchain_type(name = "toolchain_type")

cc_library(
    name = "malloc",
)

filegroup(
    name = "grep-includes",
    srcs = ["grep-includes.sh"],
)

filegroup(
    name = "empty",
    srcs = []
)

# This is the entry point for --crosstool_top.  Toolchains are found
# by lopping off the name of --crosstool_top and searching for
# "cc-compiler-${CPU}" in this BUILD file, where CPU is the target CPU
# specified in --cpu.
#
# This file group should include
#   * all cc_toolchain targets supported
#   * all file groups that said cc_toolchain might refer to

# Hardcoded toolchain, legacy behaviour.
cc_toolchain_suite(
    name = "crossx86-toolchain",
    toolchains = {
        "k8": ":cc-compiler-crossx86",
    },
)

cc_toolchain(
    name = "cc-compiler-crossx86",
    all_files = ":empty",
    ar_files = ":empty",
    as_files = ":empty",
    compiler_files = ":empty",
    dwp_files = ":empty",
    linker_files = ":empty",
    objcopy_files = ":empty",
    strip_files = ":empty",
    supports_param_files = 1,
    toolchain_config = ":local_linux",
    toolchain_identifier = "local_linux",
)

cc_toolchain_config(
    name = "local_linux"
)

toolchain(
    name = "cc-toolchain-local",
    exec_compatible_with = [
    ],
    target_compatible_with = [
    ],
    toolchain = ":cc-compiler-local",
    toolchain_type = ":toolchain_type",
)

filegroup(
    name = "srcs",
    srcs = glob(["**"]) + [
        "//tools/cpp/runfiles:srcs",
    ],
)

filegroup(
    name = "embedded_tools",
    srcs = glob(["**"]) + [
        "//tools/cpp/runfiles:embedded_tools",
    ],
)

filegroup(
    name = "interface_library_builder",
    srcs = ["build_interface_so"],
)

filegroup(
    name = "osx_wrapper",
    srcs = ["osx_cc_wrapper.sh"],
)

filegroup(
    name = "link_dynamic_library",
    srcs = ["link_dynamic_library.sh"],
)

filegroup(
    name = "lib_cc_configure",
    srcs = ["lib_cc_configure.bzl"],
)

cc_toolchain_config.bzl:

# Copyright 2019 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""A Starlark cc_toolchain configuration rule"""

load(
    "@bazel_tools//tools/cpp:cc_toolchain_config_lib.bzl",
    "action_config",
    "artifact_name_pattern",
    "env_entry",
    "env_set",
    "feature",
    "feature_set",
    "flag_group",
    "flag_set",
    "make_variable",
    "tool",
    "tool_path",
    "variable_with_value",
    "with_feature_set",
)
load(
    "@bazel_tools//tools/build_defs/cc:action_names.bzl",
    _ASSEMBLE_ACTION_NAME = "ASSEMBLE_ACTION_NAME",
    _CLIF_MATCH_ACTION_NAME = "CLIF_MATCH_ACTION_NAME",
    _CPP_COMPILE_ACTION_NAME = "CPP_COMPILE_ACTION_NAME",
    _CPP_HEADER_PARSING_ACTION_NAME = "CPP_HEADER_PARSING_ACTION_NAME",
    _CPP_LINK_DYNAMIC_LIBRARY_ACTION_NAME = "CPP_LINK_DYNAMIC_LIBRARY_ACTION_NAME",
    _CPP_LINK_EXECUTABLE_ACTION_NAME = "CPP_LINK_EXECUTABLE_ACTION_NAME",
    _CPP_LINK_NODEPS_DYNAMIC_LIBRARY_ACTION_NAME = "CPP_LINK_NODEPS_DYNAMIC_LIBRARY_ACTION_NAME",
    _CPP_LINK_STATIC_LIBRARY_ACTION_NAME = "CPP_LINK_STATIC_LIBRARY_ACTION_NAME",
    _CPP_MODULE_CODEGEN_ACTION_NAME = "CPP_MODULE_CODEGEN_ACTION_NAME",
    _CPP_MODULE_COMPILE_ACTION_NAME = "CPP_MODULE_COMPILE_ACTION_NAME",
    _C_COMPILE_ACTION_NAME = "C_COMPILE_ACTION_NAME",
    _LINKSTAMP_COMPILE_ACTION_NAME = "LINKSTAMP_COMPILE_ACTION_NAME",
    _LTO_BACKEND_ACTION_NAME = "LTO_BACKEND_ACTION_NAME",
    _LTO_INDEXING_ACTION_NAME = "LTO_INDEXING_ACTION_NAME",
    _PREPROCESS_ASSEMBLE_ACTION_NAME = "PREPROCESS_ASSEMBLE_ACTION_NAME",
    _STRIP_ACTION_NAME = "STRIP_ACTION_NAME",
)

all_compile_actions = [
    _C_COMPILE_ACTION_NAME,
    _CPP_COMPILE_ACTION_NAME,
    _LINKSTAMP_COMPILE_ACTION_NAME,
    _ASSEMBLE_ACTION_NAME,
    _PREPROCESS_ASSEMBLE_ACTION_NAME,
    _CPP_HEADER_PARSING_ACTION_NAME,
    _CPP_MODULE_COMPILE_ACTION_NAME,
    _CPP_MODULE_CODEGEN_ACTION_NAME,
    _CLIF_MATCH_ACTION_NAME,
    _LTO_BACKEND_ACTION_NAME,
]

all_cpp_compile_actions = [
    _CPP_COMPILE_ACTION_NAME,
    _LINKSTAMP_COMPILE_ACTION_NAME,
    _CPP_HEADER_PARSING_ACTION_NAME,
    _CPP_MODULE_COMPILE_ACTION_NAME,
    _CPP_MODULE_CODEGEN_ACTION_NAME,
    _CLIF_MATCH_ACTION_NAME,
]

preprocessor_compile_actions = [
    _C_COMPILE_ACTION_NAME,
    _CPP_COMPILE_ACTION_NAME,
    _LINKSTAMP_COMPILE_ACTION_NAME,
    _PREPROCESS_ASSEMBLE_ACTION_NAME,
    _CPP_HEADER_PARSING_ACTION_NAME,
    _CPP_MODULE_COMPILE_ACTION_NAME,
    _CLIF_MATCH_ACTION_NAME,
]

codegen_compile_actions = [
    _C_COMPILE_ACTION_NAME,
    _CPP_COMPILE_ACTION_NAME,
    _LINKSTAMP_COMPILE_ACTION_NAME,
    _ASSEMBLE_ACTION_NAME,
    _PREPROCESS_ASSEMBLE_ACTION_NAME,
    _CPP_MODULE_CODEGEN_ACTION_NAME,
    _LTO_BACKEND_ACTION_NAME,
]

all_link_actions = [
    _CPP_LINK_EXECUTABLE_ACTION_NAME,
    _CPP_LINK_DYNAMIC_LIBRARY_ACTION_NAME,
    _CPP_LINK_NODEPS_DYNAMIC_LIBRARY_ACTION_NAME,
]

def _impl(ctx):
    toolchain_identifier = "local_linux"
    host_system_name = "local"
    target_system_name = "local"
    target_cpu = "local"
    target_libc = "GLIBC_2.17"
    abi_libc_version = "2.17"
    compiler = "compiler"
    abi_version = "gcc-5.5-cxx11"
    cc_target_os = None
    builtin_sysroot = "/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux"
    objcopy_embed_data_action = action_config(
        action_name = "objcopy_embed_data",
        enabled = True,
        tools = [tool(path = "/usr/bin/objcopy")],
    )
    action_configs = [objcopy_embed_data_action]
    random_seed_feature = feature(name = "random_seed", enabled = True)

    default_link_flags_feature = feature(
        name = "default_link_flags",
        enabled = True,
        flag_sets = [
            flag_set(
                actions = all_link_actions,
                flag_groups = [
                    flag_group(
                        flags = [
                            "-lstdc++",
                            "-Wl,-z,relro,-z,now",
                            "-no-canonical-prefixes",
                            "-pass-exit-codes",
                        ],
                    ),
                ],
            ),
            flag_set(
                actions = all_link_actions,
                flag_groups = [flag_group(flags = ["-Wl,--gc-sections"])],
                with_features = [with_feature_set(features = ["opt"])],
            ),
        ],
    )

    unfiltered_compile_flags_feature = feature(
        name = "unfiltered_compile_flags",
        enabled = True,
        flag_sets = [
            flag_set(
                actions = [
                    _ASSEMBLE_ACTION_NAME,
                    _PREPROCESS_ASSEMBLE_ACTION_NAME,
                    _LINKSTAMP_COMPILE_ACTION_NAME,
                    _C_COMPILE_ACTION_NAME,
                    _CPP_COMPILE_ACTION_NAME,
                    _CPP_HEADER_PARSING_ACTION_NAME,
                    _CPP_MODULE_COMPILE_ACTION_NAME,
                    _CPP_MODULE_CODEGEN_ACTION_NAME,
                    _LTO_BACKEND_ACTION_NAME,
                    _CLIF_MATCH_ACTION_NAME,
                ],
                flag_groups = [
                    flag_group(
                        flags = [
                            "-no-canonical-prefixes",
                            "-fno-canonical-system-headers",
                            "-Wno-builtin-macro-redefined",
                            "-D__DATE__=\"redacted\"",
                            "-D__TIMESTAMP__=\"redacted\"",
                            "-D__TIME__=\"redacted\"",
                        ],
                    ),
                ],
            ),
        ],
    )

    supports_pic_feature = feature(name = "supports_pic", enabled = True)

    default_compile_flags_feature = feature(
        name = "default_compile_flags",
        enabled = True,
        flag_sets = [
            flag_set(
                actions = [
                    _ASSEMBLE_ACTION_NAME,
                    _PREPROCESS_ASSEMBLE_ACTION_NAME,
                    _LINKSTAMP_COMPILE_ACTION_NAME,
                    _C_COMPILE_ACTION_NAME,
                    _CPP_COMPILE_ACTION_NAME,
                    _CPP_HEADER_PARSING_ACTION_NAME,
                    _CPP_MODULE_COMPILE_ACTION_NAME,
                    _CPP_MODULE_CODEGEN_ACTION_NAME,
                    _LTO_BACKEND_ACTION_NAME,
                    _CLIF_MATCH_ACTION_NAME,
                ],
                flag_groups = [
                    flag_group(
                        flags = [
                            "-U_FORTIFY_SOURCE",
                            "-D_FORTIFY_SOURCE=1",
                            "-fstack-protector",
                            "-Wall",
                            "-Wunused-but-set-parameter",
                            "-Wno-free-nonheap-object",
                            "-fno-omit-frame-pointer",
                        ],
                    ),
                ],
            ),
            flag_set(
                actions = [
                    _ASSEMBLE_ACTION_NAME,
                    _PREPROCESS_ASSEMBLE_ACTION_NAME,
                    _LINKSTAMP_COMPILE_ACTION_NAME,
                    _C_COMPILE_ACTION_NAME,
                    _CPP_COMPILE_ACTION_NAME,
                    _CPP_HEADER_PARSING_ACTION_NAME,
                    _CPP_MODULE_COMPILE_ACTION_NAME,
                    _CPP_MODULE_CODEGEN_ACTION_NAME,
                    _LTO_BACKEND_ACTION_NAME,
                    _CLIF_MATCH_ACTION_NAME,
                ],
                flag_groups = [flag_group(flags = ["-g"])],
                with_features = [with_feature_set(features = ["dbg"])],
            ),
            flag_set(
                actions = [
                    _ASSEMBLE_ACTION_NAME,
                    _PREPROCESS_ASSEMBLE_ACTION_NAME,
                    _LINKSTAMP_COMPILE_ACTION_NAME,
                    _C_COMPILE_ACTION_NAME,
                    _CPP_COMPILE_ACTION_NAME,
                    _CPP_HEADER_PARSING_ACTION_NAME,
                    _CPP_MODULE_COMPILE_ACTION_NAME,
                    _CPP_MODULE_CODEGEN_ACTION_NAME,
                    _LTO_BACKEND_ACTION_NAME,
                    _CLIF_MATCH_ACTION_NAME,
                ],
                flag_groups = [
                    flag_group(
                        flags = [
                            "-g0",
                            "-O2",
                            "-DNDEBUG",
                            "-ffunction-sections",
                            "-fdata-sections",
                        ],
                    ),
                ],
                with_features = [with_feature_set(features = ["opt"])],
            ),
            flag_set(
                actions = [
                    _LINKSTAMP_COMPILE_ACTION_NAME,
                    _CPP_COMPILE_ACTION_NAME,
                    _CPP_HEADER_PARSING_ACTION_NAME,
                    _CPP_MODULE_COMPILE_ACTION_NAME,
                    _CPP_MODULE_CODEGEN_ACTION_NAME,
                    _LTO_BACKEND_ACTION_NAME,
                    _CLIF_MATCH_ACTION_NAME,
                ],
                flag_groups = [flag_group(flags = ["-std=c++0x"])],
            ),
        ],
    )

    opt_feature = feature(name = "opt")

    supports_dynamic_linker_feature = feature(name = "supports_dynamic_linker", enabled = True)

    objcopy_embed_flags_feature = feature(
        name = "objcopy_embed_flags",
        enabled = True,
        flag_sets = [
            flag_set(
                actions = ["objcopy_embed_data"],
                flag_groups = [flag_group(flags = ["-I", "binary"])],
            ),
        ],
    )

    dbg_feature = feature(name = "dbg")

    user_compile_flags_feature = feature(
        name = "user_compile_flags",
        enabled = True,
        flag_sets = [
            flag_set(
                actions = [
                    _ASSEMBLE_ACTION_NAME,
                    _PREPROCESS_ASSEMBLE_ACTION_NAME,
                    _LINKSTAMP_COMPILE_ACTION_NAME,
                    _C_COMPILE_ACTION_NAME,
                    _CPP_COMPILE_ACTION_NAME,
                    _CPP_HEADER_PARSING_ACTION_NAME,
                    _CPP_MODULE_COMPILE_ACTION_NAME,
                    _CPP_MODULE_CODEGEN_ACTION_NAME,
                    _LTO_BACKEND_ACTION_NAME,
                    _CLIF_MATCH_ACTION_NAME,
                ],
                flag_groups = [
                    flag_group(
                        flags = ["%{user_compile_flags}"],
                        iterate_over = "user_compile_flags",
                        expand_if_available = "user_compile_flags",
                    ),
                ],
            ),
        ],
    )

    sysroot_feature = feature(
        name = "sysroot",
        enabled = True,
        flag_sets = [
            flag_set(
                actions = [
                    _PREPROCESS_ASSEMBLE_ACTION_NAME,
                    _LINKSTAMP_COMPILE_ACTION_NAME,
                    _C_COMPILE_ACTION_NAME,
                    _CPP_COMPILE_ACTION_NAME,
                    _CPP_HEADER_PARSING_ACTION_NAME,
                    _CPP_MODULE_COMPILE_ACTION_NAME,
                    _CPP_MODULE_CODEGEN_ACTION_NAME,
                    _LTO_BACKEND_ACTION_NAME,
                    _CLIF_MATCH_ACTION_NAME,
                    _CPP_LINK_EXECUTABLE_ACTION_NAME,
                    _CPP_LINK_DYNAMIC_LIBRARY_ACTION_NAME,
                    _CPP_LINK_NODEPS_DYNAMIC_LIBRARY_ACTION_NAME,
                ],
                flag_groups = [
                    flag_group(
                        flags = ["--sysroot=/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux"],
                    ),
                ],
            ),
        ],
    )

    fastbuild_feature = feature(name = "fastbuild")


    features = [
        default_compile_flags_feature,
        default_link_flags_feature,
        supports_dynamic_linker_feature,
        supports_pic_feature,
        objcopy_embed_flags_feature,
        opt_feature,
        dbg_feature,
        user_compile_flags_feature,
        sysroot_feature,
        unfiltered_compile_flags_feature,
    ]


    cxx_builtin_include_directories = [ # NEW
        "/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/lib/gcc/x86_64-linux-gnu/5.5.0/include",
        "/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/lib/gcc/x86_64-linux-gnu/5.5.0/include-fixed",
        "/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/x86_64-linux-gnu/include"
    ]

    artifact_name_patterns = []

    make_variables = []

    tool_paths = [ # NEW
        tool_path(
            name = "gcc",
            path = "/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/bin/x86_64-linux-gnu-gcc",
        ),
        tool_path(
            name = "g++",
            path = "/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/bin/x86_64-linux-gnu-g++",
        ),
        tool_path(
            name = "ld",
            path = "/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/bin/x86_64-linux-gnu-ld",
        ),
        tool_path(
            name = "ar",
            path = "/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/bin/ar",
        ),
        tool_path(
            name = "cpp",
            path = "/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/bin/x86_64-linux-gnu-cpp",
        ),
        tool_path(
            name = "gcov",
            path = "/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/bin/x86_64-linux-gnu-gcov",
        ),
        tool_path(
            name = "nm",
            path = "/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/bin/x86_64-linux-gnu-gcc-nm",
        ),
        tool_path(
            name = "objdump",
            path = "/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/bin/x86_64-linux-gnu-objdump",
        ),
        tool_path(
            name = "strip",
            path = "/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/bin/x86_64-linux-gnu-strip",
        ),
    ]

    out = ctx.actions.declare_file(ctx.label.name)
    ctx.actions.write(out, "Fake executable")
    return [
        cc_common.create_cc_toolchain_config_info(
            ctx = ctx,
            features = features,
            action_configs = action_configs,
            artifact_name_patterns = artifact_name_patterns,
            cxx_builtin_include_directories = cxx_builtin_include_directories,
            toolchain_identifier = toolchain_identifier,
            host_system_name = host_system_name,
            target_system_name = target_system_name,
            target_cpu = target_cpu,
            target_libc = target_libc,
            compiler = compiler,
            abi_version = abi_version,
            abi_libc_version = abi_libc_version,
            tool_paths = tool_paths,
            make_variables = make_variables,
            builtin_sysroot = builtin_sysroot,
            cc_target_os = cc_target_os,
        ),
        DefaultInfo(
            executable = out,
        ),
    ]

cc_toolchain_config = rule(
    implementation = _impl,
    attrs = {},
    provides = [CcToolchainConfigInfo],
    executable = True,
)

 

 

 

6 python專案的適配

部分專案的編譯框架是python構建的,例如horovod,相關適配修改只能通過直接修改horovod的setup.py檔案。

類似cmake的交叉編譯工具鏈的配置,主要涉及CC/CXX/SYSROOT,要不然python專案中涉及的一些c或者c++程式碼的編譯無法使用交叉編譯工具鏈來編譯。

如果python專案中有子專案是cmake或者bazel編譯的話,需要參照其他cmake和bazel專案的編譯配置方法進行配置。

7 其他注意事項

7.1 不要直接修改全域性的CC或者CXX,除非知道修改的影響

特別的,TensorFlow目標編譯過程中需要host編譯器編譯的臨時檔案,在修改CC和CXX為交叉工具鏈編譯的可執行檔案之後,有可能在host上是無法執行。

但python專案又必須要修改CC和CXX,這時必須通過臨時修改環境變數的做法,確保python專案裡面使用的是交叉的CC和CXX,但其他專案還是用預設的CC和CXX。

7.2 bazel target的符號缺失問題

使用交叉編譯工具鏈編譯出來的bazel tf_cc_test目標,部分target會從動態連結變成靜態連結,如果依賴規則裡面沒有alwayslink編譯選項的話,可能導致部分.o靜態連結的時候符號缺失,這種情況下需要增加alwayslink=1的選項。

相關文章