From b0e84ec75f02f4b920084419cf2a3df98134e225 Mon Sep 17 00:00:00 2001 From: Carlos Date: Tue, 12 Mar 2024 16:17:42 +0000 Subject: [PATCH] LLVM 18 (#33) * gh: bump minor code version * third-party: remove unsupported submodules * tasks: bump to llvm 17 for clang formatting * remove unnecessary references to apps * cpython: use lllvm-17 for the native build * cpp: point to llvm-17 branch * cpython: fix func compilation * cpp: bump * cpython: set the target in the func compilation * cpp: bump dep * cpython: use llvm native version for native compilation * nits * nits: clang-format-17 checks passing * cpp: bump to latest version * cpp: bump after merge to main --- .clang-format | 12 +++++++++ .github/workflows/tests.yml | 4 +-- .gitmodules | 12 --------- Dockerfile | 7 +----- VERSION | 2 +- crossenv/tasks/modules.py | 1 + func/cpp/CMakeLists.txt | 15 ++++++++---- tasks/cpython.py | 49 ++++++++++++++++++++++--------------- tasks/format_code.py | 4 ++- third-party/cpp | 2 +- third-party/cpython | 2 +- third-party/horovod | 1 - third-party/mxnet | 1 - third-party/numpy | 1 - 14 files changed, 61 insertions(+), 52 deletions(-) delete mode 160000 third-party/horovod delete mode 160000 third-party/mxnet delete mode 160000 third-party/numpy diff --git a/.clang-format b/.clang-format index 0c2dc98..7e8982a 100644 --- a/.clang-format +++ b/.clang-format @@ -10,4 +10,16 @@ AlwaysBreakAfterReturnType: None IndentWidth: 4 DerivePointerAlignment: false + +# Precise control over braces alignment +BreakBeforeBraces: Custom +BraceWrapping: + AfterClass: true + AfterEnum: true + AfterExternBlock: true + AfterFunction: true + AfterStruct: true + AfterUnion: true + SplitEmptyFunction: false + SplitEmptyRecord: false --- diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 88f8c94..e1cddca 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -20,7 +20,7 @@ jobs: if: github.event.pull_request.draft == false runs-on: ubuntu-latest container: - image: faasm.azurecr.io/cpython:0.3.1 + image: faasm.azurecr.io/cpython:0.4.0 credentials: username: ${{ secrets.ACR_SERVICE_PRINCIPAL_ID }} password: ${{ secrets.ACR_SERVICE_PRINCIPAL_PASSWORD }} @@ -44,7 +44,7 @@ jobs: REDIS_QUEUE_HOST: redis REDIS_STATE_HOST: redis container: - image: faasm.azurecr.io/cpython:0.3.1 + image: faasm.azurecr.io/cpython:0.4.0 credentials: username: ${{ secrets.ACR_SERVICE_PRINCIPAL_ID }} password: ${{ secrets.ACR_SERVICE_PRINCIPAL_PASSWORD }} diff --git a/.gitmodules b/.gitmodules index 05f35f8..197ba36 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,22 +2,10 @@ path = third-party/cpython url = https://github.com/faasm/cpython branch = faasm -[submodule "third-party/numpy"] - path = third-party/numpy - url = https://github.com/faasm/numpy - branch = faasm [submodule "third-party/crossenv"] path = third-party/crossenv url = https://github.com/faasm/crossenv branch = faasm -[submodule "third-party/horovod"] - path = third-party/horovod - url = https://github.com/faasm/horovod - branch = faasm -[submodule "third-party/mxnet"] - path = third-party/mxnet - url = https://github.com/faasm/incubator-mxnet - branch = faasm [submodule "third-party/cpp"] path = third-party/cpp url = https://github.com/faasm/cpp diff --git a/Dockerfile b/Dockerfile index a3fba6d..191a1f4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM faasm.azurecr.io/cpp-sysroot:0.2.4 +FROM faasm.azurecr.io/cpp-sysroot:0.4.0 ARG FAASM_PYTHON_VERSION SHELL ["/bin/bash", "-c"] @@ -41,11 +41,6 @@ RUN cd /code/python \ && source ./venv/bin/activate \ && inv modules.install - -# TODO: enable these once the MXNet/ Horovod work is completed -# Build mxnet -# RUN inv mxnet - # TODO: Install experimental pacakges # RUN . ./cross_venv/bin/activate && inv libs.install --experimental diff --git a/VERSION b/VERSION index 9e11b32..1d0ba9e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.1 +0.4.0 diff --git a/crossenv/tasks/modules.py b/crossenv/tasks/modules.py index 561cc71..ea2fd01 100644 --- a/crossenv/tasks/modules.py +++ b/crossenv/tasks/modules.py @@ -25,6 +25,7 @@ MODIFIED_LIBS_EXPERIMENTAL = { # 15/11/2022 - Move Numpy back to experimental as either the upgrades # to the CPP toolchain or LLVM 13 break the build + # (experimental <=> not yet supported) "numpy": { "env": {"NPY_NUM_BUILD_JOBS": USABLE_CPUS}, }, diff --git a/func/cpp/CMakeLists.txt b/func/cpp/CMakeLists.txt index 79aeaaf..0c7da02 100644 --- a/func/cpp/CMakeLists.txt +++ b/func/cpp/CMakeLists.txt @@ -3,27 +3,32 @@ project(faasm-func) set(CMAKE_CXX_STANDARD 17) -if (CMAKE_SYSTEM_NAME STREQUAL "Wasm") +if (CMAKE_SYSTEM_NAME STREQUAL "WASI") + # Set the right target so that we look in the right sysroot + set(CMAKE_C_FLAGS "--target=$ENV{FAASM_WASM_TRIPLE} ${CMAKE_C_FLAGS}") + set(CMAKE_CXX_FLAGS "--target=$ENV{FAASM_WASM_TRIPLE} ${CMAKE_CXX_FLAGS}") + # For Faasm functions, we add `_faasm_zygote` as an exported symbol set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Xlinker --export=_faasm_zygote" ) + message(STATUS "Detected wasm build (sysroot=$ENV{FAASM_WASM_SYSROOT})") set(CMAKE_EXECUTABLE_SUFFIX ".wasm") endif () -if (CMAKE_SYSTEM_NAME STREQUAL "Wasm") +if (CMAKE_SYSTEM_NAME STREQUAL "WASI") set(FAASM_FUNC_LIBS faasm emscripten c-printscan-long-double) - set(PYTHON_LIBRARIES ${CMAKE_SYSROOT}/lib/wasm32-wasi/libpython3.8.a) - set(PYTHON_INCLUDE_DIRS ${CMAKE_SYSROOT}/include/python3.8) + set(PYTHON_LIBRARIES $ENV{FAASM_WASM_LIB_INSTALL_DIR}/libpython3.8.a) + set(PYTHON_INCLUDE_DIRS $ENV{FAASM_WASM_HEADER_INSTALL_DIR}/python3.8) else () find_package(PythonLibs) endif() add_executable(py_func pyinit.c pyinit.h py_func.cpp) -if (CMAKE_SYSTEM_NAME STREQUAL "Wasm") +if (CMAKE_SYSTEM_NAME STREQUAL "WASI") target_link_libraries(py_func ${FAASM_FUNC_LIBS} ${PYTHON_LIBRARIES} ffi) target_include_directories(py_func PUBLIC ${PYTHON_INCLUDE_DIRS}) diff --git a/tasks/cpython.py b/tasks/cpython.py index fa4b3f6..a190c02 100644 --- a/tasks/cpython.py +++ b/tasks/cpython.py @@ -1,14 +1,8 @@ from copy import copy as deep_copy from faasmctl.util.upload import upload_wasm -from faasmtools.build import ( - FAASM_BUILD_ENV_DICT, - WASM_HEADER_INSTALL, - WASM_LIB_INSTALL, - WASM_WASI_LIBC_LDFLAGS, - build_config_cmd, -) +from faasmtools.build import build_config_cmd, get_faasm_build_env_dict from faasmtools.compile_util import wasm_cmake, wasm_copy_upload -from faasmtools.env import WASM_DIR +from faasmtools.env import LLVM_NATIVE_VERSION, WASM_DIR from invoke import task from os import environ, makedirs from os.path import join, exists @@ -54,16 +48,18 @@ "PATH": PATH_ENV_VAR, } ) -ENV_VARS.update(FAASM_BUILD_ENV_DICT) +ENV_VARS.update(get_faasm_build_env_dict(is_threads=True)) LIB_SRC_DIR = join(CPYTHON_INSTALL_DIR, "lib") LIB_DEST_DIR = join(FAASM_RUNTIME_ROOT, "lib") LIBPYTHON_SRC_PATH = join(LIB_SRC_DIR, "libpython3.8.a") -LIBPYTHON_DEST_PATH = join(WASM_LIB_INSTALL, "libpython3.8.a") +LIBPYTHON_DEST_PATH = join( + ENV_VARS["FAASM_WASM_LIB_INSTALL_DIR"], "libpython3.8.a" +) INCLUDE_SRC_DIR = join(CPYTHON_INSTALL_DIR, "include", "python3.8") -INCLUDE_DEST_DIR = join(WASM_HEADER_INSTALL, "python3.8") +INCLUDE_DEST_DIR = join(ENV_VARS["FAASM_WASM_HEADER_INSTALL_DIR"], "python3.8") # See the CPython docs for more info: # - General: https://devguide.python.org/setup/#compile-and-build @@ -96,21 +92,26 @@ def wasm(ctx, clean=False, noconf=False, nobuild=False): # relevant in the module builds. # Link in extra wasi-libc long double support (see wasi-libc docs) - link_libs = ["-lfaasm"] + WASM_WASI_LIBC_LDFLAGS - link_libs = " ".join(link_libs) + link_libs = "-lfaasm " + ENV_VARS["FAASM_WASM_STATIC_LINKER_FLAGS"] + # link_libs = " ".join(link_libs) # Configure configure_cmd = build_config_cmd( + ENV_VARS, [ "CONFIG_SITE=./config.site", "READELF=true", "./configure", 'LIBS="{}"'.format(link_libs), + "--build=wasm32", + "--host={}".format(ENV_VARS["FAASM_WASM_TRIPLE"]), "--disable-ipv6", "--disable-shared", "--prefix={}".format(CPYTHON_INSTALL_DIR), "--with-system-ffi", - ] + ], + # Do not set the --host flag as we want to use the wasi-threads target + conf_args=False, ) if not noconf: @@ -133,8 +134,8 @@ def wasm(ctx, clean=False, noconf=False, nobuild=False): _run_cpython_cmd("bininstall", ["make", "bininstall"]) # Prepare destinations - makedirs(WASM_HEADER_INSTALL, exist_ok=True) - makedirs(WASM_LIB_INSTALL, exist_ok=True) + makedirs(ENV_VARS["FAASM_WASM_HEADER_INSTALL_DIR"], exist_ok=True) + makedirs(ENV_VARS["FAASM_WASM_LIB_INSTALL_DIR"], exist_ok=True) rmtree(INCLUDE_DEST_DIR, ignore_errors=True) @@ -187,12 +188,13 @@ def native(ctx, clean=False): run("wget {}".format(tar_url), shell=True, check=True, cwd=workdir) run("tar -xf {}".format(tar_name), shell=True, check=True, cwd=workdir) + llvm_native_version_major = LLVM_NATIVE_VERSION.split(".")[0] workdir = join(workdir, PYTHON_VERSION) native_configure_cmd = [ - 'CC="clang-13"', - 'CXX="clang++-13"', + 'CC="clang-{}"'.format(llvm_native_version_major), + 'CXX="clang++-{}"'.format(llvm_native_version_major), 'CFLAGS="-O3 -DANSI"', - 'LD="clang-13"', + 'LD="clang-{}"'.format(llvm_native_version_major), "./configure", "--prefix={}".format(PYTHON_INSTALL_DIR), ] @@ -230,7 +232,14 @@ def func(ctx, clean=False, debug=False): wasm_file = join(func_build_dir, "{}.wasm".format(CPYTHON_FUNC_NAME)) # Build and install the wasm - wasm_cmake(func_dir, func_build_dir, CPYTHON_FUNC_NAME, clean, debug) + wasm_cmake( + func_dir, + func_build_dir, + CPYTHON_FUNC_NAME, + clean, + debug, + is_threads=True, + ) wasm_copy_upload(CPYTHON_FUNC_USER, CPYTHON_FUNC_NAME, wasm_file) diff --git a/tasks/format_code.py b/tasks/format_code.py index 2a6dbd9..0eebd27 100644 --- a/tasks/format_code.py +++ b/tasks/format_code.py @@ -1,3 +1,4 @@ +from faasmtools.env import LLVM_NATIVE_VERSION from invoke import task from tasks.env import PROJ_ROOT from subprocess import run @@ -51,8 +52,9 @@ def format(ctx, check=False): ) clang_cmd = [ - "clang-format-13", + "clang-format-{}".format(LLVM_NATIVE_VERSION.split(".")[0]), "--dry-run --Werror" if check else "-i", + "-style=file", " ".join(files_to_check), ] clang_cmd = " ".join(clang_cmd) diff --git a/third-party/cpp b/third-party/cpp index 16102b1..d9fc3ac 160000 --- a/third-party/cpp +++ b/third-party/cpp @@ -1 +1 @@ -Subproject commit 16102b14075c61491bf4a46f79672ea687413d53 +Subproject commit d9fc3acfaa96caebbe585389af79d5e7c0959c90 diff --git a/third-party/cpython b/third-party/cpython index f6890ba..7e1130f 160000 --- a/third-party/cpython +++ b/third-party/cpython @@ -1 +1 @@ -Subproject commit f6890ba59fc1fc1c86cf397175f4f8382e85e096 +Subproject commit 7e1130ff24992c3fe365ad22fb50c170948a6bed diff --git a/third-party/horovod b/third-party/horovod deleted file mode 160000 index 9caabd5..0000000 --- a/third-party/horovod +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 9caabd58db88d3a396cc016399e2a1c0128f9770 diff --git a/third-party/mxnet b/third-party/mxnet deleted file mode 160000 index b444f05..0000000 --- a/third-party/mxnet +++ /dev/null @@ -1 +0,0 @@ -Subproject commit b444f05eb58dee8152eb744c66519417dde4ccf2 diff --git a/third-party/numpy b/third-party/numpy deleted file mode 160000 index ea6079d..0000000 --- a/third-party/numpy +++ /dev/null @@ -1 +0,0 @@ -Subproject commit ea6079db9503e54982195aea2453bc45bbca7e0f