Skip to content

Commit

Permalink
[AutoBuild] Fix default value of lazy_artifacts (#1191)
Browse files Browse the repository at this point in the history 
* [AutoBuild] Fix default value of `lazy_artifacts`

* [AutoBuild] Simplify writing of files in `.pkg` directory

* [AutoBuild] Add tests for platform augmentation
  • Loading branch information
@giordano
giordano authored Mar 23, 2022
1 parent 28af078 commit 3955444
Show file tree
Hide file tree
Showing 3 changed files with 63 additions and 42 deletions.
1 change: 1 addition & 0 deletions Project.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ RegistryTools = "d1eb7eb1-105f-429d-abf5-b0f65cb9e2c4"
SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce"
Scratch = "6c6a2e73-6563-6170-7368-637461726353"
Sockets = "6462fe0b-24de-5631-8697-dd941f90decc"
TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
ghr_jll = "07c12ed4-43bc-5495-8a2a-d5838ef8d533"

Expand Down
80 changes: 39 additions & 41 deletions src/AutoBuild.jl
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
export build_tarballs, autobuild, print_artifacts_toml, build, get_meta_json
import GitHub: gh_get_json, DEFAULT_API
import SHA: sha256, sha1
using Pkg.TOML, Dates, UUIDs
using TOML, Dates, UUIDs
using RegistryTools
import LibGit2
import PkgLicenses
Expand Down Expand Up @@ -104,7 +104,7 @@ const BUILD_HELP = (
instead of actually building. Note that this can
(and often does) output multiple JSON objects for
multiple platforms, multi-stage builds, etc...
--skip-audit Skips auditing of the output products.
--help Print out this message.
Expand Down Expand Up @@ -543,8 +543,8 @@ function register_jll(name, build_version, dependencies, julia_compat;
code_dir=joinpath(Pkg.devdir(), "$(name)_jll"),
gh_auth=Wizard.github_auth(;allow_anonymous=false),
gh_username=gh_get_json(DEFAULT_API, "/user"; auth=gh_auth)["login"],
lazy_artifacts::Bool=false,
augment_platform_block="",
augment_platform_block::String="",
lazy_artifacts::Bool=!isempty(augment_platform_block) && minimum_compat(julia_compat) < v"1.7",
kwargs...)
if !isempty(augment_platform_block) && minimum_compat(julia_compat) < v"1.6"
error("Augmentation blocks cannot be used with Julia v1.5-.\nChange `julia_compat` to require at least Julia v1.6")
Expand Down Expand Up @@ -622,9 +622,10 @@ function get_meta_json(
products::Vector{<:Product},
dependencies::Vector{<:AbstractDependency};
julia_compat::String = DEFAULT_JULIA_VERSION_SPEC,
lazy_artifacts::Bool = false,
init_block::String = "",
augment_platform_block::String = "")
augment_platform_block::String = "",
lazy_artifacts::Bool=!isempty(augment_platform_block) && minimum_compat(julia_compat) < v"1.7",
)

dict = Dict(
"name" => src_name,
Expand Down Expand Up @@ -654,7 +655,7 @@ function compose_debug_prompt(workspace)
end
end
end

if length(log_files) > 0
log_files_str = join(log_files, "\n - ")

Expand Down Expand Up @@ -1064,7 +1065,6 @@ function rebuild_jll_package(obj::Dict;
build_version = nothing,
gh_org::String = "JuliaBinaryWrappers",
verbose::Bool = false,
lazy_artifacts::Bool = false,
from_scratch::Bool = true)
if build_version === nothing
build_version = BinaryBuilder.get_next_wrapper_version(obj["name"], obj["version"])
Expand All @@ -1079,6 +1079,9 @@ function rebuild_jll_package(obj::Dict;
error("If download_dir is specified, you must specify upload_prefix as well!")
end

julia_compat = get(obj, "julia_compat", DEFAULT_JULIA_VERSION_SPEC)
augment_platform_block = get(obj, "augment_platform_block", "")
lazy_artifacts = get(obj, "lazy_artifacts", !isempty(augment_platform_block) && minimum_compat(julia_compat) < v"1.7")
return rebuild_jll_package(
obj["name"],
build_version,
Expand All @@ -1088,12 +1091,12 @@ function rebuild_jll_package(obj::Dict;
obj["dependencies"],
download_dir,
upload_prefix;
verbose=verbose,
lazy_artifacts = lazy_artifacts,
julia_compat = get(obj, "julia_compat", DEFAULT_JULIA_VERSION_SPEC),
verbose,
lazy_artifacts,
julia_compat,
init_block = get(obj, "init_block", ""),
augment_platform_block = get(obj, "augment_platform_block", ""),
from_scratch = from_scratch,
augment_platform_block,
from_scratch,
)
end

Expand Down Expand Up @@ -1196,8 +1199,8 @@ function build_jll_package(src_name::String,
bin_path::String;
verbose::Bool = false,
julia_compat::String = DEFAULT_JULIA_VERSION_SPEC,
init_block = "",
augment_platform_block = "",
init_block::String = "",
augment_platform_block::String = "",
# If we support versions older than Julia v1.7 the artifact
# should be lazy when we augment the platform.
lazy_artifacts::Bool = !isempty(augment_platform_block) && minimum_compat(julia_compat) < v"1.7",
Expand Down Expand Up @@ -1327,33 +1330,28 @@ function build_jll_package(src_name::String,
if !isempty(augment_platform_block)
pkg_dir = joinpath(code_dir, ".pkg")
!ispath(pkg_dir) && mkdir(pkg_dir)
open(joinpath(pkg_dir, "platform_augmentation.jl"), "w") do io
println(io, """
$(augment_platform_block)
""")
end
write(joinpath(pkg_dir, "platform_augmentation.jl"), augment_platform_block)

open(joinpath(pkg_dir, "select_artifacts.jl"), "w") do io
println(io, """
push!(Base.LOAD_PATH, dirname(@__DIR__))
write(joinpath(pkg_dir, "select_artifacts.jl"),
"""
push!(Base.LOAD_PATH, dirname(@__DIR__))
using TOML, Artifacts, Base.BinaryPlatforms
include("./platform_augmentation.jl")
artifacts_toml = joinpath(dirname(@__DIR__), "Artifacts.toml")
using TOML, Artifacts, Base.BinaryPlatforms
include("./platform_augmentation.jl")
artifacts_toml = joinpath(dirname(@__DIR__), "Artifacts.toml")
# Get "target triplet" from ARGS, if given (defaulting to the host triplet otherwise)
target_triplet = get(ARGS, 1, Base.BinaryPlatforms.host_triplet())
# Get "target triplet" from ARGS, if given (defaulting to the host triplet otherwise)
target_triplet = get(ARGS, 1, Base.BinaryPlatforms.host_triplet())
# Augment this platform object with any special tags we require
platform = augment_platform!(HostPlatform(parse(Platform, target_triplet)))
# Augment this platform object with any special tags we require
platform = augment_platform!(HostPlatform(parse(Platform, target_triplet)))
# Select all downloadable artifacts that match that platform
artifacts = select_downloadable_artifacts(artifacts_toml; platform, include_lazy=true)
# Select all downloadable artifacts that match that platform
artifacts = select_downloadable_artifacts(artifacts_toml; platform, include_lazy=true)
#Output the result to `stdout` as a TOML dictionary
TOML.print(stdout, artifacts)
""")
end
#Output the result to `stdout` as a TOML dictionary
TOML.print(stdout, artifacts)
""")
end

# Generate target-demuxing main source file.
Expand Down Expand Up @@ -1419,9 +1417,9 @@ function build_jll_package(src_name::String,
if is_yggdrasil()
println(io, """
The originating [`build_tarballs.jl`](https://github.com/JuliaPackaging/Yggdrasil/blob/$(yggdrasil_head())/$(ENV["PROJECT"])/build_tarballs.jl) script can be found on [`Yggdrasil`](https://github.com/JuliaPackaging/Yggdrasil/), the community build tree.")
## Bug Reports
If you have any issue, please report it to the Yggdrasil [bug tracker](https://github.com/JuliaPackaging/Yggdrasil/issues).
""")
end
Expand Down Expand Up @@ -1505,7 +1503,7 @@ function build_jll_package(src_name::String,
jllwrappers_compat = isempty(augment_platform_block) ? "1.2.0" : "1.4.0"
project = build_project_dict(src_name, build_version, dependencies, julia_compat; lazy_artifacts, jllwrappers_compat, augment_platform_block)
open(joinpath(code_dir, "Project.toml"), "w") do io
Pkg.TOML.print(io, project)
TOML.print(io, project)
end

# Add a `.gitignore`
Expand Down Expand Up @@ -1579,8 +1577,8 @@ end
function build_project_dict(name, version, dependencies::Array{Dependency},
julia_compat::String=DEFAULT_JULIA_VERSION_SPEC;
jllwrappers_compat::String=DEFAULT_JLLWRAPPERS_VERSION_SPEC,
lazy_artifacts::Bool=false,
augment_platform_block="",
augment_platform_block::String="",
lazy_artifacts::Bool=!isempty(augment_platform_block) && minimum_compat(julia_compat) < v"1.7",
kwargs...)
Pkg.Types.semver_spec(julia_compat) # verify julia_compat is valid
project = Dict(
Expand Down
24 changes: 23 additions & 1 deletion test/jll.jl
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
using TOML
using JSON
using UUIDs
using GitHub
Expand Down Expand Up @@ -48,6 +49,17 @@ end
platforms = [platform, freebsd]
# We depend on Zlib_jll only on the host platform, but not on FreeBSD
dependencies = [Dependency("Zlib_jll"; platforms=[platform])]
# Augment platform
augment_platform_block = """
using Base.BinaryPlatforms
function augment_platform!(platform::Platform)
platform["test"] = "enabled"
return platform
end
"""
# Julia compat. Include Julia v1.6 to exercise the code path which forces lazy
# artifacts when augmenting the platform
julia_compat = "1.6"
# The buffer where we'll write the JSON meta data
buff = IOBuffer()

Expand All @@ -65,6 +77,8 @@ end
# The products we expect to be build
libfoo_products,
dependencies;
julia_compat,
augment_platform_block,
)
# Generate the JSON file
println(buff, JSON.json(dict))
Expand Down Expand Up @@ -143,13 +157,21 @@ end
@test occursin("MESON_TARGET_TOOLCHAIN", json_obj["script"])
end

BinaryBuilder.rebuild_jll_package(json_obj; download_dir=download_dir, upload_prefix=upload_prefix, verbose=false, lazy_artifacts=json_obj["lazy_artifacts"], from_scratch=from_scratch)
BinaryBuilder.rebuild_jll_package(json_obj; download_dir, upload_prefix, verbose=false, from_scratch)
end

env_dir = joinpath(build_path, "foo")
mkpath(env_dir)
Pkg.activate(env_dir)
Pkg.develop(PackageSpec(path=code_dir))
@test isfile(joinpath(code_dir, ".pkg", "select_artifacts.jl"))
@test read(joinpath(code_dir, ".pkg", "platform_augmentation.jl"), String) == augment_platform_block
# Make sure the artifacts are lazy because we are augmenting the platform and
# Julia compat includes versions before v1.7.
artifacts_toml = TOML.parsefile(joinpath(code_dir, "Artifacts.toml"))
for artifact in artifacts_toml["libfoo"]
@test artifact["lazy"]
end
# Make sure we use Zlib_jll only in the wrapper for the host
# platform and not the FreeBSD one.
platform_wrapper = joinpath(code_dir, "src", "wrappers", triplet(platform) * ".jl")
Expand Down

0 comments on commit 3955444

Please sign in to comment.