Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add netboot files #21

Closed
wants to merge 3 commits into from
Closed

Add netboot files #21

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
918d37c
Add boost to deps to fix build with nix 2.x
lheckemann Jul 19, 2018
0cf0276
Make AWS upload optional
lheckemann Jul 21, 2018
96ada14
Add netboot files to channel
lheckemann Aug 1, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion default.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ stdenv.mkDerivation {
name = "nixos-channel-scripts";

buildInputs = with perlPackages;
[ pkgconfig nix sqlite makeWrapper perl FileSlurp LWP LWPProtocolHttps ListMoreUtils DBDSQLite NetAmazonS3 boehmgc nlohmann_json ];
[ pkgconfig nix sqlite makeWrapper perl FileSlurp LWP LWPProtocolHttps ListMoreUtils DBDSQLite NetAmazonS3 boehmgc nlohmann_json boost ];

buildCommand = ''
mkdir -p $out/bin
Expand Down
76 changes: 44 additions & 32 deletions mirror-nixos-branch.pl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

my $channelName = $ARGV[0];
my $releaseUrl = $ARGV[1];
my $useAWS = ($ENV{'AWS_DISABLE'} or "") eq "1";

die "Usage: $0 CHANNEL-NAME RELEASE-URL\n" unless defined $channelName && defined $releaseUrl;

Expand All @@ -31,19 +32,21 @@

$ENV{'GIT_DIR'} = "/home/hydra-mirror/nixpkgs-channels";

my $bucket;
if ($useAWS) {
# S3 setup.
my $aws_access_key_id = $ENV{'AWS_ACCESS_KEY_ID'} or die;
my $aws_secret_access_key = $ENV{'AWS_SECRET_ACCESS_KEY'} or die;

# S3 setup.
my $aws_access_key_id = $ENV{'AWS_ACCESS_KEY_ID'} or die;
my $aws_secret_access_key = $ENV{'AWS_SECRET_ACCESS_KEY'} or die;
my $s3 = Net::Amazon::S3->new(
{ aws_access_key_id => $aws_access_key_id,
aws_secret_access_key => $aws_secret_access_key,
retry => 1,
host => "s3-eu-west-1.amazonaws.com",
});

my $s3 = Net::Amazon::S3->new(
{ aws_access_key_id => $aws_access_key_id,
aws_secret_access_key => $aws_secret_access_key,
retry => 1,
host => "s3-eu-west-1.amazonaws.com",
});

my $bucket = $s3->bucket($bucketName) or die;
$bucket = $s3->bucket($bucketName) or die;
}


sub fetch {
Expand Down Expand Up @@ -80,7 +83,7 @@ sub fetch {
die "channel would go back in time from $curRelease to $releaseName, bailing out\n" if $d == 1;
}

if ($bucket->head_key("$releasePrefix")) {
if ($useAWS && $bucket->head_key("$releasePrefix")) {
print STDERR "release already exists\n";
} else {
my $tmpDir = "/data/releases/tmp/release-$channelName/$releaseName";
Expand All @@ -96,25 +99,27 @@ sub fetch {
}

sub downloadFile {
my ($jobName, $dstName) = @_;
my ($jobName, $dstBase) = @_;

my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json'));

my $srcFile = $buildInfo->{buildproducts}->{1}->{path} or die "job '$jobName' lacks a store path";
$dstName //= basename($srcFile);
my $dstFile = "$tmpDir/" . $dstName;

my $sha256_expected = $buildInfo->{buildproducts}->{1}->{sha256hash} or die;

if (! -e $dstFile) {
print STDERR "downloading $srcFile to $dstFile...\n";
write_file("$dstFile.sha256", "$sha256_expected $dstName");
system("NIX_REMOTE=https://cache.nixos.org/ nix cat-store '$srcFile' > '$dstFile.tmp'") == 0
or die "unable to fetch $srcFile\n";
rename("$dstFile.tmp", $dstFile) or die;
}
my $buildproducts = $buildInfo->{buildproducts};
for my $product (values %$buildproducts) {
my %product = %$product;
my $srcFile = $product->{path} or die "job '$jobName' lacks a store path";
my $dstName = $dstBase // basename($srcFile);
my $dstFile = "$tmpDir/" . $dstName;

my $sha256_expected = $product->{sha256hash} or die "file '$srcFile' lacks an expected hash";

if (! -e $dstFile) {
print STDERR "downloading $srcFile to $dstFile...\n";
write_file("$dstFile.sha256", "$sha256_expected $dstName\n");
system("NIX_REMOTE=https://cache.nixos.org/ nix cat-store '$srcFile' > '$dstFile.tmp'") == 0
or die "unable to fetch $srcFile\n";
rename("$dstFile.tmp", $dstFile) or die;
}

if (-e "$dstFile.sha256") {
my $sha256_actual = `nix hash-file --type sha256 '$dstFile'`;
chomp $sha256_actual;
if ($sha256_expected ne $sha256_actual) {
Expand All @@ -134,6 +139,11 @@ sub fetch {
#downloadFile("nixos.iso_graphical.i686-linux");
downloadFile("nixos.ova.x86_64-linux");
#downloadFile("nixos.ova.i686-linux");
# Netboot is currently not included in -small but maybe should be?
# This may fail because of nixos/hydra#580, hence wrapping in eval
# nixos/nixpkgs#44089 works around it, but is not backported to all channels.
eval { downloadFile("nixos.netboot.x86_64-linux"); };
warn "Could not download netboot files: $@" if $@;
}

} else {
Expand Down Expand Up @@ -175,7 +185,7 @@ sub fetch {
my $basename = basename $fn;
my $key = "$releasePrefix/" . $basename;

unless (defined $bucket->head_key($key)) {
if ($useAWS && ! (defined $bucket->head_key($key))) {
print STDERR "mirroring $fn to s3://$bucketName/$key...\n";
$bucket->add_key_filename(
$key, $fn,
Expand All @@ -196,11 +206,13 @@ sub fetch {

$html .= "</tbody></table></body></html>";

$bucket->add_key($releasePrefix, $html,
{ content_type => "text/html" })
or die $bucket->err . ": " . $bucket->errstr;
if ($useAWS) {
$bucket->add_key($releasePrefix, $html,
{ content_type => "text/html" })
or die $bucket->err . ": " . $bucket->errstr;

File::Path::remove_tree($tmpDir);
File::Path::remove_tree($tmpDir);
}
}

# Prevent concurrent writes to the channels directory.
Expand Down