diff --git a/delft/pluto/default.nix b/delft/pluto/default.nix index 0b07c055..c988cc90 100644 --- a/delft/pluto/default.nix +++ b/delft/pluto/default.nix @@ -18,6 +18,7 @@ ../../modules/netboot-serve.nix ../../modules/rfc39.nix ../../modules/tarball-mirror.nix + ../../metrics/fastly/process-raw-nix-logs.nix ]; networking = { diff --git a/metrics/fastly/cron.sh b/metrics/fastly/cron.sh index d0dd7f4b..1daa6b1d 100755 --- a/metrics/fastly/cron.sh +++ b/metrics/fastly/cron.sh @@ -1,15 +1,15 @@ #! /bin/sh -e -export AWS_PROFILE=nixos-org +export AWS_PROFILE=fastly-log-processor now=$(date +%s) #now=$((now - 86400)) prev_week=$((($now / 86400 / 7))) from_date_incl=$(date +%F --date="@$(($prev_week * 86400 * 7 - 2 * 86400))") -to_date_incl=$(date +%F --date="@$(($prev_week * 86400 * 7 + 5 * 86400))") +to_date_excl=$(date +%F --date="@$(($prev_week * 86400 * 7 + 5 * 86400))") -echo "Ingesting [$from_date_incl, $to_date_incl)." +echo "Ingesting [$from_date_incl, $to_date_excl)." marker="$HOME/weeks-done/$prev_week" @@ -21,6 +21,6 @@ fi mkdir -p "$(dirname "$marker")" touch $marker -./ingest-raw-logs.sh "$from_date_incl" "$to_date_incl" +./ingest-raw-logs.sh "$from_date_incl" "$to_date_excl" ./run-queries.sh diff --git a/metrics/fastly/flake.nix b/metrics/fastly/flake.nix deleted file mode 100644 index 8c276421..00000000 --- a/metrics/fastly/flake.nix +++ /dev/null @@ -1,28 +0,0 @@ -{ - outputs = { self, nixpkgs }: { - nixosModules.nix-metrics = { config, pkgs, ... }: { - - users.users.nix-metrics = - { isNormalUser = true; - description = "Nix Metrics Collection"; - }; - - systemd.services.process-raw-nix-logs = { - description = "Process Raw nixos.org Logs"; - serviceConfig.Type = "oneshot"; - serviceConfig.User = "nix-metrics"; - path = - [ pkgs.awscli - pkgs.jq - ]; - script = - '' - cd ${./.} - ./cron.sh - ''; - startAt = "Tue 07:30"; - }; - - }; - }; -} diff --git a/metrics/fastly/process-raw-nix-logs.nix b/metrics/fastly/process-raw-nix-logs.nix new file mode 100644 index 00000000..793b28d2 --- /dev/null +++ b/metrics/fastly/process-raw-nix-logs.nix @@ -0,0 +1,25 @@ +{ config, pkgs, ... }: { + + users.users.nix-metrics = + { isNormalUser = true; + description = "Nix Metrics Collection"; + }; + + systemd.services.process-raw-nix-logs = { + description = "Process Raw nixos.org Logs"; + serviceConfig.Type = "oneshot"; + serviceConfig.User = "nix-metrics"; + path = + [ pkgs.awscli + pkgs.jq + ]; + script = + '' + cd ${./.} + ./cron.sh + ''; + startAt = "Tue 07:30"; + }; + +} + diff --git a/terraform-iam/fastlylog/main.tf b/terraform-iam/fastlylog/main.tf index 79524003..64587852 100644 --- a/terraform-iam/fastlylog/main.tf +++ b/terraform-iam/fastlylog/main.tf @@ -31,19 +31,17 @@ resource "aws_s3_bucket_policy" "logs" { "Sid": "AllowNixOSOrgRead", "Effect": "Allow", "Principal": { - "AWS": "arn:aws:iam::008826681144:user/eelco.dolstra" + "AWS": "arn:aws:iam::008826681144:user/eelco.dolstra", + "AWS": "arn:aws:iam::008826681144:user/fastly-log-processor" }, - "Action": "s3:GetObject", - "Resource": "arn:aws:s3:::${aws_s3_bucket.logs.id}/*" - }, - { - "Sid": "AllowNixOSOrgList", - "Effect": "Allow", - "Principal": { - "AWS": "arn:aws:iam::008826681144:user/eelco.dolstra" - }, - "Action": "s3:ListBucket", - "Resource": "arn:aws:s3:::${aws_s3_bucket.logs.id}" + "Action": [ + "s3:GetObject", + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::${aws_s3_bucket.logs.id}/*", + "arn:aws:s3:::${aws_s3_bucket.logs.id}" + ] } ] }