Skip to content

Commit

Permalink
Track data in the populate and import
Browse files Browse the repository at this point in the history
  • Loading branch information
epugh committed Nov 1, 2024
1 parent 396c0a6 commit b5e28c6
Show file tree
Hide file tree
Showing 10 changed files with 72 additions and 40 deletions.
12 changes: 11 additions & 1 deletion app/controllers/api/v1/books/populate_controller.rb
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,10 @@ def update
compressed_data = Zlib::Deflate.deflate(serialized_data)
@book.populate_file.attach(io: StringIO.new(compressed_data), filename: "book_populate_#{@book.id}.bin.zip",
content_type: 'application/zip')
PopulateBookJob.perform_later current_user, @book, @case
track_book_populate_queued do
PopulateBookJob.perform_later current_user, @book, @case
end

head :no_content
end

Expand All @@ -36,6 +39,13 @@ def query_doc_pairs_params
# object
request.parameters
end

def track_book_populate_queued
@book.update(populate_job: "queued at #{Time.zone.now}")

# Yield to the block to perform the job
yield if block_given?
end
end
end
end
Expand Down
2 changes: 1 addition & 1 deletion app/controllers/api/v1/export/books_controller.rb
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ class BooksController < Api::ApiController
before_action :check_book

# rubocop:disable Metrics/MethodLength
def show
def update
# WARNING books/export_controller.rb and
# api/v1/export/books_controller.rb ARE DUPLICATED
message = nil
Expand Down
2 changes: 1 addition & 1 deletion app/controllers/books/export_controller.rb
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ class ExportController < ApplicationController
before_action :set_book
before_action :check_book

def show
def update
# WARNING books/export_controller.rb and
# api/v1/export/books_controller.rb ARE DUPLICATED
message = nil
Expand Down
62 changes: 38 additions & 24 deletions app/controllers/books/import_controller.rb
Original file line number Diff line number Diff line change
Expand Up @@ -21,41 +21,48 @@ def create
force_create_users = bool.deserialize params[:book][:force_create_users]

uploaded_file = params[:book][:import_file]
tempfile = uploaded_file.tempfile
json_data = if 'application/zip' == uploaded_file.content_type || uploaded_file.path.end_with?('.zip')
# Handle zip file
Zip::File.open(tempfile.path) do |zip_file|
# Assuming the zip contains only one JSON file
json_file_entry = zip_file.entries.find { |e| e.name.end_with?('.json') }
if json_file_entry
json_file_entry.get_input_stream { |io| read_json(io) }
else
raise 'No JSON file found in the zip.'
if uploaded_file.nil?
@book.errors.add(:base, 'You must select the file to be imported first.')
else
tempfile = uploaded_file.tempfile
json_data = if 'application/zip' == uploaded_file.content_type || uploaded_file.path.end_with?('.zip')
# Handle zip file
Zip::File.open(tempfile.path) do |zip_file|
# Assuming the zip contains only one JSON file
json_file_entry = zip_file.entries.find { |e| e.name.end_with?('.json') }
if json_file_entry
json_file_entry.get_input_stream { |io| read_json(io) }
else
raise 'No JSON file found in the zip.'
end
end
else
# Handle normal JSON file
read_json(tempfile)
end
else
# Handle normal JSON file
read_json(tempfile)
end

begin
params_to_use = json_data.deep_symbolize_keys
end
if @book.errors.empty?
begin
params_to_use = json_data.deep_symbolize_keys

@book.name = params_to_use[:name]
@book.name = params_to_use[:name]

service = ::BookImporter.new @book, current_user, params_to_use, { force_create_users: force_create_users }
service.validate
rescue JSON::ParserError => e
@book.errors.add(:base, "Invalid JSON file format: #{e.message}")
service = ::BookImporter.new @book, current_user, params_to_use, { force_create_users: force_create_users }
service.validate
rescue JSON::ParserError => e
@book.errors.add(:base, "Invalid JSON file format: #{e.message}")
end
end

if @book.errors.empty? && @book.save
serialized_data = Marshal.dump(params_to_use)
compressed_data = Zlib::Deflate.deflate(serialized_data)
@book.import_file.attach(io: StringIO.new(compressed_data), filename: "book_import_#{@book.id}.bin.zip",
content_type: 'application/zip')
@book.save
ImportBookJob.perform_later current_user, @book

track_book_import_queued do
ImportBookJob.perform_later current_user, @book
end
redirect_to @book, notice: 'Book was successfully created.'
else
render :new
Expand All @@ -70,5 +77,12 @@ def create
def read_json file
JSON.parse(file.read)
end

def track_book_import_queued
@book.update(import_job: "queued at #{Time.zone.now}")

# Yield to the block to perform the job
yield if block_given?
end
end
end
2 changes: 2 additions & 0 deletions app/jobs/import_book_job.rb
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ class ImportBookJob < ApplicationJob

# rubocop:disable Security/MarshalLoad
def perform user, book
book.update(import_job: "import started at #{Time.zone.now}")
options = {}

compressed_data = book.import_file.download
Expand All @@ -15,6 +16,7 @@ def perform user, book

service.import
book.import_file.purge
book.import_job = nil
book.save
end
# rubocop:enable Security/MarshalLoad
Expand Down
2 changes: 2 additions & 0 deletions app/jobs/populate_book_job.rb
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ def perform user, book, kase
# down the road we should be using ActiveRecord-import and first_or_initialize instead.
# See how snapshots are managed.

book.update(populate_job: "populate started at #{Time.zone.now}")
compressed_data = book.populate_file.download
serialized_data = Zlib::Inflate.inflate(compressed_data)
params = Marshal.load(serialized_data)
Expand Down Expand Up @@ -50,6 +51,7 @@ def perform user, book, kase
)
end
book.populate_file.purge
book.populate_job = nil
book.save

Analytics::Tracker.track_query_doc_pairs_bulk_updated_event user, book, is_book_empty
Expand Down
1 change: 1 addition & 0 deletions app/views/books/edit.html.erb
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
<div class="btn-group me-2">
<%= link_to 'New Book', new_book_path, class: "btn btn-sm btn-outline-secondary" %>
<button type="button" class="btn btn-sm btn-outline-secondary">Export</button>
<%= button_to 'Delete Book', @book, method: :delete, class: "btn btn-sm btn-danger" %>
</div>
<button type="button" class="btn btn-sm btn-outline-secondary dropdown-toggle">
<span data-feather="calendar" class="align-text-bottom"></span>
Expand Down
7 changes: 5 additions & 2 deletions app/views/books/show.html.erb
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
<div class="btn-toolbar mb-2 mb-md-0">
<div class="btn-group me-2">
<%= link_to 'New Book', new_book_path, class: "btn btn-sm btn-outline-secondary" %>
<%= link_to 'Export', books_export_path(@book), data: { turbo: false }, class: "btn btn-sm btn-outline-secondary" %>
<%= button_to 'Export', books_export_path(@book), method: :put, data: { turbo: false }, class: "btn btn-sm btn-outline-secondary" %>
</div>
</div>
</div>
Expand Down Expand Up @@ -71,6 +71,9 @@ This book consists of <%= @book.query_doc_pairs.count %> query document pairs an
<% end %>
<% if @book.import_file.attached? %>
<%= link_to 'Import file', rails_blob_path(@book.import_file.blob, only_path: true) %> made <%= time_ago_in_words @book.import_file.created_at %> ago.
<%if @book.import_job %>
There is currently a import job for this book. It is <code><%=@book.import_job%></code>.
<% end %>
<% end %>
<% if @book.export_file.attached? %>
<%= link_to 'Export file', rails_blob_path(@book.export_file.blob, only_path: true) %> made <%= time_ago_in_words @book.export_file.created_at %> ago.
Expand Down Expand Up @@ -105,7 +108,7 @@ This book consists of <%= @book.query_doc_pairs.count %> query document pairs an
<% if @book.export_job %>
The book is currently being exported. The status is <code><%=@book.export_job%></code>.
<% else %>
<%= button_to 'Export', books_export_path(@book), data: { turbo: false }, method: :get, class: "#{@book.export_job.nil? ? '' : 'disabled'}" %>
<%= button_to "#{@book.export_file.present? ? 'Re-Export' : 'Export'}", books_export_path(@book), method: :put, data: { turbo: false }, class: "#{@book.export_job.nil? ? '' : 'disabled'}" %>
<% end %>
<% if @book.export_file.present? %>
<p>
Expand Down
10 changes: 5 additions & 5 deletions config/routes.rb
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@

namespace :books do
resources :import, only: [ :new, :create ]
resources :export, only: [ :show ], param: :book_id
resources :export, only: [ :update ], param: :book_id
end

devise_for :users, controllers: {
Expand Down Expand Up @@ -242,11 +242,11 @@

# Exports
namespace :export do
resources :books, only: [ :show ], param: :book_id
resources :cases, only: [ :show ], param: :case_id
resources :ratings, only: [ :show ], param: :case_id
resources :books, only: [ :update ], param: :book_id
resources :cases, only: [ :show ], param: :case_id # should be post (:update)
resources :ratings, only: [ :show ], param: :case_id # should be post (:update)
namespace :queries do
resources :information_needs, only: [ :show ], param: :case_id
resources :information_needs, only: [ :show ], param: :case_id # should be post (:update)
end
end

Expand Down
12 changes: 6 additions & 6 deletions test/controllers/api/v1/export/books_controller_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -21,15 +21,15 @@ class BooksControllerTest < ActionController::TestCase
test 'the book returns a message on start' do
assert doug.books.include? book

get :show, params: { book_id: book.id }
post :update, params: { book_id: book.id }
assert_response :ok
body = response.parsed_body

assert_equal body['message'], 'Starting export of book as file.'
end

test 'duplicate calls report back in progress work' do
get :show, params: { book_id: book.id }
post :update, params: { book_id: book.id }
assert_response :ok

assert_enqueued_with(job: ExportBookJob, args: [ book ])
Expand All @@ -38,27 +38,27 @@ class BooksControllerTest < ActionController::TestCase
# assert book.job_statuses

# duplicate call
get :show, params: { book_id: book.id }
post :update, params: { book_id: book.id }
assert_response :ok
body = response.parsed_body
assert body['message'].start_with? 'Currently exporting book as file. Status is queued at'

get :show, params: { book_id: book.id }
post :update, params: { book_id: book.id }
assert_response :ok
body = response.parsed_body
assert body['message'].start_with? 'Currently exporting book as file. Status is queued at'
end

test 'running a job and waiting gives you the resulting zip file' do
get :show, params: { book_id: book.id }
post :update, params: { book_id: book.id }
assert_response :ok
body = response.parsed_body

assert_equal body['message'], 'Starting export of book as file.'

perform_enqueued_jobs

get :show, params: { book_id: book.id }
post :update, params: { book_id: book.id }
assert_response :ok
body = response.parsed_body
assert_not_nil body['download_file_url']
Expand Down

0 comments on commit b5e28c6

Please sign in to comment.