From 1eb6d8bbe19449786be183abb0b7108094c695ec Mon Sep 17 00:00:00 2001 From: Aashish Passrija <40358683+akp2603@users.noreply.github.com> Date: Sat, 24 Oct 2020 01:00:42 +0530 Subject: [PATCH] Data to csv (#1900) Co-authored-by: Aashish Passrija Co-authored-by: Aashish Passrija Co-authored-by: Julia Nguyen --- .circleci/config.yml | 7 +- Gemfile | 5 + Gemfile.lock | 16 +++ app/controllers/users/reports_controller.rb | 28 +++++ app/helpers/users/reports_helper.rb | 35 ++++++ app/models/allyship.rb | 8 ++ app/models/application_record.rb | 17 +++ app/models/care_plan_contact.rb | 8 ++ app/models/category.rb | 10 ++ app/models/group.rb | 9 ++ app/models/group_member.rb | 8 ++ app/models/medication.rb | 25 ++++- app/models/meeting_member.rb | 9 ++ app/models/moment.rb | 17 +++ app/models/mood.rb | 10 ++ app/models/notification.rb | 8 ++ app/models/strategy.rb | 14 +++ app/models/user.rb | 83 +++++++++++++- app/models/users.rb | 6 + app/models/users/data_request.rb | 103 ++++++++++++++++++ app/workers/delete_stale_data_worker.rb | 18 +++ app/workers/process_data_request_worker.rb | 13 +++ config/env/development.example.env | 4 + config/env/test.example.env | 4 + config/initializers/sidekiq.rb | 14 +++ config/routes.rb | 13 ++- config/sidekiq.yml | 12 ++ config/sidekiq_schedule.yml | 6 + ...201008170831_create_users_data_requests.rb | 23 ++++ db/schema.rb | 12 ++ spec/factories/users/data_requests.rb | 35 ++++++ spec/models/users/data_request_spec.rb | 41 +++++++ spec/requests/users/reports_request_spec.rb | 94 ++++++++++++++++ 33 files changed, 710 insertions(+), 5 deletions(-) create mode 100644 app/controllers/users/reports_controller.rb create mode 100644 app/helpers/users/reports_helper.rb create mode 100644 app/models/users.rb create mode 100644 app/models/users/data_request.rb create mode 100644 app/workers/delete_stale_data_worker.rb create mode 100644 app/workers/process_data_request_worker.rb create mode 100644 config/initializers/sidekiq.rb create mode 100644 config/sidekiq.yml create mode 100644 config/sidekiq_schedule.yml create mode 100644 db/migrate/20201008170831_create_users_data_requests.rb create mode 100644 spec/factories/users/data_requests.rb create mode 100644 spec/models/users/data_request_spec.rb create mode 100644 spec/requests/users/reports_request_spec.rb diff --git a/.circleci/config.yml b/.circleci/config.yml index bd5e40954a..c3e14aa5cc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -17,6 +17,11 @@ defaults: &defaults POSTGRES_USER: circleci POSTGRES_DB: ifme_test POSTGRES_PASSWORD: "" + - image: redis + environment: + REDIS_HOST: redis + # The default Redis port + REDIS_PORT: 6379 steps: - checkout jobs: @@ -202,4 +207,4 @@ workflows: branches: ignore: /.*/ tags: - only: /^v[0-9]+\.[0-9]+\.[0-9]+/ \ No newline at end of file + only: /^v[0-9]+\.[0-9]+\.[0-9]+/ diff --git a/Gemfile b/Gemfile index 4b2a834c75..741ed62331 100644 --- a/Gemfile +++ b/Gemfile @@ -52,6 +52,11 @@ gem 'selenium-webdriver', '~> 3.142.3' gem 'rubyzip', '~> 1.3.0' +gem 'sidekiq', '5.0.5' +gem 'sidekiq-middleware' +gem 'sidekiq-failures' +gem "sidekiq-cron", "~> 1.1" + group :development, :test do gem 'bundler-audit' gem 'dotenv-rails', '~> 2.7.2' diff --git a/Gemfile.lock b/Gemfile.lock index 4428ff1092..b8e43053a0 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -454,6 +454,18 @@ GEM faraday (>= 0.7.6, < 1.0) shoulda-matchers (4.4.1) activesupport (>= 4.2.0) + sidekiq (5.0.5) + concurrent-ruby (~> 1.0) + connection_pool (~> 2.2, >= 2.2.0) + rack-protection (>= 1.5.0) + redis (>= 3.3.4, < 5) + sidekiq-cron (1.2.0) + fugit (~> 1.1) + sidekiq (>= 4.2.1) + sidekiq-failures (1.0.0) + sidekiq (>= 4.0.0) + sidekiq-middleware (0.3.0) + sidekiq (>= 2.12.4) signet (0.14.0) addressable (~> 2.3) faraday (>= 0.17.3, < 2.0) @@ -583,6 +595,10 @@ DEPENDENCIES selenium-webdriver (~> 3.142.3) sentry-raven shoulda-matchers + sidekiq (= 5.0.5) + sidekiq-cron (~> 1.1) + sidekiq-failures + sidekiq-middleware simplecov (= 0.16.1) spring turbolinks (~> 5.2.0) diff --git a/app/controllers/users/reports_controller.rb b/app/controllers/users/reports_controller.rb new file mode 100644 index 0000000000..0de28ce1b7 --- /dev/null +++ b/app/controllers/users/reports_controller.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true +module Users + class ReportsController < ApplicationController + before_action :authenticate_user! + include Users::ReportsHelper + + def submit_request + status, response = submit_request_helper(current_user) + render json: response, status: status + end + + def fetch_request_status + status, response = fetch_request_status_helper(current_user, + params[:request_id]) + render json: response, status: status + end + + def download_data + status, response = download_data_helper(current_user, + params[:request_id]) + if status != 200 + render json: response, status: status + else + send_file(response, status: 200) + end + end + end +end diff --git a/app/helpers/users/reports_helper.rb b/app/helpers/users/reports_helper.rb new file mode 100644 index 0000000000..dfd3ea05f0 --- /dev/null +++ b/app/helpers/users/reports_helper.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true +module Users + module ReportsHelper + def submit_request_helper(user) + [200, { request_id: user.generate_data_request }] + rescue StandardError => e + [422, { error: e.message }] + end + + def fetch_request_status_helper(user, request_id) + return 400, { error: "Request id can't be blank." } if request_id.blank? + + data_request = user.data_requests.find_by(request_id: request_id) + if data_request.blank? + return 404, { error: 'No such request exists for current user.' } + end + + [200, { current_status: data_request.status_id }] + end + + def download_data_helper(user, request_id) + return 400, { error: "Request id can't be blank." } if request_id.blank? + + data_request = user.data_requests.find_by( + request_id: request_id, + status_id: Users::DataRequest::STATUS[:success] + ) + if data_request.blank? || !File.exist?(data_request.file_path.to_s) + return 404, { error: 'Requested csv not found.' } + end + + [200, data_request.file_path] + end + end +end diff --git a/app/models/allyship.rb b/app/models/allyship.rb index 096c79d9d1..284672c088 100644 --- a/app/models/allyship.rb +++ b/app/models/allyship.rb @@ -12,6 +12,14 @@ # class Allyship < ApplicationRecord + USER_DATA_ATTRIBUTES = %w[ + id + created_at + updated_at + ally_id + status + ].map!(&:freeze).freeze + enum status: { accepted: 0, pending_from_user: 1, pending_from_ally: 2 } validate :different_users diff --git a/app/models/application_record.rb b/app/models/application_record.rb index 71fbba5b32..55263e298a 100644 --- a/app/models/application_record.rb +++ b/app/models/application_record.rb @@ -2,4 +2,21 @@ class ApplicationRecord < ActiveRecord::Base self.abstract_class = true + + def self.build_csv_rows(objects) + return [] if objects.blank? + + klass = objects.klass + data = [["#{klass.name.underscore}_info"]] + attributes = if klass.const_defined?('USER_DATA_ATTRIBUTES') + klass.const_get('USER_DATA_ATTRIBUTES') + else + klass.column_names + end + data << attributes + objects.each do |object| + data << attributes.map { |attribute| object.send(attribute.to_sym) } + end + data + end end diff --git a/app/models/care_plan_contact.rb b/app/models/care_plan_contact.rb index 70bbc595a5..35fd4e2974 100644 --- a/app/models/care_plan_contact.rb +++ b/app/models/care_plan_contact.rb @@ -12,6 +12,14 @@ # class CarePlanContact < ApplicationRecord + USER_DATA_ATTRIBUTES = %w[ + id + name + phone + created_at + updated_at + ].map!(&:freeze).freeze + validates :user_id, :name, presence: true belongs_to :user end diff --git a/app/models/category.rb b/app/models/category.rb index 02a520ba33..b497b9eb4e 100644 --- a/app/models/category.rb +++ b/app/models/category.rb @@ -23,4 +23,14 @@ class Category < ApplicationRecord has_many :moments_categories, dependent: :destroy has_many :strategies_categories, dependent: :destroy validates :visible, inclusion: [true, false] + + USER_DATA_ATTRIBUTES = %w[ + id + name + description + created_at + updated_at + slug + visible + ].map!(&:freeze).freeze end diff --git a/app/models/group.rb b/app/models/group.rb index 4f3c2ff86f..4dc18e5bc0 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -23,6 +23,15 @@ class Group < ApplicationRecord through: :group_members, source: :user after_destroy :destroy_notifications + USER_DATA_ATTRIBUTES = %w[ + id + name + created_at + updated_at + description + slug + ].map!(&:freeze).freeze + def led_by?(user) leaders.include? user end diff --git a/app/models/group_member.rb b/app/models/group_member.rb index a4fa161f25..d335f0ab49 100644 --- a/app/models/group_member.rb +++ b/app/models/group_member.rb @@ -12,6 +12,14 @@ # class GroupMember < ApplicationRecord + USER_DATA_ATTRIBUTES = %w[ + id + group_id + leader + created_at + updated_at + ].map!(&:freeze).freeze + after_destroy :destroy_meeting_memberships validates :group_id, :user_id, presence: true diff --git a/app/models/medication.rb b/app/models/medication.rb index 161e26f632..889fa205dd 100644 --- a/app/models/medication.rb +++ b/app/models/medication.rb @@ -18,14 +18,35 @@ # comments :text # slug :string # add_to_google_cal :boolean default(FALSE) -# weekly_dosage :integer -# default(["0", "1", "2", "3", "4", "5", "6"]), is an Array +# rubocop:disable Layout/LineLength +# weekly_dosage :integer default(["0", "1", "2", "3", "4", "5", "6"]), is an Array +# rubocop:enable Layout/LineLength +# class Medication < ApplicationRecord # dosage: amount of medication taken at one time # total: total quantity of medication # strength: strength of medication + USER_DATA_ATTRIBUTES = %w[ + id + name + dosage + refill + created_at + updated_at + user_id + total + strength + strength_unit + dosage_unit + total_unit + comments + slug + add_to_google_cal + weekly_dosage + ].map!(&:freeze).freeze + extend FriendlyId friendly_id :name belongs_to :user, foreign_key: :user_id diff --git a/app/models/meeting_member.rb b/app/models/meeting_member.rb index 94031181c0..069c88dcf9 100644 --- a/app/models/meeting_member.rb +++ b/app/models/meeting_member.rb @@ -13,6 +13,15 @@ # class MeetingMember < ApplicationRecord + USER_DATA_ATTRIBUTES = %w[ + id + meeting_id + leader + created_at + updated_at + google_cal_event_id + ].map!(&:freeze).freeze + validates :meeting_id, :user_id, presence: true validates :leader, inclusion: [true, false] diff --git a/app/models/moment.rb b/app/models/moment.rb index f630c67490..c62b71072e 100644 --- a/app/models/moment.rb +++ b/app/models/moment.rb @@ -25,6 +25,23 @@ class Moment < ApplicationRecord include CommonMethods extend FriendlyId + USER_DATA_ATTRIBUTES = %w[ + id + name + why + fix + created_at + updated_at + viewers + comment + slug + secret_share_identifier + secret_share_expires_at + published_at + bookmarked + resource_recommendations + ].map!(&:freeze).freeze + friendly_id :name serialize :viewers, Array diff --git a/app/models/mood.rb b/app/models/mood.rb index 5ffb829025..98093167ed 100644 --- a/app/models/mood.rb +++ b/app/models/mood.rb @@ -14,6 +14,16 @@ # class Mood < ApplicationRecord + USER_DATA_ATTRIBUTES = %w[ + id + name + description + created_at + updated_at + slug + visible + ].map!(&:freeze).freeze + extend FriendlyId friendly_id :name validates :user_id, :name, presence: true diff --git a/app/models/notification.rb b/app/models/notification.rb index 6832d3bb1d..b2fc8283d2 100644 --- a/app/models/notification.rb +++ b/app/models/notification.rb @@ -12,6 +12,14 @@ # class Notification < ApplicationRecord + USER_DATA_ATTRIBUTES = %w[ + id + uniqueid + data + created_at + updated_at + ].map!(&:freeze).freeze + validates :user_id, :uniqueid, :data, presence: true belongs_to :user, foreign_key: :user_id diff --git a/app/models/strategy.rb b/app/models/strategy.rb index 03d35ab023..53c2933772 100644 --- a/app/models/strategy.rb +++ b/app/models/strategy.rb @@ -22,6 +22,20 @@ class Strategy < ApplicationRecord include CommonMethods extend FriendlyId + USER_DATA_ATTRIBUTES = %w[ + id + description + viewers + comment + created_at + updated_at + name + slug + published_at + visible + bookmarked + ].map!(&:freeze).freeze + friendly_id :name serialize :viewers, Array diff --git a/app/models/user.rb b/app/models/user.rb index 147a1fa0d8..0b1825c6ed 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -44,12 +44,38 @@ # admin :boolean default(FALSE) # third_party_avatar :text # - +# rubocop:disable Metrics/ClassLength class User < ApplicationRecord include AllyConcern OAUTH_TOKEN_URL = 'https://accounts.google.com/o/oauth2/token' + USER_DATA_ATTRIBUTES = %w[ + id + email + sign_in_count + current_sign_in_at + last_sign_in_at + current_sign_in_ip + last_sign_in_ip + created_at + updated_at + name + location + timezone + about + conditions + uid + provider + comment_notify + ally_notify + group_notify + meeting_notify + locale + banned + admin + ].map!(&:freeze).freeze + # Include default devise modules. Others available are: # :confirmable, :lockable, :timeoutable and :omniauthable devise :invitable, :database_authenticatable, :registerable, :uid, @@ -70,6 +96,9 @@ class User < ApplicationRecord has_many :moments has_many :categories has_many :care_plan_contacts + # rubocop:disable Layout/LineLength + has_many :data_requests, class_name: 'Users::DataRequest', foreign_key: :user_id + # rubocop:enable Layout/LineLength belongs_to :invited_by, class_name: 'User' after_initialize :set_defaults, unless: :persisted? @@ -132,6 +161,57 @@ def update_access_token new_access_token end + def build_csv_data + user_data = [['user_info']] + user_data << USER_DATA_ATTRIBUTES + user_data << USER_DATA_ATTRIBUTES.map { |attribute| send(attribute.to_sym) } + user_data += Group.build_csv_rows(groups) + user_data += GroupMember.build_csv_rows(group_members) + user_data += Category.build_csv_rows(categories) + user_data += Medication.build_csv_rows(medications) + user_data += Strategy.build_csv_rows(strategies) + user_data += Moment.build_csv_rows(moments) + user_data += Notification.build_csv_rows(notifications) + user_data += Mood.build_csv_rows(moods) + user_data += CarePlanContact.build_csv_rows(care_plan_contacts) + user_data += Allyship.build_csv_rows(allyships) + user_data += MeetingMember.build_csv_rows(meeting_members) + user_data + end + + def generate_data_request + ActiveRecord::Base.transaction do + lock! + data_request = data_requests + .where(status_id: Users::DataRequest::STATUS[:enqueued]) + .first_or_initialize + return data_request.request_id if data_request.request_id.present? + + data_request.request_id = SecureRandom.uuid + data_request.save! + return data_request.request_id + end + end + + def delete_stale_data_file + successful_data_requests = data_requests + .where( + status_id: Users::DataRequest::STATUS[:success] + ) + .order('updated_at desc') + return if successful_data_requests.count < 2 + + ActiveRecord::Base.transaction do + stale_data_requests = successful_data_requests.where.not( + id: successful_data_requests.first + ) + stale_data_requests.each do |dr| + File.delete(dr.file_path) if File.exist?(dr.file_path) + dr.update!(status_id: Users::DataRequest::STATUS[:deleted]) + end + end + end + private_class_method def self.update_access_token_fields(user:, access_token:) user.update!( provider: access_token.provider, @@ -152,3 +232,4 @@ def access_token_expired? !access_expires_at || Time.zone.now > access_expires_at end end +# rubocop:enable Metrics/ClassLength diff --git a/app/models/users.rb b/app/models/users.rb new file mode 100644 index 0000000000..f6d6b4a8a7 --- /dev/null +++ b/app/models/users.rb @@ -0,0 +1,6 @@ +# frozen_string_literal: true +module Users + def self.table_name_prefix + 'users_' + end +end diff --git a/app/models/users/data_request.rb b/app/models/users/data_request.rb new file mode 100644 index 0000000000..2c2fefad2b --- /dev/null +++ b/app/models/users/data_request.rb @@ -0,0 +1,103 @@ +# frozen_string_literal: true +# == Schema Information +# +# Table name: users_data_requests +# +# id :bigint not null, primary key +# request_id :string not null +# status_id :integer not null +# user_id :bigint not null +# created_at :datetime not null +# updated_at :datetime not null +# + +module Users + class DataRequest < ApplicationRecord + STATUS = { + enqueued: 1, + success: 2, + failed: 3, + deleted: 4 + }.freeze + + ASSOCIATIONS_TO_EXPORT = %i[ + allyships + group_members + groups + categories + medications + strategies + moments + notifications + moods + care_plan_contacts + meeting_members + ].freeze + + DEFAULT_FILE_PATH = Rails.root.join('tmp', 'csv_data') + + belongs_to :user, class_name: '::User', foreign_key: 'user_id' + + after_commit :after_commit_tasks + + validates :user_id, uniqueness: { + scope: :status_id, + message: 'There is already a request enqueued for this user.' + }, + if: -> { status_id == STATUS[:enqueued] } + + validates :request_id, uniqueness: { + message: 'There is already a request with this request_id.' + } + + validates :status_id, inclusion: { + in: STATUS.values, + message: proc { |request| "'#{request.status_id}' is not valid." } + }, + presence: true + + validates :request_id, presence: true + + def after_commit_tasks + return unless saved_change_to_id? && status_id == STATUS[:enqueued] + + Dir.mkdir(DEFAULT_FILE_PATH) unless File.exist?(DEFAULT_FILE_PATH) + + enqueue_download_request + end + + def enqueue_download_request + ProcessDataRequestWorker.perform_async(request_id) + end + + def create_csv + user = User.includes(*ASSOCIATIONS_TO_EXPORT).find(user_id) + begin + require 'csv' + csv_rows = user.build_csv_data + write_to_csv(csv_rows) + self.status_id = STATUS[:success] + save! + user.delete_stale_data_file + rescue StandardError + File.delete(file_path) if file_path.present? && File.exist?(file_path) + self.status_id = STATUS[:failed] + save! + end + end + + def file_path + DEFAULT_FILE_PATH.join("#{request_id}.csv").to_s + end + + private + + def write_to_csv(csv_rows) + CSV.open(file_path, 'wb') do |csv_row| + csv_rows.each do |row| + csv_row << row + end + end + end + end +end diff --git a/app/workers/delete_stale_data_worker.rb b/app/workers/delete_stale_data_worker.rb new file mode 100644 index 0000000000..47ef77dbb9 --- /dev/null +++ b/app/workers/delete_stale_data_worker.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true +class DeleteStaleDataWorker + include Sidekiq::Worker + + ACTIVE_DURATION = 30.days + + def perform + Users::DataRequest + .where('updated_at < ?', (Time.current - ACTIVE_DURATION)) + .where(status_id: Users::DataRequest::STATUS[:success]) + .each do |dr| + if dr.file_path.present? && File.exist?(dr.file_path) + File.delete(dr.file_path) + end + dr.update(status_id: Users::DataRequest::STATUS[:deleted]) + end + end +end diff --git a/app/workers/process_data_request_worker.rb b/app/workers/process_data_request_worker.rb new file mode 100644 index 0000000000..eaaa8e61bc --- /dev/null +++ b/app/workers/process_data_request_worker.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true +class ProcessDataRequestWorker + include Sidekiq::Worker + sidekiq_options queue: 'critical' + + def perform(request_id) + data_request = Users::DataRequest.find_by(request_id: request_id) + return if data_request.blank? || + data_request.status_id == Users::DataRequest::STATUS[:success] + + data_request.create_csv + end +end diff --git a/config/env/development.example.env b/config/env/development.example.env index cdc310b186..1a12422d8a 100644 --- a/config/env/development.example.env +++ b/config/env/development.example.env @@ -39,3 +39,7 @@ RAISE_DELIVERY_ERRORS="false" PSQL_HOST="" PSQL_USERNAME="" PSQL_PASSWORD="" + +# REDIS +REDIS_HOST="" +REDIS_PORT="" diff --git a/config/env/test.example.env b/config/env/test.example.env index cdc310b186..caa7dc5f9d 100644 --- a/config/env/test.example.env +++ b/config/env/test.example.env @@ -39,3 +39,7 @@ RAISE_DELIVERY_ERRORS="false" PSQL_HOST="" PSQL_USERNAME="" PSQL_PASSWORD="" + +# REDIS +REDIS_HOST="" +REDIS_PORT="" \ No newline at end of file diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb new file mode 100644 index 0000000000..f557bbb0b0 --- /dev/null +++ b/config/initializers/sidekiq.rb @@ -0,0 +1,14 @@ +Sidekiq.configure_client do |config| + config.redis = { url: "redis://#{ENV['REDIS_HOST']}:#{ENV['REDIS_PORT']}/0", namespace: 'ifme' } +end + +Sidekiq.configure_server do |config| + config.redis = { url: "redis://#{ENV['REDIS_HOST']}:#{ENV['REDIS_PORT']}/0", namespace: 'ifme' } + schedule_file = "config/sidekiq_schedule.yml" + + if File.exist?(schedule_file) + Sidekiq::Cron::Job.load_from_hash YAML.load_file(schedule_file) + end +end + +Sidekiq.default_worker_options = { :backtrace => true, :unique => :all, :failures => true } diff --git a/config/routes.rb b/config/routes.rb index df386bc259..f8bc0459ae 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true - +require 'sidekiq/web' +require 'sidekiq/cron/web' Rails.application.routes.draw do get 'errors/not_found' get 'errors/internal_server_error' @@ -7,6 +8,10 @@ get '/404' => 'errors#not_found' get '/500' => 'errors#internal_server_error' + authenticate :user, lambda { |u| u.admin? } do + mount Sidekiq::Web => '/sidekiq' + end + resources :allies, only: :index do collection do post 'add' @@ -126,6 +131,12 @@ invitations: 'users/invitations', sessions: :sessions } + namespace :users do + post "/data" => "reports#submit_request" + get "/data/status" => "reports#fetch_request_status" + get "/data/download" => "reports#download_data" + end + post 'pusher/auth' Rails.configuration.i18n.available_locales.each do |locale| diff --git a/config/sidekiq.yml b/config/sidekiq.yml new file mode 100644 index 0000000000..e471576c4d --- /dev/null +++ b/config/sidekiq.yml @@ -0,0 +1,12 @@ +--- +:concurrency: 1 +:logfile: log/sidekiq.log +staging: + :concurrency: 1 +production: + :concurrency: 1 + :timeout: 10800 +:queues: + - critical + - default + - low diff --git a/config/sidekiq_schedule.yml b/config/sidekiq_schedule.yml new file mode 100644 index 0000000000..9582ceb791 --- /dev/null +++ b/config/sidekiq_schedule.yml @@ -0,0 +1,6 @@ + +delete_stale_csv: + cron: "0 0 0 * * * *" + class: "DeleteStaleDataWorker" + queue: default + diff --git a/db/migrate/20201008170831_create_users_data_requests.rb b/db/migrate/20201008170831_create_users_data_requests.rb new file mode 100644 index 0000000000..a50ff95b38 --- /dev/null +++ b/db/migrate/20201008170831_create_users_data_requests.rb @@ -0,0 +1,23 @@ +class CreateUsersDataRequests < ActiveRecord::Migration[6.0] + def up + unless table_exists? :users_data_requests + create_table :users_data_requests do |t| + t.string :request_id, null: false + t.integer :status_id, null: false + t.references :user, foreign_key: true, null: false + t.timestamps + end + add_index :users_data_requests, :request_id, unique: true unless index_exists?(:users_data_requests, :request_id) + add_index :users_data_requests, :user_id unless index_exists?(:users_data_requests, :user_id) + execute "CREATE UNIQUE INDEX IF NOT EXISTS index_users_data_requests_on_users_id_and_status_uniq + ON users_data_requests(user_id, status_id) WHERE (status_id = #{Users::DataRequest::STATUS[:enqueued]})" + end + end + + def down + execute "DROP INDEX IF EXISTS index_users_data_requests_on_users_id_and_status_uniq" + remove_index :users_data_requests, :request_id if index_exists?(:users_data_requests, :request_id) + remove_index :users_data_requests, :user_id if index_exists?(:users_data_requests, :user_id) + drop_table :users_data_requests if table_exists? :users_data_requests + end +end diff --git a/db/schema.rb b/db/schema.rb index 4fef2d7bb8..28741c2178 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -302,6 +302,17 @@ t.index ["uid"], name: "index_users_on_uid", unique: true end + create_table "users_data_requests", force: :cascade do |t| + t.string "request_id", null: false + t.integer "status_id", null: false + t.bigint "user_id", null: false + t.datetime "created_at", precision: 6, null: false + t.datetime "updated_at", precision: 6, null: false + t.index ["request_id"], name: "index_users_data_requests_on_request_id", unique: true + t.index ["user_id", "status_id"], name: "index_users_data_requests_on_users_id_and_status_uniq", unique: true, where: "(status_id = 1)" + t.index ["user_id"], name: "index_users_data_requests_on_user_id" + end + add_foreign_key "moments_categories", "categories" add_foreign_key "moments_categories", "moments" add_foreign_key "moments_moods", "moments" @@ -310,4 +321,5 @@ add_foreign_key "moments_strategies", "strategies" add_foreign_key "strategies_categories", "categories" add_foreign_key "strategies_categories", "strategies" + add_foreign_key "users_data_requests", "users" end diff --git a/spec/factories/users/data_requests.rb b/spec/factories/users/data_requests.rb new file mode 100644 index 0000000000..0f4c8f996c --- /dev/null +++ b/spec/factories/users/data_requests.rb @@ -0,0 +1,35 @@ +# == Schema Information +# +# Table name: users_data_requests +# +# id :bigint not null, primary key +# request_id :string not null +# status_id :integer not null +# user_id :bigint not null +# created_at :datetime not null +# updated_at :datetime not null +# + +EXAMPLE_UUID = "89ebd1e0-c500-4bda-b879-370e01f7f7f9" + +FactoryBot.define do + factory :partial_data_request, class: 'Users::DataRequest' do + transient do + use_example_uuid { false } + end + user { create(:user) } + request_id { use_example_uuid ? EXAMPLE_UUID : SecureRandom.uuid } + end + + factory :enqueued_data_request, parent: :partial_data_request do + status_id {Users::DataRequest::STATUS[:enqueued]} + end + + factory :invalid_status_data_request, parent: :partial_data_request do + status_id { 10 } + end + + factory :empty_request_id_data_request, parent: :enqueued_data_request do + request_id { nil } + end +end diff --git a/spec/models/users/data_request_spec.rb b/spec/models/users/data_request_spec.rb new file mode 100644 index 0000000000..88fdfd0dc5 --- /dev/null +++ b/spec/models/users/data_request_spec.rb @@ -0,0 +1,41 @@ +# == Schema Information +# +# Table name: users_data_requests +# +# id :bigint not null, primary key +# request_id :string not null +# status_id :integer not null +# user_id :bigint not null +# created_at :datetime not null +# updated_at :datetime not null +# + +RSpec.describe Users::DataRequest, type: :model do + context 'validations' do + it 'is invalid without a request_id' do + data_request = build(:empty_request_id_data_request) + expect(data_request).to have(1).error_on(:request_id) + end + + it 'is invalid without a status_id' do + data_request = build(:partial_data_request) + expect(data_request).to have(2).error_on(:status_id) + end + + it 'is invalid with an invalid status_id' do + data_request = build(:invalid_status_data_request) + expect(data_request).to have(1).error_on(:status_id) + end + + it 'is a valid data request object' do + data_request = create(:enqueued_data_request) + expect(data_request.valid?).to be(true) + end + + it 'fails unique request_id check' do + data_request1 = create(:enqueued_data_request, use_example_uuid: true) + data_request2 = build(:enqueued_data_request, use_example_uuid: true) + expect(data_request2.valid?).to be(false) + end + end +end diff --git a/spec/requests/users/reports_request_spec.rb b/spec/requests/users/reports_request_spec.rb new file mode 100644 index 0000000000..3228e48147 --- /dev/null +++ b/spec/requests/users/reports_request_spec.rb @@ -0,0 +1,94 @@ +RSpec.describe "Users::Reports", type: :request do + let(:user) { create(:user) } + + describe '#submit_request' do + context 'when the user is logged in' do + before { sign_in user } + + it 'creates a data download request' do + post users_data_path + expect(status).to eq(200) + data_request = Users::DataRequest.last + expect(JSON.parse(response.body)['request_id']).to eq(data_request.request_id) + expect(user.id).to eq(data_request.user_id) + end + end + + context 'when the user is not logged in' do + before { post users_data_path } + it_behaves_like :with_no_logged_in_user + end + end + + describe '#fetch_request_status' do + context 'when the user is logged in' do + before { sign_in user } + + it 'fetches the status of data request with a blank request_id' do + get users_data_status_path + expect(status).to eq(400) + expect(JSON.parse(response.body)).to have_key('error') + end + + it 'fetches the status of data request with a random request_id' do + params = { request_id: SecureRandom.uuid } + get users_data_status_path, params: params + expect(status).to eq(404) + expect(JSON.parse(response.body)).to have_key('error') + end + + it "creates a data request and then fetches it's status" do + post users_data_path + expect(status).to eq(200) + params = { request_id: JSON.parse(response.body)['request_id'] } + get users_data_status_path, params: params + expect(status).to eq(200) + expect(JSON.parse(response.body)).to have_key('current_status') + end + end + + context 'when the user is not logged in' do + before { get users_data_status_path } + it_behaves_like :with_no_logged_in_user + end + end + + describe '#download_data' do + context 'when the user is logged in' do + before { sign_in user } + + it 'fetches the file with a blank request_id' do + get users_data_download_path + expect(status).to eq(400) + expect(JSON.parse(response.body)).to have_key('error') + end + + it 'fetches the file with a random request_id' do + params = { request_id: SecureRandom.uuid } + get users_data_download_path, params: params + expect(status).to eq(404) + expect(JSON.parse(response.body)).to have_key('error') + end + + it "creates a data request and then fetches it's status and then fetches the file" do + post users_data_path + expect(status).to eq(200) + params = { request_id: JSON.parse(response.body)['request_id'] } + get users_data_status_path, params: params + expect(status).to eq(200) + expect(JSON.parse(response.body)).to have_key('current_status') + ProcessDataRequestWorker.new.perform(params[:request_id]) + data_request = Users::DataRequest.find_by(request_id: params[:request_id]) + get users_data_download_path, params: params + expect(status).to eq(200) + expect(File.exist?(data_request.file_path.to_s)).to be(true) + File.delete(data_request.file_path.to_s) if File.exist?(data_request.file_path.to_s) + end + end + + context 'when the user is not logged in' do + before { get users_data_download_path } + it_behaves_like :with_no_logged_in_user + end + end +end