Merge pull request #950 from mlibrary/BLUEDOC-1096-find-and-fix-resou…

…rce-consumption-2a

 BLUEDOC-1096 - Find and fix job consuming too many resources in prod…

app/jobs/find_and_fix_job.rb

@@ -83,7 +83,7 @@ def initialize( begin_date:, end_date: )
     @end_date = to_datetime( end_date )
   end
 
-  def include?( date: )
+  def include?( date )
     return false if date.nil?
     return date >= @begin if @end_date.nil?
     return date <= @end if @begin_date.nil?

config/scheduler_jobs_prod.yml

@@ -1,6 +1,6 @@
 # config/scheduler_jobs_prod.yml
 #
-# last updated: 2020/12/03 - production deployment
+# last updated: 2021/03/25 - production deployment
 #
 heartbeat_email_job:
   # Run once a day, one minute after midnight (which is offset by 4 or [5 during daylight savings time], due to GMT)

config/scheduler_jobs_test.yml

@@ -0,0 +1,273 @@
+# config/scheduler_jobs_prod.yml
+#
+# last updated: 2021/03/25 - testing deployment
+#
+heartbeat_email_job:
+  # Run once a day, one minute after midnight (which is offset by 4 or [5 during daylight savings time], due to GMT)
+  #      M H
+  # cron: '*/5 * * * *'
+  cron: '1 5 * * *'
+  # rails_env: production
+  class: HeartbeatEmailJob
+  queue: scheduler
+  description: Heartbeat email job.
+  args:
+    hostnames:
+      - 'deepblue.lib.umich.edu'
+    subscription_service_id: heartbeat_email_job
+
+heartbeat_email_job_hourly:
+  # Run once a day, one minute after midnight (which is offset by 4 or [5 during daylight savings time], due to GMT)
+  #      M H
+  # cron: '*/5 * * * *'
+  cron: '0 * * * *'
+  # rails_env: production
+  class: HeartbeatEmailJob
+  queue: scheduler
+  description: Heartbeat email job hourly.
+  args:
+    hostnames:
+      - 'x.deepblue.lib.umich.edu'
+    subscription_service_id: heartbeat_email_job_hourly
+
+find_and_fix_job:
+  # Run once a day, 15 minutes after midnight (which is offset by 4 or [5 during daylight savings time], due to GMT)
+  #       M H D
+  cron: '15 4 * * *'
+  class: FindAndFixJob
+  queue: scheduler
+  description: Find and fix problems
+  args:
+    email_results_to:
+      - '[email protected]'
+    filter_date_begin: now - 7 days
+    filter_date_end: now
+    find_and_fix_empty_file_size: true
+    find_and_fix_over_file_sets: true
+    find_and_fix_all_ordered_members_containing_nils: true
+    #    hostnames:
+    #      - 'deepblue.lib.umich.edu'
+    #      - 'staging.deepblue.lib.umich.edu'
+    #      - 'testing.deepblue.lib.umich.edu'
+    subscription_service_id: find_and_fix_job
+    verbose: true
+
+find_and_fix_empty_file_size:
+  # Run once a day, 15 minutes after midnight (which is offset by 4 or [5 during daylight savings time], due to GMT)
+  #       M H D
+  cron: '15 4 * * *'
+  class: FindAndFixJob
+  queue: scheduler
+  description: Find and fix problems - empty file size
+  args:
+    email_results_to:
+      - '[email protected]'
+    filter_date_begin: now - 7 days
+    filter_date_end: now
+    find_and_fix_empty_file_size: true
+    find_and_fix_over_file_sets: false
+    find_and_fix_all_ordered_members_containing_nils: false
+    hostnames:
+      - 'deepblue.lib.umich.edu'
+      - 'staging.deepblue.lib.umich.edu'
+      - 'testing.deepblue.lib.umich.edu'
+    subscription_service_id: find_and_fix_job
+    verbose: true
+
+find_and_fix_over_file_sets:
+  # Run once a day, 15 minutes after midnight (which is offset by 4 or [5 during daylight savings time], due to GMT)
+  #       M H D
+  cron: '15 6 * * *'
+  class: FindAndFixJob
+  queue: scheduler
+  description: Find and fix problems - over file sets
+  args:
+    email_results_to:
+      - '[email protected]'
+    filter_date_begin: now - 7 days
+    filter_date_end: now
+    find_and_fix_empty_file_size: false
+    find_and_fix_over_file_sets: true
+    find_and_fix_all_ordered_members_containing_nils: false
+    hostnames:
+      - 'deepblue.lib.umich.edu'
+      - 'staging.deepblue.lib.umich.edu'
+      - 'testing.deepblue.lib.umich.edu'
+    subscription_service_id: find_and_fix_job
+    verbose: true
+
+find_and_fix_all_ordered_members_containing_nils:
+  # Run once a day, 15 minutes after midnight (which is offset by 4 or [5 during daylight savings time], due to GMT)
+  #       M H D
+  cron: '15 5 * * *'
+  class: FindAndFixJob
+  queue: scheduler
+  description: Find and fix problems - ordered members
+  args:
+    email_results_to:
+      - '[email protected]'
+    filter_date_begin: now - 7 days
+    filter_date_end: now
+    find_and_fix_empty_file_size: false
+    find_and_fix_over_file_sets: false
+    find_and_fix_all_ordered_members_containing_nils: true
+    hostnames:
+      - 'deepblue.lib.umich.edu'
+      - 'staging.deepblue.lib.umich.edu'
+      - 'testing.deepblue.lib.umich.edu'
+    subscription_service_id: find_and_fix_job
+    verbose: true
+
+clean_tmp:
+  # Run once a week on Saturday, 6 minutes after midnight (which is offset by 4 or [5 during daylight savings time], due to GMT)
+  #       M H D
+  cron: '6 4 * * 6'
+  class: RakeTaskJob
+  queue: scheduler
+  description: Run rake tmp:clean (Saturdays
+  args:
+    hostnames:
+      - 'deepblue.lib.umich.edu'
+      - 'staging.deepblue.lib.umich.edu'
+      - 'testing.deepblue.lib.umich.edu'
+    email_results_to:
+      - '[email protected]'
+    job_delay: 0
+    rake_task: 'tmp:clean'
+    verbose: true
+
+clean_tmp2:
+  # Run once daily at 6 minutes after noon (which is offset by 4 or [5 during daylight savings time], due to GMT)
+  #       M H D
+  cron: '6 17 * * *'
+  class: RakeTaskJob
+  queue: scheduler
+  description: Run rake tmp:clean (daily on testing)
+  args:
+    hostnames:
+      - 'testing.deepblue.lib.umich.edu'
+    email_results_to:
+      - '[email protected]'
+    job_delay: 0
+    rake_task: 'tmp:clean'
+    verbose: true
+
+deactivate_expired_embargoes_job:
+  # Run once a day, five minutes after midnight (which is offset by 4 or [5 during daylight savings time], due to GMT)
+  #      M H
+  cron: '5 4 * * *'
+  # rails_env: production
+  class: DeactivateExpiredEmbargoesJob
+  queue: scheduler
+  description: Deactivate embargoes job.
+  args:
+    email_owner: true
+    test_mode: false
+    verbose: true
+
+about_to_deactivate_embargoes_job:
+  # Run once a day, fifteen minutes after midnight (which is offset by 4 or [5 during daylight savings time], due to GMT)
+  #      M  H
+  cron: '15 4 * * *'
+  # rails_env: production
+  class: AboutToExpireEmbargoesJob
+  queue: scheduler
+  description: About to deactivate embargoes job.
+  args:
+    email_owner: true
+    test_mode: false
+    verbose: true
+
+user_stat_importer_job:
+  # Run once a day, thirty minutes after midnight (which is offset by 4 or [5 during daylight savings time], due to GMT)
+  #      M  H
+  cron: '30 4 * * *'
+  # rails_env: production
+  class: UserStatImporterJob
+  queue: scheduler
+  description: Import user stats job.
+  args:
+    hostnames:
+      - 'deepblue.lib.umich.edu'
+    verbose: false
+    test: false
+
+works_report_job_daily:
+  # Run once a day, five minutes after midnight (which is offset by 4 or [5 during daylight savings time], due to GMT)
+  #      M H D
+  cron: '5 4 * * *'
+  class: WorksReportJob
+  queue: scheduler
+  description: Daily Works report job.
+  args:
+    hostnames:
+      - 'testing.deepblue.lib.umich.edu'
+    quiet: true
+    report_file_prefix: '%date%.%hostname%.works_report'
+    report_dir: '/deepbluedata-prep/reports'
+    subscription_service_id: works_report_job_daily
+
+works_report_job_monthly:
+  # Run once a month on the 1st, five minutes after midnight (which is offset by 4 or [5 during daylight savings time], due to GMT)
+  #      M H D
+  # cron: '*/5 * * * *'
+  cron: '5 4 1 * *'
+  class: WorksReportJob
+  queue: scheduler
+  description: Monthly Works report job.
+  args:
+    hostnames:
+      - 'deepblue.lib.umich.edu'
+      - 'staging.deepblue.lib.umich.edu'
+      - 'testing.deepblue.lib.umich.edu'
+    quiet: true
+    report_file_prefix: '%date%.%hostname%.works_report'
+    report_dir: '/deepbluedata-prep/reports'
+    subscription_service_id: works_report_job_monthly
+
+about_to_deactivate_embargoes_at_2_days_job:
+  # Run once a day, twenty minutes after midnight (which is offset by 4 or [5 during daylight savings time], due to GMT)
+  #      M  H
+  cron: '20 4 * * *'
+  # rails_env: production
+  class: AboutToExpireEmbargoesJob
+  queue: scheduler
+  description: About to deactivate embargoes job.
+  args:
+    email_owner: true
+    expiration_lead_days: 2
+    test_mode: false
+    hostnames:
+      - 'testing.deepblue.lib.umich.edu'
+    verbose: true
+
+update_condensed_events_job_daily:
+  # Run once a day, twenty-five minutes after midnight (which is offset by 4 or [5 during daylight savings time], due to GMT)
+  #      M H D
+  # cron: '*/5 * * * *'
+  cron: '25 4 * * *'
+  class: UpdateCondensedEventsJob
+  queue: scheduler
+  description: Update the condensed events job.
+  args:
+    hostnames:
+      - 'deepblue.lib.umich.edu'
+      - 'staging.deepblue.lib.umich.edu'
+      - 'testing.deepblue.lib.umich.edu'
+    quiet: true
+
+monthly_events_report_job:
+  # Run once a month on the 1st, twenty-five minutes after midnight (which is offset by 4 or [5 during daylight savings time], due to GMT)
+  #      M H D
+  # cron: '*/5 * * * *'
+  cron: '25 4 1 * *'
+  class: MonthlyEventsReportJob
+  queue: scheduler
+  description: Monthly events report job.
+  args:
+    hostnames:
+      - 'deepblue.lib.umich.edu'
+      - 'staging.deepblue.lib.umich.edu'
+      - 'testing.deepblue.lib.umich.edu'
+    quiet: true
+    this_month: false

spec/jobs/find_and_fix_job_spec.rb

@@ -125,15 +125,15 @@
       let(:work)        { build(:work, date_modified: test_date_in ) }
 
       it 'filters in' do
-        expect( filter.include?( date: work.date_modified ) ).to eq true
+        expect( filter.include?( work.date_modified ) ).to eq true
       end
     end
 
     context 'out' do
       let(:work)        { build(:work, date_modified: test_date_out ) }
 
       it 'filters in' do
-        expect( filter.include?( date: work.date_modified ) ).to eq false
+        expect( filter.include?( work.date_modified ) ).to eq false
       end
     end