From df9ce17aea879c428a85b3c54f5ba4d522746b34 Mon Sep 17 00:00:00 2001
From: Tom Aldcroft <taldcroft@gmail.com>
Date: Mon, 25 Oct 2021 17:28:22 -0400
Subject: [PATCH 1/7] Remove sybase option in parse_cmd_load_gen

---
 parse_cmd_load_gen.pl | 46 ++++++++++++++++---------------------------
 1 file changed, 17 insertions(+), 29 deletions(-)

diff --git a/parse_cmd_load_gen.pl b/parse_cmd_load_gen.pl
index ddf2888..5d5b6e8 100755
--- a/parse_cmd_load_gen.pl
+++ b/parse_cmd_load_gen.pl
@@ -57,29 +57,17 @@
     print "Updating clgps tables with files newer than ", $opt{touch_file}, "\n";
 }
 
-if (-e $opt{touch_file}){    
+if (-e $opt{touch_file}){
     $touch_stat = stat($opt{touch_file});
 }
 
 
-my $load_arg;
-if ($opt{dbi} eq 'sybase'){
-    my $user = defined $opt{user} ? $opt{user}
-              :      $opt{dryrun} ? 'aca_read' 
-                                  : 'aca_ops';
-    my $database = defined $opt{database}      ? $opt{database} 
-                 : defined $ENV{SKA_DATABASE}  ? $ENV{SKA_DATABASE}
-                                               : 'aca';
-    $load_arg = sprintf("%s-%s-%s", 'sybase', $database, $user);
-}
-else{
-    $load_arg = { database  => $opt{server},
+my $load_arg = { database  => $opt{server},
                      type => 'array',
-                     raise_error => 1,  
-                     print_error => 1,  
-                     DBI_module => 'dbi:SQLite', 
+                     raise_error => 1,
+                     print_error => 1,
+                     DBI_module => 'dbi:SQLite',
                 };
-}
 
 my $load_handle = sql_connect($load_arg);
 
@@ -111,7 +99,7 @@
 	if ($max_touch_time < $mtime){
 	    $max_touch_time = $mtime;
 	    $max_touch_file = $file;
-	}    
+	}
     }
 }
 
@@ -133,9 +121,9 @@ sub update_for_file{
     my $file = shift;
 
     print "Parsing $file" if $opt{verbose};
-	
+
     my $file_stat = stat("$file");
-	
+
     my ( $week, $loads ) = parse_clgps( $file );
     my ( $dir, $filename);
     if ($file =~ /${mp_dir}(\/\d{4}\/\w{3}\d{4}\/ofls\w?\/)mps\/(C.*\.sum)/){
@@ -152,23 +140,23 @@ sub update_for_file{
     if (defined $dir and defined $filename){
 	$week->{dir} = $dir;
 	$week->{file} = $filename;
-	$week->{sumfile_modtime} = $file_stat->mtime; 
+	$week->{sumfile_modtime} = $file_stat->mtime;
 	for my $load_ref (@{$loads}){
 	    $load_ref->{file} = $filename;
 	    $load_ref->{sumfile_modtime} = $week->{sumfile_modtime};
-	    my $delete = qq( delete from tl_built_loads where year = $load_ref->{year} 
-			     and load_segment = "$load_ref->{load_segment}" 
+	    my $delete = qq( delete from tl_built_loads where year = $load_ref->{year}
+			     and load_segment = "$load_ref->{load_segment}"
 			     and file = "$filename"
                              and load_scs = $load_ref->{load_scs}
 			     and sumfile_modtime = $week->{sumfile_modtime} );
 	    sql_do( $load_handle, $delete);
 	    sql_insert_hash( $load_handle, 'tl_built_loads', $load_ref );
 	}
-	
+
 	# only bother to store if it has loads
 	sql_do( $load_handle, qq( delete from tl_processing where dir = "$dir" and file = "$filename"));
 	sql_insert_hash( $load_handle, 'tl_processing', $week );
-	
+
 	my $obsids = get_obsids("${mp_dir}/${dir}", $loads);
 	for my $obs_load (keys %{$obsids}){
 	    for my $obs_entry (@{$obsids->{$obs_load}}){
@@ -179,7 +167,7 @@ sub update_for_file{
 				load_segment => $ids[1],
 				obsid => $obs_entry->{obsid},
 				date => $obs_entry->{date},
-				); 
+				);
 		my $obs_delete = qq( delete from tl_obsids where dir = "$dir"
 				     and year = $ids[0]
 				     and load_segment = "$ids[1]"
@@ -190,7 +178,7 @@ sub update_for_file{
 	    }
 	}
     }
-    
+
     print " ... Done \n" if $opt{verbose};
     return $file_stat->mtime;
 
@@ -205,7 +193,7 @@ sub update_for_file{
 sub get_obsids{
     my $dir = shift;
     my $loads = shift;
-    
+
     my @bs_list = glob("${dir}/*.backstop");
     my $backstop = $bs_list[0];
 
@@ -235,7 +223,7 @@ sub parse_clgps {
 
     # assume this isn't a replan
     my %week = ( replan => 0 );
-		 
+
     my @rawloads;
     my $lines = io($gps)->slurp;
 

From 266fa091e77dc76ec1e2b13658912daef482ebfc Mon Sep 17 00:00:00 2001
From: Tom Aldcroft <taldcroft@gmail.com>
Date: Mon, 25 Oct 2021 17:29:03 -0400
Subject: [PATCH 2/7] Remove sybase in task_schedule, use cmd_states3 dir

---
 task_schedule.cfg | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/task_schedule.cfg b/task_schedule.cfg
index 2649ead..8d8a584 100644
--- a/task_schedule.cfg
+++ b/task_schedule.cfg
@@ -40,12 +40,9 @@ alert       aca@head.cfa.harvard.edu
 <task timelines_cmd_states>
       cron       */10 * * * *
       check_cron 15 7 * * *
-      exec 1: parse_cmd_load_gen.pl --dbi 'sybase' --server 'sybase' --verbose
-      exec 1: update_load_seg_db.py --dbi 'sybase' --server 'sybase' --verbose
-      exec 6: $ENV{SKA_SHARE}/cmd_states/update_cmd_states.py --dbi 'sybase' --server 'sybase' --h5file $ENV{SKA_DATA}/cmd_states/cmd_states.h5
-      exec 1: parse_cmd_load_gen.pl --dbi 'sqlite' --server $ENV{SKA_DATA}/cmd_states/cmd_states.db3 --touch_file $ENV{SKA_DATA}/timelines/sum_files_sqlite3.touch  --verbose 
-      exec 1: update_load_seg_db.py --dbi 'sqlite' --server $ENV{SKA_DATA}/cmd_states/cmd_states.db3 --verbose
-      exec 6: $ENV{SKA_SHARE}/cmd_states/update_cmd_states.py --dbi 'sqlite' --server $ENV{SKA_DATA}/cmd_states/cmd_states.db3 --h5file ''
+      exec 1: parse_cmd_load_gen.pl --dbi 'sqlite' --server $ENV{SKA_DATA}/cmd_states3/cmd_states.db3 --touch_file $ENV{SKA_DATA}/timelines/sum_files_sqlite3.touch  --verbose
+      exec 1: update_load_seg_db.py --dbi 'sqlite' --server $ENV{SKA_DATA}/cmd_states3/cmd_states.db3 --verbose
+      exec 6: $ENV{SKA_SHARE}/cmd_states/update_cmd_states.py --dbi 'sqlite' --server $ENV{SKA_DATA}/cmd_states3/cmd_states.db3 --h5file ''
       context 1
       <check>
         <error>

From 0132c6a5a120fa511fd5e021890e8b00c8608a6c Mon Sep 17 00:00:00 2001
From: Tom Aldcroft <taldcroft@gmail.com>
Date: Mon, 25 Oct 2021 17:30:47 -0400
Subject: [PATCH 3/7] Remove sybase option in update_load_seq_db.py

---
 update_load_seg_db.py | 137 +++++++++++++++++++++---------------------
 1 file changed, 67 insertions(+), 70 deletions(-)

diff --git a/update_load_seg_db.py b/update_load_seg_db.py
index c0f2bc0..9a73484 100755
--- a/update_load_seg_db.py
+++ b/update_load_seg_db.py
@@ -25,9 +25,6 @@ def get_options():
     from optparse import OptionParser
     parser = OptionParser(usage='update_load_seg_db.py [options]')
     parser.set_defaults()
-    parser.add_option("--dbi",
-                      default='sqlite',
-                      help="Database interface (sqlite|sybase)")
     parser.add_option("--server",
                       default='db_base.db3',
                       help="DBI server (<filename>|sybase)")
@@ -51,22 +48,22 @@ def get_options():
     parser.add_option("--loadseg_rdb_dir",
                       default=os.path.join(os.environ['SKA'], 'data', 'arc', 'iFOT_events', 'load_segment'),
                       help="directory containing iFOT rdb files of load segments")
-    
-    (opt,args) = parser.parse_args()
+
+    (opt, args) = parser.parse_args()
     return opt, args
 
 
 
 def get_built_load( run_load, dbh=None):
     """
-    Given an entry from the load_segments table, return the matching entry 
+    Given an entry from the load_segments table, return the matching entry
     from the tl_built_loads table
     :param load: run load from load_segments table
-    :rtype: dict of matching built load 
+    :rtype: dict of matching built load
 
     """
 
-    built_query = ( """select * from tl_built_loads where load_segment = '%s' 
+    built_query = ( """select * from tl_built_loads where load_segment = '%s'
                        and year = %d order by sumfile_modtime desc"""
                     % ( run_load['load_segment'], run_load['year'] ))
     built = dbh.fetchone( built_query )
@@ -76,7 +73,7 @@ def get_built_load( run_load, dbh=None):
             raise ValueError("Load name %s is in unknown form, expects /CL\d{3}:\d{4}/" %
                              run_load['load_segment'])
         like_load = match_like.group(1)
-        built_query = ( """select * from tl_built_loads where load_segment like '%s%s%s' 
+        built_query = ( """select * from tl_built_loads where load_segment like '%s%s%s'
                            and year = %d order by sumfile_modtime desc"""
                         % ( '%', like_load, '%',run_load['year'] ))
         built = dbh.fetchone( built_query )
@@ -88,12 +85,12 @@ def get_built_load( run_load, dbh=None):
 
 def get_processing( built, dbh=None ):
     """
-    Given an entry from the tl_built_loads table, return the entry for the 
+    Given an entry from the tl_built_loads table, return the entry for the
     corresponding file from tl_processing
     :param built: tl_built_loads entry/dict
     :rtype: dict of matching tl_processing entry
     """
-    processing_query = ("""select * from tl_processing where file = '%s' 
+    processing_query = ("""select * from tl_processing where file = '%s'
                            and sumfile_modtime = %s order by dir desc """
                         % ( built['file'], built['sumfile_modtime'] ))
     processed = dbh.fetchone( processing_query )
@@ -134,7 +131,7 @@ def get_replan_dir(replan_seg, run_datestart, dbh=None):
 
 
 def weeks_for_load( run_load, dbh=None, test=False ):
-    """ 
+    """
     Determine the timeline intervals that exist for a load segment
 
     How does this work?
@@ -158,7 +155,7 @@ def weeks_for_load( run_load, dbh=None, test=False ):
     entry mapping that time range to a directory is created.
 
     Gotchas:
-    
+
     If the processing summary indicates that the load was part of a
     Replan/ReOpen (when some commands actually came from a different
     file) then that directory is determined by searching for the
@@ -245,19 +242,19 @@ def get_ref_timelines( datestart, dbh=None, n=10):
 def update_timelines_db( loads, dbh, max_id, dryrun=False, test=False):
     """
     Given a list of load segments this routine determines the timelines (mission
-    planning weeks and loads etc) over the loads and inserts new timelines 
+    planning weeks and loads etc) over the loads and inserts new timelines
     into the aca timelines db table.
 
     In common use, this will just insert new timelines at the end of the table.
 
-    In case of scheduled replan, timelines will be updated when load 
+    In case of scheduled replan, timelines will be updated when load
     segments are updated.
 
-    In case of autonomous safing, individual timelines are shortened by outside 
-    processing and act as place holders.  This script will not update shortened 
-    timelines until new load segments are seen in ifot.  
+    In case of autonomous safing, individual timelines are shortened by outside
+    processing and act as place holders.  This script will not update shortened
+    timelines until new load segments are seen in ifot.
 
-    :param loads: dict or recarray of loads 
+    :param loads: dict or recarray of loads
     :param dryrun: do not update database
     :rtype: None
     """
@@ -265,7 +262,7 @@ def update_timelines_db( loads, dbh, max_id, dryrun=False, test=False):
     as_run = loads
     as_run = sorted(as_run, key=lambda k: k['datestart'])
 
-    log.info("TIMELINES INFO: Updating timelines for range %s to %s" 
+    log.info("TIMELINES INFO: Updating timelines for range %s to %s"
              % ( as_run[0]['datestart'], as_run[-1]['datestop']))
 
     timelines = []
@@ -281,11 +278,11 @@ def update_timelines_db( loads, dbh, max_id, dryrun=False, test=False):
 
     timelines = sorted(timelines, key=lambda k: k['datestart'])
     # get existing entries
-    db_timelines = dbh.fetchall("""select * from timelines 
+    db_timelines = dbh.fetchall("""select * from timelines
                                    where datestop >= '%s'
-                                   order by datestart, load_segment_id 
+                                   order by datestart, load_segment_id
                                    """ % ( timelines[0]['datestart']))
-       
+
     if len(db_timelines) > 0:
         i_diff = 0
         for timeline in timelines:
@@ -303,7 +300,7 @@ def update_timelines_db( loads, dbh, max_id, dryrun=False, test=False):
     else:
        i_diff = 0
 
-    # Mismatch occured at i_diff.  
+    # Mismatch occured at i_diff.
 
     if i_diff == len(timelines):
         log.info('TIMELINES INFO: No database update required')
@@ -316,7 +313,7 @@ def update_timelines_db( loads, dbh, max_id, dryrun=False, test=False):
             log.warn("TIMELINES WARN: short timeline at %s, %d minutes" % ( run_timeline['datestart'],
                                                                             time_length / 60. ))
     # find all db timelines that start after the last valid one [i_diff-1]
-    findcmd = ("""SELECT id from timelines 
+    findcmd = ("""SELECT id from timelines
                   WHERE datestart > '%s'
                   AND fixed_by_hand = 0
                   AND datestart <= datestop """
@@ -325,14 +322,14 @@ def update_timelines_db( loads, dbh, max_id, dryrun=False, test=False):
     if len(defunct_tl):
         for id in defunct_tl.id:
             clear_timeline( id, dbh=dbh, dryrun=dryrun )
-        
-    # Insert new timelines[i_diff:] 
+
+    # Insert new timelines[i_diff:]
     log.info('TIMELINES INFO: inserting timelines[%d:%d] to timelines' %
                   (i_diff, len(timelines)+1))
     for t_count, timeline in enumerate(timelines[i_diff:]):
         log.debug('TIMELINES DEBUG: inserting timeline:')
-        insert_string = "\t %s %s %s" % ( timeline['dir'], 
-                                             timeline['datestart'], timeline['datestop'], 
+        insert_string = "\t %s %s %s" % ( timeline['dir'],
+                                             timeline['datestart'], timeline['datestop'],
                                              )
         log.debug(insert_string)
         timeline['id'] = max_id + 1 + t_count
@@ -346,12 +343,12 @@ def rdb_to_db_schema( orig_ifot_loads ):
     the schema used by the load segments table
 
     :param orig_rdb_loads: recarray from the get_iFOT_events.pl rdb table
-    :rtype: recarray 
+    :rtype: recarray
     """
     loads = []
     for orig_load in orig_ifot_loads:
         starttime = DateTime(orig_load['TStart (GMT)'])
-        load = ( 
+        load = (
                  orig_load['LOADSEG.NAME'],
                  int(starttime.frac_year),
                  orig_load['TStart (GMT)'],
@@ -371,26 +368,26 @@ def check_load_overlap( loads, max_sep_hours=36):
     Checks command load segment overlap
 
     Logs warnings for : separation greater than max_sep_hours hours
-                        any overlap in the same SCS    
-                        
+                        any overlap in the same SCS
+
     :param loads: recarray of load segments
     :rtype: None
     """
 
-    sep_times = ( DateTime(loads[1:]['datestart']).secs 
+    sep_times = ( DateTime(loads[1:]['datestart']).secs
                   - DateTime(loads[:-1]['datestop']).secs )
     max_sep = max_sep_hours * 60 * 60
     # check for too much sep
     if (any(sep_times > max_sep )):
         for load_idx in np.flatnonzero(sep_times > max_sep):
-            log.warn('LOAD_SEG WARN: Loads %s %s separated by more than %i hours' 
+            log.warn('LOAD_SEG WARN: Loads %s %s separated by more than %i hours'
                          % (loads[load_idx]['load_segment'],
                             loads[load_idx+1]['load_segment'],
                             max_sep_hours ))
     # any SCS overlap
     for scs in (128, 129, 130, 131, 132, 133):
         scs_loads = loads[ loads['load_scs'] == scs ]
-        scs_times = ( DateTime(scs_loads[1:]['datestart']).secs 
+        scs_times = ( DateTime(scs_loads[1:]['datestart']).secs
                       - DateTime(scs_loads[:-1]['datestop']).secs)
         if (any(scs_times < 0 )):
             log.warn('LOAD_SEG WARN: Same SCS loads overlap')
@@ -414,10 +411,10 @@ def clear_timeline( id, dbh=None, dryrun=False ):
         log.debug('TIMELINES DEBUG: ' + cmd)
         if not dryrun:
             dbh.execute(cmd)
-    
+
     # remove defunct timelines
-    cmd = ("""DELETE FROM timelines 
-              WHERE id = %s 
+    cmd = ("""DELETE FROM timelines
+              WHERE id = %s
               AND fixed_by_hand = 0 """
            % id)
     log.debug('TIMELINES DEBUG: ' + cmd)
@@ -433,13 +430,13 @@ def clear_rel_timelines( load_segments, dbh=None, dryrun=False ):
     :rtype: None
     """
     for load in load_segments:
-        db_timelines = dbh.fetchall("SELECT * from timelines where load_segment_id = %i" 
+        db_timelines = dbh.fetchall("SELECT * from timelines where load_segment_id = %i"
                                    % ( load['id']))
-        
+
         if len(db_timelines) > 0:
             if (any(db_timelines['fixed_by_hand'])):
                 for timeline in db_timelines[ db_timelines['fixed_by_hand'] == 1]:
-                    log.warn("LOAD_SEG WARN: updating timelines across %i which is fixed_by_hand" 
+                    log.warn("LOAD_SEG WARN: updating timelines across %i which is fixed_by_hand"
                              % (timeline['id']))
 
             for timeline in db_timelines:
@@ -455,19 +452,19 @@ def get_last_timeline(dbh=None):
     :rtype: recarray from timelines db
     """
 
-    timeline = dbh.fetchall("""SELECT * from timelines where datestart 
+    timeline = dbh.fetchall("""SELECT * from timelines where datestart
                                   = (select max(datestart) from timelines)""")[0]
     return timeline
 
-                               
+
 def find_load_seg_changes(wants, haves, exclude=[]):
-    
+
     to_delete = []
     to_insert = []
     # Find mismatches:
     match_cols = [x[0] for x in haves.dtype.descr if 'f' not in x[1]]
     [match_cols.remove(col) for col in exclude]
-    # Use explicity increment for i_diff so that it gets set to the 
+    # Use explicity increment for i_diff so that it gets set to the
     # index of the first not-matching entry, or if wants is longer than
     # haves (the usual append condition) it gets set to the index of
     # the first new entry in wants.
@@ -478,20 +475,20 @@ def find_load_seg_changes(wants, haves, exclude=[]):
             log.info('LOAD_SEG INFO: Mismatch on these entries:')
             log.info(want_entry)
             log.info(have_entry)
-            break                
+            break
         i_diff += 1
 
     if i_diff == 0:
         raise ValueError("Unexpected mismatch at first database entry in range")
-    
+
     # if i_diff incremented past the end of both lists because
-    # they match and have the same length, the if i_diff < len() 
+    # they match and have the same length, the if i_diff < len()
     # statements below will return False.
 
     # if there is a mismatch, return entries to be removed
     if i_diff < len(haves):
         to_delete = haves[i_diff:]
-    # if there is either a mismatch or new entries, return the 
+    # if there is either a mismatch or new entries, return the
     # entries to-be-inserted
     if i_diff < len(wants):
         to_insert = wants[i_diff:]
@@ -504,13 +501,13 @@ def update_loads_db( ifot_loads, dbh=None, test=False, dryrun=False,):
     """
     Update the load_segments table with the loads from an RDB file.
 
-    :param ifot_loads: recarray of ifot run loads 
-    :param test: allow writes of < year 2009 data 
+    :param ifot_loads: recarray of ifot run loads
+    :param test: allow writes of < year 2009 data
     :param dryrun: do not write to the database
     :rtype: list of new loads
     """
-    
-    log.info("LOAD_SEG INFO: Updating Loads for range %s to %s" 
+
+    log.info("LOAD_SEG INFO: Updating Loads for range %s to %s"
              % ( ifot_loads[0]['datestart'], ifot_loads[-1]['datestop']))
 
     # raise errors on some unexpected possibilities
@@ -518,18 +515,18 @@ def update_loads_db( ifot_loads, dbh=None, test=False, dryrun=False,):
         raise ValueError("LOAD_SEG: No loads passed to update_loads_db()")
     if len(ifot_loads) == 1:
         raise ValueError("LOAD_SEG: only one load passed to update_loads_db()")
-        
+
     # Tom doesn't want < year 2009 load segments to change ids etc
     min_time_datestart ='2009:001:00:00:00.000'
     if (ifot_loads[0]['datestart'] < min_time_datestart) and not test:
-        raise ValueError("Attempting to update loads before %s" 
+        raise ValueError("Attempting to update loads before %s"
                          % min_time_datestart )
 
     max_id = dbh.fetchone('SELECT max(id) AS max_id FROM load_segments')['max_id'] or 0
     if max_id == 0 and test == False:
         raise ValueError("LOAD SEG: no load_segments in database.")
-    db_loads = dbh.fetchall("""select * from load_segments 
-                               where datestart >= '%s' 
+    db_loads = dbh.fetchall("""select * from load_segments
+                               where datestart >= '%s'
                                order by datestart, load_scs""" % (
                                ifot_loads[0]['datestart'],
                                )
@@ -566,7 +563,7 @@ def update_loads_db( ifot_loads, dbh=None, test=False, dryrun=False,):
             dbh.execute(cmd)
 
     # check for overlap in the loads... the check_load_overlap just logs warnings
-    check_load_overlap( ifot_loads )    
+    check_load_overlap( ifot_loads )
 
     # Insert new loads[i_diff:] into load_segments
     #log.info('LOAD_SEG INFO: inserting load_segs[%d:%d] to load_segments' %
@@ -583,14 +580,14 @@ def update_loads_db( ifot_loads, dbh=None, test=False, dryrun=False,):
             dbh.insert(load_dict, 'load_segments', commit=True)
     return to_insert
 
-    
 
-        
+
+
 def main(loadseg_rdb_dir, dryrun=False, test=False,
-         dbi='sqlite', server='db_base.db3' ,database=None, user=None, verbose=False):
+         server='db_base.db3' ,database=None, user=None, verbose=False):
     """
     Command Load Segment Table Updater
-    
+
     Read RDB table from SKA arc iFOT events area and update load_segments table
     Meant to be run as a cront task with no arguments.
 
@@ -605,7 +602,7 @@ def main(loadseg_rdb_dir, dryrun=False, test=False,
 
     """
 
-    dbh = DBI(dbi=dbi, server=server, database=database, user=user, verbose=verbose)
+    dbh = DBI(dbi='sqlite', server=server, database=database, user=user, verbose=verbose)
     ch = logging.StreamHandler()
     ch.setLevel(logging.WARN)
     if verbose:
@@ -632,9 +629,9 @@ def main(loadseg_rdb_dir, dryrun=False, test=False,
             'SELECT max(id) AS max_id FROM timelines')['max_id'] or 0
         if max_timelines_id == 0 and test == False:
             raise ValueError("TIMELINES: no timelines in database.")
-        update_loads_db( ifot_loads, dbh=dbh, test=test, dryrun=dryrun )    
-        db_loads = dbh.fetchall("""select * from load_segments 
-                                   where datestart >= '%s' order by datestart   
+        update_loads_db( ifot_loads, dbh=dbh, test=test, dryrun=dryrun )
+        db_loads = dbh.fetchall("""select * from load_segments
+                                   where datestart >= '%s' order by datestart
                                   """ % ( ifot_loads[0]['datestart'] )
                                 )
         update_timelines_db(loads=db_loads, dbh=dbh, max_id=max_timelines_id,
@@ -645,11 +642,11 @@ def main(loadseg_rdb_dir, dryrun=False, test=False,
 if __name__ == "__main__":
 
     (opt,args) = get_options()
-    main(opt.loadseg_rdb_dir, dryrun=opt.dryrun, 
-             test=opt.test, dbi=opt.dbi, server=opt.server,
+    main(opt.loadseg_rdb_dir, dryrun=opt.dryrun,
+             test=opt.test, server=opt.server,
              database=opt.database, user=opt.user,
              verbose=opt.verbose)
 
 
 
- 
+

From 42b230833858b8d6e964454bbd81b68c4d9996fc Mon Sep 17 00:00:00 2001
From: Tom Aldcroft <taldcroft@gmail.com>
Date: Mon, 25 Oct 2021 17:34:24 -0400
Subject: [PATCH 4/7] 2to3 changes for update_load_seq_db.py

---
 update_load_seg_db.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/update_load_seg_db.py b/update_load_seg_db.py
index 9a73484..bb23154 100755
--- a/update_load_seg_db.py
+++ b/update_load_seg_db.py
@@ -7,7 +7,7 @@
 import logging
 from logging.handlers import SMTPHandler
 import numpy as np
-from itertools import count, izip
+from itertools import count
 
 import Ska.Table
 from Ska.DBI import DBI
@@ -225,9 +225,9 @@ def get_ref_timelines( datestart, dbh=None, n=10):
     pre_query =  ("select * from timelines where datestart <= '%s' order by datestart desc"
                   %  datestart )
     pre_query_fetch = dbh.fetch(pre_query)
-    for cnt in xrange(0,n):
+    for cnt in range(0,n):
         try:
-            ref_timelines.append( pre_query_fetch.next() )
+            ref_timelines.append( next(pre_query_fetch) )
         except StopIteration:
             if (cnt == 0):
                 log.warn("""TIMELINES WARN: no timelines found before current insert""")
@@ -469,7 +469,7 @@ def find_load_seg_changes(wants, haves, exclude=[]):
     # haves (the usual append condition) it gets set to the index of
     # the first new entry in wants.
     i_diff = 0
-    for want_entry, have_entry in izip(wants, haves):
+    for want_entry, have_entry in zip(wants, haves):
         # does the db load match?
         if (any(have_entry[x] != want_entry[x] for x in match_cols) ):
             log.info('LOAD_SEG INFO: Mismatch on these entries:')

From f8765e3651fb3f3bbdd0f362a86501d700e08855 Mon Sep 17 00:00:00 2001
From: Tom Aldcroft <taldcroft@gmail.com>
Date: Tue, 26 Oct 2021 06:31:00 -0400
Subject: [PATCH 5/7] Formatting and other bigger changes

---
 fix_load_segments.py  |  69 ++++++------
 update_load_seg_db.py | 245 ++++++++++++++++++++----------------------
 2 files changed, 148 insertions(+), 166 deletions(-)

diff --git a/fix_load_segments.py b/fix_load_segments.py
index 2ac85e7..74bb964 100755
--- a/fix_load_segments.py
+++ b/fix_load_segments.py
@@ -1,9 +1,5 @@
 #!/usr/bin/env python
 
-import Ska.DBI
-from Chandra.Time import DateTime
-
-
 def repair(ifot_loads):
     """
     Make any known edits to a recarray of load segments.  This is called
@@ -12,78 +8,79 @@ def repair(ifot_loads):
 
     :param ifot_loads: numpy.recarray from the (Ska.Table) parsed rdb of load segments from arc/iFOT
     :rtype: numpy.recarray
-    
+
     """
 
     import numpy as np
 
     # delete a load that was never run
-    match = ((ifot_loads.load_segment == 'CL304:0504') &
-             (ifot_loads.year == 2003))
+    match = ((ifot_loads.load_segment == 'CL304:0504')
+             & (ifot_loads.year == 2003))
     if any(match):
         # slice to get the ones that don't match
-        ifot_loads = ifot_loads[match == False]
+        ifot_loads = ifot_loads[~match]
 
     # repair a couple of loads
     # 352 ended early at...
-    match = ((ifot_loads.load_segment == 'CL352:1208') &
-             (ifot_loads.year == 2008)) 
+    match = ((ifot_loads.load_segment == 'CL352:1208')
+             & (ifot_loads.year == 2008))
     if any(match):
         ifot_loads.datestop[match] = '2008:353:05:00:00.000'
         ifot_loads.fixed_by_hand[match] = 1
 
     # CL110:1404 was run, not CL110:1409
-    match = ((ifot_loads.load_segment == 'CL110:1409') &
-             (ifot_loads.year == 2003)) 
+    match = ((ifot_loads.load_segment == 'CL110:1409')
+             & (ifot_loads.year == 2003))
     if any(match):
-        rec_tuple = ( 'CL110:1404', 2003,
-                      '2003:110:14:07:09.439', '2003:112:00:00:31.542', 130, 1)
+        rec_tuple = ('CL110:1404', 2003,
+                     '2003:110:14:07:09.439', '2003:112:00:00:31.542', 130, 1)
         # ifot_list except the bad one
-        ifot_list = ifot_loads[match == False].tolist()
+        ifot_list = ifot_loads[~match].tolist()
         ifot_list.append(rec_tuple)
-	    # sort by datestart in third field
-        ifot_list.sort(lambda x,y: cmp(x[2],y[2]))
-        new_ifot = np.rec.fromrecords( ifot_list,
-                                       dtype=ifot_loads.dtype,
-                                       )
-        ifot_loads = new_ifot
+        # sort by datestart in third field
 
+        def cmp(a, b):
+            return (a > b) - (a < b)
+        ifot_list.sort(lambda x, y: cmp(x[2], y[2]))
+        new_ifot = np.rec.fromrecords(ifot_list,
+                                      dtype=ifot_loads.dtype,
+                                      )
+        ifot_loads = new_ifot
 
     # CL188:0402 is missing
     cmd_date = '2009:188:04:00:00.000'
 
     if ((ifot_loads[0].datestart <= cmd_date) and (ifot_loads[-1].datestart > cmd_date)):
-        if np.flatnonzero((ifot_loads.load_segment == 'CL188:0402') &
-                          (ifot_loads.year == 2009)):
+        if np.flatnonzero((ifot_loads.load_segment == 'CL188:0402')
+                          & (ifot_loads.year == 2009)):
             pass
         else:
-            rec_tuple = ( 'CL188:0402', 2009,
+            rec_tuple = ('CL188:0402', 2009,
                          '2009:188:04:00:00.000', '2009:188:20:57:33.571', 129, 1)
             ifot_list = ifot_loads.tolist()
             ifot_list.append(rec_tuple)
-            ifot_list.sort(lambda x,y: cmp(x[2],y[2]))
-            new_ifot = np.rec.fromrecords( ifot_list,
-                                           dtype=ifot_loads.dtype,
-                                           )
+            ifot_list.sort(lambda x, y: cmp(x[2], y[2]))
+            new_ifot = np.rec.fromrecords(ifot_list,
+                                          dtype=ifot_loads.dtype,
+                                          )
             ifot_loads = new_ifot
 
     # 2011 CL103:2002 is missing, interrupted by  the APR1311 replan
     cmd_date = '2011:103:20:40:00.000'
 
     if ((ifot_loads[0].datestart <= cmd_date) and (ifot_loads[-1].datestart > cmd_date)):
-        if np.flatnonzero((ifot_loads.load_segment == 'CL103:2002') &
-                          (ifot_loads.year == 2011)):
+        if np.flatnonzero((ifot_loads.load_segment == 'CL103:2002')
+                          & (ifot_loads.year == 2011)):
             pass
         else:
-            rec_tuple = ( 'CL103:2002', 2011,
+            rec_tuple = ('CL103:2002', 2011,
                          '2011:103:20:40:00.000', '2011:103:22:57:00.000', 129, 1)
             ifot_list = ifot_loads.tolist()
             ifot_list.append(rec_tuple)
-            ifot_list.sort(lambda x,y: cmp(x[2],y[2]))
-            new_ifot = np.rec.fromrecords( ifot_list,
-                                           dtype=ifot_loads.dtype,
-                                           )
+            ifot_list.sort(lambda x, y: cmp(x[2], y[2]))
+            new_ifot = np.rec.fromrecords(ifot_list,
+                                          dtype=ifot_loads.dtype,
+                                          )
             ifot_loads = new_ifot
 
-
     return ifot_loads
diff --git a/update_load_seg_db.py b/update_load_seg_db.py
index bb23154..db4e69e 100755
--- a/update_load_seg_db.py
+++ b/update_load_seg_db.py
@@ -1,21 +1,15 @@
 #!/usr/bin/env python
 
 import os
-import sys
 import glob
 import re
 import logging
-from logging.handlers import SMTPHandler
 import numpy as np
-from itertools import count
 
-import Ska.Table
+from astropy.table import Table
 from Ska.DBI import DBI
 from Chandra.Time import DateTime
 
-import fix_tl_processing
-import fix_load_segments
-
 log = logging.getLogger()
 log.setLevel(logging.DEBUG)
 MP_DIR = '/data/mpcrit1/mplogs/'
@@ -46,15 +40,15 @@ def get_options():
                       default=True,
                       help='Enable tracebacks')
     parser.add_option("--loadseg_rdb_dir",
-                      default=os.path.join(os.environ['SKA'], 'data', 'arc', 'iFOT_events', 'load_segment'),
+                      default=os.path.join(os.environ['SKA'], 'data',
+                                           'arc', 'iFOT_events', 'load_segment'),
                       help="directory containing iFOT rdb files of load segments")
 
     (opt, args) = parser.parse_args()
     return opt, args
 
 
-
-def get_built_load( run_load, dbh=None):
+def get_built_load(run_load, dbh=None):
     """
     Given an entry from the load_segments table, return the matching entry
     from the tl_built_loads table
@@ -63,27 +57,28 @@ def get_built_load( run_load, dbh=None):
 
     """
 
-    built_query = ( """select * from tl_built_loads where load_segment = '%s'
+    built_query = ("""select * from tl_built_loads where load_segment = '%s'
                        and year = %d order by sumfile_modtime desc"""
-                    % ( run_load['load_segment'], run_load['year'] ))
-    built = dbh.fetchone( built_query )
+                   % (run_load['load_segment'], run_load['year']))
+    built = dbh.fetchone(built_query)
     if built is None:
-        match_like = re.search('(CL\d{3}:\d{4})', run_load['load_segment'])
+        match_like = re.search(r'(CL\d{3}:\d{4})', run_load['load_segment'])
         if match_like is None:
             raise ValueError("Load name %s is in unknown form, expects /CL\d{3}:\d{4}/" %
                              run_load['load_segment'])
         like_load = match_like.group(1)
-        built_query = ( """select * from tl_built_loads where load_segment like '%s%s%s'
+        built_query = ("""select * from tl_built_loads where load_segment like '%s%s%s'
                            and year = %d order by sumfile_modtime desc"""
-                        % ( '%', like_load, '%',run_load['year'] ))
-        built = dbh.fetchone( built_query )
+                       % ('%', like_load, '%', run_load['year']))
+        built = dbh.fetchone(built_query)
         # if a built version *still* hasn't been found
-        if built  is None:
-            raise ValueError("Unable to find file for %s,%s" % (run_load['year'], run_load['load_segment']))
+        if built is None:
+            raise ValueError("Unable to find file for %s,%s" %
+                             (run_load['year'], run_load['load_segment']))
     return built
 
 
-def get_processing( built, dbh=None ):
+def get_processing(built, dbh=None):
     """
     Given an entry from the tl_built_loads table, return the entry for the
     corresponding file from tl_processing
@@ -92,12 +87,13 @@ def get_processing( built, dbh=None ):
     """
     processing_query = ("""select * from tl_processing where file = '%s'
                            and sumfile_modtime = %s order by dir desc """
-                        % ( built['file'], built['sumfile_modtime'] ))
-    processed = dbh.fetchone( processing_query )
+                        % (built['file'], built['sumfile_modtime']))
+    processed = dbh.fetchone(processing_query)
     if processed is None:
         raise ValueError("Unable to find processing for built file %s", built['file'])
     return processed
 
+
 def get_replan_dir(replan_seg, run_datestart, dbh=None):
     """
     Given the 'replan_cmds' entry from a processing summary find
@@ -130,7 +126,7 @@ def get_replan_dir(replan_seg, run_datestart, dbh=None):
     return replan['dir']
 
 
-def weeks_for_load( run_load, dbh=None, test=False ):
+def weeks_for_load(run_load, dbh=None, test=False):
     """
     Determine the timeline intervals that exist for a load segment
 
@@ -176,43 +172,44 @@ def weeks_for_load( run_load, dbh=None, test=False ):
 
 
     """
-    built = get_built_load( run_load, dbh=dbh )
-    processed = get_processing( built, dbh=dbh )
+    built = get_built_load(run_load, dbh=dbh)
+    processed = get_processing(built, dbh=dbh)
 
     match_load_pieces = []
 
-    match = { 'dir' : processed['dir'],
-              'load_segment_id' : run_load['id'],
-              'datestart': run_load['datestart'],
-              'datestop': run_load['datestop'],
-              'replan': processed['replan'],
-              'incomplete': 1 }
+    match = {'dir': processed['dir'],
+             'load_segment_id': run_load['id'],
+             'datestart': run_load['datestart'],
+             'datestop': run_load['datestop'],
+             'replan': processed['replan'],
+             'incomplete': 1}
 
     if processed['replan'] and processed['bcf_cmd_count'] > 0:
         if processed['replan_cmds'] is None:
-            raise ValueError("Replan/ReOpen load: %s without replan_cmd source load! " % (run_load['load_segment']))
+            raise ValueError("Replan/ReOpen load: %s without replan_cmd source load! " %
+                             (run_load['load_segment']))
         # if load is completely contained in processing (earlier load interrupted in week )
-        if  (( run_load['datestart']  >=  processed['processing_tstart'] )
-             & (  run_load['datestop'] <=  processed['processing_tstop'] )):
-            match_load_pieces.append( match.copy() )
+        if ((run_load['datestart'] >= processed['processing_tstart'])
+                & (run_load['datestop'] <= processed['processing_tstop'])):
+            match_load_pieces.append(match.copy())
         # if processing covers the end but not the beginning (this one was interruped)
         if (run_load['datestart'] < processed['processing_tstart']
-            and run_load['datestop'] <= processed['processing_tstop']):
-            replan_dir = get_replan_dir(processed['replan_cmds'], run_load['datestart'], dbh=dbh )
+                and run_load['datestop'] <= processed['processing_tstop']):
+            replan_dir = get_replan_dir(processed['replan_cmds'], run_load['datestart'], dbh=dbh)
             log.info("TIMELINES INFO: %s,%s is replan/reopen, using %s dir for imported cmds" % (
-                run_load['year'], run_load['load_segment'], replan_dir ))
+                run_load['year'], run_load['load_segment'], replan_dir))
             # the end
             match['datestart'] = processed['processing_tstart']
-            match_load_pieces.append( match.copy() )
+            match_load_pieces.append(match.copy())
             # the pre-this-processed-file chunk
             match['datestart'] = run_load['datestart']
             match['datestop'] = processed['processing_tstart']
             match['dir'] = replan_dir
-            match_load_pieces.append( match.copy() )
+            match_load_pieces.append(match.copy())
     else:
         # if the run load matches the times of the built load
-        if ( (DateTime(run_load['datestart']).secs >= ( DateTime(built['first_cmd_time']).secs))
-             & (DateTime(run_load['datestop']).secs <= ( DateTime(built['last_cmd_time']).secs))):
+        if ((DateTime(run_load['datestart']).secs >= (DateTime(built['first_cmd_time']).secs))
+                & (DateTime(run_load['datestop']).secs <= (DateTime(built['last_cmd_time']).secs))):
             match['incomplete'] = 0
         # append even if incomplete
         match_load_pieces.append(match.copy())
@@ -220,26 +217,27 @@ def weeks_for_load( run_load, dbh=None, test=False ):
     timelines = sorted(match_load_pieces, key=lambda k: (k['datestart'], k['load_segment_id']))
     return timelines
 
-def get_ref_timelines( datestart, dbh=None, n=10):
+
+def get_ref_timelines(datestart, dbh=None, n=10):
     ref_timelines = []
-    pre_query =  ("select * from timelines where datestart <= '%s' order by datestart desc"
-                  %  datestart )
+    pre_query = ("select * from timelines where datestart <= '%s' order by datestart desc"
+                 % datestart)
     pre_query_fetch = dbh.fetch(pre_query)
-    for cnt in range(0,n):
+    for cnt in range(0, n):
         try:
-            ref_timelines.append( next(pre_query_fetch) )
+            ref_timelines.append(next(pre_query_fetch))
         except StopIteration:
             if (cnt == 0):
                 log.warn("""TIMELINES WARN: no timelines found before current insert""")
             else:
                 log.warn("""TIMELINES WARN: only found %i of %i timelines before current insert"""
-                         % (cnt-1, 10))
+                         % (cnt - 1, 10))
             break
     ref_timelines = sorted(ref_timelines, key=lambda k: k['datestart'])
     return ref_timelines
 
 
-def update_timelines_db( loads, dbh, max_id, dryrun=False, test=False):
+def update_timelines_db(loads, dbh, max_id, dryrun=False, test=False):
     """
     Given a list of load segments this routine determines the timelines (mission
     planning weeks and loads etc) over the loads and inserts new timelines
@@ -263,15 +261,15 @@ def update_timelines_db( loads, dbh, max_id, dryrun=False, test=False):
     as_run = sorted(as_run, key=lambda k: k['datestart'])
 
     log.info("TIMELINES INFO: Updating timelines for range %s to %s"
-             % ( as_run[0]['datestart'], as_run[-1]['datestop']))
+             % (as_run[0]['datestart'], as_run[-1]['datestop']))
 
     timelines = []
     for run_load in as_run:
-        run_timelines = weeks_for_load( run_load,
-                                        dbh=dbh,
-                                        test=test)
+        run_timelines = weeks_for_load(run_load,
+                                       dbh=dbh,
+                                       test=test)
         if len(run_timelines) == 0:
-            raise ValueError("No timelines found for load %s" % run_load )
+            raise ValueError("No timelines found for load %s" % run_load)
         for run_timeline in run_timelines:
             # append the new timeline to the list to insert
             timelines.append(run_timeline)
@@ -281,7 +279,7 @@ def update_timelines_db( loads, dbh, max_id, dryrun=False, test=False):
     db_timelines = dbh.fetchall("""select * from timelines
                                    where datestop >= '%s'
                                    order by datestart, load_segment_id
-                                   """ % ( timelines[0]['datestart']))
+                                   """ % (timelines[0]['datestart']))
 
     if len(db_timelines) > 0:
         i_diff = 0
@@ -308,36 +306,38 @@ def update_timelines_db( loads, dbh, max_id, dryrun=False, test=False):
 
     # warn if timeline is shorter than an hour
     for run_timeline in timelines[i_diff:]:
-        time_length = DateTime(run_timeline['datestop']).secs - DateTime(run_timeline['datestart']).secs
+        time_length = DateTime(run_timeline['datestop']).secs - \
+            DateTime(run_timeline['datestart']).secs
         if time_length / 60. < 60:
-            log.warn("TIMELINES WARN: short timeline at %s, %d minutes" % ( run_timeline['datestart'],
-                                                                            time_length / 60. ))
+            log.warn("TIMELINES WARN: short timeline at %s, %d minutes" % (run_timeline['datestart'],
+                                                                           time_length / 60.))
     # find all db timelines that start after the last valid one [i_diff-1]
     findcmd = ("""SELECT id from timelines
                   WHERE datestart > '%s'
                   AND fixed_by_hand = 0
                   AND datestart <= datestop """
-               % timelines[i_diff-1]['datestart'] )
-    defunct_tl = dbh.fetchall( findcmd )
+               % timelines[i_diff - 1]['datestart'])
+    defunct_tl = dbh.fetchall(findcmd)
     if len(defunct_tl):
         for id in defunct_tl.id:
-            clear_timeline( id, dbh=dbh, dryrun=dryrun )
+            clear_timeline(id, dbh=dbh, dryrun=dryrun)
 
     # Insert new timelines[i_diff:]
     log.info('TIMELINES INFO: inserting timelines[%d:%d] to timelines' %
-                  (i_diff, len(timelines)+1))
+             (i_diff, len(timelines) + 1))
     for t_count, timeline in enumerate(timelines[i_diff:]):
         log.debug('TIMELINES DEBUG: inserting timeline:')
-        insert_string = "\t %s %s %s" % ( timeline['dir'],
-                                             timeline['datestart'], timeline['datestop'],
-                                             )
+        insert_string = "\t %s %s %s" % (timeline['dir'],
+                                         timeline['datestart'], timeline['datestop'],
+                                         )
         log.debug(insert_string)
         timeline['id'] = max_id + 1 + t_count
         timeline['fixed_by_hand'] = 0
         if not dryrun:
             dbh.insert(timeline, 'timelines')
 
-def rdb_to_db_schema( orig_ifot_loads ):
+
+def rdb_to_db_schema(orig_ifot_loads):
     """
     Convert the load segment data from a get_iFOT_events.pl rdb table into
     the schema used by the load segments table
@@ -349,21 +349,21 @@ def rdb_to_db_schema( orig_ifot_loads ):
     for orig_load in orig_ifot_loads:
         starttime = DateTime(orig_load['TStart (GMT)'])
         load = (
-                 orig_load['LOADSEG.NAME'],
-                 int(starttime.frac_year),
-                 orig_load['TStart (GMT)'],
-                 orig_load['TStop (GMT)'],
-                 orig_load['LOADSEG.SCS'],
-                 0,
-                 )
+            orig_load['LOADSEG.NAME'],
+            int(starttime.frac_year),
+            orig_load['TStart (GMT)'],
+            orig_load['TStop (GMT)'],
+            orig_load['LOADSEG.SCS'],
+            0,
+        )
         loads.append(load)
-    names = ('load_segment','year','datestart','datestop','load_scs','fixed_by_hand')
-    load_recs = np.rec.fromrecords( loads, names=names)
+    names = ('load_segment', 'year', 'datestart', 'datestop', 'load_scs', 'fixed_by_hand')
+    load_recs = np.rec.fromrecords(loads, names=names)
     load_recs = np.sort(load_recs, order=['datestart', 'load_scs'])
     return load_recs
 
 
-def check_load_overlap( loads, max_sep_hours=36):
+def check_load_overlap(loads, max_sep_hours=36):
     """
     Checks command load segment overlap
 
@@ -374,26 +374,26 @@ def check_load_overlap( loads, max_sep_hours=36):
     :rtype: None
     """
 
-    sep_times = ( DateTime(loads[1:]['datestart']).secs
-                  - DateTime(loads[:-1]['datestop']).secs )
+    sep_times = (DateTime(loads[1:]['datestart']).secs
+                 - DateTime(loads[:-1]['datestop']).secs)
     max_sep = max_sep_hours * 60 * 60
     # check for too much sep
-    if (any(sep_times > max_sep )):
+    if (any(sep_times > max_sep)):
         for load_idx in np.flatnonzero(sep_times > max_sep):
             log.warn('LOAD_SEG WARN: Loads %s %s separated by more than %i hours'
-                         % (loads[load_idx]['load_segment'],
-                            loads[load_idx+1]['load_segment'],
-                            max_sep_hours ))
+                     % (loads[load_idx]['load_segment'],
+                        loads[load_idx + 1]['load_segment'],
+                        max_sep_hours))
     # any SCS overlap
     for scs in (128, 129, 130, 131, 132, 133):
-        scs_loads = loads[ loads['load_scs'] == scs ]
-        scs_times = ( DateTime(scs_loads[1:]['datestart']).secs
-                      - DateTime(scs_loads[:-1]['datestop']).secs)
-        if (any(scs_times < 0 )):
+        scs_loads = loads[loads['load_scs'] == scs]
+        scs_times = (DateTime(scs_loads[1:]['datestart']).secs
+                     - DateTime(scs_loads[:-1]['datestop']).secs)
+        if (any(scs_times < 0)):
             log.warn('LOAD_SEG WARN: Same SCS loads overlap')
 
 
-def clear_timeline( id, dbh=None, dryrun=False ):
+def clear_timeline(id, dbh=None, dryrun=False):
     """
     Clear the commands related to a timeline and then delete the timeline.
 
@@ -407,7 +407,7 @@ def clear_timeline( id, dbh=None, dryrun=False ):
     for table in ('cmd_fltpars', 'cmd_intpars', 'cmds'):
         cmd = (""" DELETE from %s
                    WHERE timeline_id = %s """
-               % ( table, id ))
+               % (table, id))
         log.debug('TIMELINES DEBUG: ' + cmd)
         if not dryrun:
             dbh.execute(cmd)
@@ -422,7 +422,7 @@ def clear_timeline( id, dbh=None, dryrun=False ):
         dbh.execute(cmd)
 
 
-def clear_rel_timelines( load_segments, dbh=None, dryrun=False ):
+def clear_rel_timelines(load_segments, dbh=None, dryrun=False):
     """
     Remove timelines related to (fk constrained to) load_segments from the timelines db
 
@@ -431,18 +431,16 @@ def clear_rel_timelines( load_segments, dbh=None, dryrun=False ):
     """
     for load in load_segments:
         db_timelines = dbh.fetchall("SELECT * from timelines where load_segment_id = %i"
-                                   % ( load['id']))
+                                    % (load['id']))
 
         if len(db_timelines) > 0:
             if (any(db_timelines['fixed_by_hand'])):
-                for timeline in db_timelines[ db_timelines['fixed_by_hand'] == 1]:
+                for timeline in db_timelines[db_timelines['fixed_by_hand'] == 1]:
                     log.warn("LOAD_SEG WARN: updating timelines across %i which is fixed_by_hand"
                              % (timeline['id']))
 
             for timeline in db_timelines:
-                clear_timeline( timeline['id'], dbh=dbh, dryrun=dryrun )
-
-
+                clear_timeline(timeline['id'], dbh=dbh, dryrun=dryrun)
 
 
 def get_last_timeline(dbh=None):
@@ -471,7 +469,7 @@ def find_load_seg_changes(wants, haves, exclude=[]):
     i_diff = 0
     for want_entry, have_entry in zip(wants, haves):
         # does the db load match?
-        if (any(have_entry[x] != want_entry[x] for x in match_cols) ):
+        if (any(have_entry[x] != want_entry[x] for x in match_cols)):
             log.info('LOAD_SEG INFO: Mismatch on these entries:')
             log.info(want_entry)
             log.info(have_entry)
@@ -496,8 +494,7 @@ def find_load_seg_changes(wants, haves, exclude=[]):
     return to_delete, to_insert
 
 
-
-def update_loads_db( ifot_loads, dbh=None, test=False, dryrun=False,):
+def update_loads_db(ifot_loads, dbh=None, test=False, dryrun=False,):
     """
     Update the load_segments table with the loads from an RDB file.
 
@@ -508,7 +505,7 @@ def update_loads_db( ifot_loads, dbh=None, test=False, dryrun=False,):
     """
 
     log.info("LOAD_SEG INFO: Updating Loads for range %s to %s"
-             % ( ifot_loads[0]['datestart'], ifot_loads[-1]['datestop']))
+             % (ifot_loads[0]['datestart'], ifot_loads[-1]['datestop']))
 
     # raise errors on some unexpected possibilities
     if len(ifot_loads) == 0:
@@ -517,10 +514,10 @@ def update_loads_db( ifot_loads, dbh=None, test=False, dryrun=False,):
         raise ValueError("LOAD_SEG: only one load passed to update_loads_db()")
 
     # Tom doesn't want < year 2009 load segments to change ids etc
-    min_time_datestart ='2009:001:00:00:00.000'
+    min_time_datestart = '2009:001:00:00:00.000'
     if (ifot_loads[0]['datestart'] < min_time_datestart) and not test:
         raise ValueError("Attempting to update loads before %s"
-                         % min_time_datestart )
+                         % min_time_datestart)
 
     max_id = dbh.fetchone('SELECT max(id) AS max_id FROM load_segments')['max_id'] or 0
     if max_id == 0 and test == False:
@@ -528,9 +525,9 @@ def update_loads_db( ifot_loads, dbh=None, test=False, dryrun=False,):
     db_loads = dbh.fetchall("""select * from load_segments
                                where datestart >= '%s'
                                order by datestart, load_scs""" % (
-                               ifot_loads[0]['datestart'],
-                               )
-                            )
+        ifot_loads[0]['datestart'],
+    )
+    )
 
     # if no overlap on time range, check for an empty table (which should be OK)
     if len(db_loads) == 0:
@@ -558,12 +555,12 @@ def update_loads_db( ifot_loads, dbh=None, test=False, dryrun=False,):
     if not dryrun:
         for load in to_delete:
             cmd = ("DELETE FROM load_segments WHERE id = %d"
-                   % load['id'] )
+                   % load['id'])
             log.info('LOAD_SEG INFO: ' + cmd)
             dbh.execute(cmd)
 
     # check for overlap in the loads... the check_load_overlap just logs warnings
-    check_load_overlap( ifot_loads )
+    check_load_overlap(ifot_loads)
 
     # Insert new loads[i_diff:] into load_segments
     #log.info('LOAD_SEG INFO: inserting load_segs[%d:%d] to load_segments' %
@@ -571,20 +568,18 @@ def update_loads_db( ifot_loads, dbh=None, test=False, dryrun=False,):
 
     for load_count, load in enumerate(to_insert):
         log.debug('LOAD_SEG DEBUG: inserting load')
-        insert_string = "\t %s %d %s %s %d" % ( load['load_segment'], load['year'],
-                                             load['datestart'], load['datestop'], load['load_scs'] )
+        insert_string = "\t %s %d %s %s %d" % (load['load_segment'], load['year'],
+                                               load['datestart'], load['datestop'], load['load_scs'])
         log.debug(insert_string)
         load_dict = dict(zip(load.dtype.names, load))
-        load_dict['id'] = max_id + 1 + load_count;
+        load_dict['id'] = max_id + 1 + load_count
         if not dryrun:
             dbh.insert(load_dict, 'load_segments', commit=True)
     return to_insert
 
 
-
-
 def main(loadseg_rdb_dir, dryrun=False, test=False,
-         server='db_base.db3' ,database=None, user=None, verbose=False):
+         server='db_base.db3', database=None, user=None, verbose=False):
     """
     Command Load Segment Table Updater
 
@@ -615,38 +610,28 @@ def main(loadseg_rdb_dir, dryrun=False, test=False,
     all_rdb_files = glob.glob(os.path.join(loadseg_dir, "*"))
     rdb_file = max(all_rdb_files)
     log.debug("LOAD_SEG DEBUG: Updating from %s" % rdb_file)
-    orig_rdb_loads = Ska.Table.read_ascii_table(rdb_file, datastart=3)
-    ifot_loads = rdb_to_db_schema( orig_rdb_loads )
+    orig_rdb_loads = Table.read(rdb_file)
+    ifot_loads = rdb_to_db_schema(orig_rdb_loads)
     if len(ifot_loads):
-        # make any scripted edits to the tables of parsed files to override directory
-        # mapping
-        import fix_tl_processing
-        fix_tl_processing.repair(dbh)
-        # make any scripted edits to the load segments table
-        import fix_load_segments
-        ifot_loads = fix_load_segments.repair(ifot_loads)
         max_timelines_id = dbh.fetchone(
             'SELECT max(id) AS max_id FROM timelines')['max_id'] or 0
         if max_timelines_id == 0 and test == False:
             raise ValueError("TIMELINES: no timelines in database.")
-        update_loads_db( ifot_loads, dbh=dbh, test=test, dryrun=dryrun )
+        update_loads_db(ifot_loads, dbh=dbh, test=test, dryrun=dryrun)
         db_loads = dbh.fetchall("""select * from load_segments
                                    where datestart >= '%s' order by datestart
-                                  """ % ( ifot_loads[0]['datestart'] )
+                                  """ % (ifot_loads[0]['datestart'])
                                 )
         update_timelines_db(loads=db_loads, dbh=dbh, max_id=max_timelines_id,
                             dryrun=dryrun, test=test)
 
     log.removeHandler(ch)
 
+
 if __name__ == "__main__":
 
-    (opt,args) = get_options()
+    (opt, args) = get_options()
     main(opt.loadseg_rdb_dir, dryrun=opt.dryrun,
-             test=opt.test, server=opt.server,
-             database=opt.database, user=opt.user,
-             verbose=opt.verbose)
-
-
-
-
+         test=opt.test, server=opt.server,
+         database=opt.database, user=opt.user,
+         verbose=opt.verbose)

From 21ab108a83cca3aed7e464eef852608dbee185bf Mon Sep 17 00:00:00 2001
From: Tom Aldcroft <taldcroft@gmail.com>
Date: Tue, 26 Oct 2021 06:42:48 -0400
Subject: [PATCH 6/7] Formatting

---
 fix_tl_processing.py | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/fix_tl_processing.py b/fix_tl_processing.py
index 03cccce..989ba94 100755
--- a/fix_tl_processing.py
+++ b/fix_tl_processing.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 import Ska.DBI
-from Chandra.Time import DateTime
+
 
 def get_options():
     from optparse import OptionParser
@@ -17,27 +17,28 @@ def get_options():
     return opt, args
 
 
-def repair( dbh ):
+def repair(dbh):
     """
     Make any necessary edits to the tl_processing table.  This is called in
     update_load_seg_db.py before loads are inserted and timelines are determined.
 
     It is not expected that any edits will need to be created, but this mechanism
     is scripted into the suite to allow the possibility.
-    
+
     If, at some point, we need to manually override mapping of replan commands
     to a specific directory instead of the one determined by parse_cmd_load_gen.pl,
     we could use something like the commented-out code in this routine.
     """
     dbh.verbose = True
-    #dbh.execute("delete from tl_processing where dir = '/2008/FEB1808/oflsb/'")
-    #dbh.execute("""insert into tl_processing
-    #( year, dir, file )
-    #values 
-    #(2008, '/2008/FEB1808/oflsb/', 'C048_0802.sum')
-    #""")
+    # dbh.execute("delete from tl_processing where dir = '/2008/FEB1808/oflsb/'")
+    # dbh.execute("""insert into tl_processing
+    # ( year, dir, file )
+    # values
+    # (2008, '/2008/FEB1808/oflsb/', 'C048_0802.sum')
+    # """)
     dbh.verbose = False
 
+
 def main():
     opt, args = get_options()
     dbh = Ska.DBI.DBI(dbi=opt.dbi, server=opt.server)

From 043ed8811cfbf6ced98b7488947ee2cbf90c1493 Mon Sep 17 00:00:00 2001
From: Tom Aldcroft <taldcroft@gmail.com>
Date: Tue, 26 Oct 2021 06:45:37 -0400
Subject: [PATCH 7/7] More format / flake8

---
 update_load_seg_db.py | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/update_load_seg_db.py b/update_load_seg_db.py
index db4e69e..eb02b3e 100755
--- a/update_load_seg_db.py
+++ b/update_load_seg_db.py
@@ -64,7 +64,7 @@ def get_built_load(run_load, dbh=None):
     if built is None:
         match_like = re.search(r'(CL\d{3}:\d{4})', run_load['load_segment'])
         if match_like is None:
-            raise ValueError("Load name %s is in unknown form, expects /CL\d{3}:\d{4}/" %
+            raise ValueError("Load name %s is in unknown form, expects /CL\\d{3}:\\d{4}/" %
                              run_load['load_segment'])
         like_load = match_like.group(1)
         built_query = ("""select * from tl_built_loads where load_segment like '%s%s%s'
@@ -103,9 +103,9 @@ def get_replan_dir(replan_seg, run_datestart, dbh=None):
     :param run_datestart: datestart of the new/replanned load
     :rtype: directory name string
     """
-    match_like = re.search('C(\d{3}).?(\d{4})', replan_seg)
+    match_like = re.search(r'C(\d{3}).?(\d{4})', replan_seg)
     if match_like is None:
-        raise ValueError("Replan load seg %s is in unknown form, expects /C\d{3}?\d{4}/" %
+        raise ValueError(r"Replan load seg %s is in unknown form, expects /C\d{3}?\d{4}/" %
                          replan_seg)
     # get_replan_dirs is called on a text string from the command load generation processing
     # summary that was the "replan source" and then this is trying to match up that source to
@@ -296,7 +296,7 @@ def update_timelines_db(loads, dbh, max_id, dryrun=False, test=False):
                 break
             i_diff += 1
     else:
-       i_diff = 0
+        i_diff = 0
 
     # Mismatch occured at i_diff.
 
@@ -309,8 +309,8 @@ def update_timelines_db(loads, dbh, max_id, dryrun=False, test=False):
         time_length = DateTime(run_timeline['datestop']).secs - \
             DateTime(run_timeline['datestart']).secs
         if time_length / 60. < 60:
-            log.warn("TIMELINES WARN: short timeline at %s, %d minutes" % (run_timeline['datestart'],
-                                                                           time_length / 60.))
+            log.warn("TIMELINES WARN: short timeline at %s, %d minutes"
+                     % (run_timeline['datestart'], time_length / 60.))
     # find all db timelines that start after the last valid one [i_diff-1]
     findcmd = ("""SELECT id from timelines
                   WHERE datestart > '%s'
@@ -520,7 +520,7 @@ def update_loads_db(ifot_loads, dbh=None, test=False, dryrun=False,):
                          % min_time_datestart)
 
     max_id = dbh.fetchone('SELECT max(id) AS max_id FROM load_segments')['max_id'] or 0
-    if max_id == 0 and test == False:
+    if max_id == 0 and not test:
         raise ValueError("LOAD SEG: no load_segments in database.")
     db_loads = dbh.fetchall("""select * from load_segments
                                where datestart >= '%s'
@@ -563,13 +563,14 @@ def update_loads_db(ifot_loads, dbh=None, test=False, dryrun=False,):
     check_load_overlap(ifot_loads)
 
     # Insert new loads[i_diff:] into load_segments
-    #log.info('LOAD_SEG INFO: inserting load_segs[%d:%d] to load_segments' %
+    # log.info('LOAD_SEG INFO: inserting load_segs[%d:%d] to load_segments' %
     #              (i_diff, len(ifot_loads)+1))
 
     for load_count, load in enumerate(to_insert):
         log.debug('LOAD_SEG DEBUG: inserting load')
         insert_string = "\t %s %d %s %s %d" % (load['load_segment'], load['year'],
-                                               load['datestart'], load['datestop'], load['load_scs'])
+                                               load['datestart'], load['datestop'],
+                                               load['load_scs'])
         log.debug(insert_string)
         load_dict = dict(zip(load.dtype.names, load))
         load_dict['id'] = max_id + 1 + load_count
@@ -615,7 +616,7 @@ def main(loadseg_rdb_dir, dryrun=False, test=False,
     if len(ifot_loads):
         max_timelines_id = dbh.fetchone(
             'SELECT max(id) AS max_id FROM timelines')['max_id'] or 0
-        if max_timelines_id == 0 and test == False:
+        if max_timelines_id == 0 and not test:
             raise ValueError("TIMELINES: no timelines in database.")
         update_loads_db(ifot_loads, dbh=dbh, test=test, dryrun=dryrun)
         db_loads = dbh.fetchall("""select * from load_segments