#!/usr/bin/env python

"""
eis_do_quicklook.py - automated quicklook processing

    One Program to rule them all, One Program to find them,
    One Program to bring them all and in the darkness bind them
    In the Land of ISAS where the Shadows lie.

SYNOPSIS

    ./eis_do_quicklook.py [dd-mon-yyyy]

DESCRIPTION

    This Python script runs the EIS quicklook processing software for
    either the selected day or by default the previous day. The
    created FITS files are copied to the appropriate area on DARTS. In
    its present form, this script can only be run from this
    directory. The script is normally invoked by the cron script
    ~/bin/eis_quicklook.csh. When that script is used, a log file is
    written to the EIS staging area on DARTS:
    
    (http://darts.isas.jaxa.jp/pub/solar/solarb/eis/staging\
    /logs/quicklook).

    The script attempts to recover from a number of failure modes
    exhibited by the quicklook software. If for some reason it
    discovers a new failure mode, the script will give up after
    restarting the quicklook software 15 times.

EXAMPLES

    ./eis_do_quicklook.py 12-Jun-2016

    ./eis_do_quicklook.py 12-06-2016

    ./eis_do_quicklook.py

TODO

    Add the ability to run from any directoy.

HISTORY

    (2009-Jun-24) JTM: Initial production version.
    (2010-Mar-03) JTM: Additional cleanup.
    (2014-Sep-02) JTM: Added kill_subs call when restarts too many times.
    (2017-Jan-11) JTM: Make date handling more robust.

CONTACT

    John Mariska, jmariska@gmu.edu, jtmariska@gmail.com
"""

import os
import sys
import time
from dateutil import parser
import commands
import shutil


def mk_batch_script():
    cmd1 = "eis_quick_look, '%s', /get_data\n\n" % (day)
    cmd2 = "eis_quick_look, '%s', /restart\n\n" % (day)

    ip = open('eis_do_quicklook_in.txt', mode='w')
    ip.write('; eis_do_quicklook_in.txt\n')
    ip.write('; count: %d\n' % (count))
    ip.write('; ' + time.asctime() + '\n\n')
    ip.write('print, "* * * Batch start * * *"\n\n')
    if restart:
        ip.write(cmd2)
    else:
        ip.write(cmd1)
    ip.write('print, "* * * Batch end * * *"\n\n')
    ip.write('exit\n')
    ip.close()


def kill_subs(pid):
    # For now, assume there are just two processes below pid to
    # worry about, the SSW script and the IDL process. This could
    # be a mess if there are more.
    pscmd = 'ps --format pid --no-headers --ppid %s'
    kstring = '/usr/bin/kill -s KILL '

    pid1 = commands.getoutput(pscmd % (str(pid)))
    if pid1:
        pid2 = commands.getoutput(pscmd % (pid1))
    else:
        pid2 = ''

    # Kill in reverse order one at a time for now
    print 'Killing processes %s %s %s' % (str(pid), pid1, pid2)
    if pid2: os.system(kstring + pid2)
    if pid1: os.system(kstring + pid1)
    if str(pid): os.system(kstring + str(pid))

    # Remove IDL start script
    sswfile = '/home/sbukeis/ssw_idl.' + pid1.strip()
    if os.path.exists(sswfile):
        print 'Removing ' + sswfile
        os.remove(sswfile)


# * * * M A I N * * *

print 'Started at ', time.asctime()

if len(sys.argv) == 2:
    dt = parser.parse(sys.argv[1], dayfirst=True)
    day = dt.strftime('%d-%b-%Y')
else:
    yesterday = time.time() - 24*60*60 - 4*60*60 # adjust as needed for cron use
    day = time.strftime('%d-%b-%Y', time.localtime(yesterday))

# Remove any leftover push scripts
if os.path.exists('push_files.sh'): os.remove('push_files.sh')

# First script gets data
count = 1
maxcount = 15
restart = False
mk_batch_script()

pid = os.spawnlp(os.P_NOWAIT, './eis_do_quicklook.sh', 'eis_do_quicklook.sh')
print 'PID: ', pid

tlim = 90.0
len = 0
while True:
    time.sleep(tlim)

    newlen = os.stat('eis_do_quicklook_out.txt').st_size
    print time.asctime()
    print 'Size: ', newlen
    line = commands.getoutput('tail -1 eis_do_quicklook_out.txt')

    if newlen > len: # still growing, keep going
        len = newlen
        continue
    elif count > maxcount: # may be stuck in a loop
        print 'Too many restarts. Aborting.'
        kill_subs(pid)
        break
    elif line == '* * * Batch end * * *':
        if os.path.exists('push_files.sh'): # really are done!
            print 'Completed'
            print 'Script called %d times' % (count)
            pushlen = os.stat('push_files.sh').st_size
            if pushlen > 0:
                print 'Running push_files.sh'
                os.system('./push_files.sh')
            else:
                print 'Problem with push_files.sh: zero length'
            break
        else: # might have crashed back to shell, restart
            len = 0 # outfile started again
            restart = True
            count = count + 1
            mk_batch_script()
            pid = os.spawnlp(os.P_NOWAIT, './eis_do_quicklook.sh',
                             'eis_do_quicklook.sh')
            print 'Restarting after apparent IDL exit'
            print 'New PID: ', pid
            continue
    elif line == 'Segmentation fault': # crashed back to shell, restart
        len = 0 # outfile started again
        restart = True
        count = count + 1
        mk_batch_script()
        pid = os.spawnlp(os.P_NOWAIT, './eis_do_quicklook.sh', 'eis_do_quicklook.sh')
        print 'Restarting after IDL exit on Segmentation fault'
        print 'New PID: ', pid
        continue
    else: # must be stalled, restart
        kill_subs(pid)
        len = 0 # outfile started again
        restart = True
        count = count + 1
        mk_batch_script()
        pid = os.spawnlp(os.P_NOWAIT, './eis_do_quicklook.sh', 'eis_do_quicklook.sh')
        print 'Restarting after stalling'
        print 'New PID: ', pid
        continue

# Clean up old directories and log files
ndays = 10
day_to_rm = time.time() - ndays*24*60*60
day_to_rm_str = time.strftime('eis_fits_%Y%m%d', time.localtime(day_to_rm))
if os.path.exists(day_to_rm_str): shutil.rmtree(day_to_rm_str)

lfile = time.strftime('quicklook_log_%Y%m%d.txt', time.localtime(day_to_rm))
if os.path.exists(lfile): os.remove(lfile)

dir_str = time.strftime('%Y%m%d', time.localtime(day_to_rm))
full_path = '/home/sbukeis/work/eis/localdata/sdtp/md/' + dir_str
if os.path.exists(full_path): shutil.rmtree(full_path)

full_path = '/home/sbukeis/work/eis/localdata/sdtp/fits/mission/' + dir_str
if os.path.exists(full_path): shutil.rmtree(full_path)

print 'Finished at ', time.asctime()
print 'Restarted %d times' % (count - 1)