Commit 51b4f609 authored by renzo's avatar renzo Committed by Renzo Marini

Se loguea por dia en una misma carpeta

parent a18e76c3
import fcntl
import inspect
import os
import pickle
import argparse
from datetime import datetime, date
......@@ -43,21 +42,24 @@ def get_downloader_args(f, terminal_args):
" ni se encuentra en el archivo de configuración del proyecto")
return args
# Starts logging and lock the downloader name so only once process can access
def start_logging(source):
def start_logging(args, subdir):
global log_handle
log_path = metadata.logs_path + "/" + source + ".log"
# Stops logging on previous file
if log_handle:
stop_logging()
# Starts logging on new file
log_dir = metadata.logs_path + "/" + datetime.strftime(args["date"], '%Y%m%d') + "/" + subdir + "/"
if not os.path.exists(log_dir):
os.makedirs(log_dir)
log_path = log_dir + args["source"] + ".log"
log_handle = open(log_path, "a+")
sys.stderr = log_handle
sys.stdout = log_handle
# Ends logging and unlocks the downloader
def stop_logging(source):
# Stops logging
def stop_logging():
log_handle.close()
clean_log(source)
def disable_same_calls(args):
lock_name = "".join([str(args[arg]) for arg in args if arg != "date"])
......@@ -72,52 +74,23 @@ def enable_same_calls(handle):
fcntl.flock(handle, fcntl.LOCK_UN)
handle.close()
def log_time(source):
error_stamps_path = metadata.logs_path + "/error_stamps.pickle"
with open(error_stamps_path, "ba+") as handle_state:
fcntl.flock(handle_state, fcntl.LOCK_EX)
handle_state.seek(0)
if os.path.getsize(error_stamps_path) == 0:
error_stamps = {}
else:
error_stamps = dict(pickle.load(handle_state))
error_stamps[source] = date.today()
pickle.dump(error_stamps, handle_state, protocol=pickle.HIGHEST_PROTOCOL)
fcntl.flock(handle_state, fcntl.LOCK_UN)
def clean_log(source):
error_stamps_path = metadata.logs_path + "/error_stamps.pickle"
with open(error_stamps_path, "ba+") as handle_state:
fcntl.flock(handle_state, fcntl.LOCK_EX)
handle_state.seek(0)
error_stamps = dict(pickle.load(handle_state))
if source in error_stamps:
day_diff = (date.today() - error_stamps[source]).days
if day_diff >= metadata.logs_expiration:
f = open(metadata.logs_path + "/" + source + ".log", 'r+')
f.truncate(0)
fcntl.flock(handle_state, fcntl.LOCK_UN)
def start(args):
# Try to execute downloading and processing scripts
try:
lock_handle = disable_same_calls(args)
source = args["source"]
start_logging(source)
if not args["only_process"]:
source_config = getattr(metadata, source)
downloader_args = get_downloader_args(source_config["download"], args)
start_logging(args, "download")
try_execute(source_config["max_tries"], source_config["wait"], source_config["download"], downloader_args)
if not args["only_download"]:
start_logging(args, "process")
exec.process(source, args)
except Exception as e:
log_time(source)
logging.exception(str(datetime.now()) + "\n" + repr(e))
stop_logging(source)
mail.send(source)
stop_logging()
finally:
enable_same_calls(lock_handle)
......
......@@ -2,7 +2,6 @@ import subprocess
import datetime
import os
import metadata
......@@ -10,12 +9,16 @@ def run_model(source):
if source == "hycom":
os.chdir(metadata.telemac_dir)
for filename in os.listdir(metadata.hycom["output_path"]):
date = filename[:8]
try:
datetime.strptime(date, '%Y%m%d')
date = filename[:8]
if not [dir for dir in os.listdir(metadata.telemac_dir + "/RESULTS") if dir[7:15] == date and "HYCOM_" in dir]:
subprocess.run(metadata.telemac_script + " 'HYCOM' " + date + " " + date + " 1 8", shell=True, check=True)
except:
log_dir = metadata.logs_path + "/" + date + "/models"
if not os.path.exists(log_dir):
os.makedirs(log_dir)
with open(log_dir + "/telemac.log", "a+") as log_file:
subprocess.check_call(metadata.telemac_script + " 'HYCOM' " + date + " " + date + " 1 8", shell=True, stderr=log_file, stdout=log_file)
except Exception as e:
print(e)
pass # file is not valid
if __name__ == "__main__":
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment