Commit 51b4f609 authored by renzo's avatar renzo Committed by Renzo Marini

Se loguea por dia en una misma carpeta

parent a18e76c3
import fcntl import fcntl
import inspect import inspect
import os import os
import pickle
import argparse import argparse
from datetime import datetime, date from datetime import datetime, date
...@@ -43,21 +42,24 @@ def get_downloader_args(f, terminal_args): ...@@ -43,21 +42,24 @@ def get_downloader_args(f, terminal_args):
" ni se encuentra en el archivo de configuración del proyecto") " ni se encuentra en el archivo de configuración del proyecto")
return args return args
def start_logging(args, subdir):
# Starts logging and lock the downloader name so only once process can access
def start_logging(source):
global log_handle global log_handle
log_path = metadata.logs_path + "/" + source + ".log" # Stops logging on previous file
if log_handle:
stop_logging()
# Starts logging on new file
log_dir = metadata.logs_path + "/" + datetime.strftime(args["date"], '%Y%m%d') + "/" + subdir + "/"
if not os.path.exists(log_dir):
os.makedirs(log_dir)
log_path = log_dir + args["source"] + ".log"
log_handle = open(log_path, "a+") log_handle = open(log_path, "a+")
sys.stderr = log_handle sys.stderr = log_handle
sys.stdout = log_handle sys.stdout = log_handle
# Ends logging and unlocks the downloader # Stops logging
def stop_logging(source): def stop_logging():
log_handle.close() log_handle.close()
clean_log(source)
def disable_same_calls(args): def disable_same_calls(args):
lock_name = "".join([str(args[arg]) for arg in args if arg != "date"]) lock_name = "".join([str(args[arg]) for arg in args if arg != "date"])
...@@ -72,52 +74,23 @@ def enable_same_calls(handle): ...@@ -72,52 +74,23 @@ def enable_same_calls(handle):
fcntl.flock(handle, fcntl.LOCK_UN) fcntl.flock(handle, fcntl.LOCK_UN)
handle.close() handle.close()
def log_time(source):
error_stamps_path = metadata.logs_path + "/error_stamps.pickle"
with open(error_stamps_path, "ba+") as handle_state:
fcntl.flock(handle_state, fcntl.LOCK_EX)
handle_state.seek(0)
if os.path.getsize(error_stamps_path) == 0:
error_stamps = {}
else:
error_stamps = dict(pickle.load(handle_state))
error_stamps[source] = date.today()
pickle.dump(error_stamps, handle_state, protocol=pickle.HIGHEST_PROTOCOL)
fcntl.flock(handle_state, fcntl.LOCK_UN)
def clean_log(source):
error_stamps_path = metadata.logs_path + "/error_stamps.pickle"
with open(error_stamps_path, "ba+") as handle_state:
fcntl.flock(handle_state, fcntl.LOCK_EX)
handle_state.seek(0)
error_stamps = dict(pickle.load(handle_state))
if source in error_stamps:
day_diff = (date.today() - error_stamps[source]).days
if day_diff >= metadata.logs_expiration:
f = open(metadata.logs_path + "/" + source + ".log", 'r+')
f.truncate(0)
fcntl.flock(handle_state, fcntl.LOCK_UN)
def start(args): def start(args):
# Try to execute downloading and processing scripts # Try to execute downloading and processing scripts
try: try:
lock_handle = disable_same_calls(args) lock_handle = disable_same_calls(args)
source = args["source"] source = args["source"]
start_logging(source)
if not args["only_process"]: if not args["only_process"]:
source_config = getattr(metadata, source) source_config = getattr(metadata, source)
downloader_args = get_downloader_args(source_config["download"], args) downloader_args = get_downloader_args(source_config["download"], args)
start_logging(args, "download")
try_execute(source_config["max_tries"], source_config["wait"], source_config["download"], downloader_args) try_execute(source_config["max_tries"], source_config["wait"], source_config["download"], downloader_args)
if not args["only_download"]: if not args["only_download"]:
start_logging(args, "process")
exec.process(source, args) exec.process(source, args)
except Exception as e: except Exception as e:
log_time(source)
logging.exception(str(datetime.now()) + "\n" + repr(e)) logging.exception(str(datetime.now()) + "\n" + repr(e))
stop_logging(source)
mail.send(source) mail.send(source)
stop_logging()
finally: finally:
enable_same_calls(lock_handle) enable_same_calls(lock_handle)
......
...@@ -2,7 +2,6 @@ import subprocess ...@@ -2,7 +2,6 @@ import subprocess
import datetime import datetime
import os import os
import metadata import metadata
...@@ -10,13 +9,17 @@ def run_model(source): ...@@ -10,13 +9,17 @@ def run_model(source):
if source == "hycom": if source == "hycom":
os.chdir(metadata.telemac_dir) os.chdir(metadata.telemac_dir)
for filename in os.listdir(metadata.hycom["output_path"]): for filename in os.listdir(metadata.hycom["output_path"]):
date = filename[:8]
try: try:
datetime.strptime(date, '%Y%m%d') date = filename[:8]
if not [dir for dir in os.listdir(metadata.telemac_dir + "/RESULTS") if dir[7:15] == date and "HYCOM_" in dir]: if not [dir for dir in os.listdir(metadata.telemac_dir + "/RESULTS") if dir[7:15] == date and "HYCOM_" in dir]:
subprocess.run(metadata.telemac_script + " 'HYCOM' " + date + " " + date + " 1 8", shell=True, check=True) log_dir = metadata.logs_path + "/" + date + "/models"
except: if not os.path.exists(log_dir):
os.makedirs(log_dir)
with open(log_dir + "/telemac.log", "a+") as log_file:
subprocess.check_call(metadata.telemac_script + " 'HYCOM' " + date + " " + date + " 1 8", shell=True, stderr=log_file, stdout=log_file)
except Exception as e:
print(e)
pass # file is not valid pass # file is not valid
if __name__ == "__main__": if __name__ == "__main__":
run_model("hycom") run_model("hycom")
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment