Commit f8bc6785 authored by renzo's avatar renzo

beta 1.4

parent 9a1c1691
import sys
import argparse
import os
import subprocess
......@@ -19,7 +21,7 @@ def download_copernicus(tmp_path):
call = call.replace("DATE-MIN", (date.today() - timedelta(days=22)).strftime("%Y-%m-%d"))
call = call.replace("DATE-MAX", (date.today() + timedelta(days=9)).strftime("%Y-%m-%d"))
subprocess.run(call, shell=True, check=True)
subprocess.run(call, shell=True, check=True, stdout=sys.stdout, stderr=sys.stderr)
# Se hace lo siguiente porque a veces motuclient retorna 0 cuando hubo error
if not os.path.isfile(download_dir + "/" + out_name):
raise RuntimeError("Error en la descarga")
......
......@@ -14,6 +14,7 @@ import process.process as exec
log_handle = 0
def try_execute(max_tries, waiting_time, f, args):
tries = 0
while tries < max_tries:
......@@ -46,9 +47,10 @@ def get_downloader_args(f, terminal_args):
def start_logging(source):
global log_handle
log_path = metadata.logs_path + "/" + source + ".log"
log_handle = open(log_path, "a+")
log_handle = open(log_path, "w")
fcntl.flock(log_handle, fcntl.LOCK_EX)
sys.stderr = log_handle
sys.stdout = log_handle
# Ends logging and unlocks the downloader
......@@ -56,6 +58,7 @@ def stop_logging(source):
log_handle.close()
clean_log(source)
def disable_same_calls(args):
lock_name = "".join([str(args[arg]) for arg in args])
lock_name = lock_name.replace("/", "").replace(" ", "") + ".lock"
......@@ -64,9 +67,11 @@ def disable_same_calls(args):
fcntl.flock(lock_handle, fcntl.LOCK_EX)
return lock_handle
def enable_same_calls(handle):
fcntl.flock(handle, fcntl.LOCK_UN)
def log_time(source):
error_stamps_path = metadata.logs_path + "/error_stamps.pickle"
with open(error_stamps_path, "ba+") as handle_state:
......@@ -80,6 +85,7 @@ def log_time(source):
pickle.dump(error_stamps, handle_state, protocol=pickle.HIGHEST_PROTOCOL)
fcntl.flock(handle_state, fcntl.LOCK_UN)
def clean_log(source):
error_stamps_path = metadata.logs_path + "/error_stamps.pickle"
with open(error_stamps_path, "ba+") as handle_state:
......@@ -93,10 +99,13 @@ def clean_log(source):
f.truncate(0)
fcntl.flock(handle_state, fcntl.LOCK_UN)
if __name__ == "__main__":
# Parse arguments
parser = argparse.ArgumentParser()
parser.add_argument("source", type=str, help="Fuente de datos", choices=metadata.sources)
parser.add_argument('--only_download', action='store_true')
parser.add_argument('--only_process', action='store_true')
for arg in metadata.args:
final_arg = "--" + arg
if "choices" in metadata.args[arg]:
......@@ -112,14 +121,16 @@ if __name__ == "__main__":
try:
source = args["source"]
start_logging(source)
downloader_path = os.path.abspath(inspect.getfile(getattr(metadata, source)["download"]))
source_config = getattr(metadata, source)
downloader_args = get_downloader_args(source_config["download"], args)
try_execute(source_config["max_tries"], source_config["wait"], source_config["download"], downloader_args)
exec.process(source, args)
if not args["only_process"]:
downloader_path = os.path.abspath(inspect.getfile(getattr(metadata, source)["download"]))
source_config = getattr(metadata, source)
downloader_args = get_downloader_args(source_config["download"], args)
try_execute(source_config["max_tries"], source_config["wait"], source_config["download"], downloader_args)
if not args["only_download"]:
exec.process(source, args)
except Exception as e:
log_time(source)
logging.exception(repr(e))
logging.exception(datetime.datetime.now() + "\n" + repr(e))
stop_logging(source)
finally:
enable_same_calls(lock_handle)
\ No newline at end of file
enable_same_calls(lock_handle)
......@@ -14,12 +14,12 @@ logs_expiration = 7 # Measured in days
processing_error_msg = "El script de procesamiento ha fallado, chequear el archivo de log"
# 1.2 - PATHS
root_path = "/home/renzo/Desktop/test"
nctoolbox_path = ""
matlab_scripts = "/home/renzo/Desktop/automatizacion-de-bajadas/process/matlab_scripts"
root_path = "/export/flyma/PronosticoRPFM/DATOS/scripts"
nctoolbox_path = "/home/rmarini2/.matlab/nctoolbox-1.1.3/setup_nctoolbox.m"
matlab_scripts = root_path + "/process/matlab_scripts"
logs_path = root_path + "/logs"
tmp_path = root_path + "/tmp"
output_root_path = root_path + "/output"
output_root_path = "/export/flyma/PronosticoRPFM/DATOS"
# 2 - MAIN SCRIPT ARGUMENTS
args = {}
......@@ -38,7 +38,7 @@ args["parallel_downloads"]["type"] = int
shn = {"max_tries": 45, "wait": 60}
mins = (shn["wait"] * shn["max_tries"]) / 60
shn["download"] = download_shn
shn["process"] = []
shn["process"] = ["filtro_ARGENTINA"]
shn["output_path"] = output_root_path + "/ARCHIVOS_SHN"
shn["message"] = "Se considera un fallo cuando el script intenta descargar los datos sin éxito " \
+ str(shn["max_tries"]) + " veces, a lo largo de " + str(mins) \
......@@ -56,7 +56,7 @@ anp["message"] = "Este mensaje se envía únicamente cuando el script intenta re
hycom = {"max_tries": 72, "wait": 600}
mins = (hycom["wait"] * hycom["max_tries"]) / 60
hycom["download"] = download_hycom
hycom["process"] = ["procesar_Hycom", "filtro_ARGENTINA"]
hycom["process"] = ["procesar_Hycom"]
hycom["output_path"] = output_root_path + "/ARCHIVOS_HYCOM"
hycom["message"] = "Este mensaje se envía únicamente cuando el script intenta descargar sin éxito un archivo " \
+ str(hycom["max_tries"]) + " veces, a lo largo de " + str(mins / 60) \
......
......@@ -69,10 +69,7 @@ function []=procesar_copernicus(output_path)
save(strcat(output_path, "/", num2str(name),'fnaCOP.mat'),'name','fecha','XX','YY','sshData','tempData','u','v')
FILES=dir('*fnaCOP.mat');
file_name=FILES.name;
load(file_name);
load(char(strcat(output_path, "/", num2str(name),'fnaCOP.mat')));
%La Paloma
IndAproxLON=192;%-54.0833
......
import os
import sys
import subprocess
from datetime import date, timedelta
import re
......@@ -53,7 +55,7 @@ def process(source, optional_args):
copy(script_path, data_dir)
os.chdir(data_dir)
matlab_call = create_matlab_call(script, source, optional_args)
subprocess.run(matlab_call, shell=True, check=True, stderr=main.log_handle)
subprocess.run(matlab_call, shell=True, check=True, stderr=sys.stderr, stdout=sys.stdout)
clean(metadata.tmp_path)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment