Commit 16278489 authored by renzo's avatar renzo

agregada funcion de download and process para pasarle a try_execute

parent d90c792f
......@@ -76,20 +76,30 @@ def enable_same_calls(handle):
fcntl.flock(handle, fcntl.LOCK_UN)
handle.close()
def download_and_process(source, source_config, downloader_args):
start_logging(args, "download")
source_config["download"](*downloader_args)
start_logging(args, "process")
exec.process(source, args)
def start(args):
# Try to execute downloading and processing scripts
try:
lock_handle = disable_same_calls(args)
source = args["source"]
if not args["only_process"]:
source_config = getattr(metadata, source)
downloader_args = get_downloader_args(source_config["download"], args)
source_config = getattr(metadata, source)
downloader_args = get_downloader_args(source_config["download"], args)
# Normal case, process and download
if not args["only_process"] and not args["only_download"]:
try_execute(source_config["max_tries"], source_config["wait"], download_and_process, [source, source_config, downloader_args])
elif args["only_download"]:
start_logging(args, "download")
try_execute(source_config["max_tries"], source_config["wait"], source_config["download"], downloader_args)
if not args["only_download"]:
elif args["only_process"]:
start_logging(args, "process")
exec.process(source, args)
try_execute(source_config["max_tries"], source_config["wait"], exec.process, [source, args])
except Exception as e:
logging.exception(str(datetime.now()) + "\n" + repr(e))
mail.send(source)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment