Commit 8dee4301 authored by renzo's avatar renzo

funcion de clean mejorada + shn downloader baja los 4 dias anteriores

parent 20161c70
......@@ -3,21 +3,11 @@ import requests
import argparse
import os
import csv
from datetime import datetime
from datetime import datetime, timedelta
import metadata
def download_shn(tmp_path, date):
destination_dir = tmp_path + "/" + datetime.strftime(date, '%Y%m%d')
if destination_dir and not os.path.exists(destination_dir):
os.makedirs(destination_dir)
destination_path = destination_dir + "/data.csv"
if os.path.exists(destination_path):
return
def downloader(date, destination_path):
# Download data from hidro API
ids = ["MDPL", "STER", "SCLE", "OYAR", "ATAL", "LPLA", "PNOR", "BSAS", "SFER"]
names = ['Mar del Plata', 'Santa Teresita', 'San Clemente', 'Oyarvide', 'Atalaya',
......@@ -35,7 +25,7 @@ def download_shn(tmp_path, date):
rows.append([names[i]] + ['{0:.2f}'.format(x["altura"]) if x["altura"] else "F/S" for x in readings])
timestamps = [x["fecha"].split("T")[1][:5] for x in readings[:24]]
# Se escribe el CSV
# The data is dumped into a CSV
with open(destination_path, 'w') as csvfile:
writer = csv.writer(csvfile, delimiter=';', dialect='excel')
writer.writerow([""] + len(timestamps) * [datetime.strftime(date, '%d/%m/%Y')])
......@@ -43,6 +33,21 @@ def download_shn(tmp_path, date):
writer.writerow([""] + timestamps)
writer.writerows(rows)
def download_shn(tmp_path, date):
# The matlab script that process the data needs the data from the previous 4 days,
# so that data is also downloaded in case it wasn't downloaded before
for day in range(0, 5):
cur_date = date - timedelta(days=day)
destination_dir = tmp_path + "/" + datetime.strftime(cur_date, '%Y%m%d')
if destination_dir and not os.path.exists(destination_dir):
os.makedirs(destination_dir)
destination_path = destination_dir + "/data.csv"
if os.path.exists(destination_path):
continue
downloader(cur_date, destination_path)
if __name__ == "__main__":
parser = argparse.ArgumentParser()
......
......@@ -100,8 +100,8 @@ def clean_log(source):
f.truncate(0)
fcntl.flock(handle_state, fcntl.LOCK_UN)
def download_and_process(args):
def start(args):
# Try to execute downloading and processing scripts
try:
lock_handle = disable_same_calls(args)
......@@ -121,6 +121,7 @@ def download_and_process(args):
finally:
enable_same_calls(lock_handle)
def parse_arguments():
# Parse arguments
parser = argparse.ArgumentParser()
......@@ -133,6 +134,7 @@ def parse_arguments():
args = vars(parser.parse_args())
return args
if __name__ == "__main__":
args = parse_arguments()
download_and_process(args)
\ No newline at end of file
start(args)
......@@ -2,11 +2,11 @@ import os
import sys
import subprocess
from datetime import date, timedelta
import re
import shutil
from datetime import datetime
import metadata
import main
from shutil import copy
......@@ -61,11 +61,21 @@ def process(source, args):
def clean(root):
delete = []
for path, dirnames, filenames in os.walk(root):
delete.append(path)
for file in filenames:
if file.split(".")[-1] not in ["m", "pickle"]:
delete = delete[:-1]
break
try:
delete.append(path)
date = datetime.strptime(path.split("/")[-1], '%Y%m%d')
today = datetime.today()
for file in filenames:
if file.split(".")[-1] not in ["m", "pickle", "csv"] or (today - date).days < 7:
delete = delete[:-1]
break
except ValueError:
pass
for path in delete:
if path != root:
shutil.rmtree(path)
\ No newline at end of file
shutil.rmtree(path)
# Remove locks
for filename in os.listdir(root):
if filename[-4:] == "lock" and path.split("/")[-1] in filename:
os.remove(os.path.join(root, filename))
\ No newline at end of file
......@@ -3,7 +3,7 @@ from os import listdir
from os.path import isfile, join
import metadata
from main import download_and_process, parse_arguments
from main import start, parse_arguments
def download_all(args):
......@@ -17,13 +17,7 @@ def download_all(args):
if (cur_date not in dates):
print(cur_date)
args["date"] = cur_date
if (args["source"] == "shn" and args["only_download"]):
print(args["source"] + str(args["only_download"]))
for day in range(0, 6):
args["date"] = cur_date - timedelta(days=day)
download_and_process(args)
else:
download_and_process(args)
start(args)
cur_date += timedelta(days=1)
if __name__ == "__main__":
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment