a_adli/colouring-montreal
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

treat invalid future dates as invalid

Browse Source 
Brent-87_0946 has "valid_date": "23/04/9187"
yes, they apparently recorded filing in Anno Dominin 9187
This commit is contained in:
Mateusz Konieczny 2022-11-08 23:10:25 +01:00
parent da4b198555
commit e7f67cbe8f
1 changed files with 18 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
etl/planning_data/obtain_livestream_data_and_load_into_database.py
View File

@ -66,6 +66,19 @@ def load_data_into_database(cursor, data):
"data_source": "Greater London Authority's Planning London DataHub",
"data_source_link": None
}
if date_in_future(entry["registered_with_local_authority_date"]):
print("registered_with_local_authority_date is treated as invalid:", entry["registered_with_local_authority_date"])
# Brent-87_0946 has "valid_date": "23/04/9187"
entry["registered_with_local_authority_date"] = None
if date_in_future(entry["decision_date"]):
print("decision_date is treated as invalid:", entry["decision_date"])
entry["decision_date"] = None
if date_in_future(entry["last_synced_date"]):
print("last_synced_date is treated as invalid:", entry["last_synced_date"])
entry["last_synced_date"] = None
if "Hackney" in entry["application_id"]:
if entry["application_url"] != None:
if "https://" not in entry["application_url"]:
@ -80,6 +93,11 @@ def load_data_into_database(cursor, data):
print(json.dumps(entry, indent = 4))
raise e
def date_in_future(date):
if date == None:
return False
return date > datetime.datetime.now()
def query(search_after):
headers = {
'X-API-AllowRequest': os.environ['PLANNNING_DATA_API_ALLOW_REQUEST_CODE'],