a_adli/colouring-montreal
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

drop dead code

Browse Source
This commit is contained in:
Mateusz Konieczny 2022-09-14 12:08:06 +02:00
parent f995a92a82
commit ac7a11a805
1 changed files with 1 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
etl/planning_data/load_into_database.py
View File

@ -27,11 +27,6 @@ def insert_entry(connection, e):
''', (e["application_id"], application_url, e["description"], e["registered_with_local_authority_date"], e["decision_date"], e["last_synced_date"], e["status"], e["data_source"], e["data_source_link"], e["uprn"]))
connection.commit()
return """INSERT INTO planning_data
(planning_application_id, planning_application_link, description, decision_date, last_synced_date, status, data_source, data_source_link, uprn)
VALUES""" + ",\n".join(elements) + ";"
def parse_date_string_into_datestring(incoming):
date = None
try:
@ -40,23 +35,6 @@ def parse_date_string_into_datestring(incoming):
date = datetime.datetime.strptime(incoming, "%Y-%m-%dT%H:%M:%S.%fZ") # '2022-08-08T20:07:22.238Z'
return datetime.datetime.strftime(date, "%Y-%m-%d")
def shorten_description(original_description):
description = original_description.strip()
limit = 400
if len(description) > limit:
description = ""
for entry in original_description.split():
extended = description
if extended != "":
extended += " "
extended += entry
if len(extended) <= limit:
description = extended
if description == "":
description = description[0:limit]
description += "... <i>(show more)</i>"
return description
def main():
connection = get_connection()
with connection.cursor() as cur:
@ -70,7 +48,7 @@ def main():
if data['is_running']:
raise Exception("query getting livestream data has failed")
for entry in data['rawResponse']['hits']['hits']:
description = shorten_description(entry['_source']['description'])
description = entry['_source']['description'].strip()
application_id = entry['_source']['id']
decision_date = parse_date_string_into_datestring(entry['_source']['decision_date'])
last_synced_date = parse_date_string_into_datestring(entry['_source']['last_synced'])
@ -89,7 +67,6 @@ def main():
status = "Appeal In Progress"
if (status not in ["Approved", "Rejected", "Appeal In Progress", "Withdrawn", ]):
raise Exception("Unexpected status " + status)
description = entry['_source']['description'].strip()
if uprn == None:
continue
entry = {