a_adli/colouring-montreal
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

delete no longer used code

Browse Source
This commit is contained in:
Mateusz Konieczny 2022-11-08 15:59:47 +01:00
parent e6b6187c24
commit 8a037b2ebc
1 changed files with 0 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
etl/planning_data/make_query.py
View File

@ -1,55 +0,0 @@
from requests.structures import CaseInsensitiveDict
import requests
import time
import json
def obtain_data(data):
url = "https://planningdata.london.gov.uk/dashboard/internal/search/es"
headers = headers_of_query()
response = requests.post(url, headers=headers, data=json.dumps(data))
# typically initially return something like that
# {'id': 'Fmo0RW9DX0k5U3UtLWJIVlEtMzRwR3cfdGwtYkJaaHNUeG1GdF9kRHFtQldaUToxODczMzM5Nw==', 'is_partial': True, 'is_running': True, 'rawResponse': {'took': 100, 'timed_out': False, 'terminated_early': False, 'num_reduce_phases': 0, '_shards': {'total': 1, 'successful': 0, 'skipped': 0, 'failed': 0}, 'hits': {'total': 0, 'max_score': None, 'hits': []}}, 'total': 1, 'loaded': 0}
if response.status_code != 200:
raise Exception("unexpected status code " + str(response.status_code))
output = response.content.decode('utf-8')
output = json.loads(output)
if output["is_partial"]:
identifier = output["id"]
while output["is_partial"]:
time.sleep(3)
response = reask_for_query_results(identifier)
output = json.loads(response.content.decode('utf-8'))
if response.status_code != 200:
raise Exception("unexpected status code " +
str(response.status_code))
return output
def headers_of_query():
headers = CaseInsensitiveDict()
headers["User-Agent"] = "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:93.0) Gecko/20100101 Firefox/93.0"
headers["Accept"] = "*/*"
headers["Referer"] = "https://planningdata.london.gov.uk/dashboard/app/discover"
headers["Content-Type"] = "application/json"
headers["kbn-version"] = "7.9.3"
headers["Origin"] = "https://planningdata.london.gov.uk"
headers["Connection"] = "keep-alive"
headers["Sec-Fetch-Dest"] = "empty"
headers["Sec-Fetch-Mode"] = "cors"
headers["Sec-Fetch-Site"] = "same-origin"
headers["TE"] = "trailers"
return headers
def reask_for_query_results(identifier):
data = {'id': identifier}
return requests.post(
'https://planningdata.london.gov.uk/dashboard/internal/search/es',
data=data,
timeout=100000,
headers={
'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:93.0) Gecko/20100101 Firefox/93.0',
'kbn-version': '7.9.3',
}
)