87 lines
2.5 KiB
Python
Executable File
87 lines
2.5 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
import requests
|
|
import json
|
|
import time
|
|
import os
|
|
import copy
|
|
|
|
# Set to your cookie
|
|
cookie=os.getenv('PRPT_COOKIE', '')
|
|
|
|
def run(projects = ""):
|
|
endpoint = "https://www.ura.gov.sg/realEstateIIWeb/transaction/submitSearch.action"
|
|
# endpoint = "https://httpbin.org/post"
|
|
headersTR = {
|
|
'User-Agent': 'Mozilla/5.0 (Linux) Gecko/20100101 Firefox/60.0',
|
|
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
|
'Accept-Encoding': 'gzip, deflate, br',
|
|
'Accept-Language': 'en-US,en;q=0.5',
|
|
'Pragma': 'no-cache',
|
|
'Cache-Control': 'no-cache',
|
|
'Referer': 'https://www.ura.gov.sg/',
|
|
'Content-Type': 'application/x-www-form-urlencoded',
|
|
'Cookie': cookie
|
|
}
|
|
headersT = {
|
|
'User-Agent': 'Mozilla/5.0 (Linux) Gecko/20100101 Firefox/60.0',
|
|
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
|
'Accept-Encoding': 'gzip, deflate, br',
|
|
'Accept-Language': 'en-US,en;q=0.5',
|
|
'Pragma': 'no-cache',
|
|
'Cache-Control': 'no-cache',
|
|
'Referer': 'https://www.ura.gov.sg/',
|
|
'Cookie': cookie
|
|
}
|
|
dataT = {
|
|
'submissionType': 'pn',
|
|
'selectedFromPeriodProjectName': 'JUL 2016',
|
|
'selectedToPeriodProjectName': 'JUL 2019',
|
|
'saleType': ['1', '2', '3'],
|
|
'selectedProjects1': [projects],
|
|
'__multiselect_selectedProjects1': '',
|
|
'selectedFromPeriodPostalDistrict': 'JUL 2016',
|
|
'selectedToPeriodPostalDistrict': 'JUL 2019',
|
|
'propertyType': 'lp',
|
|
'postalDistrictList': '28',
|
|
'__multiselect_selectedPostalDistricts1': ''
|
|
}
|
|
|
|
r = requests.post(endpoint, data=dataT, headers=headersTR)
|
|
if 'Missing' in r.text:
|
|
print(r.text)
|
|
print('Error')
|
|
exit(1)
|
|
time.sleep(2)
|
|
|
|
r = requests.get('https://www.ura.gov.sg/realEstateIIWeb/transaction/submitExcelResults.action', headers=headersT)
|
|
|
|
if 'html' in r.text:
|
|
print(r.text)
|
|
print('CSV Download Error')
|
|
exit(1)
|
|
|
|
filepath = os.path.join(os.path.dirname(__file__), 'data', projects.replace('/', '+') + '.csv')
|
|
|
|
f = open(filepath, 'w')
|
|
print("Writing \"" + filepath)
|
|
f.write(r.text)
|
|
f.close()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
places = []
|
|
# open places file
|
|
with open('input/places.json', 'r') as f:
|
|
places = json.loads(f.read())
|
|
|
|
i = 0
|
|
|
|
for place in places:
|
|
i += 1
|
|
print('Query #' + str(i) + ': ' + place)
|
|
run(place)
|
|
time.sleep(5)
|
|
|