Skip to content
Snippets Groups Projects
Commit 53c6f51a authored by Hoa Ben The Nguyen's avatar Hoa Ben The Nguyen
Browse files

fix: type and json format of json

parent b0f3cebd
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 297 additions and 138 deletions
server/data_processing/__pycache__/area_processing.cpython-39.pyc
+ 0
− 0
View file @ 53c6f51a
No preview for this file type
server/data_processing/__pycache__/process_lidar_data.cpython-39.pyc
+ 0
− 0
View file @ 53c6f51a
No preview for this file type
server/data_processing/process_lidar_data.py
+ 7
− 36
View file @ 53c6f51a
...@@ -36,15 +36,9 @@ def find_folder_files(direcory): ...@@ -36,15 +36,9 @@ def find_folder_files(direcory):
files.append(os.path.join(root, fileName)) files.append(os.path.join(root, fileName))
return files return files
print(find_folder_files())
# find the height of an area based on the coordinates of it's center # find the height of an area based on the coordinates of it's center
# and it's affiliations (subId and groupId) (soon to be implemented # and it's affiliations (subId and groupId) (soon to be implemented
def calculate_area_data(center, body_of_water): def calculate_area_data(center, body_of_water, path):
# test dataset
laz_root_path = "server\\lidar_data\\" + body_of_water
laz_data_paths = find_folder_files(laz_root_path)
# container for all the heights in area # container for all the heights in area
area_heights = [] area_heights = []
...@@ -79,42 +73,20 @@ def calculate_area_data(center, body_of_water): ...@@ -79,42 +73,20 @@ def calculate_area_data(center, body_of_water):
map_data = list(filter(lambda point_position: inArea((point_position['properties']['sub_div_center'][0], point_position['properties']['sub_div_center'][1], 0.0), map_zones), map_data)) map_data = list(filter(lambda point_position: inArea((point_position['properties']['sub_div_center'][0], point_position['properties']['sub_div_center'][1], 0.0), map_zones), map_data))
# Refactor lidar data to a readable format # Refactor lidar data to a readable format
iceOver = laspy.read(laz_data_paths[0]) iceOver = laspy.read(path)
iceUnder = laspy.read(laz_data_paths[1])
ice_points = list(zip(iceOver.X, iceOver.Y, iceOver.Z))
# add two files together temporary test data(soon to be removed)
ice_points = list(zip(iceOver.X,iceOver.Y,iceOver.Z)) + list(zip(iceUnder.X,iceUnder.Y,iceUnder.Z))
print(len(ice_points))
ice_points = list(filter(lambda point_position: inArea((point_position[0],point_position[1],0.0),[(-3300000, 4500000), (-3200000, 5000000)]), ice_points))
print(len(ice_points))
# only for visualizating the testing data
#print("max",max(ice_points))
#print("min",min(ice_points))
#x = [points[0] for points in ice_points]
#y = [points[1] for points in ice_points]
# Plot the points
#plt.plot(x, y, 'bo') # 'bo' specifies blue color and circle markers
#plt.xlabel('X axis')
#plt.ylabel('Y axis')
#plt.title('Plotting Points')
#plt.grid(True) # Add grid
#plt.show()
max_point = max(ice_points) max_point = max(ice_points)
min_point = min(ice_points) min_point = min(ice_points)
# define all the sub-areas within the area, local coordinates # define all the sub-areas within the area, local coordinates
grid_sub_area = define_gridareas(center[0], center[1], (cell_x, cell_y),grid_size) grid_sub_area = define_gridareas(center[0], center[1], (cell_x, cell_y),grid_size)
# define all the sub-areas within the area, lidar coordinates # define all the sub-areas within the area, lidar coordinates
grid_area_lidar_heights = define_grid_lidardata((min_point, max_point), grid_size, ice_points) grid_area_lidar_heights = define_grid_lidardata((min_point, max_point), grid_size, ice_points)
print('grid size: ', grid_sub_area) print('grid height: ', grid_area_lidar_heights)
print('grid size: ', grid_area_lidar_heights)
sub_area_heights = list(zip(grid_sub_area, grid_area_lidar_heights)) sub_area_heights = list(zip(grid_sub_area, grid_area_lidar_heights))
print('sub_area_heights: ', sub_area_heights[0])
# find the heights of each sub-area => area-heights # find the heights of each sub-area => area-heights
if len(map_data) > 0: if len(map_data) > 0:
...@@ -136,8 +108,7 @@ def calculate_area_data(center, body_of_water): ...@@ -136,8 +108,7 @@ def calculate_area_data(center, body_of_water):
sub_center = ((xs + xe)/2, (ys + ye)/2) sub_center = ((xs + xe)/2, (ys + ye)/2)
# check if area is part of water body # check if area is part of water body
part_of_subarea_of_waterbody = list(filter(lambda pos: inArea((pos['properties']['sub_div_center'][0], pos['properties']['sub_div_center'][1], 0.0), [start,end]), map_data)) part_of_subarea_of_waterbody = list(filter(lambda pos: inArea((pos['properties']['sub_div_center'][0], pos['properties']['sub_div_center'][1], 0.0), [start,end]), map_data))
print('sub_area: ', sub_area)
print('part_of_subarea_of_waterbody: ', part_of_subarea_of_waterbody)
if(len(part_of_subarea_of_waterbody) > 0): if(len(part_of_subarea_of_waterbody) > 0):
current_map_zone = closest_points(sub_center, part_of_subarea_of_waterbody, taken_coords) current_map_zone = closest_points(sub_center, part_of_subarea_of_waterbody, taken_coords)
sub_center = current_map_zone['properties']['sub_div_center'] sub_center = current_map_zone['properties']['sub_div_center']
...@@ -162,4 +133,4 @@ def calculate_area_data(center, body_of_water): ...@@ -162,4 +133,4 @@ def calculate_area_data(center, body_of_water):
return [] # return [0] if no data collected from lidar return [] # return [0] if no data collected from lidar
#print(calculate_area_data((61, 11), 'mj\u00f8sa')) # print(calculate_area_data((61, 11), 'mj\u00f8sa', "server\\lidar_data\\mj\u00f8sa\\measurement_id_2.laz"))
\ No newline at end of file
server/database/icedb
+ 0
− 0
View file @ 53c6f51a
No preview for this file type
server/main.py
+ 5
− 3
View file @ 53c6f51a
...@@ -117,20 +117,22 @@ class IceHTTP(BaseHTTPRequestHandler): ...@@ -117,20 +117,22 @@ class IceHTTP(BaseHTTPRequestHandler):
self.send_header('Content-type', 'application/json') self.send_header('Content-type', 'application/json')
self.end_headers() self.end_headers()
def do_POST(self): elif self.path.startswith('/new_lidar_data'):
if self.path == '/new_lidar_data':
parsed_path = urlparse(self.path) parsed_path = urlparse(self.path)
query_params = parse_qs(parsed_path.query) query_params = parse_qs(parsed_path.query)
lake_name_param = query_params.get('lake', [''])[0] lake_name_param = query_params.get('lake', [''])[0]
lake_name = unquote(lake_name_param) # Decode url param lake_name = unquote(lake_name_param) # Decode url param
if lake_name_param:
if lake_name:
input_new_Lidar_data(self, self.cursor, 1, lake_name) # hardcoded body of water must change later input_new_Lidar_data(self, self.cursor, 1, lake_name) # hardcoded body of water must change later
else: else:
self.send_response(400) self.send_response(400)
self.send_header('Content-type', 'application/json') self.send_header('Content-type', 'application/json')
self.end_headers() self.end_headers()
#def do_POST(self):
# Start a server on port 8443 using self defined HTTP class # Start a server on port 8443 using self defined HTTP class
if __name__ == "__main__": if __name__ == "__main__":
try: try:
......
server/map_handler/__pycache__/input_new_data.cpython-39.pyc
+ 0
− 0
View file @ 53c6f51a
No preview for this file type
server/map_handler/input_new_data.py
+ 128
− 97
View file @ 53c6f51a
import json import json
import os import os
from datetime import datetime from datetime import datetime
from server.data_processing.process_lidar_data import calculate_area_data, about_laz_file from server.data_processing.process_lidar_data import calculate_area_data, about_laz_file, find_folder_files
# input_new_Lidar_data send new data gathered from the lidar and send it to the database (from the drone, most likely) # input_new_Lidar_data send new data gathered from the lidar and send it to the database (from the drone, most likely)
def input_new_Lidar_data(self, cursor, sensorId, bodyOfWater): def input_new_Lidar_data(self, cursor, sensorId, bodyOfWater):
...@@ -14,115 +14,141 @@ def input_new_Lidar_data(self, cursor, sensorId, bodyOfWater): ...@@ -14,115 +14,141 @@ def input_new_Lidar_data(self, cursor, sensorId, bodyOfWater):
:return: :return:
""" """
try: try:
print("name=",bodyOfWater) # print("name=",bodyOfWater)
# hard coded coordinates # laz_root_path = "server\\lidar_data\\" + bodyOfWater
latitude = 60.816848 # laz_data_paths = find_folder_files(laz_root_path)
longitude = 10.723823
# read json data with path to data for specific water body
# data about the file read from file_path = "server/map_handler/lake_relations/" + bodyOfWater + "_lidar_data.json"
# about_laz = about_laz_file() cannot do this if data is changed if os.path.exists(file_path):
# scale_factor = max(about_laz[2]) with open(file_path) as data:
scale_factor = max([0.01]) measurement_data = json.load(data)
time_now = datetime.now().utcnow().replace(microsecond=0) if len(measurement_data) < 1:
print("no coordinates found")
# create a new measurement with the time the data is sent, sensor type, where self.send_response(404) # Sending 404 Not Found response
# and an estimate of average thickness of ice on water body self.end_headers()
cursor.execute(''' # Stop the server
INSERT INTO Measurement(MeasurementID, SensorID, TimeMeasured, WaterBodyName, CenterLat, CenterLon) VALUES self.server.shutdown()
(?,?,?,?,?,?);
''', (1, sensorId, time_now, bodyOfWater, latitude, longitude)) lidar_json_data = []
subdiv_json_data = []
# auto generate new measurement id
measurement_id = cursor.lastrowid for measurement in measurement_data:
# calculate the area of to be calculated based on the coordinates given to the calculation model measurement_id = measurement["MeasurementID"]
areas_data = calculate_area_data((latitude, longitude), bodyOfWater)
latitude = measurement["CenterLat"]
lidar_json_data = [] longitude = measurement["CenterLon"]
subdiv_json_data = []
laz_file_path = "server\\lidar_data\\" + bodyOfWater + "\\measurement_id_" + str(measurement_id)+".laz"
measurement_data = {
'measurement_id': measurement_id, # data about the file read from
'TimeMeasured': time_now, # about_laz = about_laz_file() cannot do this if data is changed
'CenterLat': latitude, # scale_factor = max(about_laz[2])
'CenterLon': longitude, scale_factor = max([0.01])
'Sensor': {
'SensorId': 2,
'SensorType': "LiDar",
"Active": True,
}
}
if(areas_data):
# store lidar data in jason formate
# calculate data for each zone within the area
for area in areas_data:
# lng and lat relative to map
subId = int(area[0])
map_lat, map_lng = area[1]
heights = area[2]
if(len(heights) != 0 or sum(heights) == 0):
average = sum(heights)/len(heights)
minimum_thickness = min(heights)
else:
average = 0
minimum_thickness = 0
time_measured = datetime.utcnow().replace(microsecond=0) time_now = datetime.now().utcnow().replace(microsecond=0)
# input the data into the database
# create a new measurement with the time the data is sent, sensor type, where
# and an estimate of average thickness of ice on water body
cursor.execute(''' cursor.execute('''
INSERT INTO SubdivisionMeasurementData(MeasurementID, TimeMeasured, SubdivID, WaterBodyName, MinimumThickness, AverageThickness, CalculatedSafety, Accuracy) VALUES INSERT INTO Measurement(MeasurementID, SensorID, TimeMeasured, WaterBodyName, CenterLat, CenterLon) VALUES
(?,?,?,?,?,?,?,?); (?,?,?,?,?,?);
''', (measurement_id, time_measured, subId, bodyOfWater, float(minimum_thickness), float(average), float(0.0), scale_factor)) ''', (measurement_id, sensorId, time_now, bodyOfWater, latitude, longitude))
sub_center = (map_lat, map_lng)
# set up json formate # calculate the area of to be calculated based on the coordinates given to the calculation model
lidar_read = { areas_data = calculate_area_data((latitude, longitude), bodyOfWater, laz_file_path)
'SubdivId': subId,
'MinThickness': minimum_thickness, if(areas_data):
'AvgThickness': average, # store lidar data in jason formate
'CenLatitude': sub_center[0], # calculate data for each zone within the area
'CenLongitude': sub_center[1], for area in areas_data:
'Accuracy': scale_factor, # lng and lat relative to map
subId = int(area[0])
map_lat, map_lng = area[1]
heights = area[2]
if(len(heights) != 0 or sum(heights) != 0):
average = sum(heights)/len(heights)
minimum_thickness = min(heights)
else:
average = 0
minimum_thickness = 0
# input the data into the database
cursor.execute('''
INSERT INTO SubdivisionMeasurementData(MeasurementID, TimeMeasured, SubdivID, WaterBodyName, MinimumThickness, AverageThickness, CalculatedSafety, Accuracy) VALUES
(?,?,?,?,?,?,?,?);
''', (measurement_id, time_now, subId, bodyOfWater, float(minimum_thickness), float(average), float(0.0), scale_factor))
sub_center = (map_lat, map_lng)
# set up json formate
lidar_read = {
'SubdivID': subId,
'MinThickness': float(minimum_thickness),
'AvgThickness': float(average),
'CenLatitude': float(sub_center[0]),
'CenLongitude': float(sub_center[1]),
'Accuracy': scale_factor,
}
subdiv_json_data.append(lidar_read)
else:
print('No data found, line 79')
measurement_data = {
'MeasurementID': measurement_id,
'TimeMeasured': str(time_now),
'CenterLat': latitude,
'CenterLon': longitude,
'Sensor': {
'SensorId': 2,
'SensorType': "LiDar",
"Active": True,
},
'Subdivisions': subdiv_json_data,
} }
subdiv_json_data.append(lidar_read) lidar_json_data.append(measurement_data)
else:
print('No data found, line 79')
measurement_json_data = { # send the changes to the database
'Measurement': measurement_data, cursor.connection.commit()
'Subdivisions': subdiv_json_data,
}
lidar_json_data.append(measurement_json_data) # Send response
self.send_response(200)
self.send_header('Content-type', "application/json")
self.end_headers()
content = None
# send the changes to the database current_directory = os.getcwd()
cursor.connection.commit() print("Current working directory:", current_directory)
# Send response if len(lidar_json_data) > 0:
self.send_response(200) if os.path.exists(file_path):
self.send_header('Content-type', "application/json") os.remove(file_path)
self.end_headers()
file_path = "./server/map_handler/lake_relations/"+bodyOfWater+"_lidar_data.json" # convert list of lidar data to json
content = None content = json.dumps(lidar_json_data, indent=4)
with open(file_path, "w") as file:
file.write(content)
else:
print('No data found, line 101')
content = json.dumps([])
current_directory = os.getcwd()
print("Current working directory:", current_directory)
if len(lidar_json_data) > 0:
if os.path.exists(file_path):
os.remove(file_path)
# convert list of lidar data to json
content = json.dumps(lidar_json_data)
with open(file_path, "w") as file:
file.write(content)
else: else:
print('No data found, line 101') # Temporary not gonna use:
content = json.dumps([]) with open(file_path, "w") as file:
file.write("[{\"MeasurementID\": 1,\"TimeMeasured\": \"2024-04-15 16:23:28.620516\",\"CenterLat\": 60.841532,\"CenterLon\": 10.717878,\"Sensor\": {\"SensorID\": 2,\"SensorType\": \"LiDar\",\"Active\": true},\"Subdivisions\": []},{\"MeasurementID\": 2,\"TimeMeasured\": \"2024-04-15 16:23:28.620516\",\"CenterLat\": 60.841532,\"CenterLon\": 10.717878,\"Sensor\": {\"SensorID\": 2,\"SensorType\": \"LiDar\",\"Active\": true},\"Subdivisions\": []}]") # Writing an empty JSON object
print("file does not exist", file_path)
# Send response
self.send_response(404)
self.send_header('Content-type', "application/json")
self.end_headers()
content = None
# Write content data to response object # Write content data to response object
self.wfile.write(content.encode('utf-8')) self.wfile.write(content.encode('utf-8'))
...@@ -132,3 +158,8 @@ def input_new_Lidar_data(self, cursor, sensorId, bodyOfWater): ...@@ -132,3 +158,8 @@ def input_new_Lidar_data(self, cursor, sensorId, bodyOfWater):
print("An error occurred: ", e) print("An error occurred: ", e)
# rollback in case of error # rollback in case of error
cursor.connection.rollback() cursor.connection.rollback()
# laz_root_path = "server\\lidar_data\\mj\u00f8sa"
# laz_data_paths = find_folder_files(laz_root_path)
# print(laz_data_paths)
server/map_handler/lake_relations/mjøsa_lidar_data.json
+ 157
− 2
View file @ 53c6f51a
[{"MeasurementId": "7", "SubId": "36", "SubCenter": "(60.841532, 10.717878)", "Heights": "[1]"}, {"MeasurementId": "7", "SubId": "83", "SubCenter": "(60.828326, 10.982563)", "Heights": "[1, 27]"}, {"MeasurementId": "7", "SubId": "33", "SubCenter": "(60.771059, 10.698341)", "Heights": "[1]"}, {"MeasurementId": "7", "SubId": "136", "SubCenter": "(60.396856, 11.220933)", "Heights": "[1, 4, 6, 27, 7]"}] [
\ No newline at end of file {
"MeasurementID": 1,
"TimeMeasured": "2024-04-25 13:14:17",
"CenterLat": 60.841532,
"CenterLon": 10.717878,
"Sensor": {
"SensorId": 2,
"SensorType": "LiDar",
"Active": true
},
"Subdivisions": [
{
"SubdivID": 37,
"MinThickness": 0.0,
"AvgThickness": 0.0,
"CenLatitude": 60.864282,
"CenLongitude": 10.717878,
"Accuracy": 0.01
},
{
"SubdivID": 148,
"MinThickness": 0.0,
"AvgThickness": 0.0,
"CenLatitude": 60.851609,
"CenLongitude": 10.951163,
"Accuracy": 0.01
},
{
"SubdivID": 33,
"MinThickness": 0.0,
"AvgThickness": 0.0,
"CenLatitude": 60.771059,
"CenLongitude": 10.698341,
"Accuracy": 0.01
},
{
"SubdivID": 136,
"MinThickness": 0.0,
"AvgThickness": 0.0,
"CenLatitude": 60.396856,
"CenLongitude": 11.220933,
"Accuracy": 0.01
},
{
"SubdivID": 37,
"MinThickness": 1.0,
"AvgThickness": 1.0,
"CenLatitude": 60.864282,
"CenLongitude": 10.717878,
"Accuracy": 0.01
},
{
"SubdivID": 148,
"MinThickness": 1.0,
"AvgThickness": 14.0,
"CenLatitude": 60.851609,
"CenLongitude": 10.951163,
"Accuracy": 0.01
},
{
"SubdivID": 33,
"MinThickness": 1.0,
"AvgThickness": 1.0,
"CenLatitude": 60.771059,
"CenLongitude": 10.698341,
"Accuracy": 0.01
},
{
"SubdivID": 136,
"MinThickness": 1.0,
"AvgThickness": 9.0,
"CenLatitude": 60.396856,
"CenLongitude": 11.220933,
"Accuracy": 0.01
}
]
},
{
"MeasurementID": 2,
"TimeMeasured": "2024-04-25 13:15:23",
"CenterLat": 60.841532,
"CenterLon": 10.717878,
"Sensor": {
"SensorId": 2,
"SensorType": "LiDar",
"Active": true
},
"Subdivisions": [
{
"SubdivID": 37,
"MinThickness": 0.0,
"AvgThickness": 0.0,
"CenLatitude": 60.864282,
"CenLongitude": 10.717878,
"Accuracy": 0.01
},
{
"SubdivID": 148,
"MinThickness": 0.0,
"AvgThickness": 0.0,
"CenLatitude": 60.851609,
"CenLongitude": 10.951163,
"Accuracy": 0.01
},
{
"SubdivID": 33,
"MinThickness": 0.0,
"AvgThickness": 0.0,
"CenLatitude": 60.771059,
"CenLongitude": 10.698341,
"Accuracy": 0.01
},
{
"SubdivID": 136,
"MinThickness": 0.0,
"AvgThickness": 0.0,
"CenLatitude": 60.396856,
"CenLongitude": 11.220933,
"Accuracy": 0.01
},
{
"SubdivID": 37,
"MinThickness": 1.0,
"AvgThickness": 1.0,
"CenLatitude": 60.864282,
"CenLongitude": 10.717878,
"Accuracy": 0.01
},
{
"SubdivID": 148,
"MinThickness": 1.0,
"AvgThickness": 14.0,
"CenLatitude": 60.851609,
"CenLongitude": 10.951163,
"Accuracy": 0.01
},
{
"SubdivID": 33,
"MinThickness": 1.0,
"AvgThickness": 1.0,
"CenLatitude": 60.771059,
"CenLongitude": 10.698341,
"Accuracy": 0.01
},
{
"SubdivID": 136,
"MinThickness": 1.0,
"AvgThickness": 9.0,
"CenLatitude": 60.396856,
"CenLongitude": 11.220933,
"Accuracy": 0.01
}
]
}
]
\ No newline at end of file
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment