Merge branch 'main' of gitlab.stud.idi.ntnu.no:sarasdj/prog2900 into connect_data_to_map

server/data_processing/process_lidar_data.py

 import json
 import math
+import os
 
 import laspy
 import numpy as np
@@ -7,12 +8,10 @@ import numpy as np
 from server.data_processing.area_processing import calculate_corners, define_gridareas, inArea, find_height, \
     closest_points, position_relative_to_pointcloud, define_grid_lidardata
 
-# hard coded files for test datas
-lazData_path = ["server/example_lidar_data/ot_N_000005_1.laz", "server/example_lidar_data/ot_N_000033_1.laz"]
 
 # Info about data
-def about_laz_file():
-    with laspy.open(lazData_path[0]) as fh:
+def about_laz_file(path):
+    with laspy.open(path) as fh:
         # Print metadata properties
         print("File Version:", fh.header.version)
         print("Point Count:", fh.header.point_count)
@@ -30,9 +29,16 @@ def about_laz_file():
 
     return [las.header.version, las.header.point_count, las.header.scale, las.header.offset]
 
+def find_folder_files(direcory):
+    files = []
+    for root, _, fileNames in os.walk(direcory):
+        for fileName in fileNames:
+            files.append(os.path.join(root, fileName))
+    return files
+
 # find the height of an area based on the coordinates of it's center
 # and it's affiliations (subId and groupId) (soon to be implemented
-def calculate_area_data(center, body_of_water):
+def calculate_area_data(center, body_of_water, path):
     # container for all the heights in area
     area_heights = []
 
@@ -67,42 +73,20 @@ def calculate_area_data(center, body_of_water):
     map_data = list(filter(lambda point_position: inArea((point_position['properties']['sub_div_center'][0], point_position['properties']['sub_div_center'][1], 0.0), map_zones), map_data))
 
     # Refactor lidar data to a readable format
-    iceOver = laspy.read(lazData_path[0])
-    iceUnder = laspy.read(lazData_path[1])
-
-    # add two files together temporary test data(soon to be removed)
-    ice_points = list(zip(iceOver.X,iceOver.Y,iceOver.Z)) + list(zip(iceUnder.X,iceUnder.Y,iceUnder.Z))
-    print(len(ice_points))
-    ice_points = list(filter(lambda point_position: inArea((point_position[0],point_position[1],0.0),[(-3300000, 4500000), (-3200000, 5000000)]), ice_points))
-    print(len(ice_points))
-    # only for visualizating the testing data
-    #print("max",max(ice_points))
-    #print("min",min(ice_points))
-
-    #x = [points[0] for points in ice_points]
-    #y = [points[1] for points in ice_points]
-
-    # Plot the points
-    #plt.plot(x, y, 'bo')  # 'bo' specifies blue color and circle markers
-    #plt.xlabel('X axis')
-    #plt.ylabel('Y axis')
-    #plt.title('Plotting Points')
-    #plt.grid(True)  # Add grid
-    #plt.show()
+    iceOver = laspy.read(path)
+
+    ice_points = list(zip(iceOver.X, iceOver.Y, iceOver.Z))
 
     max_point = max(ice_points)
     min_point = min(ice_points)
 
-
     # define all the sub-areas within the area, local coordinates
     grid_sub_area = define_gridareas(center[0], center[1], (cell_x, cell_y),grid_size)
     # define all the sub-areas within the area, lidar coordinates
     grid_area_lidar_heights = define_grid_lidardata((min_point, max_point), grid_size, ice_points)
 
-    print('grid size: ', grid_sub_area)
-    print('grid size: ', grid_area_lidar_heights)
+    print('grid height: ', grid_area_lidar_heights)
     sub_area_heights = list(zip(grid_sub_area, grid_area_lidar_heights))
-    print('sub_area_heights: ', sub_area_heights[0])
 
     # find the heights of each sub-area => area-heights
     if len(map_data) > 0:
@@ -124,8 +108,7 @@ def calculate_area_data(center, body_of_water):
             sub_center = ((xs + xe)/2, (ys + ye)/2)
             # check if area is part of water body
             part_of_subarea_of_waterbody = list(filter(lambda pos: inArea((pos['properties']['sub_div_center'][0], pos['properties']['sub_div_center'][1], 0.0), [start,end]), map_data))
-            print('sub_area: ', sub_area)
-            print('part_of_subarea_of_waterbody: ', part_of_subarea_of_waterbody)
+
             if(len(part_of_subarea_of_waterbody) > 0):
                 current_map_zone = closest_points(sub_center, part_of_subarea_of_waterbody, taken_coords)
                 sub_center = current_map_zone['properties']['sub_div_center']
@@ -150,4 +133,4 @@ def calculate_area_data(center, body_of_water):
         return [] # return [0] if no data collected from lidar
 
 
-#print(calculate_area_data((61, 11), 'mj\u00f8sa'))
+# print(calculate_area_data((61, 11), 'mj\u00f8sa', "server\\lidar_data\\mj\u00f8sa\\measurement_id_2.laz"))
\ No newline at end of file

server/main.py

@@ -117,10 +117,21 @@ class IceHTTP(BaseHTTPRequestHandler):
                 self.send_header('Content-type', 'application/json')
                 self.end_headers()
 
-    def do_POST(self):
-        if self.path == '/new_lidar_data':
-            input_new_Lidar_data(self, self.cursor, 1, 'mj\u00f8sa')  # hardcoded body of water must change later
+        elif  self.path.startswith('/new_lidar_data'):
+            parsed_path = urlparse(self.path)
+            query_params = parse_qs(parsed_path.query)
+
+            lake_name_param = query_params.get('lake', [''])[0]
+            lake_name = unquote(lake_name_param)  # Decode url param
+
+            if lake_name:
+                input_new_Lidar_data(self, self.cursor, 1, lake_name)  # hardcoded body of water must change later
+            else:
+                self.send_response(400)
+                self.send_header('Content-type', 'application/json')
+                self.end_headers()
 
+    #def do_POST(self):
 
 # Start a server on port 8443 using self defined HTTP class
 if __name__ == "__main__":

server/map_handler/input_new_data.py

 import json
 import os
 from datetime import datetime
-from server.data_processing.process_lidar_data import calculate_area_data, about_laz_file
+from server.data_processing.process_lidar_data import calculate_area_data, about_laz_file, find_folder_files
 
 # input_new_Lidar_data send new data gathered from the lidar and send it to the database (from the drone, most likely)
 def input_new_Lidar_data(self, cursor, sensorId, bodyOfWater):
+    """
+
+    :param self:
+    :param cursor:
+    :param sensorId:
+    :param bodyOfWater:
+    :return:
+    """
     try:
-        print("name=",bodyOfWater)
-        # hard coded coordinates
-        latitude = 60.816848
-        longitude = 10.723823
+        # print("name=",bodyOfWater)
+        # laz_root_path = "server\\lidar_data\\" + bodyOfWater
+        # laz_data_paths = find_folder_files(laz_root_path)
+
+        # read json data with path to data for specific water body
+        file_path = "server/map_handler/lake_relations/" + bodyOfWater + "_lidar_data.json"
+        if os.path.exists(file_path):
+            with open(file_path) as data:
+                measurement_data = json.load(data)
+
+            if len(measurement_data) < 1:
+                print("no coordinates found")
+                self.send_response(404)  # Sending 404 Not Found response
+                self.end_headers()
+                # Stop the server
+                self.server.shutdown()
+
+            lidar_json_data = []
+            subdiv_json_data = []
+
+            for measurement in measurement_data:
+
+                measurement_id = measurement["MeasurementID"]
+
+                latitude = measurement["CenterLat"]
+                longitude = measurement["CenterLon"]
+
+                laz_file_path = "server\\lidar_data\\" + bodyOfWater + "\\measurement_id_" + str(measurement_id)+".laz"
 
                 # data about the file read from
-        about_laz = about_laz_file()
-        scale_factor = max(about_laz[2])
+                # about_laz = about_laz_file() cannot do this if data is changed
+                # scale_factor = max(about_laz[2])
+                scale_factor = max([0.01])
+
+                time_now = datetime.now().utcnow().replace(microsecond=0)
 
                 # create a new measurement with the time the data is sent, sensor type, where
                 # and an estimate of average thickness of ice on water body
                 cursor.execute('''
                     INSERT INTO Measurement(MeasurementID, SensorID, TimeMeasured, WaterBodyName, CenterLat, CenterLon) VALUES 
                         (?,?,?,?,?,?);
-        ''', (1, sensorId, datetime.utcnow().replace(microsecond=0), bodyOfWater, latitude, longitude))
-
-        # auto generate new measurement id
-        measurement_id = cursor.lastrowid
+                ''', (measurement_id, sensorId, time_now, bodyOfWater, latitude, longitude))
 
                 # calculate the area of to be calculated based on the coordinates given to the calculation model
-        areas_data = calculate_area_data((latitude, longitude), bodyOfWater)
-
-        lidar_json_data = []
+                areas_data = calculate_area_data((latitude, longitude), bodyOfWater, laz_file_path)
 
                 if(areas_data):
                     # store lidar data in jason formate
@@ -39,39 +69,48 @@ def input_new_Lidar_data(self, cursor, sensorId, bodyOfWater):
                         map_lat, map_lng = area[1]
                         heights = area[2]
 
-                if(len(heights) != 0 or sum(heights) == 0):
+                        if(len(heights) != 0 or sum(heights) != 0):
                             average = sum(heights)/len(heights)
                             minimum_thickness = min(heights)
                         else:
                             average = 0
                             minimum_thickness = 0
 
-                time_measured = datetime.utcnow().replace(microsecond=0)
                         # input the data into the database
                         cursor.execute('''
                         INSERT INTO SubdivisionMeasurementData(MeasurementID, TimeMeasured, SubdivID, WaterBodyName, MinimumThickness, AverageThickness, CalculatedSafety, Accuracy) VALUES
                             (?,?,?,?,?,?,?,?);
-                ''', (measurement_id, time_measured, subId, bodyOfWater, float(minimum_thickness), float(average), float(0.0), scale_factor))
+                        ''', (measurement_id, time_now, subId, bodyOfWater, float(minimum_thickness), float(average), float(0.0), scale_factor))
                         sub_center = (map_lat, map_lng)
                         # set up json formate
                         lidar_read = {
-                    'MeasurementId': str(measurement_id),
-                    'SubId': str(subId),
-                    'SubCenter': str(sub_center),
-                    'Heights': str(heights)
+                            'SubdivID': subId,
+                            'MinThickness': float(minimum_thickness),
+                            'AvgThickness': float(average),
+                            'CenLatitude': float(sub_center[0]),
+                            'CenLongitude': float(sub_center[1]),
+                            'Accuracy': scale_factor,
                         }
 
-                lidar_json_data.append(lidar_read)
-
-            # input the newly generated measurement_id and whole average thickness
-            cursor.execute('''
-                UPDATE Measurement
-                SET measurementID = ?
-                WHERE MeasurementID IS NULL;
-            ''', (int(measurement_id),))
+                        subdiv_json_data.append(lidar_read)
                 else:
                     print('No data found, line 79')
 
+                measurement_data = {
+                    'MeasurementID': measurement_id,
+                    'TimeMeasured': str(time_now),
+                    'CenterLat': latitude,
+                    'CenterLon': longitude,
+                    'Sensor': {
+                        'SensorId': 2,
+                        'SensorType': "LiDar",
+                        "Active": True,
+                    },
+                    'Subdivisions': subdiv_json_data,
+                }
+
+                lidar_json_data.append(measurement_data)
+
             # send the changes to the database
             cursor.connection.commit()
 
@@ -79,7 +118,6 @@ def input_new_Lidar_data(self, cursor, sensorId, bodyOfWater):
             self.send_response(200)
             self.send_header('Content-type', "application/json")
             self.end_headers()
-        file_path = "./server/map_handler/lake_relations/"+bodyOfWater+"_lidar_data.json"
             content = None
 
             current_directory = os.getcwd()
@@ -90,7 +128,7 @@ def input_new_Lidar_data(self, cursor, sensorId, bodyOfWater):
                     os.remove(file_path)
 
                 # convert list of lidar data to json
-            content = json.dumps(lidar_json_data)
+                content = json.dumps(lidar_json_data, indent=4)
                 with open(file_path, "w") as file:
                     file.write(content)
             else:
@@ -98,6 +136,20 @@ def input_new_Lidar_data(self, cursor, sensorId, bodyOfWater):
                 content = json.dumps([])
 
 
+
+        else:
+            # Temporary not gonna use:
+            with open(file_path, "w") as file:
+                file.write("[{\"MeasurementID\": 1,\"TimeMeasured\": \"2024-04-15 16:23:28.620516\",\"CenterLat\": 60.841532,\"CenterLon\": 10.717878,\"Sensor\": {\"SensorID\": 2,\"SensorType\": \"LiDar\",\"Active\": true},\"Subdivisions\": []},{\"MeasurementID\": 2,\"TimeMeasured\": \"2024-04-15 16:23:28.620516\",\"CenterLat\": 60.841532,\"CenterLon\": 10.717878,\"Sensor\": {\"SensorID\": 2,\"SensorType\": \"LiDar\",\"Active\": true},\"Subdivisions\": []}]")  # Writing an empty JSON object
+
+            print("file does not exist", file_path)
+
+            # Send response
+            self.send_response(404)
+            self.send_header('Content-type', "application/json")
+            self.end_headers()
+            content = None
+
         # Write content data to response object
         self.wfile.write(content.encode('utf-8'))
 
@@ -106,3 +158,8 @@ def input_new_Lidar_data(self, cursor, sensorId, bodyOfWater):
         print("An error occurred: ", e)
         # rollback in case of error
         cursor.connection.rollback()
+
+# laz_root_path = "server\\lidar_data\\mj\u00f8sa"
+# laz_data_paths = find_folder_files(laz_root_path)
+# print(laz_data_paths)
+

server/map_handler/lake_relations/mjøsa_lidar_data.json

-[{"MeasurementId": "7", "SubId": "36", "SubCenter": "(60.841532, 10.717878)", "Heights": "[1]"}, {"MeasurementId": "7", "SubId": "83", "SubCenter": "(60.828326, 10.982563)", "Heights": "[1, 27]"}, {"MeasurementId": "7", "SubId": "33", "SubCenter": "(60.771059, 10.698341)", "Heights": "[1]"}, {"MeasurementId": "7", "SubId": "136", "SubCenter": "(60.396856, 11.220933)", "Heights": "[1, 4, 6, 27, 7]"}]
+[
\ No newline at end of file
+    {
+        "MeasurementID": 1,
+        "TimeMeasured": "2024-04-25 13:14:17",
+        "CenterLat": 60.841532,
+        "CenterLon": 10.717878,
+        "Sensor": {
+            "SensorId": 2,
+            "SensorType": "LiDar",
+            "Active": true
+        },
+        "Subdivisions": [
+            {
+                "SubdivID": 37,
+                "MinThickness": 0.0,
+                "AvgThickness": 0.0,
+                "CenLatitude": 60.864282,
+                "CenLongitude": 10.717878,
+                "Accuracy": 0.01
+            },
+            {
+                "SubdivID": 148,
+                "MinThickness": 0.0,
+                "AvgThickness": 0.0,
+                "CenLatitude": 60.851609,
+                "CenLongitude": 10.951163,
+                "Accuracy": 0.01
+            },
+            {
+                "SubdivID": 33,
+                "MinThickness": 0.0,
+                "AvgThickness": 0.0,
+                "CenLatitude": 60.771059,
+                "CenLongitude": 10.698341,
+                "Accuracy": 0.01
+            },
+            {
+                "SubdivID": 136,
+                "MinThickness": 0.0,
+                "AvgThickness": 0.0,
+                "CenLatitude": 60.396856,
+                "CenLongitude": 11.220933,
+                "Accuracy": 0.01
+            },
+            {
+                "SubdivID": 37,
+                "MinThickness": 1.0,
+                "AvgThickness": 1.0,
+                "CenLatitude": 60.864282,
+                "CenLongitude": 10.717878,
+                "Accuracy": 0.01
+            },
+            {
+                "SubdivID": 148,
+                "MinThickness": 1.0,
+                "AvgThickness": 14.0,
+                "CenLatitude": 60.851609,
+                "CenLongitude": 10.951163,
+                "Accuracy": 0.01
+            },
+            {
+                "SubdivID": 33,
+                "MinThickness": 1.0,
+                "AvgThickness": 1.0,
+                "CenLatitude": 60.771059,
+                "CenLongitude": 10.698341,
+                "Accuracy": 0.01
+            },
+            {
+                "SubdivID": 136,
+                "MinThickness": 1.0,
+                "AvgThickness": 9.0,
+                "CenLatitude": 60.396856,
+                "CenLongitude": 11.220933,
+                "Accuracy": 0.01
+            }
+        ]
+    },
+    {
+        "MeasurementID": 2,
+        "TimeMeasured": "2024-04-25 13:15:23",
+        "CenterLat": 60.841532,
+        "CenterLon": 10.717878,
+        "Sensor": {
+            "SensorId": 2,
+            "SensorType": "LiDar",
+            "Active": true
+        },
+        "Subdivisions": [
+            {
+                "SubdivID": 37,
+                "MinThickness": 0.0,
+                "AvgThickness": 0.0,
+                "CenLatitude": 60.864282,
+                "CenLongitude": 10.717878,
+                "Accuracy": 0.01
+            },
+            {
+                "SubdivID": 148,
+                "MinThickness": 0.0,
+                "AvgThickness": 0.0,
+                "CenLatitude": 60.851609,
+                "CenLongitude": 10.951163,
+                "Accuracy": 0.01
+            },
+            {
+                "SubdivID": 33,
+                "MinThickness": 0.0,
+                "AvgThickness": 0.0,
+                "CenLatitude": 60.771059,
+                "CenLongitude": 10.698341,
+                "Accuracy": 0.01
+            },
+            {
+                "SubdivID": 136,
+                "MinThickness": 0.0,
+                "AvgThickness": 0.0,
+                "CenLatitude": 60.396856,
+                "CenLongitude": 11.220933,
+                "Accuracy": 0.01
+            },
+            {
+                "SubdivID": 37,
+                "MinThickness": 1.0,
+                "AvgThickness": 1.0,
+                "CenLatitude": 60.864282,
+                "CenLongitude": 10.717878,
+                "Accuracy": 0.01
+            },
+            {
+                "SubdivID": 148,
+                "MinThickness": 1.0,
+                "AvgThickness": 14.0,
+                "CenLatitude": 60.851609,
+                "CenLongitude": 10.951163,
+                "Accuracy": 0.01
+            },
+            {
+                "SubdivID": 33,
+                "MinThickness": 1.0,
+                "AvgThickness": 1.0,
+                "CenLatitude": 60.771059,
+                "CenLongitude": 10.698341,
+                "Accuracy": 0.01
+            },
+            {
+                "SubdivID": 136,
+                "MinThickness": 1.0,
+                "AvgThickness": 9.0,
+                "CenLatitude": 60.396856,
+                "CenLongitude": 11.220933,
+                "Accuracy": 0.01
+            }
+        ]
+    }
+]
\ No newline at end of file