change: add the data to its own json file, must reconfigure the offsett of the range

server/data_processing/area_processing.py

@@ -18,14 +18,14 @@ def position_relative_to_pointcloud(l1, l2, p1, p2, center_lim, max_lim):
         ((p2[0] - center_lim[0]) * ((l1[0] - l2[0]) / (max_lim[2][0] - max_lim[0][0])) + center_l[0],
             (p2[1] - center_lim[1]) * ((l1[1] - l2[1]) / (max_lim[2][1] - max_lim[0][1])) + center_l[1]),
     ]
-print(position_relative_to_pointcloud((-20,20), (-10,30), (1,3), (2,2), (2.5,2.5), [(5,5),(0,5),(0,0),(5,0)]))
-print(position_relative_to_pointcloud((-3299999, 4608018), (-3200001, 4687153), (61.47620866851029, 8.961138281887507), (61.95241733702057, 6.8508373935926645), (61, 11), [(61.95241733702057, 15.125349549679886), (60.04758266297943, 15.125349549679886), (60.04758266297943, 6.8746504503201145), (61.95241733702057, 6.8746504503201145)]))
+#print(position_relative_to_pointcloud((-20,20), (-10,30), (1,3), (2,2), (2.5,2.5), [(5,5),(0,5),(0,0),(5,0)]))
+#print(position_relative_to_pointcloud((-3299999, 4608018), (-3200001, 4687153), (61.47620866851029, 8.961138281887507), (61.95241733702057, 6.8508373935926645), (61, 11), [(61.95241733702057, 15.125349549679886), (60.04758266297943, 15.125349549679886), (60.04758266297943, 6.8746504503201145), (61.95241733702057, 6.8746504503201145)]))
 
 # check if lidar points is within range of the area selected
 def inArea(position, areaRange):
     x, y, _ = position # position to be checked
-    print((areaRange[0][0])," < ",x," < ",(areaRange[1][0])," and ",(areaRange[0][1])," > ",(y)," > ",(areaRange[1][1])," ",((areaRange[0][0]) < x < (areaRange[1][0])) and ((areaRange[0][1]) > (y) > (areaRange[1][1])))
-    if ((areaRange[0][0]) < x < (areaRange[1][0])) and ((areaRange[0][1]) > y > (areaRange[1][1])):
+    #print((areaRange[0][0])," < ",x," < ",(areaRange[1][0])," and ",(areaRange[0][1])," < ",(y)," < ",(areaRange[1][1])," ",((areaRange[0][0]) < x < (areaRange[1][0])) and ((areaRange[0][1]) < (y) < (areaRange[1][1])))
+    if ((areaRange[0][0]) < x < (areaRange[1][0])) and ((areaRange[0][1]) < y < (areaRange[1][1])):
         return True
     else:
         return False
@@ -49,7 +49,8 @@ def closest_points(point, list, coords_taken):
 
 # Calculation of height in an area from las files
 def find_height(points):
-    print(points, " 5 ")
+    if len(points) == 0:
+        return [0]
     height_differences = [] # final container for height data
 
     # sort the points
@@ -116,8 +117,6 @@ def define_gridareas(lat, lng, area_offset, grid_size):
     subarea_offset_lng = offset_lng/grid_size
     subarea_offset_lat = offset_lat/grid_size
 
-    group_id = 0
-    i = 0
     # find the center coordinates of each area in grid to find the corner areas
     for y in (range(grid_size)):
         relative_size_lat = y / grid_size
@@ -125,19 +124,43 @@ def define_gridareas(lat, lng, area_offset, grid_size):
             relative_size_lng = x / grid_size
             lat_pos = main_area[3][0] + relative_size_lat * area_size[0] + dist_to_subcenter[0]
             lng_pos = main_area[3][1] + relative_size_lng * area_size[1] + dist_to_subcenter[1]
-            # id of each subarea
-            #sub_id = x % grid_size + grid_size * y
-
-            if (x+(y*grid_size)) % ((grid_size*2) + 1) == grid_size * 2:
-                group_id += 2
-
-            if (x+(y*grid_size)) % 2 == 0 and (x+(y*grid_size)) != 0:
-                i += 1
 
             # use the center of sub areas to find the corner of each subarea
             subarea_offset = (subarea_offset_lng, subarea_offset_lat)
             corners = calculate_corners(lat_pos, lng_pos, subarea_offset)
-            grided_area.append((group_id + i%2, corners))
+            grided_area.append(corners)
+
+    return grided_area
+
+# separate the zones into smaller area, into a grid
+def define_grid_lidardata(max_area, grid_size, points):
+    # container for an area turned into a grid of areas
+    grided_area = []
+
+    bottom_left, top_right = max_area
+
+    # find subareas size relative to the full size of data
+    subarea_offset_lng = (top_right[0]-bottom_left[0])/grid_size
+    subarea_offset_lat = (top_right[1]-bottom_left[1])/grid_size
+
+    # find the center coordinates of each area in grid to find the corner areas
+    for y in (range(grid_size)):
+        relative_size_lat = y / grid_size
+        for x in (range(grid_size)):
+            relative_size_lng = x / grid_size
+            bottom_left_range = (subarea_offset_lng * (relative_size_lng - 1 / 2) + bottom_left[0],
+                                 subarea_offset_lat * (relative_size_lat - 1 / 2) + bottom_left[1])
+
+            top_right_range = (subarea_offset_lng * relative_size_lng + bottom_left[0],
+                               subarea_offset_lat * relative_size_lat + bottom_left[1])
+
+            area_range = [bottom_left_range, top_right_range]
+
+            area = list(filter(lambda point_position: inArea(point_position, area_range), points))
+
+            height_in_area = find_height(area)
+
+            grided_area.append(height_in_area)
 
     return grided_area
 #[1,2,2,3,4,5,6,3,4,6,8,9,5,3,5.7,8,5,3]

server/data_processing/process_lidar_data.py

@@ -5,7 +5,7 @@ import laspy
 import numpy as np
 
 from server.data_processing.area_processing import calculate_corners, define_gridareas, inArea, find_height, \
-    closest_points, position_relative_to_pointcloud
+    closest_points, position_relative_to_pointcloud, define_grid_lidardata
 
 # hard coded files for test datas
 lazData_path = ["server/example_lidar_data/ot_N_000005_1.laz", "server/example_lidar_data/ot_N_000033_1.laz"]
@@ -45,12 +45,17 @@ def calculate_area_data(center, body_of_water):
         map_data = json.load(data)
 
     # grid cell offset
+    # NB: make these a global var
+    grid_size = 4
     #cell_x, cell_y = map_data['tile_width'], map_data['tile_height']
+    #cell_x, cell_y = (400*grid_size, 400*grid_size)
     cell_x, cell_y = (2000, 1000)
 
     # convert the offset to meter
+    #cell_x = 111.320 * cell_x * 1000
+    #cell_y = (40075 * math.cos(cell_y) * 1000) / 360
     cell_x = 111.320 * cell_x
-    cell_y = 111.320 * math.cos(60) * cell_y
+    cell_y = (111.320 * math.cos(60)*cell_y)
 
     # set the limit of the area compared to local coordinates
     area_limit = calculate_corners(center[0], center[1], (cell_x, cell_y))
@@ -58,7 +63,7 @@ def calculate_area_data(center, body_of_water):
 
     # grid data
     map_data = map_data['features']
-    map_zones = [area_limit[1], area_limit[3]]
+    map_zones = [area_limit[2], area_limit[0]]
     map_data = list(filter(lambda point_position: inArea((point_position['properties']['sub_div_center'][0], point_position['properties']['sub_div_center'][1], 0.0), map_zones), map_data))
 
     # Refactor lidar data to a readable format
@@ -67,8 +72,9 @@ def calculate_area_data(center, body_of_water):
 
     # add two files together temporary test data(soon to be removed)
     ice_points = list(zip(iceOver.X,iceOver.Y,iceOver.Z)) + list(zip(iceUnder.X,iceUnder.Y,iceUnder.Z))
-    ice_points = list(filter(lambda point_position: inArea((point_position[0],point_position[1],0.0),[(-3300000,5000000), (-3200000,4500000)]),ice_points))
-
+    print(len(ice_points))
+    ice_points = list(filter(lambda point_position: inArea((point_position[0],point_position[1],0.0),[(-3300000, 4500000), (-3200000, 5000000)]), ice_points))
+    print(len(ice_points))
     # only for visualizating the testing data
     #print("max",max(ice_points))
     #print("min",min(ice_points))
@@ -87,55 +93,61 @@ def calculate_area_data(center, body_of_water):
     max_point = max(ice_points)
     min_point = min(ice_points)
 
+
     # define all the sub-areas within the area, local coordinates
-    grid_area_heights = define_gridareas(center[0], center[1], (cell_x, cell_y),4)
+    grid_sub_area = define_gridareas(center[0], center[1], (cell_x, cell_y),grid_size)
+    # define all the sub-areas within the area, lidar coordinates
+    grid_area_lidar_heights = define_grid_lidardata((min_point, max_point), grid_size, ice_points)
+
+    print('grid size: ', grid_sub_area)
+    print('grid size: ', grid_area_lidar_heights)
+    sub_area_heights = list(zip(grid_sub_area, grid_area_lidar_heights))
+    print('sub_area_heights: ', sub_area_heights[0])
+
     # find the heights of each sub-area => area-heights
     if len(map_data) > 0:
-        for sub_area in grid_area_heights:
-            start = (sub_area[1][2])
-            end = (sub_area[1][0])
+        for sub_area in sub_area_heights:
+            start = (sub_area[0][2])
+            end = (sub_area[0][0])
 
             #test data
             # zone coordinates sett to be relative to the lidar data's point cloud
-            print("l1 = ", min_point, " l2 = ", max_point)
-            print("p1 = ", start, " p2 = ", end)
-            print("center_lim = ", center)
-            print("max_lim = ", area_limit)
-            areazone = position_relative_to_pointcloud(min_point, max_point, start, end, center, area_limit)
-            print("area",areazone)
+            #print("l1 = ", min_point, " l2 = ", max_point)
+            #print("p1 = ", start, " p2 = ", end)
+            #print("center_lim = ", center)
+            #print("max_lim = ", area_limit)
+            #areazone = position_relative_to_pointcloud(min_point, max_point, start, end, center, area_limit)
+            #print("area",areazone)
             # calculate map zones height
             ys, xs = start
             ye, xe = end
             sub_center = ((xs + xe)/2, (ys + ye)/2)
             # check if area is part of water body
             part_of_subarea_of_waterbody = list(filter(lambda pos: inArea((pos['properties']['sub_div_center'][0], pos['properties']['sub_div_center'][1], 0.0), [start,end]), map_data))
+            print('sub_area: ', sub_area)
+            print('part_of_subarea_of_waterbody: ', part_of_subarea_of_waterbody)
             if(len(part_of_subarea_of_waterbody) > 0):
                 current_map_zone = closest_points(sub_center, part_of_subarea_of_waterbody, taken_coords)
                 sub_center = current_map_zone['properties']['sub_div_center']
                 taken_coords.append(sub_center)
                 print("item added", sub_center, " len ", len(taken_coords))
             else:
-                print("nothing here")
+                print("sub area not found on map")
                 continue
 
             current_zone_id = current_map_zone['properties']['sub_div_id']
 
             # filter data within sub-area zones
-            print(areazone[0][0]," < ",ice_points[0][0]," < ",areazone[1][0])
-            print(areazone[0][1]," > ",ice_points[0][1]," > ",areazone[1][1])
-            points_in_area = list(filter(lambda point_position: inArea(point_position, areazone), ice_points))
+            #print(areazone[0][0]," < ",ice_points[0][0]," < ",areazone[1][0])
+            #print(areazone[0][1]," > ",ice_points[0][1]," > ",areazone[1][1])
+            #points_in_area = list(filter(lambda point_position: inArea(point_position, areazone), ice_points))
 
-            if(len(points_in_area) > 0):
-                heights = find_height(points_in_area)
-
-                if current_map_zone is not None:
-                    area_heights.append((current_zone_id, sub_area[0], sub_center, heights))
-            else:
-                print("no points in area")
+            if current_map_zone is not None:
+                #                     sub_id        center         heights
+                area_heights.append((current_zone_id, sub_center, sub_area[1]))
         return area_heights
     else:
         return [] # return [0] if no data collected from lidar
 
 
-print(calculate_area_data((61, 11), 'mj\u00f8sa'))
-
+#print(calculate_area_data((61, 11), 'mj\u00f8sa'))

server/map_handler/input_new_data.py

@@ -11,8 +11,6 @@ def input_new_Lidar_data(self, cursor, sensorId, bodyOfWater):
         latitude = 60.816848
         longitude = 10.723823
 
-        total_measurement_average = 0  # the total average of a measurement
-
         # data about the file read from
         about_laz = about_laz_file()
         scale_factor = max(about_laz[2])
@@ -20,18 +18,16 @@ def input_new_Lidar_data(self, cursor, sensorId, bodyOfWater):
         # create a new measurement with the time the data is sent, sensor type, where
         # and an estimate of average thickness of ice on water body
         cursor.execute('''
-            INSERT INTO Measurement(  SensorID, TimeMeasured, WaterBodyName, 
-                                        WholeAverageThickness, CenterLat, CenterLon) VALUES 
+            INSERT INTO Measurement(MeasurementID, SensorID, TimeMeasured, WaterBodyName, CenterLat, CenterLon) VALUES 
                 (?,?,?,?,?,?);
-        ''', (sensorId, datetime.utcnow().replace(microsecond=0), bodyOfWater, 0, latitude, longitude))
+        ''', (1, sensorId, datetime.utcnow().replace(microsecond=0), bodyOfWater, latitude, longitude))
 
         # auto generate new measurement id
         measurement_id = cursor.lastrowid
 
         # calculate the area of to be calculated based on the coordinates given to the calculation model
         areas_data = calculate_area_data((latitude, longitude), bodyOfWater)
-        print(bodyOfWater)
-        print(len(areas_data))
+
         lidar_json_data = []
 
         if(areas_data):
@@ -39,42 +35,40 @@ def input_new_Lidar_data(self, cursor, sensorId, bodyOfWater):
             # calculate data for each zone within the area
             for area in areas_data:
                 # lng and lat relative to map
-                map_lat, map_lng = area[2]
+                subId = int(area[0])
+                map_lat, map_lng = area[1]
+                heights = area[2]
 
-                if(len(area[3]) != 0):
-                    average = sum(area[3])/len(area[3])
-                    minimum_thickness = min(area[3])
+                if(len(heights) != 0 or sum(heights) == 0):
+                    average = sum(heights)/len(heights)
+                    minimum_thickness = min(heights)
                 else:
                     average = 0
                     minimum_thickness = 0
 
-                total_measurement_average += average
-
+                time_measured = datetime.utcnow().replace(microsecond=0)
                 # input the data into the database
                 cursor.execute('''
-                INSERT INTO SubDivision(MeasurementID, SubDivisionID, GroupID, MinimumThickness, AverageThickness, CenterLatitude, CenterLongitude, Accuracy) VALUES
+                INSERT INTO SubdivisionMeasurementData(MeasurementID, TimeMeasured, SubdivID, WaterBodyName, MinimumThickness, AverageThickness, CalculatedSafety, Accuracy) VALUES
                     (?,?,?,?,?,?,?,?);
-                ''', (measurement_id, area[0], area[1], float(minimum_thickness), float(average), float(map_lat), float(map_lng), scale_factor))
-
+                ''', (measurement_id, time_measured, subId, bodyOfWater, float(minimum_thickness), float(average), float(0.0), scale_factor))
+                sub_center = (map_lat, map_lng)
                 # set up json formate
                 lidar_read = {
-                    'MeasurementId': measurement_id,
-                    'SubId': area[0],
-                    'GroupId': area[1],
-                    'SubCenter': (map_lat, map_lng),
-                    'Heights': area[3]
+                    'MeasurementId': str(measurement_id),
+                    'SubId': str(subId),
+                    'SubCenter': str(sub_center),
+                    'Heights': str(heights)
                 }
 
                 lidar_json_data.append(lidar_read)
 
-            total_measurement_average = total_measurement_average / len(areas_data)
-
             # input the newly generated measurement_id and whole average thickness
             cursor.execute('''
                 UPDATE Measurement
-                SET measurementID = ?, WholeAverageThickness = ?
-                WHERE MeasurementID IS NULL AND WholeAverageThickness = 0;
-            ''', (int(measurement_id), total_measurement_average), )
+                SET measurementID = ?
+                WHERE MeasurementID IS NULL;
+            ''', (int(measurement_id),))
         else:
             print('No data found, line 79')
 
@@ -85,10 +79,12 @@ def input_new_Lidar_data(self, cursor, sensorId, bodyOfWater):
         self.send_response(200)
         self.send_header('Content-type', "application/json")
         self.end_headers()
-
-        file_path = "./lake_relation/newest_lidar_data.json"
+        file_path = "./server/map_handler/lake_relations/newest_lidar_data.json"
         content = None
 
+        current_directory = os.getcwd()
+        print("Current working directory:", current_directory)
+
         if len(lidar_json_data) > 0:
             if os.path.exists(file_path):
                 os.remove(file_path)
@@ -101,11 +97,12 @@ def input_new_Lidar_data(self, cursor, sensorId, bodyOfWater):
             print('No data found, line 101')
             content = json.dumps([])
 
+
         # Write content data to response object
         self.wfile.write(content.encode('utf-8'))
 
     # error handling
     except Exception as e:
-        print("An error occurred", e)
+        print("An error occurred: ", e)
         # rollback in case of error
-        cursor.connection.rollback()
\ No newline at end of file
+        cursor.connection.rollback()

server/map_handler/lake_relations/newest_lidar_data.json

+[{"MeasurementId": "17", "SubId": "48", "SubCenter": "(60.816856, 10.723847)", "Heights": "[1]"}, {"MeasurementId": "17", "SubId": "100", "SubCenter": "(60.816856, 10.990419)", "Heights": "[1, 27]"}, {"MeasurementId": "17", "SubId": "36", "SubCenter": "(60.756856, 10.710419)", "Heights": "[1]"}, {"MeasurementId": "17", "SubId": "168", "SubCenter": "(60.396856, 11.230419)", "Heights": "[1, 4, 6, 27, 7]"}]
\ No newline at end of file