change: working function for adding lidar data, but slow

server/data_processing/process_lidar_data.py

@@ -62,7 +62,8 @@ def height_in_area(area):
     # ]
 
     # NB: is only for the test data should be removed after
-    area = (area[0] * ((-3671212/60.815356) - (-3200175/60.774878)), area[1] * ((7898422/10.672022) - (4699978/10.768867)))
+    areazone = [(((area[0]) * ((-3671212/60.815356) - (-3200175/60.774878))) - 3200175, (area[1] * ((7898422/10.768867) - (4699978/10.672022))) + 4699978),
+                (((area[0] - 100) * ((-3671212/60.815356) - (-3200175/60.774878))) - 3200175, ((area[1] - 1) * ((7898422/10.768867) - (4699978/10.672022))) + 4699978)]
 
     # Refactor data format
     iceOver = laspy.read(lazData_path[0])
@@ -72,7 +73,8 @@ def height_in_area(area):
 
     area_heights = []
 
-    ice_points = list(filter(lambda position: inArea(position, area), ice_points))
-    area_heights.append(find_height(ice_points))
+    ice_points = list(filter(lambda position: inArea(position, areazone), ice_points))
+
+    area_heights = find_height(ice_points)
 
     return area_heights

server/main.py

@@ -29,7 +29,7 @@ class IceHTTP(BaseHTTPRequestHandler):
 
     def do_GET(self):
         # Root path
-        if self.path == '/': # NB: temporary root path behavior
+        if self.path == '/':  # NB: temporary root path behavior
             self.send_response(200)
             self.send_header("Content-type", "text/plain")
             self.end_headers()
@@ -37,17 +37,18 @@ class IceHTTP(BaseHTTPRequestHandler):
             self.wfile.write(b"Root path hit!")
 
         elif self.path == '/update_map':  # NB: should be POST?
-            get_all_markers(self, self.cursor, False)   # Get all markers
-
+            get_all_markers(self, self.cursor, False, 'Mjosa')  # Get all markers
+            # NB: temporary hardcoded waterBodyName
         elif self.path == '/get_valid_markers':  # NB: should be POST?
-            get_all_markers(self, self.cursor, True)  # Get only valid markers
+            get_all_markers(self, self.cursor, True, 'Mjosa')  # Get only valid markers
+            # NB: temporary hardcoded waterBodyName
 
     def do_POST(self):
         if self.path == '/get_weather_data':
             get_weather(self)
 
         elif self.path == '/new_lidar_data':
-            input_new_Lidar_data(self,self.cursor, 1, 'Mjøsa') # hardcoded body of water must change later
+            input_new_Lidar_data(self,self.cursor, 1, 'Mjosa') # hardcoded body of water must change later
 
 # Terminate server on key press q
 def on_key_press(server, event, cursor, conn):

server/map/get_markers.py

 import json
 
+
 # get_markers requests all marker data or valid markers, converts the data to json, and writes
 # the data to the response object
-def get_all_markers(self, cursor, valid: bool):
+def get_all_markers(self, cursor, valid: bool, waterBodyName):
     try:
-        # NB: interval temporarily hard coded to 5 days
-        if valid:
-            cursor.execute('''
-                SELECT m.MeasurementID, m.SensorID, m.TimeMeasured, d.Latitude, d.Longitude,
-                       d.IceTop, d.IceBottom, d.CalculatedThickness, d.Accuracy, s.SensorType, s.Active,
-                       c.CornerID, c.CornerLatitude, c.CornerLongitude, b.Name
-                FROM Measurement m
-                INNER JOIN Sensor s ON m.SensorID = s.SensorID
-                INNER JOIN Data d ON m.MeasurementID = d.MeasurementID
-                LEFT JOIN Corner c ON m.MeasurementID = c.MeasurementID
-                INNER JOIN BodyOfWater b ON m.WaterBodyName = b.Name
-                WHERE m.TimeMeasured > DATE_SUB(NOW(), INTERVAL 5 DAY);
-            ''')
-        else:
-            cursor.execute('''
-                SELECT m.MeasurementID, m.SensorID, m.TimeMeasured, d.Latitude, d.Longitude,
-                       d.IceTop, d.IceBottom, d.CalculatedThickness, d.Accuracy, s.SensorType, s.Active,
-                       c.CornerID, c.CornerLatitude, c.CornerLongitude, b.Name
-                FROM Measurement m
-                INNER JOIN Sensor s ON m.SensorID = s.SensorID
-                INNER JOIN Data d ON m.MeasurementID = d.MeasurementID
-                LEFT JOIN Corner c ON m.MeasurementID = c.MeasurementID
-                INNER JOIN BodyOfWater b ON m.WaterBodyName = b.Name
-            ''')
+        sql_query = '''
+            SELECT m.MeasurementID, m.SensorID, m.TimeMeasured, 
+                   s.SensorType, s.Active, 
+                   b.Name,
+                   d.SubDivisionID, d.GroupID, d.MinimumThickness, 
+                   d.AverageThickness, d.CenterLatitude, d.CenterLongitude, 
+                   d.Accuracy
+            FROM Measurement m
+            INNER JOIN Sensor s ON m.SensorID = s.SensorID
+            INNER JOIN BodyOfWater b ON m.WaterBodyName = b.Name
+            LEFT JOIN SubDivision d ON m.MeasurementID = d.MeasurementID
+            WHERE b.Name = 'Mjosa'
+        '''
+
+        if valid:  # Append time restriction to WHERE statement, NB temporarily hardcoded to 5 days
+            sql_query += ' AND m.TimeMeasured > DATE_SUB(NOW(), INTERVAL 5 DAY)'
+
+        cursor.execute(sql_query)
 
         rows = cursor.fetchall()
 
+        # Container for all fetched measurement objects
         measurement_data = {}
 
-        # Iterate over the fetched rows
+        # Iterate over all fetched rows
         for row in rows:
             measurement_id = row[0]
 
-            # Create a data object for current row
-            data_object = {
-                'Latitude': row[3],
-                'Longitude': row[4],
-                'IceTop': row[5],
-                'IceBottom': row[6],
-                'CalculatedThickness': row[7],
-                'Accuracy': row[8]
-            }
-
-            # Create a corner object for current row
-            corner_object = {
-                'Latitude': row[12],
-                'Longitude': row[13]
+            # Create subdivision new object
+            sub_division = {
+                'SubdivID': row[6],
+                'GroupID': row[7],
+                'MinThickness': row[8],
+                'AvgThickness': row[9],
+                'CenLatitude': row[10],
+                'CenLongitude': row[11],
+                'Accuracy': row[12]
             }
 
             # Check if measurement ID already exists in measurement_data
             if measurement_id in measurement_data:
                 # Check if the data object already exists in the list
-                if data_object not in measurement_data[measurement_id]['Data']:
-                    measurement_data[measurement_id]['Data'].append(data_object)
+                if sub_division not in measurement_data[measurement_id]['Subdivisions']:
+                    measurement_data[measurement_id]['Subdivisions'].append(sub_division)
 
-                # Check if the corner object already exists in the list
-                if corner_object not in measurement_data[measurement_id]['Corners']:
-                    measurement_data[measurement_id]['Corners'].append(corner_object)
             else:
                 # Create a new entry for measurement_id if it does not already exist in the list
                 measurement_data[measurement_id] = {
                     'MeasurementID': measurement_id,
                     'TimeMeasured': row[2],
-                    'Sensor': {
+                    'Sensor': {  # Each measurement only has one related sensor
                         'SensorID': row[1],
-                        'SensorType': row[9],
-                        'Active': bool(row[10])
+                        'SensorType': row[3],
+                        'Active': bool(row[4])
                     },
-                    'Data': [data_object],
-                    'Corners': [corner_object]
+                    'Subdivisions': [sub_division],  # Array of sub_division objects
                 }
 
         # Convert dictionary values to list of measurements
         data = list(measurement_data.values())
 
         if len(rows) == 0 or len(data) == 0:  # Return 500 and empty list if no data is found
-            print(f"An error occurred while querying the database")
+            print(f"Error in querying database")
             resp_code = 500
             marker_data = '[]'
         else:
@@ -89,7 +76,7 @@ def get_all_markers(self, cursor, valid: bool):
             marker_data = json.dumps(data, indent=4)
 
     except Exception as e:
-        print(f"An error occurred while querying the database: {e}")
+        print(f"Error in querying database: {e}")
         resp_code = 500
         marker_data = '[]'
 
@@ -100,4 +87,3 @@ def get_all_markers(self, cursor, valid: bool):
 
     # Write marker data to response object
     self.wfile.write(marker_data.encode('utf-8'))
-

server/map/input_new_data.py

@@ -6,19 +6,23 @@ def input_new_Lidar_data(self, cursor, sensorId, bodyOfWater):
     try:
 
         cursor.execute('''
-            INSERT INTO Measurement( SensorID, TimeMeasured, WaterBodyName) VALUES 
-                (?,?,?);
-        ''',( sensorId, datetime.utcnow(), bodyOfWater))
+            INSERT INTO Measurement(  SensorID, TimeMeasured, WaterBodyName, WholeAverageThickness) VALUES 
+                (?,?,?,?);
+        ''', ( sensorId, datetime.utcnow().replace(microsecond=0), bodyOfWater,0))
 
         # auto generate new measurement id
         measurement_id = cursor.lastrowid
 
         cursor.execute('''
-            SELECT CenterLatitude, CenterLongitude, SubDivision, GroupID
+            UPDATE Measurement
+            SET measurementID = ?
+            WHERE MeasurementID IS NULL;
+        ''', (int(measurement_id),))
+
+        cursor.execute('''
+            SELECT CenterLatitude, CenterLongitude, SubDivisionID, GroupID
             FROM SubDivision
-            where MeasurementID = ? AND (CenterLatitude, CenterLongitude) 
-                       IN (SELECT DISTINCT CenterLongitude, CenterLatitiude
-                            FROM SubDivision);
+            GROUP BY CenterLatitude, CenterLongitude
         ''')
         position_data = cursor.fetchall()
 
@@ -26,11 +30,17 @@ def input_new_Lidar_data(self, cursor, sensorId, bodyOfWater):
             for row in position_data:
                 latitude, longitude, subID, groupID = row
                 heights = height_in_area((latitude,longitude))
+                if(len(heights) > 0):
+                    print(heights)
+                else:
+                    print("No height found")
                 average = sum(heights)/len(heights)
                 cursor.execute('''
                 INSERT INTO SubDivision(MeasurementID, SubDivisionID, GroupID, MinimumThickness, AverageThickness, CenterLatitude, CenterLongitude, Accuracy) VALUES
                     (?,?,?,?,?,?,?,?);
-                ''',(measurement_id, subID, groupID, min(heights), average, latitude, longitude, 1))
+                ''',(measurement_id, subID, groupID, float(min(heights)), float(average), float(latitude), float(longitude), float(1)))
+        else:
+            print('No data')
 
         cursor.connection.commit()
 

server/sql_db/schema.sql

@@ -61,6 +61,3 @@ INSERT INTO SubDivision (MeasurementID, SubDivisionID, GroupID, MinimumThickness
 (2, 2, 2, 3.5, 5.0, 60.749, 10.783, 1.4),
 (3, 1, 1, 4.1, 5.7, 60.768, 10.845, 1.3),
 (3, 2, 2, 3.5, 5.0, 60.749, 10.783, 1.4);
-
-
-