diff --git a/convert_geo_data.py b/convert_geo_data.py new file mode 100644 index 0000000..e7f1eb2 --- /dev/null +++ b/convert_geo_data.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 +""" +Script to convert geographic data in MongoDB from E6 format to GeoJSON Point format. +This script updates all documents in the ingress collection to use proper MongoDB GeoJSON format. +""" + +from pymongo import MongoClient +from pymongo.errors import PyMongoError +import sys + + +def convert_e6_to_degrees(e6_value): + """ + Convert E6 format to degrees. + + Args: + e6_value: Integer value in E6 format (multiply by 1e-6 to get degrees) + + Returns: + Float value in degrees + """ + return e6_value / 1_000_000 if e6_value else 0.0 + + +def create_geojson_point(lat_e6, lng_e6): + """ + Create a GeoJSON Point from E6 coordinates. + + Args: + lat_e6: Latitude in E6 format + lng_e6: Longitude in E6 format + + Returns: + GeoJSON Point object or None if coordinates are invalid + """ + if lat_e6 == 0 and lng_e6 == 0: + return None + + lat = convert_e6_to_degrees(lat_e6) + lng = convert_e6_to_degrees(lng_e6) + + # GeoJSON Point format: [longitude, latitude] + return { + "type": "Point", + "coordinates": [lng, lat] + } + + +def convert_document(doc): + """ + Convert a single document to use GeoJSON format. + + Args: + doc: MongoDB document + + Returns: + Updated document with GeoJSON coordinates + """ + updated = False + updates = {} + + # Convert coordinates field + if "coordinates" in doc and isinstance(doc["coordinates"], dict): + coords = doc["coordinates"] + if "lat" in coords and "lng" in coords: + geojson = create_geojson_point(coords["lat"], coords["lng"]) + if geojson: + updates["coordinates"] = geojson + updated = True + + # Convert markup array + if "markup" in doc and isinstance(doc["markup"], list): + new_markup = [] + for item in doc["markup"]: + new_item = item.copy() + if "latE6" in item and "lngE6" in item: + geojson = create_geojson_point(item["latE6"], item["lngE6"]) + if geojson: + new_item["location"] = geojson + updated = True + new_markup.append(new_item) + if updated: + updates["markup"] = new_markup + + return updates if updated else None + + +def main(): + """ + Main function to convert all documents in the collection. + """ + # Connect to MongoDB + mongo_uri = "mongodb://root:root@localhost:27017/?authSource=admin" + db_name = "ingress" + collection_name = "ingress" + + print(f"Connecting to MongoDB at {mongo_uri}...") + print(f"Database: {db_name}, Collection: {collection_name}") + + try: + client = MongoClient(mongo_uri) + db = client[db_name] + collection = db[collection_name] + + # Count total documents + total_docs = collection.count_documents({}) + print(f"Total documents in collection: {total_docs}") + + if total_docs == 0: + print("No documents found. Exiting.") + return + + # Process documents in batches + batch_size = 100 + processed = 0 + updated = 0 + skipped = 0 + + print(f"\nProcessing documents in batches of {batch_size}...") + + cursor = collection.find({}) + + for doc in cursor: + processed += 1 + + # Convert document + updates = convert_document(doc) + + if updates: + # Update the document + collection.update_one( + {"_id": doc["_id"]}, + {"$set": updates} + ) + updated += 1 + else: + skipped += 1 + + # Progress indicator + if processed % batch_size == 0: + print(f"Processed: {processed}/{total_docs} | Updated: {updated} | Skipped: {skipped}") + + print(f"\nConversion complete!") + print(f"Total processed: {processed}") + print(f"Total updated: {updated}") + print(f"Total skipped: {skipped}") + + except PyMongoError as e: + print(f"Error: {e}", file=sys.stderr) + sys.exit(1) + finally: + client.close() + + +if __name__ == "__main__": + main() diff --git a/schedule.py b/schedule.py index efda47c..bf9d3d8 100644 --- a/schedule.py +++ b/schedule.py @@ -17,6 +17,78 @@ COLLECTION_NAME = os.getenv("COLLECTION_NAME") TIMEZONE = ZoneInfo("Europe/Rome") +def convert_e6_to_degrees(e6_value): + """ + Convert E6 format to degrees. + + Args: + e6_value: Integer value in E6 format (multiply by 1e-6 to get degrees) + + Returns: + Float value in degrees + """ + return e6_value / 1_000_000 if e6_value else 0.0 + + +def create_geojson_point(lat_e6, lng_e6): + """ + Create a GeoJSON Point from E6 coordinates. + + Args: + lat_e6: Latitude in E6 format + lng_e6: Longitude in E6 format + + Returns: + GeoJSON Point object or None if coordinates are invalid + """ + if lat_e6 == 0 and lng_e6 == 0: + return None + + lat = convert_e6_to_degrees(lat_e6) + lng = convert_e6_to_degrees(lng_e6) + + # GeoJSON Point format: [longitude, latitude] + return { + "type": "Point", + "coordinates": [lng, lat] + } + + +def transform_plext(plext): + """ + Transform a plext to use GeoJSON format for coordinates. + + Args: + plext: Plext dictionary to transform + + Returns: + Transformed plext with GeoJSON coordinates + """ + transformed = plext.copy() + + # Transform coordinates field + if "coordinates" in transformed and isinstance(transformed["coordinates"], dict): + coords = transformed["coordinates"] + if "lat" in coords and "lng" in coords: + geojson = create_geojson_point(coords["lat"], coords["lng"]) + if geojson: + transformed["coordinates"] = geojson + + # Transform markup array + if "markup" in transformed and isinstance(transformed["markup"], list): + new_markup = [] + for item in transformed["markup"]: + new_item = item.copy() + if "latE6" in item and "lngE6" in item: + geojson = create_geojson_point(item["latE6"], item["lngE6"]) + if geojson: + new_item["location"] = geojson + new_markup.append(new_item) + transformed["markup"] = new_markup + + return transformed + + def get_time_range(): """ Calculate the time range for the current execution. @@ -67,7 +139,7 @@ def fetch_plexts(min_timestamp, max_timestamp): def save_to_mongodb(plexts): """ - Save plexts to MongoDB. + Save plexts to MongoDB with GeoJSON coordinates. Args: plexts: List of plext dictionaries to save @@ -84,8 +156,11 @@ def save_to_mongodb(plexts): db = client[DB_NAME] collection = db[COLLECTION_NAME] - # Insert all plexts - result = collection.insert_many(plexts) + # Transform plexts to use GeoJSON format + transformed_plexts = [transform_plext(plext) for plext in plexts] + + # Insert all transformed plexts + result = collection.insert_many(transformed_plexts) print(f"Inserted {len(result.inserted_ids)} plexts to MongoDB") return True except PyMongoError as e: