Loads and loads of data

app/services/metadata_processor.py

@@ -72,12 +72,31 @@ class MetadataProcessor:
 
     def generate_json(self):
         """
-        Generate the JSON structure and save it to the output file.
+        Generate the JSON structure, deduplicate and sort categories by 'path',
+        then save it to the output file.
         """
-        self._process_directory()
+        self._process_directory()  # Extract all markdown data into self.data
 
-        # Save JSON to file
-        with open(self.output_file, "w", encoding="utf-8") as json_file:
-            json.dump(self.data, json_file, indent=4, ensure_ascii=False)
+        # Ensure 'categories' exists and is a list
+        if "categories" not in self.data:
+            self.data["categories"] = []
 
-        print(f"Generated JSON saved to {self.output_file}")
+        # Deduplicate 'categories' using 'path' as the unique key
+        unique_categories = { }
+        for category in self.data["categories"]:
+            if isinstance( category, dict ):  # Ensure valid category structure
+                path = category.get( "path", "unknown" )  # Use 'path' as the unique key
+                if path not in unique_categories:
+                    unique_categories[path] = category
+
+        # Replace the 'categories' list with a sorted version by 'path'
+        self.data["categories"] = sorted(
+            unique_categories.values(),
+            key = lambda x: x.get( "path", "unknown" )
+        )
+
+        # Save the updated JSON to file
+        with open( self.output_file, "w", encoding = "utf-8" ) as json_file:
+            json.dump( self.data, json_file, indent = 4, ensure_ascii = False )
+
+        print( f"Generated JSON saved to {self.output_file}" )