Merge pull request #6 from openagri-eu/48-bug-report

bug: Update usage of data streams

Author: JoleVLF

app/api/api_v1/endpoints/report.py

@@ -80,7 +80,8 @@ async def generate_irrigation_report(
             status_code=400,
             detail=f"Data file must be provided if gatekeeper is not used.",
         )
-
+    if data:
+        data = data.file.read()
     background_tasks.add_task(
         process_irrigation_data,
         data=data,
@@ -119,7 +120,8 @@ async def generate_generic_observation_report(
         else token.id
     )
     uuid_of_pdf = f"{user_id}/{uuid_v4}"
-
+    if data:
+        data = data.file.read()
     background_tasks.add_task(
         process_farm_calendar_data,
         calendar_activity_type=calendar_activity_type,
@@ -178,7 +180,8 @@ async def generate_animal_report(
             params["status"] = status
         if parcel_id:
             params['parcel'] = parcel_id
-
+    if data:
+        data = data.file.read()
     background_tasks.add_task(
         process_animal_data,
         data=data,

app/schemas/compost.py

@@ -35,7 +35,7 @@ class CropObservation(BaseModel):
 
     type: str = Field(alias="@type")
     id: str = Field(alias="@id")
-    activityType: dict
+    activityType: Optional[dict] = None
     title: Optional[str] = ""
     details: Optional[str] = ""
     hasStartDatetime: Optional[datetime] = None

app/schemas/irrigation.py

@@ -1,5 +1,5 @@
 from datetime import datetime
-from typing import List, Optional
+from typing import List, Optional, Union
 
 from pydantic import BaseModel, Field
 
@@ -16,13 +16,13 @@ class IrrigationOperation(BaseModel):
 
     type: str = Field(alias="@type")
     id: str = Field(alias="@id")
-    activityType: dict
+    activityType: Optional[dict] = None
     title: Optional[str] = ""
     details: Optional[str] = ""
     hasStartDatetime: Optional[datetime] = None
     hasEndDatetime: Optional[datetime] = None
     responsibleAgent: Optional[str] = None
     usesAgriculturalMachinery: List[dict] = []
     hasAppliedAmount: QuantityValue
-    usesIrrigationSystem: str
+    usesIrrigationSystem: Optional[Union[str, dict]] = None
     operatedOn: Optional[dict] = None

app/utils/animals_report.py

@@ -240,7 +240,8 @@ def process_animal_data(
 
         else:
             if not settings.REPORTING_USING_GATEKEEPER:
-                json_data = json.load(data.file)
+                data = json.loads(data)
+                json_data = data.get("@graph")
             else:
                 json_data = make_get_request(
                     url=f'{settings.REPORTING_FARMCALENDAR_BASE_URL}{settings.REPORTING_FARMCALENDAR_URLS["animals"]}',

app/utils/farm_calendar_report.py

@@ -1,4 +1,5 @@
 import datetime
+import itertools
 import json
 import logging
 import os
@@ -496,13 +497,37 @@ def process_farm_calendar_data(
                 materials=materials,
             )
         else:
-            dt = json.load(data.file)
-            calendar_data = FarmCalendarData(
-                activity_type_info=calendar_activity_type,
-                observations=dt["observations"],
-                farm_activities=dt["operations"],
-                materials=dt["materials"],
-            )
+            dt = json.loads(data)
+            if dt:
+                farm_act = dt.get("@graph", [])
+                obs = [x.get("hasMeasurement") for x in farm_act]
+                materials = [x.get("hasNestedOperation") for x in farm_act]
+
+                obs =  list(itertools.chain.from_iterable(obs)) if obs else []
+                materials = list(itertools.chain.from_iterable(materials)) if obs else []
+
+                try:
+                    obs = [o['hasMember'] for o in obs]
+                    obs = list(itertools.chain.from_iterable(obs)) if obs else []
+                except Exception:
+                    # If this occurs it means we use new data type (not old)
+                    # if Except is not enter data used is using old format (old JSONLD example)
+                    # This way it is backward compatible (logger.info should stay, because it is not error)
+                    logger.info("Data is new format type, proceed as normal.")
+
+                calendar_data = FarmCalendarData(
+                    activity_type_info=calendar_activity_type,
+                    observations=obs,
+                    farm_activities=farm_act,
+                    materials=materials
+                )
+            else:
+                calendar_data = FarmCalendarData(
+                    activity_type_info=calendar_activity_type,
+                    observations=[],
+                    farm_activities=[],
+                    materials=[],
+                )
 
         pdf = create_farm_calendar_pdf(calendar_data, token)
         pdf_dir = f"{settings.PDF_DIRECTORY}{pdf_file_name}"

app/utils/irrigation_report.py

@@ -30,7 +30,8 @@ def parse_irrigation_operations(data: dict) -> Optional[List[IrrigationOperation
 
 
 def create_pdf_from_operations(
-    operations: List[IrrigationOperation], token: dict[str, str] = None
+    operations: List[IrrigationOperation], token: dict[str, str] = None,
+    data_used: bool = False,
 ):
     """
     Create PDF report from irrigation operations
@@ -118,7 +119,8 @@ def create_pdf_from_operations(
         )
 
     if len(operations) > 1:
-        operations.sort(key=lambda x: x.hasStartDatetime)
+        if not data_used:
+            operations.sort(key=lambda x: x.hasStartDatetime)
         pdf.set_fill_color(0, 255, 255)
         with pdf.table(text_align="CENTER", padding=0.5) as table:
             row = table.row()
@@ -164,7 +166,11 @@ def create_pdf_from_operations(
                     f"{op.hasAppliedAmount.numericValue} ({op.hasAppliedAmount.unit})",
                 )
 
-                row.cell(op.usesIrrigationSystem)
+                if isinstance(op.usesIrrigationSystem, dict):
+                    local_sys = op.usesIrrigationSystem.get("name")
+                else:
+                    local_sys = op.usesIrrigationSystem
+                row.cell(local_sys)
                 pdf.ln(10)
 
     return pdf
@@ -182,7 +188,7 @@ def process_irrigation_data(
     """
     Process irrigation data and generate PDF report
     """
-
+    data_used = False
     if irrigation_id:
         json_data = make_get_request(
             url=f'{settings.REPORTING_FARMCALENDAR_BASE_URL}{settings.REPORTING_FARMCALENDAR_URLS["irrigations"]}{irrigation_id}/',
@@ -206,15 +212,18 @@ def process_irrigation_data(
             )
 
         else:
-            json_data = json.load(data.file)
+            data_used = True
+            json_data = json.loads(data)
+            if json_data:
+                json_data = json_data['@graph']
 
     if json_data:
         operations = parse_irrigation_operations(json_data)
     else:
         operations = []
 
     try:
-        pdf = create_pdf_from_operations(operations, token)
+        pdf = create_pdf_from_operations(operations, token, data_used)
     except Exception:
         raise HTTPException(
             status_code=400, detail="PDF generation of irrigation report failed."