[Fixes #14042] deprecate overwrite_existing_layers in favor of action…

.env_dev

@@ -36,7 +36,7 @@ GEONODE_DB_CONN_TOUT=5
 DEFAULT_BACKEND_DATASTORE=datastore
 BROKER_URL=redis://localhost:6379/0
 CELERY_BEAT_SCHEDULER=celery.beat:PersistentScheduler
-ASYNC_SIGNALS=False
+ASYNC_SIGNALS=True
 
 # Harvesting Monitoring configuration
 HARVESTING_MONITOR_ENABLED=False

geonode/upload/api/serializer.py

@@ -51,12 +51,8 @@ class Meta:
         ref_name = "OverwriteImporterSerializer"
         model = ResourceBase
         view_name = "importer_upload"
-        fields = ImporterSerializer.Meta.fields + (
-            "overwrite_existing_layer",
-            "resource_pk",
-        )
+        fields = ImporterSerializer.Meta.fields + ("resource_pk",)
 
-    overwrite_existing_layer = serializers.BooleanField(required=True)
     resource_pk = serializers.IntegerField(required=True)
 
 

geonode/upload/celery_tasks.py

@@ -399,7 +399,7 @@ def publish_resource(
         )
         _exec = orchestrator.get_execution_object(execution_id)
         _files = _exec.input_params.get("files")
-        _overwrite = _exec.input_params.get("overwrite_existing_layer")
+        _overwrite = action == ira.REPLACE.value
 
         _publisher = DataPublisher(handler_module_path)
         kwargs.update({"exec_id": execution_id})
@@ -509,7 +509,7 @@ def create_geonode_resource(
         handler_module_path = handler_module_path or _exec.input_params.get("handler_module_path")
 
         handler = import_string(handler_module_path)()
-        _overwrite = _exec.input_params.get("overwrite_existing_layer")
+        _overwrite = action == ira.REPLACE.value
 
         if _overwrite:
             resource = handler.overwrite_geonode_resource(
@@ -975,7 +975,7 @@ def rollback(self, *args, **kwargs):
     )
 
     handler = import_string(handler_module_path)()
-    if exec_object.input_params.get("overwrite_existing_layer"):
+    if exec_object.action == ira.REPLACE.value:
         logger.warning("Rollback is skipped for the overwrite")
     else:
         handler.rollback(exec_id, rollback_from_step, action_to_rollback, *args, **kwargs)

geonode/upload/handlers/common/metadata.py

@@ -55,7 +55,6 @@ def extract_params_from_data(_data, action=None):
         return {
             "dataset_title": _data.pop("dataset_title", None),
             "skip_existing_layers": _data.pop("skip_existing_layers", "False"),
-            "overwrite_existing_layer": _data.pop("overwrite_existing_layer", False),
             "resource_pk": _data.pop("resource_pk", None),
             "store_spatial_file": _data.pop("store_spatial_files", "True"),
             "action": _data.pop("action"),

geonode/upload/handlers/common/raster.py

@@ -38,7 +38,7 @@
 from geonode.upload.handlers.utils import create_alternate, should_be_imported
 from geonode.upload.models import ResourceHandlerInfo
 from geonode.upload.orchestrator import orchestrator
-from geonode.upload.utils import find_key_recursively
+from geonode.upload.utils import find_key_recursively, ImporterRequestAction as ira
 from osgeo import gdal
 from geonode.upload.celery_app import importer_app
 from geonode.storage.manager import storage_manager
@@ -123,7 +123,6 @@ def extract_params_from_data(_data, action=None):
 
         return {
             "skip_existing_layers": _data.pop("skip_existing_layers", "False"),
-            "overwrite_existing_layer": _data.pop("overwrite_existing_layer", False),
             "resource_pk": _data.pop("resource_pk", None),
             "store_spatial_file": _data.pop("store_spatial_files", "True"),
             "action": _data.pop("action", "upload"),
@@ -276,7 +275,7 @@ def import_resource(self, files: dict, execution_id: str, **kwargs) -> str:
             # start looping on the layers available
             layer_name = self.fixup_name(filename)
 
-            should_be_overwritten = _exec.input_params.get("overwrite_existing_layer")
+            should_be_overwritten = _exec.action == ira.REPLACE.value
             # should_be_imported check if the user+layername already exists or not
             if should_be_imported(
                 layer_name,
@@ -350,7 +349,7 @@ def create_geonode_resource(
             getattr(settings, "CASCADE_WORKSPACE", "geonode"),
         )
 
-        _overwrite = _exec.input_params.get("overwrite_existing_layer", False)
+        _overwrite = _exec.action == ira.REPLACE.value
         # if the layer exists, we just update the information of the dataset by
         # let it recreate the catalogue
         if not saved_dataset.exists() and _overwrite:
@@ -398,7 +397,7 @@ def overwrite_geonode_resource(
 
         dataset = resource_type.objects.filter(alternate__icontains=alternate, owner=_exec.user)
 
-        _overwrite = _exec.input_params.get("overwrite_existing_layer", False)
+        _overwrite = _exec.action == ira.REPLACE.value
         # if the layer exists, we just update the information of the dataset by
         # let it recreate the catalogue
 

geonode/upload/handlers/common/remote.py

@@ -115,7 +115,6 @@ def extract_params_from_data(_data, action=None):
             "title": _data.pop("title", None),
             "url": _data.pop("url", None),
             "type": _data.pop("type", None),
-            "overwrite_existing_layer": _data.pop("overwrite_existing_layer", False),
         }, _data
 
     def pre_validation(self, files, execution_id, **kwargs):
@@ -144,7 +143,7 @@ def import_resource(self, files: dict, execution_id: str, **kwargs) -> str:
             # start looping on the layers available
             layer_name = self.fixup_name(title)
 
-            should_be_overwritten = _exec.input_params.get("overwrite_existing_layer")
+            should_be_overwritten = _exec.action == ira.REPLACE.value
 
             payload_alternate = params.get("remote_resource_id", None)
 
@@ -278,7 +277,7 @@ def overwrite_geonode_resource(
         _exec = self._get_execution_request_object(execution_id)
         resource = resource_type.objects.filter(alternate__icontains=alternate, owner=_exec.user)
 
-        _overwrite = _exec.input_params.get("overwrite_existing_layer", False)
+        _overwrite = _exec.action == ira.REPLACE.value
         # if the layer exists, we just update the information of the dataset by
         # let it recreate the catalogue
         if resource.exists() and _overwrite:

geonode/upload/handlers/common/serializer.py

@@ -27,7 +27,7 @@ class Meta:
         ref_name = "RemoteResourceSerializer"
         model = ResourceBase
         view_name = "importer_upload"
-        fields = ("url", "title", "type", "action", "overwrite_existing_layer")
+        fields = ("url", "title", "type", "action")
 
     url = serializers.URLField(required=True, help_text="URL of the remote service / resource")
     title = serializers.CharField(required=True, help_text="Title of the resource. Can be None or Empty")
@@ -36,5 +36,3 @@ class Meta:
         help_text="Remote resource type, for example wms or 3dtiles. Is used by the handler to understand if can handle the resource",
     )
     action = serializers.CharField(required=False, default=exa.UPLOAD.value)
-
-    overwrite_existing_layer = serializers.BooleanField(required=False, default=False)

geonode/upload/handlers/common/vector.py

@@ -193,7 +193,6 @@ def extract_params_from_data(_data, action=None):
 
         return {
             "skip_existing_layers": _data.pop("skip_existing_layers", "False"),
-            "overwrite_existing_layer": _data.pop("overwrite_existing_layer", False),
             "resource_pk": _data.pop("resource_pk", None),
             "store_spatial_file": _data.pop("store_spatial_files", "True"),
             "action": _data.pop("action", "upload"),
@@ -456,7 +455,7 @@ def import_resource(self, files: dict, execution_id: str, **kwargs) -> str:
             for index, layer in enumerate(layers, start=1):
                 layer_name = self.fixup_name(layer.GetName())
 
-                should_be_overwritten = _exec.input_params.get("overwrite_existing_layer")
+                should_be_overwritten = _exec.action == ira.REPLACE.value
                 # should_be_imported check if the user+layername already exists or not
                 if (
                     should_be_imported(
@@ -779,7 +778,7 @@ def create_geonode_resource(
             getattr(settings, "CASCADE_WORKSPACE", "geonode"),
         )
 
-        _overwrite = _exec.input_params.get("overwrite_existing_layer", False)
+        _overwrite = _exec.action == ira.REPLACE.value
         # if the layer exists, we just update the information of the dataset by
         # let it recreate the catalogue
         if not saved_dataset.exists() and _overwrite:
@@ -832,7 +831,7 @@ def overwrite_geonode_resource(
 
         dataset = resource_type.objects.filter(pk=_exec.input_params.get("resource_pk"), owner=_exec.user)
 
-        _overwrite = _exec.input_params.get("overwrite_existing_layer", False)
+        _overwrite = _exec.action == ira.REPLACE.value
         # if the layer exists, we just update the information of the dataset by
         # let it recreate the catalogue
         if dataset.exists() and _overwrite:

geonode/upload/handlers/shapefile/handler.py

@@ -85,7 +85,7 @@ def has_serializer(data) -> bool:
         if data.get("action") == ira.UPSERT.value:
             return False
         if _base.endswith("shp") if isinstance(_base, str) else _base.name.endswith("shp"):
-            is_overwrite_flow = data.get("overwrite_existing_layer", False)
+            is_overwrite_flow = data.get("action") == ira.REPLACE.value
             if isinstance(is_overwrite_flow, str):
                 is_overwrite_flow = ast.literal_eval(is_overwrite_flow.title())
             return OverwriteShapeFileSerializer if is_overwrite_flow else ShapeFileSerializer
@@ -103,7 +103,6 @@ def extract_params_from_data(_data, action=None):
 
         additional_params = {
             "skip_existing_layers": _data.pop("skip_existing_layers", "False"),
-            "overwrite_existing_layer": _data.pop("overwrite_existing_layer", False),
             "resource_pk": _data.pop("resource_pk", None),
             "store_spatial_file": _data.pop("store_spatial_files", "True"),
             "action": _data.pop("action", "upload"),

geonode/upload/handlers/shapefile/serializer.py

@@ -35,7 +35,6 @@ class Meta:
             "xml_file",
             "sld_file",
             "store_spatial_files",
-            "overwrite_existing_layer",
             "skip_existing_layers",
             "action",
         )
@@ -47,7 +46,6 @@ class Meta:
     xml_file = serializers.FileField(required=False)
     sld_file = serializers.FileField(required=False)
     store_spatial_files = serializers.BooleanField(required=False, default=True)
-    overwrite_existing_layer = serializers.BooleanField(required=False, default=False)
     skip_existing_layers = serializers.BooleanField(required=False, default=False)
     action = serializers.CharField(required=False, default=exa.UPLOAD.value)
 
@@ -57,10 +55,6 @@ class Meta:
         ref_name = "ShapeFileSerializer"
         model = ResourceBase
         view_name = "importer_upload"
-        fields = ShapeFileSerializer.Meta.fields + (
-            "overwrite_existing_layer",
-            "resource_pk",
-        )
+        fields = ShapeFileSerializer.Meta.fields + ("resource_pk",)
 
-    overwrite_existing_layer = serializers.BooleanField(required=True)
     resource_pk = serializers.IntegerField(required=True)

geonode/upload/handlers/tiles3d/handler.py

@@ -178,7 +178,6 @@ def extract_params_from_data(_data, action=None):
             "store_spatial_file": _data.pop("store_spatial_files", "True"),
             "action": _data.pop("action", "upload"),
             "original_zip_name": _data.pop("original_zip_name", None),
-            "overwrite_existing_layer": _data.pop("overwrite_existing_layer", False),
         }, _data
 
     def import_resource(self, files: dict, execution_id: str, **kwargs) -> str:
@@ -192,7 +191,7 @@ def import_resource(self, files: dict, execution_id: str, **kwargs) -> str:
         filename = _exec.input_params.get("original_zip_name") or Path(files.get("base_file")).stem
         # start looping on the layers available
         layer_name = self.fixup_name(filename)
-        should_be_overwritten = _exec.input_params.get("overwrite_existing_layer")
+        should_be_overwritten = _exec.action == ira.REPLACE.value
         # should_be_imported check if the user+layername already exists or not
         if should_be_imported(
             layer_name,

geonode/upload/handlers/xml/serializer.py

@@ -28,9 +28,8 @@ class Meta:
         ref_name = "MetadataFileSerializer"
         model = ResourceBase
         view_name = "importer_upload"
-        fields = ("overwrite_existing_layer", "resource_pk", "base_file", "action")
+        fields = ("resource_pk", "base_file", "action")
 
     base_file = serializers.FileField()
-    overwrite_existing_layer = serializers.BooleanField(required=False, default=True)
     resource_pk = serializers.IntegerField(required=True)
     action = serializers.CharField(required=False, default=exa.UPLOAD.value)

geonode/upload/orchestrator.py

@@ -16,7 +16,6 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 #########################################################################
-import ast
 import logging
 from typing import Optional
 from uuid import UUID
@@ -85,12 +84,11 @@ def get_serializer(self, _data) -> serializers.Serializer:
             if _serializer:
                 return _serializer
         logger.info("specific serializer not found, fallback on the default one")
-        is_overwrite_flow = _data.get("overwrite_existing_layer", False)
         if _data.get("action") == ira.UPSERT.value:
             return UpsertImporterSerializer
-        if isinstance(is_overwrite_flow, str):
-            is_overwrite_flow = ast.literal_eval(is_overwrite_flow.title())
-        return OverwriteImporterSerializer if is_overwrite_flow else ImporterSerializer
+        elif _data.get("action") == ira.REPLACE.value:
+            return OverwriteImporterSerializer
+        return ImporterSerializer
 
     def load_handler(self, module_path):
         try:

geonode/upload/tests/end2end/test_end2end.py

@@ -192,8 +192,7 @@ def test_import_gpkg_overwrite(self):
         payload = {"base_file": open(self.valid_gkpg, "rb"), "action": "upload"}
         prev_dataset = self._assertimport(payload, initial_name, keep_resource=True)
 
-        payload = {"base_file": open(self.valid_gkpg, "rb"), "action": "upload"}
-        payload["overwrite_existing_layer"] = True
+        payload = {"base_file": open(self.valid_gkpg, "rb"), "action": "replace"}
         payload["resource_pk"] = prev_dataset.pk
         self._assertimport(payload, initial_name, overwrite=True, last_update=prev_dataset.last_updated)
         self._cleanup_layers(name="stazioni_metropolitana")
@@ -286,8 +285,7 @@ def test_import_geojson_overwrite(self):
         payload = {"base_file": open(self.valid_geojson, "rb"), "action": "upload"}
         initial_name = "valid"
         prev_dataset = self._assertimport(payload, initial_name, keep_resource=True)
-        payload = {"base_file": open(self.valid_geojson, "rb"), "action": "upload"}
-        payload["overwrite_existing_layer"] = True
+        payload = {"base_file": open(self.valid_geojson, "rb"), "action": "replace"}
         payload["resource_pk"] = prev_dataset.pk
         self._assertimport(payload, initial_name, overwrite=True, last_update=prev_dataset.last_updated)
 
@@ -313,9 +311,8 @@ def test_import_csv_overwrite(self):
         initial_name = "valid"
         prev_dataset = self._assertimport(payload, initial_name, keep_resource=True)
 
-        payload = {"base_file": open(self.valid_csv, "rb"), "action": "upload"}
+        payload = {"base_file": open(self.valid_csv, "rb"), "action": "replace"}
         initial_name = "valid"
-        payload["overwrite_existing_layer"] = True
         payload["resource_pk"] = prev_dataset.pk
         self._assertimport(payload, initial_name, overwrite=True, last_update=prev_dataset.last_updated)
         self._cleanup_layers(name="valid")
@@ -340,8 +337,7 @@ def test_import_kml_overwrite(self):
         payload = {"base_file": open(self.valid_kml, "rb"), "action": "upload"}
         prev_dataset = self._assertimport(payload, initial_name, keep_resource=True)
 
-        payload = {"base_file": open(self.valid_kml, "rb"), "action": "upload"}
-        payload["overwrite_existing_layer"] = True
+        payload = {"base_file": open(self.valid_kml, "rb"), "action": "replace"}
         payload["resource_pk"] = prev_dataset.pk
         self._assertimport(payload, initial_name, overwrite=True, last_update=prev_dataset.last_updated)
         self._cleanup_layers(name="sample_point_dataset")
@@ -368,9 +364,8 @@ def test_import_shapefile_overwrite(self):
         initial_name = "air_Runways"
         prev_dataset = self._assertimport(payload, initial_name, keep_resource=True)
         payload = {_filename: open(_file, "rb") for _filename, _file in self.valid_shp.items()}
-        payload["overwrite_existing_layer"] = True
         payload["resource_pk"] = prev_dataset.pk
-        payload["action"] = "upload"
+        payload["action"] = "replace"
         self._assertimport(
             payload, initial_name, overwrite=True, last_update=prev_dataset.last_updated, keep_resource=True
         )
@@ -397,9 +392,8 @@ def test_import_raster_overwrite(self):
         payload = {"base_file": open(self.valid_tif, "rb"), "action": "upload"}
         prev_dataset = self._assertimport(payload, initial_name, keep_resource=True)
 
-        payload = {"base_file": open(self.valid_tif, "rb"), "action": "upload"}
+        payload = {"base_file": open(self.valid_tif, "rb"), "action": "replace"}
         initial_name = "test_raster"
-        payload["overwrite_existing_layer"] = True
         payload["resource_pk"] = prev_dataset.pk
         self._assertimport(payload, initial_name, overwrite=True, last_update=prev_dataset.last_updated)
         self._cleanup_layers(name="test_raster")
@@ -452,7 +446,7 @@ def test_import_3dtiles_overwrite(self):
             "subtype": "3dtiles",
             "resource_type": "dataset",
         }
-        payload["overwrite_existing_layer"] = True
+        payload["action"] = "replace"
         self._assertimport(
             payload,
             initial_name,

geonode/upload/tests/unit/test_task.py

@@ -80,7 +80,6 @@ def setUp(self):
             step="dummy_step",
             input_params={
                 "files": {"base_file": self.existing_file},
-                # "overwrite_existing_layer": True,
                 "store_spatial_files": True,
             },
         )
@@ -217,9 +216,9 @@ def test_publish_resource_if_overwrite_should_call_the_publishing(
                 user=get_user_model().objects.get(username=self.user),
                 func_name="dummy_func",
                 step="dummy_step",
+                action="replace",
                 input_params={
                     "files": {"base_file": self.existing_file},
-                    "overwrite_existing_layer": True,
                     "store_spatial_files": True,
                 },
             )
@@ -269,9 +268,9 @@ def test_publish_resource_if_overwrite_should_not_call_the_publishing(
                     user=get_user_model().objects.get(username=self.user),
                     func_name="dummy_func",
                     step="dummy_step",
+                    action="replace",
                     input_params={
                         "files": {"base_file": self.existing_file},
-                        "overwrite_existing_layer": True,
                         "store_spatial_files": True,
                     },
                 )
@@ -308,7 +307,6 @@ def test_create_geonode_resource(self, import_orchestrator):
             ExecutionRequest.objects.filter(exec_id=self.exec_id).update(
                 input_params={
                     "files": {"base_file": f"{_dir.name}/valid.gpkg"},
-                    # "overwrite_existing_layer": True,
                     "store_spatial_files": True,
                 }
             )
@@ -612,7 +610,6 @@ def setUp(self):
             step="dummy_step",
             input_params={
                 "files": {"base_file": self.existing_file},
-                # "overwrite_existing_layer": True,
                 "store_spatial_files": True,
             },
         )