Merge pull request #85 from pitangainnovare/add/metrics-exportation

Add/metrics exportation

Author: pitangainnovare

VERSION

@@ -1 +1 @@
-1.6.0
+1.7.0

article/models.py

@@ -91,7 +91,7 @@ class Article(CommonControlField):
     )
 
     def __str__(self):
-        return f'{self.collection.acron3} - {self.scielo_issn} - {self.pid_v2}'
+        return f'{self.collection.acron3} - {self.scielo_issn} - {self.pid_v2 or self.pid_v3 or self.pid_generic}'
 
     @classmethod
     def metadata(cls, collection=None):

config/settings/base.py

@@ -108,7 +108,6 @@
     "wagtailcaptcha",
     "wagtailmenus",
     "rest_framework",
-    "haystack",
 ]
 
 LOCAL_APPS = [
@@ -403,35 +402,11 @@
     "REFRESH_TOKEN_LIFETIME": timedelta(days=1),
 }
 
-# django haystack
-# ------------------------------------------------------------------------------
-HAYSTACK_CONNECTIONS = {
-    "default": {
-        "ENGINE": "haystack.backends.solr_backend.SolrEngine",
-        "URL": "%s%s" % (env("SOLR_URL", default="http://solr:8983/solr/"), "metrics"),
-        "ADMIN_URL": "http://solr:8983/solr/admin/cores",
-        "SILENTLY_FAIL": False,
-        "SOLR_TIMEOUT": 10,
-    },
-    "metrics": {
-        "ENGINE": "haystack.backends.solr_backend.SolrEngine",
-        "URL": "%s%s" % (env("SOLR_URL", default="http://solr:8983/solr/"), "metrics"),
-        "ADMIN_URL": "http://solr:8983/solr/admin/cores",
-        "SILENTLY_FAIL": False,
-        "SOLR_TIMEOUT": 10,
-    },
-    "log_manager": {
-        "ENGINE": "haystack.backends.solr_backend.SolrEngine",
-        "URL": "%s%s" % (env("SOLR_URL", default="http://solr:8983/solr/"), "log_manager"),
-        "ADMIN_URL": "http://solr:8983/solr/admin/cores",
-        "SILENTLY_FAIL": False,
-        "SOLR_TIMEOUT": 10,
-    },
-}
-
-USE_SOLR = env.bool("USE_SOLR", default=False)
-
-if USE_SOLR:
-    HAYSTACK_SIGNAL_PROCESSOR = "haystack.signals.RealtimeSignalProcessor"
-
 SEARCH_PAGINATION_ITEMS_PER_PAGE = 10
+
+# Elasticsearch
+# ------------------------------------------------------------------------------
+ES_URL = env("ES_URL", default="http://192.168.0.33:9200/")
+ES_INDEX_NAME = env("ES_INDEX_NAME", default="usage-daily")
+ES_API_KEY = env("ES_API_KEY", default="")
+ES_BASIC_AUTH = env("ES_BASIC_AUTH", default=("elastic", "iHktg66E"))

config/settings/production.py

@@ -198,10 +198,3 @@
     environment=env("SENTRY_ENVIRONMENT", default="production"),
     traces_sample_rate=env.float("SENTRY_TRACES_SAMPLE_RATE", default=0.0),
 )
-
-# Your stuff...
-# ------------------------------------------------------------------------------
-USE_SOLR = env.bool("USE_SOLR", default=True)
-
-if USE_SOLR:
-    HAYSTACK_SIGNAL_PROCESSOR = "haystack.signals.RealtimeSignalProcessor"

log_manager/migrations/0003_collectionlogfiledatecount_is_usage_metric_computed.py

@@ -0,0 +1,19 @@
+# Generated by Django 5.0.7 on 2025-05-18 15:48
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("log_manager", "0002_alter_collectionconfig_unique_together_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="collectionlogfiledatecount",
+            name="is_usage_metric_computed",
+            field=models.BooleanField(
+                default=False, verbose_name="Is Usage Metric Computed"
+            ),
+        ),
+    ]

log_manager/models.py

@@ -123,12 +123,18 @@ class CollectionLogFileDateCount(CommonControlField):
         blank=True,
         null=True,
     )
+
+    is_usage_metric_computed = models.BooleanField(
+        verbose_name=_('Is Usage Metric Computed'),
+        default=False,
+    )
 
     status = models.CharField(
         verbose_name=_('Status'),
         choices=choices.COLLECTION_LOG_FILE_DATE_COUNT,
         max_length=3,
     )
+
     def set_status(self):
         if self.found_log_files < self.expected_log_files:
             self.status = choices.COLLECTION_LOG_FILE_DATE_COUNT_MISSING_FILES
@@ -176,6 +182,7 @@ class Meta:
         FieldPanel('found_log_files'),
         FieldPanel('expected_log_files'),
         FieldPanel('status'),
+        FieldPanel('is_usage_metric_computed'),
     ]
 
     def __str__(self):

log_manager/tasks.py

@@ -10,7 +10,6 @@
 from core.utils.utils import _get_user
 from config import celery_app
 from collection.models import Collection
-from log_manager_config.models import CollectionValidationParameters
 from log_manager_config import exceptions as lmc_exceptions, models as lmc_models
 
 from . import (

log_manager/wagtail_hooks.py

@@ -37,15 +37,15 @@ class CollectionLogFileDateCountViewSet(SnippetViewSet):
     list_display = (
         "collection",
         "date",
-        "year",
-        "month",
         "found_log_files",
         "expected_log_files",
         "status",
+        "is_usage_metric_computed",
     )
     list_filter = (
         "collection",
         "status",
+        "is_usage_metric_computed",
         "year",
         "month"
     )

metrics/es.py

@@ -0,0 +1,240 @@
+from elasticsearch import Elasticsearch, helpers, NotFoundError
+from django.conf import settings
+
+import logging
+
+
+def get_elasticsearch_client(url=None, basic_auth=None, api_key=None):
+    """
+    Create an Elasticsearch client instance using Django settings.
+
+    :param url: Elasticsearch URL. If None, it will be taken from Django settings.
+    :param basic_auth: Basic authentication credentials. If None, it will be taken from Django settings.
+    :param api_key: API key. If None, it will be taken from Django settings.
+    """
+    if not url:
+        url = getattr(settings, "ES_URL", None)
+
+    if not basic_auth:
+        basic_auth = getattr(settings, "ES_BASIC_AUTH", None)
+
+    if not api_key:
+        api_key = getattr(settings, "ES_API_KEY", None)
+
+    if basic_auth:
+        client = Elasticsearch(url, basic_auth=basic_auth)
+    elif api_key:
+        client = Elasticsearch(url, api_key=api_key)
+    else:
+        client = Elasticsearch(url)
+
+    return client
+
+
+def create_index(index_name, mappings=None, client=None, url=None, basic_auth=None, api_key=None):
+    """
+    Create an Elasticsearch index. 
+
+    :param index_name: Name of the index to create.
+    :param mappings: Mappings for the index. If None, default mappings will be used.
+    :param client: Elasticsearch client instance. If None, a new client will be created.
+    :param url: Elasticsearch URL. If None, it will be taken from Django settings.
+    :param basic_auth: Basic authentication credentials. If None, it will be taken from Django settings.
+    :param api_key: API key. If None, it will be taken from Django settings.
+    """
+    if not client:
+        client = get_elasticsearch_client(url, basic_auth, api_key)
+
+    if not mappings:
+        mappings = {
+            "properties": {
+                "collection": {
+                    "type": "keyword"
+                },
+                "journal": {
+                    "type": "keyword"
+                },
+                "pid_v2": {
+                    "type": "keyword"
+                },
+                "pid_v3": {
+                    "type": "keyword"
+                },
+                "pid_generic": {
+                    "type": "keyword"
+                },
+                "media_language": {
+                    "type": "keyword"
+                },
+                "country_code": {
+                    "type": "keyword"
+                },
+                "date": {
+                    "type": "date",
+                    "format": "yyyy-MM-dd"
+                },
+                "year": {
+                    "type": "integer"
+                },
+                "month": {
+                    "type": "integer"
+                },
+                "day": {
+                    "type": "integer"
+                },
+                "total_requests": {
+                    "type": "integer"
+                },
+                "total_investigations": {
+                    "type": "integer"
+                },
+                "unique_requests": {
+                    "type": "integer"
+                },
+                "unique_investigations": {
+                    "type": "integer"
+                }
+            }
+        }
+
+    resp = client.indices.create(
+        index=index_name,
+        mappings=mappings,
+    )
+    logging.info(f"Index {index_name} created: {resp}")
+
+
+def delete_index(index_name, client=None, url=None, basic_auth=None, api_key=None):
+    """
+    Delete an Elasticsearch index.
+
+    :param index_name: Name of the index to delete.
+    :param client: Elasticsearch client instance. If None, a new client will be created.
+    :param url: Elasticsearch URL. If None, it will be taken from Django settings.
+    :param basic_auth: Basic authentication credentials. If None, it will be taken from Django settings.
+    :param api_key: API key. If None, it will be taken from Django settings.
+    """
+    if not client:
+        client = get_elasticsearch_client(url, basic_auth, api_key)
+    client.indices.delete(index=index_name)
+
+
+def index_document(index_name, doc_id, document, client=None, url=None, basic_auth=None, api_key=None):
+    """
+    Index a document in Elasticsearch.
+
+    :param index_name: Name of the index.
+    :param doc_id: ID of the document.
+    :param document: Document to index.
+    :param client: Elasticsearch client instance. If None, a new client will be created.
+    :param url: Elasticsearch URL. If None, it will be taken from Django settings.
+    :param basic_auth: Basic authentication credentials. If None, it will be taken from Django settings.
+    :param api_key: API key. If None, it will be taken from Django settings.
+    """
+    if not client:
+        client = get_elasticsearch_client(url, basic_auth, api_key)
+    client.index(index=index_name, id=doc_id, document=document)
+
+
+def index_documents(index_name, documents, client=None, url=None, basic_auth=None, api_key=None):
+    """
+    Index multiple documents in Elasticsearch.
+
+    :param index_name: Name of the index.
+    :param documents: Dictionary of documents to index, where keys are document IDs and values are the documents.
+    :param client: Elasticsearch client instance. If None, a new client will be created.
+    :param url: Elasticsearch URL. If None, it will be taken from Django settings.
+    :param basic_auth: Basic authentication credentials. If None, it will be taken from Django settings.
+    :param api_key: API key. If None, it will be taken from Django settings.
+    """
+    if not client:
+        client = get_elasticsearch_client(url, basic_auth, api_key)
+    
+    helpers.bulk(
+        client,
+        (
+            {
+                "_index": index_name,
+                "_id": doc_id,
+                "_source": document,
+            }
+            for doc_id, document in documents.items()
+        ),
+    )
+
+
+def delete_document(index_name, doc_id, client=None, url=None, basic_auth=None, api_key=None):
+    """
+    Delete a document from Elasticsearch.
+
+    :param index_name: Name of the index.
+    :param doc_id: ID of the document to delete.
+    :param client: Elasticsearch client instance. If None, a new client will be created.
+    :param url: Elasticsearch URL. If None, it will be taken from Django settings.
+    :param basic_auth: Basic authentication credentials. If None, it will be taken from Django settings.
+    :param api_key: API key. If None, it will be taken from Django settings.
+    """
+    if not client:
+        client = get_elasticsearch_client(url, basic_auth, api_key)
+
+    try:
+        client.delete(index=index_name, id=doc_id)
+    except NotFoundError as e:
+        logging.error(f"Failed to delete document {doc_id} from Elasticsearch: {e}")
+
+
+def delete_documents(index_name, doc_ids, client=None, url=None, basic_auth=None, api_key=None):
+    """
+    Delete multiple documents from Elasticsearch using bulk.
+    :param index_name: Name of the index.
+    :param doc_ids: List of document IDs to delete.
+    :param client: Elasticsearch client instance. If None, a new client will be created.
+    :param url: Elasticsearch URL. If None, it will be taken from Django settings.
+    :param basic_auth: Basic authentication credentials. If None, it will be taken from Django settings.
+    :param api_key: API key. If None, it will be taken from Django settings.
+    """
+    if not client:
+        client = get_elasticsearch_client(url, basic_auth, api_key)
+    
+    actions = (
+        {
+            "_op_type": "delete",
+            "_index": index_name,
+            "_id": doc_id,
+        }
+        for doc_id in doc_ids
+    )
+
+    try:
+        helpers.bulk(client, actions)
+    except helpers.BulkIndexError as e:
+        logging.error(f"BulkIndexError occurred: {e.errors}")
+
+
+def delete_documents_by_key(index_name, key, values, client=None, url=None, basic_auth=None, api_key=None):
+    """
+    Delete multiple documents from Elasticsearch based on a specific key and its values.
+
+    :param index_name: Name of the index.
+    :param key: Key to search for in the documents.
+    :param values: List of values to match against the key.
+    :param client: Elasticsearch client instance. If None, a new client will be created.
+    :param url: Elasticsearch URL. If None, it will be taken from Django settings.
+    :param basic_auth: Basic authentication credentials. If None, it will be taken from Django settings.
+    :param api_key: API key. If None, it will be taken from Django settings.
+    """
+    if not client:
+        client = get_elasticsearch_client(url, basic_auth, api_key)
+
+    query = {
+        "query": {
+            "terms": {
+                key: values
+            }
+        }
+    }
+
+    try:
+        client.delete_by_query(index=index_name, body=query)
+    except Exception as e:
+        logging.error(f"Failed to delete documents by key {key} with values {values}: {e}")