diff --git a/.gitignore b/.gitignore index 8bb0bd24..0a5f1aa8 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,7 @@ target/classes/* target/test-classes/* target/* -src/main/resources/lib/*.jar +src/main/resources/jars/*.jar siesta-query-processor.iml experiments/* diff --git a/Dockerfile b/Dockerfile index a91766f4..d1679aca 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,11 +2,11 @@ FROM ubuntu:20.04 #ENV JAVA_HOME="/usr/lib/jvm/default-jvm/" -RUN apt-get update && apt-get install -y openjdk-17-jdk maven && \ +RUN apt-get update && apt-get install -y openjdk-17-jdk maven wget && \ echo "export JAVA_HOME=$(dirname $(dirname $(readlink -f $(which java))))" >> /etc/profile.d/java.sh ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64 ENV PATH=$PATH:${JAVA_HOME}/bin - +ENV JARS_DIR=/code/src/main/resources/jars # Install maven @@ -20,8 +20,15 @@ RUN mvn dependency:resolve # Adding source, compile and package into a fat jar ADD src /code/src +RUN test -f ${JARS_DIR}/hadoop-aws-3.3.4.jar || wget -P ${JARS_DIR} https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.3.4/hadoop-aws-3.3.4.jar +RUN test -f ${JARS_DIR}/aws-java-sdk-bundle-1.12.262.jar || wget -P ${JARS_DIR} https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/1.12.262/aws-java-sdk-bundle-1.12.262.jar +RUN test -f ${JARS_DIR}/hadoop-client-3.3.4.jar || wget -P ${JARS_DIR} https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-client/3.3.4/hadoop-client-3.3.4.jar +RUN test -f ${JARS_DIR}/delta-spark_2.12-3.3.0.jar || wget -P ${JARS_DIR} https://repo1.maven.org/maven2/io/delta/delta-spark_2.12/3.3.0/delta-spark_2.12-3.3.0.jar +RUN test -f ${JARS_DIR}/delta-storage-3.3.0.jar || wget -P ${JARS_DIR} https://repo1.maven.org/maven2/io/delta/delta-storage/3.3.0/delta-storage-3.3.0.jar RUN mvn clean compile package -f pom.xml -DskipTests +# Making sure jars are where they should be + CMD ["java", "--add-exports", "java.base/sun.nio.ch=ALL-UNNAMED" , "-jar", "target/siesta-query-processor-3.0.jar"] #ENTRYPOINT ["tail", "-f", "/dev/null"] diff --git a/README.md b/README.md index 546d63df..93e75041 100644 --- a/README.md +++ b/README.md @@ -41,37 +41,37 @@ To run it locally specify the properties in the application.properties file in t and then run the project after adding in the ``Add VM options`` under ``Run/Edit configurations`` the following line \ ``--add-opens=java.base/sun.nio.ch=ALL-UNNAMED `` -### Running in docker -To run it locally open a terminal inside SequenceDetectionQueryExecutor file and run: -```bash -docker-compose build -docker-compose up -d -``` +### Running with Docker + +#### Single Machine Deployment (Development/Testing) + +To run everything on one machine: + Ensure that this docker and the database can communicate, either run database on a public ip or connect these two on the same network. You need to specify these environment variables (you can keep the default ones if you want) in the docker-compose file before running the QueryExecutor. + +#### Distributed Cluster Deployment (Production) + +For distributed processing with workers on different machines, see: +- **[QUICKSTART.md](QUICKSTART.md)** - Quick setup guide +- **[CLUSTER_DEPLOYMENT.md](CLUSTER_DEPLOYMENT.md)** - Comprehensive deployment documentation + +Quick setup: + +**On Master Machine:** +```bash +./setup-master.sh ``` - master.uri: local[*] - database: s3 - delta: false # True for streaming, False for batching - #for s3 (minio) - s3.endpoint: http://minio:9000 - s3.user: minioadmin - s3.key: minioadmin - s3.timeout: 600000 - server.port: 8090 -``` -### SIESTA Query type list -Below there is a list of all the possible SIESTA queries along with an example JSON or an example url assuming the -Query Processor is running on localhost:8090 -* GET /health/check (Checks if the application is up and running) -* GET /lognames (Returns the names of the different log databases) -* POST /eventTypes (Returns the names of the different event types for a specific log database) \ - Example JSON: + +**On Worker Machines:** +```bash +./setup-worker.sh ``` -{ - "log_name" : "test" + +This will deploy a true distributed Spark cluster with workers running on separate physical machines for better scalability and performance. + } ``` * GET /refreshData (Reloads metadata, this should run after a new log file is appended) diff --git a/docker-compose-swarm.yml b/docker-compose-swarm.yml new file mode 100644 index 00000000..9922e127 --- /dev/null +++ b/docker-compose-swarm.yml @@ -0,0 +1,154 @@ +version: '3' + +networks: + siesta-swarm-net: + +volumes: + maven-cache: + minio-storage: + +services: + minio: + image: minio/minio:latest + container_name: minio + environment: + MINIO_ROOT_USER: minioadmin + MINIO_ROOT_PASSWORD: minioadmin + command: server /data + ports: + - "9000:9000" + - "9001:9001" + volumes: + - minio_storage:/data + networks: + - siesta-swarm-net + deploy: + replicas: 1 + placement: + constraints: + - 'node.labels.host == m1' + + scylla: + container_name: scylla + image: scylladb/scylla:latest + ports: + - "9042:9042" + command: --smp 8 --memory 32G --reserve-memory 2G --overprovisioned 1 --api-address 0.0.0.0 + volumes: + - scylla_data:/var/lib/scylla + networks: + - siesta-net + healthcheck: + test: ["CMD-SHELL", "nodetool status"] + interval: 15s + timeout: 15s + retries: 5 + deploy: + replicas: 1 + restart_policy: + condition: on-failure + max_attempts: 3 + placement: + constraints: + - 'node.labels.host == m1' + + query: + build: . + image: siesta-query:latest + stdin_open: true + networks: + - siesta-swarm-net + ports: + - '8090:8090' + environment: + master.uri: spark://spark:7077 + server.port: 8090 + database: s3 # cassandra or s3 + delta: "false" # True for streaming, False for batching + spring.mvc.pathmatch.matching-strategy: ANT_PATH_MATCHER + #for s3 (minio) + s3.endpoint: http://minio:9000 + s3.user: minioadmin + s3.key: minioadmin + s3.timeout: 600000 + # Scylla/Cassandra configuration + cassandra.contact.points: scylla + cassandra.port: 9042 + cassandra.keyspace: siesta + volumes: + - maven-cache:/root/.m2 + deploy: + replicas: 1 + placement: + constraints: + - 'node.labels.host == m1' + + spark: + image: spark-base:3.5.4 + networks: + - siesta-swarm-net + environment: + - SPARK_MODE=master + - SPARK_MASTER_MEMORY=16G + - SPARK_MASTER_CORES=4 + - SPARK_RPC_AUTHENTICATION_ENABLED=no + - SPARK_RPC_ENCRYPTION_ENABLED=no + - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no + - SPARK_SSL_ENABLED=no + - SPARK_USER=spark + - LOGS_PATH=/tmp/spark-events + - RESULTS_PATH=/tmp/output/ + - SPARK_MASTER_WEBUI_PORT=8085 + ports: + - '8085:8085' + - '4040:4040' + - '8081:8081' + deploy: + replicas: 1 + placement: + constraints: + - 'node.labels.host == m1' + + spark-worker-1: + image: spark-base:3.5.4 + networks: + - siesta-swarm-net + environment: + - SPARK_MODE=worker + - SPARK_MASTER_URL=spark://spark:7077 + - SPARK_WORKER_MEMORY=8G + - SPARK_WORKER_CORES=4 + - SPARK_RPC_AUTHENTICATION_ENABLED=no + - SPARK_RPC_ENCRYPTION_ENABLED=no + - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no + - SPARK_SSL_ENABLED=no + - SPARK_USER=spark + #- LOGS_PATH=/tmp/spark-events + #- RESULTS_PATH=/tmp/output/ + deploy: + replicas: 1 + placement: + constraints: + - 'node.labels.host == m2' + + spark-worker-anaconda-2: + image: spark-base:3.5.4 + networks: + - siesta-swarm-net + environment: + - SPARK_MODE=worker + - SPARK_MASTER_URL=spark://spark:7077 + - SPARK_WORKER_MEMORY=8G + - SPARK_WORKER_CORES=4 + - SPARK_RPC_AUTHENTICATION_ENABLED=no + - SPARK_RPC_ENCRYPTION_ENABLED=no + - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no + - SPARK_SSL_ENABLED=no + - SPARK_USER=spark + #- LOGS_PATH=/tmp/spark-events + #- RESULTS_PATH=/tmp/output/ + deploy: + replicas: 1 + placement: + constraints: + - 'node.labels.host == m2' diff --git a/docker-compose.yml b/docker-compose.yml index ac1a1ce9..8eca92b9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,78 +1,116 @@ -version: '3.7' +version: '3.8' networks: - cluster_net: - external: true + siesta-net: name: siesta-net + external: true + cluster-net: + name: cluster-net + driver: bridge +volumes: + minio_storage: {} services: - funnel: + minio: + image: minio/minio:latest + container_name: minio + environment: + MINIO_ROOT_USER: minioadmin + MINIO_ROOT_PASSWORD: minioadmin + command: server /data + ports: + - "9000:9000" + - "9001:9001" + volumes: + - minio_storage:/data + networks: + - siesta-net + + scylla: + container_name: scylla + image: scylladb/scylla:latest + ports: + - "9042:9042" + command: --smp 8 --memory 32G --reserve-memory 2G --overprovisioned 1 --api-address 0.0.0.0 + volumes: + - scylla_data:/var/lib/scylla + networks: + - siesta-net + healthcheck: + test: ["CMD-SHELL", "nodetool status"] + interval: 15s + timeout: 15s + retries: 5 + + query: build: . + container_name: query stdin_open: true environment: - master.uri: local[*] - database: s3 # cassandra-rdd or s3 - delta: false # True for streaming, False for batching - #for s3 (minio) + master.uri: spark://spark-master:7077 + server.port: 8090 # port of the application + database: s3 # cassandra or s3 + delta: "false" # True for streaming, False for batching + spring.mvc.pathmatch.matching-strategy: ANT_PATH_MATCHER + # S3 (minio) s3.endpoint: http://minio:9000 s3.user: minioadmin s3.key: minioadmin s3.timeout: 600000 - #for cassandra - cassandra.max_requests_per_local_connection: 32768 - cassandra.max_requests_per_remote_connection: 22000 - cassandra.connections_per_host: 1000 - cassandra.max_queue_size: 1024 - cassandra.connection_timeout: 30000 - cassandra.read_timeout: 30000 - spring.data.cassandra.contact-points: cassandra - spring.data.cassandra.port: 9042 - spring.data.cassandra.user: cassandra - spring.data.cassandra.password: cassandra - server.port: 8090 # port of the application + # Scylla/Cassandra + cassandra.contact.points: scylla + cassandra.port: 9042 + cassandra.keyspace: siesta volumes: - ./build:/root/.m2 ports: - - '8090:8090' + - '8090:8090' # Application port + networks: + - siesta-net + - cluster-net + + spark-master: + build: + context: . + dockerfile: spark-base.Dockerfile + image: spark-base:3.5.4 + container_name: spark-master + hostname: spark-master + environment: + - SPARK_MODE=master + - SPARK_MASTER_HOST=0.0.0.0 + - SPARK_MASTER_PORT=7077 + - SPARK_MASTER_WEBUI_PORT=8080 + - SPARK_RPC_AUTHENTICATION_ENABLED=no + - SPARK_RPC_ENCRYPTION_ENABLED=no + ports: + - "7077:7077" + - "8080:8080" networks: - - cluster_net + - cluster-net + - siesta-net -# spark-master: -# image: bitnami/spark:3.5.4 -# container_name: spark-master -# environment: -# - SPARK_MODE=master -# - SPARK_RPC_AUTHENTICATION_ENABLED=no -# - SPARK_RPC_ENCRYPTION_ENABLED=no -# - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no -# - SPARK_SSL_ENABLED=no -# ports: -# - "7077:7077" # Spark master port -# - "8080:8080" # Spark Web UI -# networks: -# - cluster_net -# -# spark-worker-1: -# image: bitnami/spark:3.5.4 -# container_name: spark-worker-1 -# environment: -# - SPARK_MODE=worker -# - SPARK_MASTER_URL=spark://spark-master:7077 -# - SPARK_WORKER_MEMORY=1G -# - SPARK_WORKER_CORES=1 -# depends_on: -# - spark-master -# networks: -# - cluster_net -# -# spark-worker-2: -# image: bitnami/spark:3.5.4 -# container_name: spark-worker-2 -# environment: -# - SPARK_MODE=worker -# - SPARK_MASTER_URL=spark://spark-master:7077 -# - SPARK_WORKER_MEMORY=1G -# - SPARK_WORKER_CORES=1 -# depends_on: -# - spark-master -# networks: -# - cluster_net + spark-worker-1: + image: spark-base:3.5.4 + container_name: spark-worker-1 + environment: + - SPARK_MODE=worker + - SPARK_MASTER_URL=spark://spark-master:7077 + - SPARK_WORKER_MEMORY=2G + - SPARK_WORKER_CORES=2 + depends_on: + - spark-master + networks: + - cluster-net + + spark-worker-2: + image: spark-base:3.5.4 + container_name: spark-worker-2 + environment: + - SPARK_MODE=worker + - SPARK_MASTER_URL=spark://spark-master:7077 + - SPARK_WORKER_MEMORY=2G + - SPARK_WORKER_CORES=2 + depends_on: + - spark-master + networks: + - cluster-net diff --git a/generate_queries.py b/generate_queries.py new file mode 100644 index 00000000..eef45356 --- /dev/null +++ b/generate_queries.py @@ -0,0 +1,433 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Query Generation and Transformation Pipeline + +This script provides a complete pipeline for generating queries from XES event logs +and transforming them into JSON format with random symbol assignments. + +Features: + - Generate query patterns from XES log files + - Extract sequences of specified lengths from log traces + - Transform queries to JSON format with symbol assignments + - Configurable query lengths and sample sizes + +Usage: + # Generate queries from XES files and convert to JSON + python3 generate_queries.py --mode both --input-dir input --lengths 3,10 --samples 100 + + # Only generate .q query files + python3 generate_queries.py --mode generate --input-dir input --lengths 3,10 --samples 100 + + # Only convert existing .q files to JSON + python3 generate_queries.py --mode transform --query-dir queries --output-dir queries_json + +Arguments: + --mode: Operation mode ('generate', 'transform', or 'both') + --input-dir: Directory containing .xes files (default: 'input') + --query-dir: Directory for .q query files (default: 'queries') + --output-dir: Directory for .jsonl output files (default: 'queries_json') + --lengths: Comma-separated list of query lengths (default: '3,10') + --samples: Number of samples per length (default: 100) + +Output Formats: + .q files: LENGTH,EVENT1,EVENT2,... + .jsonl files: {"log_name": "...", "pattern": {"eventsWithSymbols": [...]}} +""" + +import argparse +import json +import os +import random +from typing import List, Tuple +from statistics import mean, stdev + +try: + from pm4py.objects.log.importer.xes import importer as xes_import_factory + PM4PY_AVAILABLE = True +except ImportError: + PM4PY_AVAILABLE = False + print("Warning: pm4py not available. Query generation from XES files will not work.") + + +# Symbol options for query transformation +SYMBOLS = ["_", "*", "||", "+"] + + +# ============================================================================ +# QUERY GENERATION FROM XES FILES +# ============================================================================ + +def generate_query_file(lengths: List[int], samples_per_length: int, log, log_filename: str, output_dir: str): + """ + Generate query file from an XES event log. + + For each specified length, extracts 'samples_per_length' random sequences + from traces in the log that are at least that long. + + Args: + lengths: List of sequence lengths to generate + samples_per_length: Number of query samples to generate per length + log: Parsed XES log object from pm4py + log_filename: Name of the source log file (used for output naming) + output_dir: Directory where .q files will be written + + Returns: + Path to the generated query file + """ + os.makedirs(output_dir, exist_ok=True) + query_file_path = os.path.join(output_dir, log_filename + ".q") + + total_queries = 0 + with open(query_file_path, "w", encoding="utf-8") as file: + for length in lengths: + queries_generated = 0 + attempts = 0 + max_attempts = samples_per_length * 100 # Prevent infinite loops + + while queries_generated < samples_per_length and attempts < max_attempts: + attempts += 1 + # Pick a random trace + trace_idx = random.randint(0, len(log) - 1) + trace = log[trace_idx] + + # Check if trace is long enough + if len(trace) >= length: + # Extract event names for the first 'length' events + event_names = [event["concept:name"] for event in trace][:length] + # Write to file: LENGTH,EVENT1,EVENT2,... + file.write(str(length) + "," + ",".join(event_names) + "\n") + queries_generated += 1 + total_queries += 1 + + if queries_generated < samples_per_length: + print(f" Warning: Only generated {queries_generated}/{samples_per_length} queries " + f"for length {length} (insufficient long traces)") + + return query_file_path, total_queries + + +def generate_queries_from_xes_files(input_dir: str, query_dir: str, lengths: List[int], + samples_per_length: int, force: bool = False): + """ + Process all XES files in input directory and generate query files. + + Args: + input_dir: Directory containing .xes files + query_dir: Directory where .q query files will be written + lengths: List of query lengths to generate + samples_per_length: Number of samples per length + force: If True, regenerate even if query file already exists + + Returns: + Number of query files generated + """ + if not PM4PY_AVAILABLE: + print("Error: pm4py is required for query generation. Install with: pip install pm4py") + return 0 + + if not os.path.isdir(input_dir): + print(f"Input directory does not exist: {input_dir}") + return 0 + + xes_files = [f for f in os.listdir(input_dir) if f.endswith('.xes')] + if not xes_files: + print(f"No .xes files found in {input_dir}") + return 0 + + print(f"Found {len(xes_files)} XES file(s) in {input_dir}") + print(f"Generating queries with lengths {lengths}, {samples_per_length} samples per length\n") + + files_generated = 0 + for xes_file in sorted(xes_files): + query_file_path = os.path.join(query_dir, xes_file + ".q") + + if os.path.exists(query_file_path) and not force: + print(f"Skipping {xes_file} (query file already exists)") + continue + + print(f"Processing: {xes_file}") + try: + # Import XES log + log_path = os.path.join(input_dir, xes_file) + log = xes_import_factory.apply(log_path) + print(f" Loaded log with {len(log)} traces") + + # Generate queries + output_path, total_queries = generate_query_file( + lengths, samples_per_length, log, xes_file, query_dir + ) + print(f" Generated {total_queries} queries -> {output_path}\n") + files_generated += 1 + + except Exception as e: + print(f" Error processing {xes_file}: {e}\n") + continue + + return files_generated + + +# ============================================================================ +# QUERY TRANSFORMATION TO JSON +# ============================================================================ + +def transform_query_line(line: str) -> dict: + """ + Parse a single query line and return the pattern object with symbols. + + Input format: LENGTH,EVENT1,EVENT2,... + We use the events for ordering/positions and assign random symbols. + + Symbol assignment rules: + - Majority of events get "_" (underscore) + - At most 2 events get non-underscore symbols ("+", "*", "||") + - Ensures semantic variety in query patterns + + Args: + line: A line from a .q file + + Returns: + Dictionary with "eventsWithSymbols" list, or None if line is invalid + """ + line = line.strip() + if not line: + return None + + parts = line.split(",") + if len(parts) < 2: + return None + + # First part is length, rest are event names + events = parts[1:] + + # Build initial pattern with all "_" symbols + pattern_events = [] + for idx, event_name in enumerate(events): + event_name = event_name.strip() + if event_name == "": + continue + pattern_events.append({ + "name": event_name, + "position": idx, + "symbol": "_", + }) + + if not pattern_events: + return None + + # Assign non-underscore symbols to at most 2 events + # Ensure majority remain "_" + num_events = len(pattern_events) + max_non_underscore = min(2, (num_events - 1) // 2) + + if max_non_underscore > 0: + # Randomly select 1 to max_non_underscore positions + num_to_change = random.randint(1, max_non_underscore) + positions_to_change = random.sample(range(num_events), num_to_change) + non_underscore_symbols = [s for s in SYMBOLS if s != "_"] + + for pos in positions_to_change: + pattern_events[pos]["symbol"] = random.choice(non_underscore_symbols) + + return {"eventsWithSymbols": pattern_events} + + +def transform_query_file(src_path: str, dst_path: str) -> int: + """ + Transform a single .q query file into .jsonl format. + + Args: + src_path: Path to input .q file + dst_path: Path to output .jsonl file + + Returns: + Number of queries written + """ + # Derive log_name from filename + basename = os.path.basename(src_path) + # Strip .q extension + log_name = basename[:-2] if basename.endswith('.q') else basename + # Also strip .xes if present (e.g., 'bpi_2017.xes.q' -> 'bpi_2017') + if log_name.endswith('.xes'): + log_name = log_name[:-4] + + written = 0 + with open(src_path, "r", encoding="utf-8") as src, \ + open(dst_path, "w", encoding="utf-8") as dst: + + for line in src: + pattern = transform_query_line(line) + if pattern is None: + continue + + # Create JSON object + obj = { + "log_name": log_name, + "pattern": pattern, + } + dst.write(json.dumps(obj) + "\n") + written += 1 + + return written + + +def transform_all_query_files(query_dir: str, output_dir: str) -> Tuple[int, int]: + """ + Transform all .q files in query directory to .jsonl format. + + Args: + query_dir: Directory containing .q files + output_dir: Directory where .jsonl files will be written + + Returns: + Tuple of (files_processed, total_queries_transformed) + """ + if not os.path.isdir(query_dir): + print(f"Query directory does not exist: {query_dir}") + return 0, 0 + + os.makedirs(output_dir, exist_ok=True) + + q_files = [f for f in os.listdir(query_dir) if f.endswith('.q')] + if not q_files: + print(f"No .q files found in {query_dir}") + return 0, 0 + + print(f"Found {len(q_files)} .q file(s) in {query_dir}") + print(f"Transforming to JSON format with random symbols\n") + + files_processed = 0 + total_queries = 0 + + for q_file in sorted(q_files): + src_path = os.path.join(query_dir, q_file) + dst_filename = q_file + ".jsonl" + dst_path = os.path.join(output_dir, dst_filename) + + try: + written = transform_query_file(src_path, dst_path) + print(f"Transformed {written} queries: {q_file} -> {dst_filename}") + files_processed += 1 + total_queries += written + except Exception as e: + print(f"Error processing {q_file}: {e}") + continue + + return files_processed, total_queries + + +# ============================================================================ +# MAIN PIPELINE +# ============================================================================ + +def main(): + parser = argparse.ArgumentParser( + description="Generate queries from XES logs and transform to JSON format", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Full pipeline: generate and transform + python3 generate_queries.py --mode both --input-dir input --lengths 3,10 --samples 100 + + # Only generate .q files from XES logs + python3 generate_queries.py --mode generate --input-dir input + + # Only transform existing .q files to JSON + python3 generate_queries.py --mode transform --query-dir queries + + # Custom configuration + python3 generate_queries.py --mode both --lengths 3,5,10,15 --samples 50 --force + """ + ) + + parser.add_argument( + '--mode', + choices=['generate', 'transform', 'both'], + default='both', + help='Operation mode: generate .q files, transform to JSON, or both (default: both)' + ) + parser.add_argument( + '--input-dir', + default='input', + help='Directory containing .xes files (default: input)' + ) + parser.add_argument( + '--query-dir', + default='queries', + help='Directory for .q query files (default: queries)' + ) + parser.add_argument( + '--output-dir', + default='queries_json', + help='Directory for .jsonl output files (default: queries_json)' + ) + parser.add_argument( + '--lengths', + default='3,10', + help='Comma-separated list of query lengths (default: 3,10)' + ) + parser.add_argument( + '--samples', + type=int, + default=100, + help='Number of query samples per length (default: 100)' + ) + parser.add_argument( + '--force', + action='store_true', + help='Force regeneration of existing query files' + ) + + args = parser.parse_args() + + # Parse lengths + try: + lengths = [int(x.strip()) for x in args.lengths.split(',')] + except ValueError: + print(f"Error: Invalid lengths format '{args.lengths}'. Use comma-separated integers.") + return 1 + + # Convert to absolute paths + input_dir = os.path.abspath(args.input_dir) + query_dir = os.path.abspath(args.query_dir) + output_dir = os.path.abspath(args.output_dir) + + print("=" * 70) + print("QUERY GENERATION AND TRANSFORMATION PIPELINE") + print("=" * 70) + print(f"Mode: {args.mode}") + print(f"Input directory: {input_dir}") + print(f"Query directory: {query_dir}") + print(f"Output directory: {output_dir}") + print(f"Query lengths: {lengths}") + print(f"Samples per length: {args.samples}") + print("=" * 70) + print() + + # Execute based on mode + if args.mode in ['generate', 'both']: + print("STEP 1: Generating queries from XES files") + print("-" * 70) + files_generated = generate_queries_from_xes_files( + input_dir, query_dir, lengths, args.samples, args.force + ) + print(f"Summary: Generated query files from {files_generated} XES log(s)") + print() + + if args.mode in ['transform', 'both']: + print("STEP 2: Transforming queries to JSON format") + print("-" * 70) + files_processed, total_queries = transform_all_query_files(query_dir, output_dir) + print() + print(f"Summary: Processed {files_processed} file(s), transformed {total_queries} queries") + print() + + print("=" * 70) + print("Pipeline completed successfully!") + print("=" * 70) + + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/pom.xml b/pom.xml index ae366779..8292b553 100644 --- a/pom.xml +++ b/pom.xml @@ -223,11 +223,36 @@ - - - - - + + io.delta + delta-core_2.12 + 2.1.0 + + + + + com.datastax.spark + spark-cassandra-connector_2.12 + 3.4.1 + + + + com.datastax.oss + java-driver-core + 4.17.0 + + + + com.amazonaws + aws-java-sdk-bundle + 1.12.262 + + + + com.datastax.oss + java-driver-query-builder + 4.17.0 + io.delta @@ -239,6 +264,16 @@ + + org.apache.maven.plugins + maven-compiler-plugin + 3.11.0 + + 17 + 17 + 17 + + org.springframework.boot spring-boot-maven-plugin diff --git a/spark-base.Dockerfile b/spark-base.Dockerfile new file mode 100644 index 00000000..68f840f6 --- /dev/null +++ b/spark-base.Dockerfile @@ -0,0 +1,13 @@ +FROM public.ecr.aws/bitnami/spark:3.5.4 + +USER root + +# Install curl (Bitnami images are Debian-based) +RUN apt-get update && apt-get install -y curl && rm -rf /var/lib/apt/lists/* + +# Download AWS + Hadoop JARs for S3 support +RUN mkdir -p /opt/bitnami/spark/jars && \ + curl -L -o /opt/bitnami/spark/jars/hadoop-aws-3.3.4.jar https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.3.4/hadoop-aws-3.3.4.jar && \ + curl -L -o /opt/bitnami/spark/jars/aws-java-sdk-bundle-1.12.262.jar https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/1.12.262/aws-java-sdk-bundle-1.12.262.jar + +USER 1001 diff --git a/src/main/java/com/datalab/siesta/queryprocessor/SaseConnection/SaseConnector.java b/src/main/java/com/datalab/siesta/queryprocessor/SaseConnection/SaseConnector.java index 9971a781..476d1519 100644 --- a/src/main/java/com/datalab/siesta/queryprocessor/SaseConnection/SaseConnector.java +++ b/src/main/java/com/datalab/siesta/queryprocessor/SaseConnection/SaseConnector.java @@ -64,10 +64,18 @@ public List evaluate(SIESTAPattern pattern, Map Occurrences ocs = new Occurrences(); ocs.setTraceID(e.getKey()); for (Match m : ec.getMatches()) { - ocs.addOccurrence(new Occurrence(Arrays.stream(m.getEvents()).parallel() - .map(x -> (SaseEvent) x) - .map(SaseEvent::getEventBoth) - .collect(Collectors.toList()))); + List eventBothList = new ArrayList<>(); + try { + for (edu.umass.cs.sase.stream.Event event : m.getEvents()) { + SaseEvent sEvent = (SaseEvent) event; + EventBoth both = sEvent.getEventBoth(); + eventBothList.add(both); + } + Occurrence oc = new Occurrence(eventBothList); + ocs.addOccurrence(oc); + }catch (Exception ignored){ + + } } occurrences.add(ocs); } @@ -98,10 +106,18 @@ public List evaluateGroups(SIESTAPattern pattern, Map (SaseEvent) x) - .map(SaseEvent::getEventBoth) - .collect(Collectors.toList()))); + List eventBothList = new ArrayList<>(); + try { + for (edu.umass.cs.sase.stream.Event event : m.getEvents()) { + SaseEvent sEvent = (SaseEvent) event; + EventBoth both = sEvent.getEventBoth(); + eventBothList.add(both); + } + Occurrence oc = new Occurrence(eventBothList); + ocs.addOccurrence(oc); + }catch (Exception ignored){ + + } } occurrences.add(ocs); } @@ -132,10 +148,18 @@ public List evaluateSmallPatterns(SIESTAPattern pattern, Map (SaseEvent) x) - .map(SaseEvent::getEventBoth) - .collect(Collectors.toList()))); + List eventBothList = new ArrayList<>(); + try { + for (edu.umass.cs.sase.stream.Event event : m.getEvents()) { + SaseEvent sEvent = (SaseEvent) event; + EventBoth both = sEvent.getEventBoth(); + eventBothList.add(both); + } + Occurrence oc = new Occurrence(eventBothList); + ocs.addOccurrence(oc); + }catch (Exception ignored){ + + } } occurrences.add(ocs); } diff --git a/src/main/java/com/datalab/siesta/queryprocessor/controllers/PatternAnalysisController.java b/src/main/java/com/datalab/siesta/queryprocessor/controllers/PatternAnalysisController.java index 6742273e..305dbfcb 100644 --- a/src/main/java/com/datalab/siesta/queryprocessor/controllers/PatternAnalysisController.java +++ b/src/main/java/com/datalab/siesta/queryprocessor/controllers/PatternAnalysisController.java @@ -176,7 +176,7 @@ private Tuple2, List> splitLogInstances(String log_da List legitimateInstances = new ArrayList<>(); List illegitimateInstances = new ArrayList<>(); indexRecords.forEach((et, instances) -> instances.forEach(instance -> { - if (legitimateTraces.contains(instance.getTraceId())) { + if (legitimateTraces.contains(instance.getTrace_id())) { legitimateInstances.add(instance); } else { illegitimateInstances.add(instance); @@ -192,8 +192,8 @@ private Tuple2, List> splitLogInstances(String log_da */ private double getCV(Count pair) { double norm_factor = 1.0 / pair.getCount(); - double mean = (double) pair.getSum_duration() / pair.getCount(); - double var = norm_factor * (pair.getSum_squares() - norm_factor * Math.pow(pair.getSum_duration(), 2)); + double mean = (double) pair.getSumDuration() / pair.getCount(); + double var = norm_factor * (pair.getSumSquares() - norm_factor * Math.pow(pair.getSumDuration(), 2)); return var / mean; } diff --git a/src/main/java/com/datalab/siesta/queryprocessor/declare/DeclareDBConnector.java b/src/main/java/com/datalab/siesta/queryprocessor/declare/DeclareDBConnector.java index e61f9834..cd446600 100644 --- a/src/main/java/com/datalab/siesta/queryprocessor/declare/DeclareDBConnector.java +++ b/src/main/java/com/datalab/siesta/queryprocessor/declare/DeclareDBConnector.java @@ -1,31 +1,22 @@ package com.datalab.siesta.queryprocessor.declare; -import com.datalab.siesta.queryprocessor.declare.model.EventPairToTrace; -import com.datalab.siesta.queryprocessor.declare.model.EventSupport; -import com.datalab.siesta.queryprocessor.declare.model.OccurrencesPerTrace; -import com.datalab.siesta.queryprocessor.declare.model.UniqueTracesPerEventPair; -import com.datalab.siesta.queryprocessor.declare.model.UniqueTracesPerEventType; +import com.datalab.siesta.queryprocessor.declare.model.*; import com.datalab.siesta.queryprocessor.declare.model.declareState.ExistenceState; import com.datalab.siesta.queryprocessor.declare.model.declareState.NegativeState; import com.datalab.siesta.queryprocessor.declare.model.declareState.OrderState; import com.datalab.siesta.queryprocessor.declare.model.declareState.PositionState; import com.datalab.siesta.queryprocessor.declare.model.declareState.UnorderStateI; import com.datalab.siesta.queryprocessor.declare.model.declareState.UnorderStateU; -import com.datalab.siesta.queryprocessor.model.DBModel.IndexPair; -import com.datalab.siesta.queryprocessor.model.DBModel.Trace; -import com.datalab.siesta.queryprocessor.model.Events.EventBoth; -import com.datalab.siesta.queryprocessor.model.Events.EventPos; +import com.datalab.siesta.queryprocessor.storage.model.EventTypeTracePositions; +import com.datalab.siesta.queryprocessor.storage.model.Trace; import com.datalab.siesta.queryprocessor.storage.DatabaseRepository; -import org.apache.spark.api.java.JavaPairRDD; -import org.apache.spark.api.java.JavaRDD; + +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.functions; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import scala.Tuple2; -import scala.Tuple3; - -import java.util.List; -import java.util.Map; @Service public class DeclareDBConnector { @@ -37,62 +28,60 @@ public DeclareDBConnector(DatabaseRepository databaseRepository){ this.db=databaseRepository; } - public JavaRDD querySequenceTableDeclare(String logName){ + public Dataset querySequenceTableDeclare(String logName){ return db.querySequenceTableDeclare(logName); } - public JavaRDD querySingleTableDeclare(String logname){ + public Dataset querySingleTableDeclare(String logname){ return this.db.querySingleTableDeclare(logname); } - public JavaRDD querySingleTable(String logname){ + public Dataset querySingleTable(String logname){ return this.db.querySingleTable(logname); } - public JavaRDD queryIndexTableDeclare(String logname){ + public Dataset queryIndexTableDeclare(String logname){ return this.db.queryIndexTableDeclare(logname); } - public JavaRDD queryIndexTableAllDeclare(String logname){ - return this.db.queryIndexTableAllDeclare(logname); - } - - public JavaPairRDD, List> querySingleTableAllDeclare(String logname){ + public Dataset querySingleTableAllDeclare(String logname){ return this.db.querySingleTableAllDeclare(logname); } - public JavaRDD queryIndexOriginalDeclare(String logname){ + public Dataset queryIndexOriginalDeclare(String logname){ return this.db.queryIndexOriginalDeclare(logname); } - public Map extractTotalOccurrencesPerEventType(String logname){ - return this.querySingleTableDeclare(logname) - .map(x -> { - long all = x.getOccurrences().stream().mapToLong(OccurrencesPerTrace::getOccurrences).sum(); - return new Tuple2<>(x.getEventType(), all); - }).keyBy(x -> x._1).mapValues(x -> x._2).collectAsMap(); + public Dataset extractTotalOccurrencesPerEventType(String logname){ + Dataset eventTypeOccurrencesDataset= this.querySingleTableDeclare(logname) + .withColumn("numberOfTraces", functions.expr( + "aggregate(occurrences, 0, (acc, a) -> acc + a.occs)" + )) + .selectExpr("eventName", "numberOfTraces") + .as(Encoders.bean(EventTypeOccurrences.class)); + return eventTypeOccurrencesDataset; } - public JavaRDD queryPositionState(String logname){ + public Dataset queryPositionState(String logname){ return this.db.queryPositionState(logname); } - public JavaRDD queryExistenceState(String logname){ + public Dataset queryExistenceState(String logname){ return this.db.queryExistenceState(logname); } - public JavaRDD queryUnorderStateI(String logname){ + public Dataset queryUnorderStateI(String logname){ return this.db.queryUnorderStateI(logname); } - public JavaRDD queryUnorderStateU(String logname){ + public Dataset queryUnorderStateU(String logname){ return this.db.queryUnorderStateU(logname); } - public JavaRDD queryOrderState(String logname){ + public Dataset queryOrderState(String logname){ return this.db.queryOrderState(logname); } - public JavaRDD queryNegativeState(String logname){ + public Dataset queryNegativeState(String logname){ return this.db.queryNegativeState(logname); } diff --git a/src/main/java/com/datalab/siesta/queryprocessor/declare/DeclareUtilities.java b/src/main/java/com/datalab/siesta/queryprocessor/declare/DeclareUtilities.java index 5d198108..645e7e71 100644 --- a/src/main/java/com/datalab/siesta/queryprocessor/declare/DeclareUtilities.java +++ b/src/main/java/com/datalab/siesta/queryprocessor/declare/DeclareUtilities.java @@ -1,9 +1,9 @@ package com.datalab.siesta.queryprocessor.declare; import com.datalab.siesta.queryprocessor.declare.model.EventPairToNumberOfTrace; -import com.datalab.siesta.queryprocessor.model.Events.Event; -import com.datalab.siesta.queryprocessor.model.Events.EventPair; -import org.apache.spark.api.java.JavaRDD; +import com.datalab.siesta.queryprocessor.model.DBModel.EventTypes; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoders; import org.jvnet.hk2.annotations.Service; import org.springframework.stereotype.Component; @@ -22,21 +22,24 @@ public DeclareUtilities() { * @param joined a rdd containing all the event pairs that occurred * @return a set of all the event pairs that did not appear in the log database */ - public Set extractNotFoundPairs(Set eventTypes,JavaRDD joined) { + public Set extractNotFoundPairs(Set eventTypes, Dataset joined) { //calculate all the event pairs (n^2) and store them in a set //event pairs of type (eventA,eventA) are excluded - Set allEventPairs = new HashSet<>(); + Set allEventPairs = new HashSet<>(); for (String eventA : eventTypes) { for (String eventB : eventTypes) { if (!eventA.equals(eventB)) { - allEventPairs.add(new EventPair(new Event(eventA), new Event(eventB))); + allEventPairs.add(new EventTypes(eventA, eventB)); } } } //removes from the above set all the event pairs that have at least one occurrence in the - List foundEventPairs = joined.map(x -> new EventPair(new Event(x.getEventA()), new Event(x.getEventB()))) - .collect(); + List foundEventPairs = joined + .select("eventA","eventB") + .as(Encoders.bean(EventTypes.class)) + .collectAsList(); + foundEventPairs.forEach(allEventPairs::remove); return allEventPairs; } diff --git a/src/main/java/com/datalab/siesta/queryprocessor/declare/model/EventSupport.java b/src/main/java/com/datalab/siesta/queryprocessor/declare/model/EventSupport.java index b0398ac9..a75d929b 100644 --- a/src/main/java/com/datalab/siesta/queryprocessor/declare/model/EventSupport.java +++ b/src/main/java/com/datalab/siesta/queryprocessor/declare/model/EventSupport.java @@ -3,7 +3,15 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.annotation.JsonSerialize; - +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@Getter +@Setter +@AllArgsConstructor +@NoArgsConstructor public class EventSupport { @JsonProperty("ev") @@ -11,28 +19,5 @@ public class EventSupport { @JsonProperty("support") @JsonSerialize(using = SupportSerializer.class) protected Double support; - - public EventSupport(String event, double support) { - this.event = event; - this.support = support; - } - - public EventSupport() { - } - - public String getEvent() { - return event; - } - - public void setEvent(String event) { - this.event = event; - } - - public double getSupport() { - return support; - } - - public void setSupport(double support) { - this.support = support; - } } + diff --git a/src/main/java/com/datalab/siesta/queryprocessor/declare/model/EventTypeOccurrences.java b/src/main/java/com/datalab/siesta/queryprocessor/declare/model/EventTypeOccurrences.java new file mode 100644 index 00000000..be8a52ea --- /dev/null +++ b/src/main/java/com/datalab/siesta/queryprocessor/declare/model/EventTypeOccurrences.java @@ -0,0 +1,17 @@ +package com.datalab.siesta.queryprocessor.declare.model; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +import java.io.Serializable; + +@Getter +@Setter +@NoArgsConstructor +@AllArgsConstructor +public class EventTypeOccurrences implements Serializable { + private String eventName; + private long numberOfTraces; +} diff --git a/src/main/java/com/datalab/siesta/queryprocessor/declare/model/OccurrencesPerTrace.java b/src/main/java/com/datalab/siesta/queryprocessor/declare/model/OccurrencesPerTrace.java index e8f9adac..437f28f7 100644 --- a/src/main/java/com/datalab/siesta/queryprocessor/declare/model/OccurrencesPerTrace.java +++ b/src/main/java/com/datalab/siesta/queryprocessor/declare/model/OccurrencesPerTrace.java @@ -17,5 +17,5 @@ @NoArgsConstructor public class OccurrencesPerTrace implements Serializable { private String traceId; - private int occurrences; + private int occs; } diff --git a/src/main/java/com/datalab/siesta/queryprocessor/declare/model/UniqueTracesPerEventPair.java b/src/main/java/com/datalab/siesta/queryprocessor/declare/model/UniqueTracesPerEventPair.java index 5aa70b6a..e3254033 100644 --- a/src/main/java/com/datalab/siesta/queryprocessor/declare/model/UniqueTracesPerEventPair.java +++ b/src/main/java/com/datalab/siesta/queryprocessor/declare/model/UniqueTracesPerEventPair.java @@ -1,47 +1,24 @@ package com.datalab.siesta.queryprocessor.declare.model; import com.datalab.siesta.queryprocessor.model.Events.EventPair; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; import scala.Tuple2; import java.io.Serializable; import java.util.List; - +@Getter +@Setter +@AllArgsConstructor +@NoArgsConstructor public class UniqueTracesPerEventPair implements Serializable { private String eventA; private String eventB; private List uniqueTraces; - public UniqueTracesPerEventPair(String eventA, String eventB, List uniqueTraces) { - this.eventA = eventA; - this.eventB = eventB; - this.uniqueTraces = uniqueTraces; - } - - public String getEventA() { - return eventA; - } - - public void setEventA(String eventA) { - this.eventA = eventA; - } - - public String getEventB() { - return eventB; - } - - public void setEventB(String eventB) { - this.eventB = eventB; - } - - public List getUniqueTraces() { - return uniqueTraces; - } - - public void setUniqueTraces(List uniqueTraces) { - this.uniqueTraces = uniqueTraces; - } - public Tuple2 getKey(){ return new Tuple2<>(this.eventA,this.eventB); } diff --git a/src/main/java/com/datalab/siesta/queryprocessor/declare/model/UniqueTracesPerEventType.java b/src/main/java/com/datalab/siesta/queryprocessor/declare/model/UniqueTracesPerEventType.java index 43c3cae8..e395bc0b 100644 --- a/src/main/java/com/datalab/siesta/queryprocessor/declare/model/UniqueTracesPerEventType.java +++ b/src/main/java/com/datalab/siesta/queryprocessor/declare/model/UniqueTracesPerEventType.java @@ -19,7 +19,7 @@ @Getter @Setter public class UniqueTracesPerEventType implements Serializable { - private String eventType; + private String eventName; private List occurrences; @@ -31,10 +31,10 @@ public class UniqueTracesPerEventType implements Serializable { public HashMap groupTimes(){ HashMap groups = new HashMap<>(); this.occurrences.forEach(x->{ - if(groups.containsKey(x.getOccurrences())){ - groups.put(x.getOccurrences(),groups.get(x.getOccurrences())+1L); + if(groups.containsKey(x.getOccs())){ + groups.put(x.getOccs(),groups.get(x.getOccs())+1L); }else{ - groups.put(x.getOccurrences(),1L); + groups.put(x.getOccs(),1L); } }); return groups; diff --git a/src/main/java/com/datalab/siesta/queryprocessor/declare/model/declareState/ExistenceState.java b/src/main/java/com/datalab/siesta/queryprocessor/declare/model/declareState/ExistenceState.java index ae33579b..68699fce 100644 --- a/src/main/java/com/datalab/siesta/queryprocessor/declare/model/declareState/ExistenceState.java +++ b/src/main/java/com/datalab/siesta/queryprocessor/declare/model/declareState/ExistenceState.java @@ -3,18 +3,16 @@ import java.io.Serializable; import lombok.Getter; +import lombok.NoArgsConstructor; import lombok.Setter; @Setter @Getter +@NoArgsConstructor public class ExistenceState implements Serializable{ private String event_type; private int occurrences; private long contained; - public ExistenceState(){ - - } - } diff --git a/src/main/java/com/datalab/siesta/queryprocessor/declare/queryPlans/QueryPlanDeclareAll.java b/src/main/java/com/datalab/siesta/queryprocessor/declare/queryPlans/QueryPlanDeclareAll.java index 8703e294..54eb5327 100644 --- a/src/main/java/com/datalab/siesta/queryprocessor/declare/queryPlans/QueryPlanDeclareAll.java +++ b/src/main/java/com/datalab/siesta/queryprocessor/declare/queryPlans/QueryPlanDeclareAll.java @@ -18,16 +18,15 @@ import com.datalab.siesta.queryprocessor.model.Queries.QueryResponses.QueryResponse; import com.datalab.siesta.queryprocessor.model.Queries.Wrapper.QueryWrapper; +import com.datalab.siesta.queryprocessor.storage.model.EventTypeTracePositions; import lombok.Setter; -import org.apache.spark.api.java.JavaPairRDD; -import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; -import org.apache.spark.broadcast.Broadcast; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.functions; import org.apache.spark.storage.StorageLevel; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.springframework.web.context.annotation.RequestScope; -import scala.Tuple2; import java.util.*; @@ -72,51 +71,47 @@ public QueryResponse execute(QueryWrapper qw) { // run existences this.queryPlanExistences.setMetadata(metadata); this.queryPlanExistences.initResponse(); - Broadcast bSupport = javaSparkContext.broadcast(support); - Broadcast bTotalTraces = javaSparkContext.broadcast(metadata.getTraces()); // if existence, absence or exactly in modes - JavaRDD uEventType = declareDBConnector + Dataset uEventType = declareDBConnector .querySingleTableDeclare(this.metadata.getLogname()); uEventType.persist(StorageLevel.MEMORY_AND_DISK()); Map> groupTimes = this.queryPlanExistences.createMapForSingle(uEventType); Map singleUnique = this.queryPlanExistences.extractUniqueTracesSingle(groupTimes); - Broadcast> bUniqueSingle = javaSparkContext.broadcast(singleUnique); - JavaRDD uPairs = declareDBConnector + Dataset uPairs = declareDBConnector .queryIndexTableDeclare(this.metadata.getLogname()); - JavaRDD joined = this.queryPlanExistences.joinUnionTraces(uPairs); + Dataset joined = this.queryPlanExistences.joinUnionTraces(uPairs); QueryResponseExistence queryResponseExistence = this.queryPlanExistences.runAll(groupTimes, support, joined, - bSupport, bTotalTraces, bUniqueSingle, uEventType); + metadata.getTraces(), singleUnique, uEventType); uEventType.unpersist(); // create joined table for order relations // load data from query table - JavaRDD indexRDD = declareDBConnector.queryIndexOriginalDeclare(this.metadata.getLogname()) - .filter(x -> !x.getEventA().equals(x.getEventB())); - - JavaPairRDD, List> singleRDD = declareDBConnector + Dataset indexRDD = declareDBConnector.queryIndexOriginalDeclare(this.metadata.getLogname()) + .filter(functions.col("eventA").notEqual(functions.col("eventB"))); +// + Dataset singleRDD = declareDBConnector .querySingleTableAllDeclare(this.metadata.getLogname()); singleRDD.persist(StorageLevel.MEMORY_AND_DISK()); - +// this.queryPlanOrderedRelations.initQueryResponse(); this.queryPlanOrderedRelations.setMetadata(metadata); // join records from single table with records from the index table - JavaRDD joinedOrder = this.queryPlanOrderedRelations.joinTables(indexRDD, singleRDD); + Dataset joinedOrder = this.queryPlanOrderedRelations.joinTables(indexRDD, singleRDD); joinedOrder.persist(StorageLevel.MEMORY_AND_DISK()); singleRDD.unpersist(); - // extract simple ordered - JavaRDD cSimple = this.queryPlanOrderedRelations.evaluateConstraint(joinedOrder, - "succession"); +// extract simple ordered + Dataset cSimple = this.queryPlanOrderedRelations.evaluateConstraint(joinedOrder); cSimple.persist(StorageLevel.MEMORY_AND_DISK()); // extract additional information required for pruning - Map uEventType2 = declareDBConnector.extractTotalOccurrencesPerEventType(this.metadata.getLogname()); - Broadcast> bUEventTypes = javaSparkContext.broadcast(uEventType2); + Dataset uEventType2 = declareDBConnector + .extractTotalOccurrencesPerEventType(this.metadata.getLogname()); // extract no-succession constraints from event pairs that do not exist in the // database log this.queryPlanOrderedRelations.extendNotSuccession(uEventType2, this.metadata.getLogname(), cSimple); // filter based on support - this.queryPlanOrderedRelations.filterBasedOnSupport(cSimple, bUEventTypes, support); + this.queryPlanOrderedRelations.filterBasedOnSupport(cSimple, uEventType2, support); // load extracted constraints to the response QueryResponseOrderedRelations qSimple = this.queryPlanOrderedRelations.getQueryResponseOrderedRelations(); cSimple.unpersist(); @@ -124,24 +119,23 @@ public QueryResponse execute(QueryWrapper qw) { // similar process to extract alternate ordered this.queryPlanOrderedRelationsAlternate.initQueryResponse(); this.queryPlanOrderedRelationsAlternate.setMetadata(metadata); - JavaRDD cAlternate = this.queryPlanOrderedRelationsAlternate - .evaluateConstraint(joinedOrder, "succession"); - this.queryPlanOrderedRelationsAlternate.filterBasedOnSupport(cAlternate, bUEventTypes, support); + Dataset cAlternate = this.queryPlanOrderedRelationsAlternate + .evaluateConstraint(joinedOrder); + this.queryPlanOrderedRelationsAlternate.filterBasedOnSupport(cAlternate, uEventType2, support); QueryResponseOrderedRelations qAlternate = this.queryPlanOrderedRelationsAlternate .getQueryResponseOrderedRelations(); // similar process to extract chain ordered this.queryPlanOrderedRelationsChain.initQueryResponse(); this.queryPlanOrderedRelationsChain.setMetadata(metadata); - JavaRDD cChain = this.queryPlanOrderedRelationsChain.evaluateConstraint(joinedOrder, - "succession"); + Dataset cChain = this.queryPlanOrderedRelationsChain.evaluateConstraint(joinedOrder); cChain.persist(StorageLevel.MEMORY_AND_DISK()); this.queryPlanOrderedRelationsChain.extendNotSuccession(uEventType2, this.metadata.getLogname(), cChain); - this.queryPlanOrderedRelationsChain.filterBasedOnSupport(cChain, bUEventTypes, support); + this.queryPlanOrderedRelationsChain.filterBasedOnSupport(cChain, uEventType2, support); QueryResponseOrderedRelations qChain = this.queryPlanOrderedRelationsChain.getQueryResponseOrderedRelations(); cChain.unpersist(); joinedOrder.unpersist(); - // combine all responses +// // combine all responses QueryResponseDeclareAll queryResponseAll = new QueryResponseDeclareAll(queryResponseExistence, queryResponsePosition, qSimple, qAlternate, qChain); return queryResponseAll; diff --git a/src/main/java/com/datalab/siesta/queryprocessor/declare/queryPlans/QueryPlanDeclareAllState.java b/src/main/java/com/datalab/siesta/queryprocessor/declare/queryPlans/QueryPlanDeclareAllState.java index 86ee9f5b..b4a53a3f 100644 --- a/src/main/java/com/datalab/siesta/queryprocessor/declare/queryPlans/QueryPlanDeclareAllState.java +++ b/src/main/java/com/datalab/siesta/queryprocessor/declare/queryPlans/QueryPlanDeclareAllState.java @@ -4,10 +4,11 @@ import java.util.Arrays; import java.util.List; -import org.apache.spark.api.java.JavaPairRDD; -import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; -import org.apache.spark.broadcast.Broadcast; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.SparkSession; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.springframework.web.context.annotation.RequestScope; @@ -15,11 +16,8 @@ import com.datalab.siesta.queryprocessor.declare.model.PairConstraint; import com.datalab.siesta.queryprocessor.declare.queryPlans.position.QueryPlanPositionsState; import com.datalab.siesta.queryprocessor.declare.queryResponses.QueryResponseDeclareAllState; -import com.datalab.siesta.queryprocessor.declare.queryResponses.QueryResponseExistence; import com.datalab.siesta.queryprocessor.declare.queryResponses.QueryResponseExistenceState; -import com.datalab.siesta.queryprocessor.declare.queryResponses.QueryResponseOrderedRelations; import com.datalab.siesta.queryprocessor.declare.queryResponses.QueryResponseOrderedRelationsState; -import com.datalab.siesta.queryprocessor.declare.queryResponses.QueryResponsePosition; import com.datalab.siesta.queryprocessor.declare.queryResponses.QueryResponsePositionState; import com.datalab.siesta.queryprocessor.declare.queryPlans.existence.*; import com.datalab.siesta.queryprocessor.declare.queryPlans.orderedRelations.QueryPlanOrderRelationsState; @@ -38,59 +36,62 @@ public class QueryPlanDeclareAllState extends QueryPlanState{ private QueryPlanExistancesState queryPlanExistencesState; private QueryPlanOrderRelationsState queryPlanOrderedRelationsState; private DBConnector dbConnector; + private SparkSession sparkSession; @Autowired public QueryPlanDeclareAllState(DeclareDBConnector declareDBConnector, JavaSparkContext javaSparkContext, - DBConnector dbConnector, - QueryPlanPositionsState queryPlanPositionsState, QueryPlanExistancesState queryPlanExistencesState, - QueryPlanOrderRelationsState queryPlanOrderedRelationsState){ + DBConnector dbConnector, + QueryPlanPositionsState queryPlanPositionsState, QueryPlanExistancesState queryPlanExistencesState, + QueryPlanOrderRelationsState queryPlanOrderedRelationsState, SparkSession sparkSession){ super(declareDBConnector,javaSparkContext); this.queryPlanPositionsState = queryPlanPositionsState; this.queryPlanExistencesState = queryPlanExistencesState; this.queryPlanOrderedRelationsState = queryPlanOrderedRelationsState; this.dbConnector = dbConnector; + this.sparkSession = sparkSession; } @Override public QueryResponse execute(QueryWrapper qw) { QueryWrapperDeclare qdw = (QueryWrapperDeclare) qw; - + double support = qdw.getSupport(); + long traces = this.metadata.getTraces(); // run extract positions - Broadcast bSupport = this.javaSparkContext.broadcast(qdw.getSupport()); - Broadcast bTraces = this.javaSparkContext.broadcast(metadata.getTraces()); QueryPositionWrapper qpw = new QueryPositionWrapper(qdw.getSupport()); qpw.setMode("both"); this.queryPlanPositionsState.setMetadata(this.metadata); - QueryResponsePositionState prpState = this.queryPlanPositionsState.extractConstraintsFunction(bSupport,bTraces,qpw); - + QueryResponsePositionState prpState = this.queryPlanPositionsState.extractConstraintsFunction(support, + traces,qpw); + // run extract existences QueryExistenceWrapper qew = new QueryExistenceWrapper(qdw.getSupport()); String[] modes = {"existence","absence","exactly","co-existence","not-co-existence", "choice", "exclusive-choice", "responded-existence"}; qew.setModes(new ArrayList<>(Arrays.asList(modes))); List activities = dbConnector.getEventNames(metadata.getLogname()); - JavaRDD activityRDD = javaSparkContext.parallelize(activities); - JavaPairRDD activityMatrix = activityRDD.cartesian(activityRDD); + Dataset activityDF = sparkSession.createDataset(activities, Encoders.STRING()); + Dataset activityMatrix = activityDF.crossJoin(activityDF).toDF("activityA", "activityB"); + this.queryPlanExistencesState.setMetadata(this.metadata); - QueryResponseExistenceState preState = this.queryPlanExistencesState.extractConstraintsFunction(bSupport,bTraces,activityMatrix,qew); + QueryResponseExistenceState preState = this.queryPlanExistencesState.extractConstraintsFunction(support,traces,activityMatrix,qew); this.extractStatistics(qdw); // run extract ordered relations this.queryPlanOrderedRelationsState.setMetadata(this.metadata); - List pairConstraints = this.queryPlanOrderedRelationsState.extractAll(bSupport, activityMatrix); + List pairConstraints = this.queryPlanOrderedRelationsState.extractAll(support, activityMatrix); QueryResponseOrderedRelationsState simple = this.queryPlanOrderedRelationsState.extractConstraintFunction(pairConstraints, "simple"); QueryResponseOrderedRelationsState alternate = this.queryPlanOrderedRelationsState.extractConstraintFunction(pairConstraints, "alternate"); QueryResponseOrderedRelationsState chain = this.queryPlanOrderedRelationsState.extractConstraintFunction(pairConstraints, "chain"); - + QueryResponseDeclareAllState response = new QueryResponseDeclareAllState(prpState.getQueryResponsePosition(), preState.getQueryResponseExistence() , simple.getQueryResponseOrderedRelations(), alternate.getQueryResponseOrderedRelations(), chain.getQueryResponseOrderedRelations()); this.extractStatistics(qdw); response.setUpToDate(qdw.isStateUpToDate()); if(!qdw.isStateUpToDate()){ - response.setEventsPercentage((qdw.getIndexedEvents()/metadata.getEvents())*100); - response.setTracesPercentage((qdw.getIndexedTraces()/metadata.getTraces())*100); + response.setEventsPercentage(((double) qdw.getIndexedEvents() /metadata.getEvents())*100); + response.setTracesPercentage(((double) qdw.getIndexedTraces() /metadata.getTraces())*100); response.setMessage("State is not fully updated. Consider re-running the preprocess to get 100% accurate constraints"); }else{ response.setEventsPercentage(100); diff --git a/src/main/java/com/datalab/siesta/queryprocessor/declare/queryPlans/QueryPlanState.java b/src/main/java/com/datalab/siesta/queryprocessor/declare/queryPlans/QueryPlanState.java index 259d44b6..7d637dab 100644 --- a/src/main/java/com/datalab/siesta/queryprocessor/declare/queryPlans/QueryPlanState.java +++ b/src/main/java/com/datalab/siesta/queryprocessor/declare/queryPlans/QueryPlanState.java @@ -1,9 +1,9 @@ package com.datalab.siesta.queryprocessor.declare.queryPlans; -import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; -import org.apache.spark.api.java.function.Function; -import org.apache.spark.api.java.function.Function2; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.functions; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.springframework.web.context.annotation.RequestScope; @@ -44,7 +44,6 @@ public QueryPlanState(DeclareDBConnector declareDBConnector, JavaSparkContext ja @Override public QueryResponse execute(QueryWrapper qw) { - // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'execute'"); } @@ -60,24 +59,23 @@ public void setMetadata(Metadata metadata) { * used by the QueryPlanner to specify how accurate the extracted constraints are. */ public void extractStatistics(QueryWrapperDeclare qwd){ - JavaRDD ps = declareDBConnector.queryPositionState(qwd.getLog_name()); - int traces_stated = (int) ps.filter((Function) x->{ - return x.getRule().equals("first"); - }) - .map((Function)x->{ - return ((int)x.getOccurrences()); - }) - .reduce((Function2)(x,y)->{ - return x + y; - }); + Dataset ps = declareDBConnector.queryPositionState(qwd.getLog_name()); + int traces_stated = ps.filter(functions.col("rule").equalTo("first")) + .select("occurrences") + .agg(functions.sum("occurrences").cast("int")) + .as(Encoders.INT()) + .collectAsList() + .get(0); qwd.setIndexedTraces(traces_stated); - JavaRDD es = declareDBConnector.queryExistenceState(qwd.getLog_name()); - int events_stated = es.map((Function)x->{ - return (int)((int)x.getOccurrences()*x.getContained()); - }).reduce((Function2)(x,y)->{ - return x+y; - }); + Dataset es = declareDBConnector.queryExistenceState(qwd.getLog_name()); + int events_stated = es.withColumn("total",functions.col("occurrences") + .multiply(functions.col("contained"))) + .agg(functions.sum("total").cast("int")) + .as(Encoders.INT()) + .collectAsList() + .get(0); + qwd.setIndexedEvents(events_stated); if(qwd.getIndexedEvents()==metadata.getEvents()){ diff --git a/src/main/java/com/datalab/siesta/queryprocessor/declare/queryPlans/existence/QueryPlanExistancesState.java b/src/main/java/com/datalab/siesta/queryprocessor/declare/queryPlans/existence/QueryPlanExistancesState.java index d50dd5eb..5c62465f 100644 --- a/src/main/java/com/datalab/siesta/queryprocessor/declare/queryPlans/existence/QueryPlanExistancesState.java +++ b/src/main/java/com/datalab/siesta/queryprocessor/declare/queryPlans/existence/QueryPlanExistancesState.java @@ -5,14 +5,10 @@ import java.util.Collections; import java.util.Comparator; import java.util.List; +import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; -import org.apache.spark.api.java.JavaPairRDD; -import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; -import org.apache.spark.api.java.function.FlatMapFunction; -import org.apache.spark.api.java.function.Function; -import org.apache.spark.broadcast.Broadcast; import com.datalab.siesta.queryprocessor.declare.DeclareDBConnector; import com.datalab.siesta.queryprocessor.declare.model.EventN; @@ -31,48 +27,46 @@ import com.datalab.siesta.queryprocessor.declare.model.UnorderedHelper; import com.datalab.siesta.queryprocessor.declare.model.PairConstraint; +import org.apache.spark.sql.*; import org.springframework.stereotype.Component; import org.springframework.web.context.annotation.RequestScope; import org.springframework.beans.factory.annotation.Autowired; -import scala.Tuple2; - @Component @RequestScope public class QueryPlanExistancesState extends QueryPlanState { + private final SparkSession sparkSession; private DBConnector dbConnector; @Autowired - public QueryPlanExistancesState(DeclareDBConnector declareDBConnector, JavaSparkContext javaSparkContext, DBConnector dbConnector) { + public QueryPlanExistancesState(DeclareDBConnector declareDBConnector, JavaSparkContext javaSparkContext, DBConnector dbConnector, SparkSession sparkSession) { super(declareDBConnector, javaSparkContext); this.dbConnector = dbConnector; + this.sparkSession = sparkSession; } @Override public QueryResponse execute(QueryWrapper qw) { QueryExistenceWrapper qew = (QueryExistenceWrapper) qw; - - - Broadcast bTraces = javaSparkContext.broadcast(metadata.getTraces()); - Broadcast bSupport = javaSparkContext.broadcast(qew.getSupport()); //get all possible activities from database to create activity matrix List activities = dbConnector.getEventNames(metadata.getLogname()); - JavaRDD activityRDD = javaSparkContext.parallelize(activities); - JavaPairRDD activityMatrix = activityRDD.cartesian(activityRDD); - - QueryResponseExistenceState response = this.extractConstraintsFunction(bSupport, bTraces, activityMatrix, qew); + Dataset activityDF = sparkSession.createDataset(activities, Encoders.STRING()); + Dataset activityMatrix = activityDF.crossJoin(activityDF).toDF("activityA", "activityB"); + + QueryResponseExistenceState response = this.extractConstraintsFunction(qew.getSupport(), metadata.getTraces(), + activityMatrix, qew); this.extractStatistics(qew); response.setUpToDate(qew.isStateUpToDate()); - if(!qew.isStateUpToDate()){ - response.setEventsPercentage((qew.getIndexedEvents()/metadata.getEvents())*100); - response.setTracesPercentage((qew.getIndexedTraces()/metadata.getTraces())*100); + if (!qew.isStateUpToDate()) { + response.setEventsPercentage(((double) qew.getIndexedEvents() / metadata.getEvents()) * 100); + response.setTracesPercentage(((double) qew.getIndexedTraces() / metadata.getTraces()) * 100); response.setMessage("State is not fully updated. Consider re-running the preprocess to get 100% accurate constraints"); - }else{ + } else { response.setEventsPercentage(100); response.setTracesPercentage(100); } @@ -80,181 +74,182 @@ public QueryResponse execute(QueryWrapper qw) { return response; } - public QueryResponseExistenceState extractConstraintsFunction(Broadcast bSupport, Broadcast bTraces, JavaPairRDD activityMatrix, - QueryExistenceWrapper qew){ + public QueryResponseExistenceState extractConstraintsFunction(double support, long traces, Dataset activityMatrix, + QueryExistenceWrapper qew) { QueryResponseExistenceState response = new QueryResponseExistenceState(); - String[] existenceConstraints = {"existence","absence","exactly"}; - if(Arrays.stream(existenceConstraints).anyMatch(qew.getModes()::contains)){ - this.calculateExistence(qew.getModes(), response, bTraces, bSupport); + String[] existenceConstraints = {"existence", "absence", "exactly"}; + if (Arrays.stream(existenceConstraints).anyMatch(qew.getModes()::contains)) { + this.calculateExistence(qew.getModes(), response, traces, support); } - - String[] unorderedConstraints = {"co-existence","not-co-existence", "choice", + + String[] unorderedConstraints = {"co-existence", "not-co-existence", "choice", "exclusive-choice", "responded-existence"}; - if(Arrays.stream(unorderedConstraints).anyMatch(qew.getModes()::contains)){ - this.calculateUnordered(qew.getModes(), response, bTraces, bSupport, activityMatrix); + if (Arrays.stream(unorderedConstraints).anyMatch(qew.getModes()::contains)) { + this.calculateUnordered(qew.getModes(), response, traces, support, activityMatrix); } return response; } - private void calculateExistence(List modes, QueryResponseExistence qre, Broadcast bTraces, Broadcast bSupport){ - JavaRDD existences = declareDBConnector.queryExistenceState(metadata.getLogname()); - JavaRDD c = existences.groupBy((Function) x -> { - return x.getEvent_type(); - }) - .flatMap((FlatMapFunction>, ExistenceConstraint>) x -> { - List l = new ArrayList<>(); - String eventType = x._1; - Iterable activities = x._2; - // Convert Iterable to List for sorting - List sortedActivities = new ArrayList<>(); - activities.forEach(sortedActivities::add); - sortedActivities.sort(Comparator.comparingInt(ExistenceState::getOccurrences)); - - // Exactly constraint - for (ExistenceState activity : sortedActivities) { - if ((double) activity.getContained() / bTraces.getValue() >= bSupport.getValue()) { - l.add(new ExistenceConstraint("exactly", eventType, activity.getOccurrences(), - (double) activity.getContained() / bTraces.getValue())); - } - } - - //Existence Constraints - Collections.reverse(sortedActivities); //reverse the list so higher is lower - long cumulativeOccurrences = sortedActivities.get(0).getContained(); - int pos = 0; - //this loop will start from the higher number of occurrences and go down to one. In each iteration - //will calculate the number of traces that contain at least x instances of the activity - for(int occurrences = sortedActivities.get(0).getOccurrences();occurrences>=1;occurrences--){ - //the occurrences are equal to the next one smaller occurrence - if(pos+1= bSupport.getValue()) { - l.add(new ExistenceConstraint("existence", eventType, occurrences, - (double) cumulativeOccurrences / bTraces.getValue())); - } - } - - // Absence constraint - Collections.reverse(sortedActivities); // smaller to greater - pos=-1; - int cumulativeAbsence = (int) (bTraces.getValue() - sortedActivities.stream().mapToLong(ExistenceState::getContained).sum()); - for (int absence = 1; absence<=3;absence++){ - if ((double) cumulativeAbsence / bTraces.getValue() >= bSupport.getValue()) { - l.add(new ExistenceConstraint("absence", eventType, absence, - (double) cumulativeAbsence / bTraces.getValue())); - } - if(pos+1 constraints = c.collect(); - if(modes.contains("existence")){ - qre.setExistence(getExistenceToResponse("existence", constraints)); + private void calculateExistence(List modes, QueryResponseExistence qre, long traces, double support) { + List existences = declareDBConnector.queryExistenceState(metadata.getLogname()) + .collectAsList(); + + List c = existences.parallelStream() + .collect(Collectors.groupingByConcurrent(ExistenceState::getEvent_type, + ConcurrentHashMap::new, Collectors.toList())) + .entrySet().parallelStream() + .flatMap(entry -> { + List constraints = new ArrayList<>(); + String eventType = entry.getKey(); + List sortedActivities = entry.getValue(); + // sort ascending by occurrences + sortedActivities.sort(Comparator.comparingInt(ExistenceState::getOccurrences)); + + for (ExistenceState activity : sortedActivities) { + if ((double) activity.getContained() / traces >= support) { + constraints.add(new ExistenceConstraint("exactly", eventType, activity.getOccurrences(), + (double) activity.getContained() / traces)); + } + } + + //Existence Constraints + Collections.reverse(sortedActivities); //reverse the list so higher is lower + long cumulativeOccurrences = sortedActivities.get(0).getContained(); + int pos = 0; + //this loop will start from the higher number of occurrences and go down to one. In each iteration + //will calculate the number of traces that contain at least x instances of the activity + for (int occurrences = sortedActivities.get(0).getOccurrences(); occurrences >= 1; occurrences--) { + //the occurrences are equal to the next one smaller occurrence + if (pos + 1 < sortedActivities.size() && occurrences == sortedActivities.get(pos + 1).getOccurrences()) { + pos += 1; + cumulativeOccurrences += sortedActivities.get(pos).getContained(); + } + if ((double) cumulativeOccurrences / traces >= support) { + constraints.add(new ExistenceConstraint("existence", eventType, occurrences, + (double) cumulativeOccurrences / traces)); + } + } + + // Absence constraint + Collections.reverse(sortedActivities); // smaller to greater + pos = -1; + int cumulativeAbsence = (int) (traces - sortedActivities.stream().mapToLong(ExistenceState::getContained).sum()); + for (int absence = 1; absence <= 3; absence++) { + if ((double) cumulativeAbsence / traces >= support) { + constraints.add(new ExistenceConstraint("absence", eventType, absence, + (double) cumulativeAbsence / traces)); + } + if (pos + 1 < sortedActivities.size() && absence == sortedActivities.get(pos + 1).getOccurrences()) { + pos += 1; + cumulativeAbsence += sortedActivities.get(pos).getContained(); + } + + } + return constraints.parallelStream(); + }) + .toList(); + + if (modes.contains("existence")) { + qre.setExistence(getExistenceToResponse("existence", c)); } - if(modes.contains("absence")){ - qre.setAbsence(getExistenceToResponse("absence", constraints)); + if (modes.contains("absence")) { + qre.setAbsence(getExistenceToResponse("absence", c)); } - if(modes.contains("exactly")){ - qre.setExactly(getExistenceToResponse("exactly", constraints)); + if (modes.contains("exactly")) { + qre.setExactly(getExistenceToResponse("exactly", c)); } } - private List getExistenceToResponse(String rule, List constraints){ - return constraints.stream().filter(x->{ + private List getExistenceToResponse(String rule, List constraints) { + return constraints.stream().filter(x -> { return x.getRule().equals(rule); - }).map(x->{ + }).map(x -> { return new EventN(x.getEvent_type(), x.getN(), x.getOccurrences()); }).collect(Collectors.toList()); } - private void calculateUnordered(List modes, QueryResponseExistence qre, Broadcast bTraces, Broadcast bSupport, JavaPairRDD activityMatrix){ - + private void calculateUnordered(List modes, QueryResponseExistence qre, long traces, + double support, Dataset activityMatrix) { + - JavaRDD iTable = declareDBConnector.queryUnorderStateI(metadata.getLogname()); - JavaRDD uTable = declareDBConnector.queryUnorderStateU(metadata.getLogname()); + Dataset iTable = declareDBConnector.queryUnorderStateI(metadata.getLogname()); + Dataset uTable = declareDBConnector.queryUnorderStateU(metadata.getLogname()).as("uTable"); + Dataset iTablePrepared = iTable + .withColumn("key_i", functions.concat(functions.col("`_1`"), functions.col("`_2`"))) + .as("iTable"); + //activity matrix is // Extract the unordered constraints by merging the activity matrix - iTable - uTable - List unorderedConstraints = activityMatrix - .keyBy((Function, String>) x-> x._1()) - .leftOuterJoin(uTable.keyBy((Function) x-> x.get_1())) - .map(x -> { - String eventA = x._2()._1()._1(); - String eventB = x._2()._1()._2(); - String key = eventA.compareTo(eventB) < 0 ? eventA + eventB : eventB + eventA; - return new UnorderedHelper(eventA, eventB, x._2()._2().orElse(new UnorderStateU("", 0L)).get_2(), 0L, 0L, key); - }) - .keyBy((Function) x-> x.getEventB()) - .leftOuterJoin(uTable.keyBy((Function) x-> x.get_1())) - .map(x->{ - return new UnorderedHelper(x._2()._1().getEventA(), x._2()._1().getEventB(), x._2()._1().getUa(), x._2()._2().orElse(new UnorderStateU("", 0L)).get_2(), 0L, x._2()._1().getKey()); - }) - .keyBy((Function) x-> x.getKey()) - .leftOuterJoin(iTable.keyBy((Function) x-> x.get_1() + x.get_2())) - .map(x -> { - UnorderedHelper p = x._2()._1(); - return new UnorderedHelper(p.getEventA(), p.getEventB(), p.getUa(), p.getUb(), x._2()._2().orElse(new UnorderStateI("", "", 0L)).get_3(), p.getKey()); - }) - .distinct() - .flatMap((FlatMapFunction) x ->{ - List l = new ArrayList<>(); - long r = bTraces.getValue() - x.getUa() + x.getPairs(); - l.add(new PairConstraint(new EventPairSupport(x.getEventA(), x.getEventB(), r), "responded-existence")); - if(x.getEventA().compareTo(x.getEventB()) < 0){ - r = x.getUa() + x.getUb() - x.getPairs(); - l.add(new PairConstraint(new EventPairSupport(x.getEventA(), x.getEventB(), r), "choice")); - r = bTraces.getValue() - x.getUa() - x.getUb() + 2 * x.getPairs(); - l.add(new PairConstraint(new EventPairSupport(x.getEventA(), x.getEventB(), r), "co-existence")); - //exclusive_choice = total - co-existen - l.add(new PairConstraint(new EventPairSupport(x.getEventA(), x.getEventB(), bTraces.getValue()-r), "exclusive-choice")); - //not-existence : traces where a exist and not b, traces where b exists and not a, traces where neither occur - r = bTraces.getValue() - x.getPairs(); - l.add(new PairConstraint(new EventPairSupport(x.getEventA(), x.getEventB(), r), "not-co-existence")); - } - return l.iterator(); - } ) - .filter((Function) x -> (x.getEventPairSupport().getSupport() / (double) bTraces.getValue()) >= bSupport.getValue()) - .map(x->{ - EventPairSupport eps = new EventPairSupport(); - eps.setEventA(x.getEventPairSupport().getEventA()); - eps.setEventB(x.getEventPairSupport().getEventB()); - eps.setSupport(x.getEventPairSupport().getSupport()/(double)bTraces.getValue()); - return new PairConstraint(eps, x.getRule()); - }) - .collect(); - - if(modes.contains("responded-existence")){ - qre.setRespondedExistence(getUnorderToResponse("responded-existence", unorderedConstraints)); - } - if(modes.contains("choice")){ - qre.setChoice(getUnorderToResponse("choice", unorderedConstraints)); - } - if(modes.contains("co-existence")){ - qre.setCoExistence(getUnorderToResponse("co-existence", unorderedConstraints)); - } - if(modes.contains("exclusive-choice")){ - qre.setExclusiveChoice(getUnorderToResponse("exclusive-choice", unorderedConstraints)); - } - if(modes.contains("not-co-existence")){ - qre.setNotCoExistence(getUnorderToResponse("not-co-existence", unorderedConstraints)); - } - + List unorderedHelpers = activityMatrix.as("activityMatrix") + .join(uTable.as("uTable"), + functions.col("activityMatrix.activityA").equalTo(functions.col("uTable.`_1`")), "left") + .selectExpr("activityMatrix.activityA as eventA", "activityMatrix.activityB as eventB", "uTable.`_2` as ua") + .join(uTable.as("uTable2"), + functions.col("eventB").equalTo(functions.col("uTable2.`_1`")), "left") + .selectExpr("eventA", "eventB", "ua", "uTable2.`_2` as ub") + .withColumn("key", functions.when(functions.col("eventA").geq("eventB"), + functions.concat(functions.col("eventA"), functions.col("eventB"))) + .otherwise(functions.concat(functions.col("eventB"), functions.col("eventA")))) + .join(iTablePrepared.as("iTable"), functions.col("key").equalTo(functions.col("key_i")), "left") + .selectExpr("eventA", "eventB", "key", "ua", "ub", "iTable.`_3` as pairs") + .na().fill(0) + .distinct() + .as(Encoders.bean(UnorderedHelper.class)) + .collectAsList(); + + List unorderedConstraints = unorderedHelpers.parallelStream() + .flatMap(x -> { + List l = new ArrayList<>(); + long r = traces - x.getUa() + x.getPairs(); + l.add(new PairConstraint(new EventPairSupport(x.getEventA(), x.getEventB(), r), "responded-existence")); + if (x.getEventA().compareTo(x.getEventB()) < 0) { + r = x.getUa() + x.getUb() - x.getPairs(); + l.add(new PairConstraint(new EventPairSupport(x.getEventA(), x.getEventB(), r), "choice")); + r = traces - x.getUa() - x.getUb() + 2 * x.getPairs(); + l.add(new PairConstraint(new EventPairSupport(x.getEventA(), x.getEventB(), r), "co-existence")); + //exclusive_choice = total - co-existen + l.add(new PairConstraint(new EventPairSupport(x.getEventA(), x.getEventB(), traces - r), "exclusive-choice")); + //not-existence : traces where a exist and not b, traces where b exists and not a, traces where neither occur + r = traces - x.getPairs(); + l.add(new PairConstraint(new EventPairSupport(x.getEventA(), x.getEventB(), r), "not-co-existence")); + } + return l.parallelStream(); + }) + .filter(x -> (x.getEventPairSupport().getSupport() / + (double) traces) >= support) + .map(x -> { + EventPairSupport eps = new EventPairSupport(); + eps.setEventA(x.getEventPairSupport().getEventA()); + eps.setEventB(x.getEventPairSupport().getEventB()); + eps.setSupport(x.getEventPairSupport().getSupport() / (double) traces); + return new PairConstraint(eps, x.getRule()); + }).toList(); + + if (modes.contains("responded-existence")) { + qre.setRespondedExistence(getUnorderToResponse("responded-existence", unorderedConstraints)); + } + if (modes.contains("choice")) { + qre.setChoice(getUnorderToResponse("choice", unorderedConstraints)); + } + if (modes.contains("co-existence")) { + qre.setCoExistence(getUnorderToResponse("co-existence", unorderedConstraints)); + } + if (modes.contains("exclusive-choice")) { + qre.setExclusiveChoice(getUnorderToResponse("exclusive-choice", unorderedConstraints)); + } + if (modes.contains("not-co-existence")) { + qre.setNotCoExistence(getUnorderToResponse("not-co-existence", unorderedConstraints)); + } + } - private List getUnorderToResponse(String rule, List constraints){ - return constraints.stream().filter(x->{ - return x.getRule().equals(rule); - }) - .map(x->{ - return x.getEventPairSupport(); - }) - .collect(Collectors.toList()); + private List getUnorderToResponse(String rule, List constraints) { + return constraints.stream().filter(x -> { + return x.getRule().equals(rule); + }) + .map(PairConstraint::getEventPairSupport) + .collect(Collectors.toList()); } } diff --git a/src/main/java/com/datalab/siesta/queryprocessor/declare/queryPlans/existence/QueryPlanExistences.java b/src/main/java/com/datalab/siesta/queryprocessor/declare/queryPlans/existence/QueryPlanExistences.java index 29b26b36..c3fcbee5 100644 --- a/src/main/java/com/datalab/siesta/queryprocessor/declare/queryPlans/existence/QueryPlanExistences.java +++ b/src/main/java/com/datalab/siesta/queryprocessor/declare/queryPlans/existence/QueryPlanExistences.java @@ -5,18 +5,15 @@ import com.datalab.siesta.queryprocessor.declare.model.*; import com.datalab.siesta.queryprocessor.declare.queryResponses.QueryResponseExistence; import com.datalab.siesta.queryprocessor.declare.queryWrappers.QueryExistenceWrapper; +import com.datalab.siesta.queryprocessor.model.DBModel.EventTypes; import com.datalab.siesta.queryprocessor.model.DBModel.Metadata; -import com.datalab.siesta.queryprocessor.model.Events.Event; -import com.datalab.siesta.queryprocessor.model.Events.EventPair; import com.datalab.siesta.queryprocessor.model.Queries.QueryPlans.QueryPlan; import com.datalab.siesta.queryprocessor.model.Queries.QueryResponses.QueryResponse; import com.datalab.siesta.queryprocessor.model.Queries.Wrapper.QueryWrapper; import lombok.Setter; -import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; -import org.apache.spark.api.java.function.FlatMapFunction; -import org.apache.spark.broadcast.Broadcast; +import org.apache.spark.sql.*; import org.apache.spark.storage.StorageLevel; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -28,10 +25,11 @@ @Component @RequestScope -public class QueryPlanExistences implements QueryPlan{ +public class QueryPlanExistences implements QueryPlan { private final DeclareDBConnector declareDBConnector; private final JavaSparkContext javaSparkContext; + private final SparkSession sparkSession; //initialize a protected variable of the required events private QueryResponseExistence queryResponseExistence; @Setter @@ -42,32 +40,35 @@ public class QueryPlanExistences implements QueryPlan{ @Autowired public QueryPlanExistences(DeclareDBConnector declareDBConnector, JavaSparkContext javaSparkContext, - DeclareUtilities declareUtilities) { + DeclareUtilities declareUtilities, SparkSession sparkSession) { this.declareDBConnector = declareDBConnector; this.javaSparkContext = javaSparkContext; this.queryResponseExistence = new QueryResponseExistence(); this.declareUtilities = declareUtilities; + this.sparkSession = sparkSession; } @Override public QueryResponse execute(QueryWrapper qw) { QueryExistenceWrapper qew = (QueryExistenceWrapper) qw; - Broadcast bSupport = javaSparkContext.broadcast(qew.getSupport()); - Broadcast bTotalTraces = javaSparkContext.broadcast(metadata.getTraces()); - //if existence, absence or exactly in modes - JavaRDD uEventType = declareDBConnector.querySingleTableDeclare(metadata.getLogname()); + Dataset uEventType = declareDBConnector.querySingleTableDeclare(metadata.getLogname()); uEventType.persist(StorageLevel.MEMORY_AND_DISK()); Map> groupTimes = this.createMapForSingle(uEventType); + //Make it available to the spark context Map singleUnique = this.extractUniqueTracesSingle(groupTimes); - Broadcast> bUniqueSingle = javaSparkContext.broadcast(singleUnique); + List uniqueSingleRows = singleUnique.entrySet().stream() + .map(entry -> new EventTypeOccurrences(entry.getKey(), entry.getValue())) + .toList(); + Dataset uniqueSingleDf = sparkSession.createDataset(uniqueSingleRows, + Encoders.bean(EventTypeOccurrences.class)); - JavaRDD uPairs = declareDBConnector.queryIndexTableDeclare(metadata.getLogname()); + Dataset uPairs = declareDBConnector.queryIndexTableDeclare(metadata.getLogname()); uPairs.persist(StorageLevel.MEMORY_AND_DISK()); - JavaRDD joined = joinUnionTraces(uPairs); + Dataset joined = joinUnionTraces(uPairs); joined.persist(StorageLevel.MEMORY_AND_DISK()); - Set notFoundPairs = declareUtilities.extractNotFoundPairs(groupTimes.keySet(),joined); + Set notFoundPairs = declareUtilities.extractNotFoundPairs(groupTimes.keySet(), joined); for (String m : qew.getModes()) { switch (m) { @@ -81,19 +82,19 @@ public QueryResponse execute(QueryWrapper qw) { exactly(groupTimes, qew.getSupport(), metadata.getTraces()); break; case "co-existence": - coExistence(joined, bUniqueSingle, bSupport, bTotalTraces,notFoundPairs); + coExistence(joined, uniqueSingleDf, qew.getSupport(), metadata.getTraces()); break; case "not-co-existence": - notCoExistence(joined, bUniqueSingle, bSupport, bTotalTraces,notFoundPairs); + notCoExistence(joined, uniqueSingleDf, qew.getSupport(), metadata.getTraces(), notFoundPairs); break; case "choice": - choice(uEventType, bSupport, bTotalTraces); + choice(uEventType, qew.getSupport(), metadata.getTraces()); break; case "exclusive-choice": - exclusiveChoice(joined, bUniqueSingle, bSupport, bTotalTraces,notFoundPairs); + exclusiveChoice(joined, uniqueSingleDf, singleUnique, qew.getSupport(), metadata.getTraces(), notFoundPairs); break; case "responded-existence": - respondedExistence(joined, bUniqueSingle, bSupport, bTotalTraces); + respondedExistence(joined, uniqueSingleDf,qew.getSupport(), metadata.getTraces()); break; } @@ -107,19 +108,26 @@ public QueryResponse execute(QueryWrapper qw) { } public QueryResponseExistence runAll(Map> groupTimes, double support, - JavaRDD joined, Broadcast bSupport, - Broadcast bTotalTraces, Broadcast> bUniqueSingle, - JavaRDD uEventType) { - Set notFoundPairs = declareUtilities.extractNotFoundPairs(groupTimes.keySet(),joined); + Dataset joined, + long totalTraces, Map uniqueSingle, + Dataset uEventType) { + Set notFoundPairs = declareUtilities.extractNotFoundPairs(groupTimes.keySet(),joined); + + //Make it available to the spark context + List uniqueSingleRows = uniqueSingle.entrySet().stream() + .map(entry -> new EventTypeOccurrences(entry.getKey(), entry.getValue())) + .toList(); + Dataset uniqueSingleDf = sparkSession.createDataset(uniqueSingleRows, + Encoders.bean(EventTypeOccurrences.class)); //all event pairs will contain only those that weren't found in the dataset existence(groupTimes, support, metadata.getTraces()); absence(groupTimes, support, metadata.getTraces()); exactly(groupTimes, support, metadata.getTraces()); - coExistence(joined, bUniqueSingle, bSupport, bTotalTraces,notFoundPairs); - choice(uEventType, bSupport, bTotalTraces); - exclusiveChoice(joined, bUniqueSingle, bSupport, bTotalTraces,notFoundPairs); - respondedExistence(joined, bUniqueSingle, bSupport, bTotalTraces); + coExistence(joined, uniqueSingleDf, support, totalTraces); + choice(uEventType, support, totalTraces); + exclusiveChoice(joined, uniqueSingleDf,uniqueSingle, support, totalTraces,notFoundPairs); + respondedExistence(joined, uniqueSingleDf, support, totalTraces); return this.queryResponseExistence; } @@ -128,24 +136,29 @@ public QueryResponseExistence runAll(Map> groupTi * (Event Type) -> (number of occurrences) -> # of traces that contain that much amount of occurrences of this * event type. e.g. searching how many traces have exactly 2 instances of the event type 'c' * ('c')->(2) -> response + * * @param uEventType an RDD containing for each event type the unique traces and their corresponding occurrences * of this event type * @return a map of the form (Event Type) -> (number of occurrences) -> # of traces that contain that * much amount of occurrences of this event type. */ - public Map> createMapForSingle(JavaRDD uEventType) { - return uEventType - .map(x -> new Tuple2<>(x.getEventType(), x.groupTimes())) - .keyBy(x -> x._1) - .mapValues(x -> x._2) - .collectAsMap(); + public Map> createMapForSingle(Dataset uEventType) { + List uEventTypeList = uEventType.collectAsList(); + Map> response = uEventTypeList.parallelStream().collect(Collectors.toMap( + UniqueTracesPerEventType::getEventName, // Key: eventType (String) + UniqueTracesPerEventType::groupTimes, // Value: groupTimes() -> HashMap + // Resolves key conflicts (shouldn't happen) + (existing, replacement) -> existing + )); + return response; } /** * Based on the output of the above function, this code extracts the number of traces that contain a particular * event type, i.e. the response will contain information (event type) -> #traces containing it + * * @param groupTimes a map of the form (Event Type) -> (number of occurrences) -> # of traces that contain that - * * much amount of occurrences of this event type. + * * much amount of occurrences of this event type. * @return a map in the form (event type) -> #traces containing it */ public Map extractUniqueTracesSingle(Map> groupTimes) { @@ -161,28 +174,44 @@ public Map extractUniqueTracesSingle(Map joinUnionTraces(JavaRDD uPairs) { - - return uPairs - .keyBy(UniqueTracesPerEventPair::getKey) - .leftOuterJoin(uPairs.keyBy(UniqueTracesPerEventPair::getKeyReverse)) - .map(x -> { - UniqueTracesPerEventPair right = x._2._2. - orElse(new UniqueTracesPerEventPair(x._1._2, x._1._1, new ArrayList<>())); - // find union of the 2 lists - Set set = new LinkedHashSet<>(x._2._1.getUniqueTraces()); - set.addAll(right.getUniqueTraces()); - ArrayList combinedList = new ArrayList<>(set); - return new EventPairToNumberOfTrace(x._1._1, x._1._2, combinedList.size()); - }); + public Dataset joinUnionTraces(Dataset uPairs) { + + // Create reversed key dataset (eventB, eventA) + Dataset reversedPairs = uPairs + .withColumnRenamed("eventA", "eventB-2") // Swap column names + .withColumnRenamed("eventB", "eventA-2") + .withColumnRenamed("uniqueTraces", "reversedUniqueTraces"); + + // Perform Left Outer Join on (eventA, eventB) with (eventB, eventA) + Dataset joinedPairs = uPairs + .join(reversedPairs, + functions.col("eventA").equalTo(functions.col("eventA-2")) + .and(functions.col("eventB").equalTo(functions.col("eventB-2"))), + "left_outer") + .select(functions.col("eventA"), + functions.col("eventB"), + functions.col("uniqueTraces"), + functions.col("reversedUniqueTraces")); + + Dataset result = joinedPairs + .withColumn("mergedTraces", functions.expr( + "array_union(uniqueTraces, reversedUniqueTraces)" + )) + .withColumn("numberOfTraces", functions.size(functions.col("mergedTraces"))) + .selectExpr("eventA", "eventB", "numberOfTraces") + .as(Encoders.bean(EventPairToNumberOfTrace.class)); + + return result; } + /** * Extract the constraints that correspond to the 'existence' template. - * @param groupTimes a map of the form (Event Type) -> (number of occurrences) -> # of traces that contain that - * * much amount of occurrences of this event type. - * @param support minimum support that a pattern should have in order to be included in the result set + * + * @param groupTimes a map of the form (Event Type) -> (number of occurrences) -> # of traces that contain that + * * much amount of occurrences of this event type. + * @param support minimum support that a pattern should have in order to be included in the result set * @param totalTraces the total number of traces in this log database */ private void existence(Map> groupTimes, double support, long totalTraces) { @@ -192,7 +221,7 @@ private void existence(Map> groupTimes, double su HashMap t = groupTimes.get(et); List times = new ArrayList<>(t.keySet()).stream().sorted(Comparator.reverseOrder()) .collect(Collectors.toList()); - for (int time=3;time>0;time--) { + for (int time = 3; time > 0; time--) { int finalTime = time; double s = (double) times.stream().filter(x -> x >= finalTime).mapToLong(t::get).sum() / totalTraces; if (s >= support) { @@ -205,9 +234,10 @@ private void existence(Map> groupTimes, double su /** * Extract the constraints that correspond to the 'absence' template. - * @param groupTimes a map of the form (Event Type) -> (number of occurrences) -> # of traces that contain that - * * much amount of occurrences of this event type. - * @param support minimum support that a pattern should have in order to be included in the result set + * + * @param groupTimes a map of the form (Event Type) -> (number of occurrences) -> # of traces that contain that + * * much amount of occurrences of this event type. + * @param support minimum support that a pattern should have in order to be included in the result set * @param totalTraces the total number of traces in this log database */ private void absence(Map> groupTimes, double support, long totalTraces) { @@ -219,7 +249,7 @@ private void absence(Map> groupTimes, double supp t.put(0, totalTraces - totalSum); List times = new ArrayList<>(t.keySet()).stream().sorted().collect(Collectors.toList()); if (!times.contains(2)) times.add(2); //to be sure that it will run at least once - for (int time=3;time>=2;time--) { + for (int time = 3; time >= 2; time--) { int finalTime = time; double s = (double) times.stream().filter(x -> x < finalTime).map(t::get) .filter(Objects::nonNull).mapToLong(x -> x).sum() / totalTraces; @@ -233,9 +263,10 @@ private void absence(Map> groupTimes, double supp /** * Extract the constraints that correspond to the 'exactly' template. - * @param groupTimes a map of the form (Event Type) -> (number of occurrences) -> # of traces that contain that - * * much amount of occurrences of this event type. - * @param support minimum support that a pattern should have in order to be included in the result set + * + * @param groupTimes a map of the form (Event Type) -> (number of occurrences) -> # of traces that contain that + * * much amount of occurrences of this event type. + * @param support minimum support that a pattern should have in order to be included in the result set * @param totalTraces the total number of traces in this log database */ private void exactly(Map> groupTimes, double support, long totalTraces) { @@ -246,7 +277,7 @@ private void exactly(Map> groupTimes, double supp long totalSum = t.values().stream().mapToLong(x -> x).sum(); if (!t.containsKey(0)) t.put(0, totalTraces - totalSum); for (Map.Entry x : t.entrySet()) { - if (x.getValue() >= (support * totalTraces) && x.getKey()>0) { + if (x.getValue() >= (support * totalTraces) && x.getKey() > 0) { response.add(new EventN(et, x.getKey(), x.getValue().doubleValue() / totalTraces)); } } @@ -256,17 +287,16 @@ private void exactly(Map> groupTimes, double supp /** * Extract the constraints that correspond to the 'exactly' template. - * @param joinedUnion A RDD that contains objects of the form (eventA,eventB,traceID), i.e. which traces - * contain at least one occurrence of the pair (eventA, eventB) - * @param bUniqueSingle A spark broadcast map, that contains a map of the form (event type) -> # traces + * + * @param joinedUnion A Dataset that contains objects of the form (eventA,eventB,traceID), i.e. which traces + * contain at least one occurrence of the pair (eventA, eventB) + * @param uniqueSingleDf A Dataset that contains a map of the form (event type) -> # traces * that contain this event type - * @param bSupport A spark broadcast variable, that corresponds to the user-defined support - * @param bTotalTraces Total traces in this log database - * @param notFound A set of the event pairs that have 0 occurrence in the log database + * @param support the user-defined support + * @param totalTraces Total traces in this log database */ - private void coExistence(JavaRDD joinedUnion, - Broadcast> bUniqueSingle, Broadcast bSupport, - Broadcast bTotalTraces, Set notFound) { + private void coExistence(Dataset joinedUnion, + Dataset uniqueSingleDf, double support, long totalTraces) { // |A| = |IndexTable(a,b) U IndexTable(b,a)|, i.e., unique traces where a and b co-exist // total_traces = |A| + (non-of them exist) + (only 'a' exist) + (only b exist) (1) @@ -274,44 +304,54 @@ private void coExistence(JavaRDD joinedUnion, // where the co-existence is true is when |A|+(non-of them exist) >= support (2) // (1)+(2)=> total_traces - (unique traces of a) + |A| - (unique traces of b) + |A| >= support* total_traces // total_traces - (unique traces of a) - (unique traces of b) - |A| >= support* total_traces - List coExistence = joinedUnion - .filter(x-> !x.getEventA().equals(x.getEventB())) //remove pairs with the same event type - .filter(x -> x.getEventA().compareTo(x.getEventB()) <= 0) - .filter(x -> x.getNumberOfTraces() >= (bSupport.getValue()) * bTotalTraces.getValue()) - .map(x -> { - double sup = (bTotalTraces.getValue() - bUniqueSingle.getValue().get(x.getEventA()) - - bUniqueSingle.getValue().get(x.getEventB()) + 2L * x.getNumberOfTraces()); - return new Abstract2DeclareConstraint(x.getEventA(), x.getEventB(), x.getNumberOfTraces(), sup); - }) - .filter(x -> x.getSupport() >= (bSupport.getValue() * bTotalTraces.getValue())) - .collect() - .stream().map(x -> new EventPairSupport(x.getEventA(), x.getEventB(), - x.getSupport() / bTotalTraces.value())) - .collect(Collectors.toList()); + Dataset initialFiltered = joinedUnion + .filter(functions.col("eventA").notEqual(functions.col("eventB"))) // Remove self pairs + .filter(functions.col("eventA").leq(functions.col("eventB"))) // Order pairs + .filter(functions.col("numberOfTraces").geq(support * totalTraces)); // Minimum support check + + Dataset joinedWithUnique = initialFiltered + .join(uniqueSingleDf.withColumnRenamed("eventName", "eventA") + .withColumnRenamed("numberOfTraces", "uniqueA"), "eventA", "left") + .join(uniqueSingleDf.withColumnRenamed("eventName", "eventB") + .withColumnRenamed("numberOfTraces", "uniqueB"), "eventB", "left"); + + Dataset filteredAndComputed = joinedWithUnique + .withColumn("computedSupport", functions.expr( + totalTraces + " - uniqueA - uniqueB + 2 * numberOfTraces" + )) + .filter(functions.col("computedSupport").geq(support * totalTraces)) // Filter based on computed support + .withColumn("support", functions.col("computedSupport").divide(totalTraces)) // Normalize support + .select("eventA", "eventB", "support"); + + List coExistence = filteredAndComputed + .as(Encoders.bean(EventPairSupport.class)) + .collectAsList(); queryResponseExistence.setCoExistence(coExistence); } - private void notCoExistence(JavaRDD joinedUnion, - Broadcast> bUniqueSingle, Broadcast bSupport, - Broadcast bTotalTraces, Set notFound) { + private void notCoExistence(Dataset joinedUnion, + Dataset uniqueSingleDf, double support, long totalTraces, + Set notFound) { //valid event types can be used as first in a pair (since they have support greater than the user-defined) - List notCoExistence = joinedUnion - .filter(x-> !x.getEventA().equals(x.getEventB())) //remove pairs with the same event type - .filter(x -> x.getEventA().compareTo(x.getEventB()) <= 0)//filter same pair that appears in both ways - .filter(x -> x.getNumberOfTraces() <= ((1 - bSupport.getValue()) * bTotalTraces.getValue()))//filter based on support - .map(x -> new EventPairSupport(x.getEventA(), x.getEventB(), - 1-(double) x.getNumberOfTraces() / bTotalTraces.getValue())) - .collect(); + Dataset initialFiltered = joinedUnion + .filter(functions.col("eventA").notEqual(functions.col("eventB"))) // Remove self pairs + .filter(functions.col("eventA").leq(functions.col("eventB"))) // Order pairs + .filter(functions.col("numberOfTraces").leq(1-support * totalTraces)); // Minimum support check + + List notCoExistence =initialFiltered + .selectExpr("eventA", "eventB", String.format("1-numberOfTraces/%s as support", totalTraces)) + .as(Encoders.bean(EventPairSupport.class)) + .collectAsList(); //Add all the pairs in the notFound that their reverse is also in this set. Meaning that these two //events never co-exist in the entire database Set notCoExist = new HashSet<>(); - for(EventPair ep:notFound){ - if(notFound.contains(new EventPair(new Event(ep.getEventB().getName()),new Event(ep.getEventA().getName())))){ - if(ep.getEventA().getName().compareTo(ep.getEventB().getName())>0) { - notCoExist.add(new EventPairSupport(ep.getEventA().getName(),ep.getEventB().getName(),1)); - }else{ - notCoExist.add(new EventPairSupport(ep.getEventB().getName(),ep.getEventA().getName(),1)); + for (EventTypes ep : notFound) { + if (notFound.contains(new EventTypes(ep.getEventB(), ep.getEventA()))) { + if (ep.getEventA().compareTo(ep.getEventB()) > 0) { + notCoExist.add(new EventPairSupport(ep.getEventA(), ep.getEventB(), 1)); + } else { + notCoExist.add(new EventPairSupport(ep.getEventB(), ep.getEventA(), 1)); } } } @@ -324,113 +364,157 @@ private void notCoExistence(JavaRDD joinedUnion, /** * Extract the constraints that correspond to the 'choice' template. - * @param uEventType a RDD in the form (event type, [(traceId,#occurrences)]) - * @param bSupport A spark broadcast variable, that corresponds to the user-defined support - * @param bTotalTraces Total traces in this log database + * + * @param uEventType a Dataset in the form (event type, [(traceId,#occurrences)]) + * @param support the user-defined support + * @param totalTraces Total traces in this log database */ - private void choice(JavaRDD uEventType, Broadcast bSupport, Broadcast bTotalTraces) { + private void choice(Dataset uEventType, double support, long totalTraces) { //create possible pairs without duplication - List choice = uEventType.keyBy(UniqueTracesPerEventType::getEventType) - .cartesian(uEventType.keyBy(UniqueTracesPerEventType::getEventType)) - .filter(x -> x._1._1.compareTo(x._2._1) < 0) //remove duplicate pairs - //filter based on the total number of traces that contain either of the two event types (early pruning) - .filter(x -> x._1._2.getOccurrences().size() + x._2._2.getOccurrences().size() >= - (bSupport.getValue()) * bTotalTraces.getValue()) - //actual count the traces in which either of them exists (removing the duplicate counts - traces where - //both exist) - .map(x -> { - LinkedHashSet listA = x._1._2.getOccurrences().stream() - .map(OccurrencesPerTrace::getTraceId).collect(Collectors.toCollection(LinkedHashSet::new)); - listA.addAll(x._2._2.getOccurrences().stream().map(OccurrencesPerTrace::getTraceId) - .collect(Collectors.toList())); - return new EventPairSupport(x._1._1, x._2._1, (double) listA.size() / bTotalTraces.getValue()); - }) - //filter based on the support (correct filtering) - .filter(x -> x.getSupport() >= bSupport.getValue()) - .collect(); - //add them to the result set + + Dataset eventPairs = uEventType.alias("a") + .crossJoin(uEventType.alias("b")) // Generates ALL event combinations + // Avoid duplicate pairs (A, B) & (B, A) + .filter(functions.col("a.eventName").lt(functions.col("b.eventName"))) + .select( + functions.col("a.eventName").alias("eventA"), + functions.col("b.eventName").alias("eventB"), + functions.col("a.occurrences").alias("occurrencesA"), + functions.col("b.occurrences").alias("occurrencesB") + ); + + // Apply early pruning: only keep event pairs where the total occurrences meet the threshold + Dataset filteredPairs = eventPairs + .withColumn("totalOccurrences", functions.expr("size(occurrencesA) + size(occurrencesB)")) + .filter(functions.col("totalOccurrences").geq(support * totalTraces)); // Early filtering + + //actual count the traces in which either of them exists (removing the duplicate counts - traces where + //both exist) + Dataset calculateSupport = filteredPairs + .withColumn("uniqueTraces", functions.expr( + "array_union(transform(occurrencesA, x -> x.traceId), transform(occurrencesB, x -> x.traceId))" + )) + .withColumn("traceCount", functions.size(functions.col("uniqueTraces"))) // Count unique traces + .withColumn("support", functions.col("traceCount").divide(totalTraces)) // Compute support + .filter(functions.col("support").geq(support)) // Final filtering based on support + .select("eventA", "eventB", "support") + .as(Encoders.bean(EventPairSupport.class)); + + List choice = calculateSupport.collectAsList(); queryResponseExistence.setChoice(choice); } /** * Extract the constraints that correspond to the 'exclusive choice' template. - * @param joined A RDD that contains objects of the form (eventA,eventB,traceID), i.e. which traces - * contain at least one occurrence of the pair (eventA, eventB) - * @param bUniqueSingle A spark broadcast map, that contains a map of the form (event type) -> # traces - * that contain this event type - * @param bSupport A spark broadcast variable, that corresponds to the user-defined support - * @param bTotalTraces Total traces in this log database - * @param notFound A set of the event pairs that have 0 occurrence in the log database + * + * @param joinedUnion A Dataset that contains objects of the form (eventA,eventB,traceID), i.e. which traces + * contain at least one occurrence of the pair (eventA, eventB) + * @param uniqueSingleDf A Dataset that contains a map of the form (event type) -> # traces + * * that contain this event type + * @param uniqueSingle The same as uniqueSingleDF but located in master + * @param support the user-defined support + * @param totalTraces Total traces in this log database + * @param notFound A set of the event pairs that have 0 occurrence in the log database */ - private void exclusiveChoice(JavaRDD joined, Broadcast> bUniqueSingle, - Broadcast bSupport, Broadcast bTotalTraces, Set notFound) { + private void exclusiveChoice(Dataset joinedUnion, Dataset uniqueSingleDf, + Map uniqueSingle,double support, long totalTraces, Set notFound) { + + Dataset initialFiltered = joinedUnion + .filter(functions.col("eventA").notEqual(functions.col("eventB"))) // Remove self pairs + .filter(functions.col("eventA").leq(functions.col("eventB"))); // Order pairs + + Dataset joinedWithUnique = initialFiltered + .join(uniqueSingleDf.withColumnRenamed("eventName", "eventA") + .withColumnRenamed("numberOfTraces", "uniqueA"), "eventA", "left") + .join(uniqueSingleDf.withColumnRenamed("eventName", "eventB") + .withColumnRenamed("numberOfTraces", "uniqueB"), "eventB", "left"); // detects exclusive choice in pairs that appear at least once in the log database - List exclusiveChoice = joined.filter(x -> x.getEventA().compareTo(x.getEventB()) < 0) - .filter(x->!x.getEventA().equals(x.getEventB())) - .map(x -> { - double sup = (double) (bUniqueSingle.getValue().get(x.getEventA()) - + bUniqueSingle.getValue().get(x.getEventB()) - 2 * x.getNumberOfTraces()) / bTotalTraces.getValue(); - return new EventPairSupport(x.getEventA(), x.getEventB(), sup); - }).filter(x -> x.getSupport() >= bSupport.getValue()) - .collect(); + Dataset exclusiveChoiceDF = joinedWithUnique + .withColumn("support", functions.expr( + "(uniqueA + uniqueB - 2 * numberOfTraces) / " + totalTraces + )) + .filter(functions.col("support").geq(support)) + .select("eventA","eventB","support") + .as(Encoders.bean(EventPairSupport.class)); + + List exclusiveChoice = exclusiveChoiceDF.collectAsList(); // detects exclusive choice in pairs that do not appear in the log database // therefore it checks if both (a,b) and (b,a) are in the notFound set Set notCoExist = new HashSet<>(); - for(EventPair ep:notFound){ - if(notFound.contains(new EventPair(new Event(ep.getEventB().getName()),new Event(ep.getEventA().getName())))){ - //check the order of the names in order to add each pair only once - if(ep.getEventA().getName().compareTo(ep.getEventB().getName())>0) { - notCoExist.add(new EventPairSupport(ep.getEventA().getName(),ep.getEventB().getName(),1)); - }else{ - notCoExist.add(new EventPairSupport(ep.getEventB().getName(),ep.getEventA().getName(),1)); + for (EventTypes ep : notFound) { + if (notFound.contains(new EventTypes(ep.getEventB(), ep.getEventA()))) { + if (ep.getEventA().compareTo(ep.getEventB()) > 0) { + notCoExist.add(new EventPairSupport(ep.getEventA(), ep.getEventB(), 1)); + } else { + notCoExist.add(new EventPairSupport(ep.getEventB(), ep.getEventA(), 1)); } } } + + + //Calculates the support of the constraints detected from the not found pairs, as this behavior //should describe at least 'support'% of the total traces - List exclusiveChoice2 = notCoExist.stream().map(x->{ - double sup = (double) (bUniqueSingle.getValue().get(x.getEventA()) + - bUniqueSingle.getValue().get(x.getEventB())) / bTotalTraces.getValue(); - return new EventPairSupport(x.getEventA(), x.getEventB(), sup); - }).filter(x -> x.getSupport() >= bSupport.getValue()) - .collect(Collectors.toList()); + List exclusiveChoice2 = notCoExist.stream().map(x -> { + double sup = (double) (uniqueSingle.get(x.getEventA()) + + uniqueSingle.get(x.getEventB())) / totalTraces; + return new EventPairSupport(x.getEventA(), x.getEventB(), sup); + }).filter(x -> x.getSupport() >= support) + .collect(Collectors.toList()); //add both together and pass them to the response exclusiveChoice2.addAll(exclusiveChoice); queryResponseExistence.setExclusiveChoice(exclusiveChoice2); } + /** - * Extract the constraints that correspond to the 'exclusive choice' template. - * @param joined A RDD that contains objects of the form (eventA,eventB,traceID), i.e. which traces - * contain at least one occurrence of the pair (eventA, eventB) - * @param bUniqueSingle A spark broadcast map, that contains a map of the form (event type) -> # traces - * that contain this event type - * @param bSupport A spark broadcast variable, that corresponds to the user-defined support - * @param bTotalTraces Total traces in this log database + * Extract the constraints that correspond to the 'responeddexistence' template. + * + * @param joinedUnion A Dataset that contains objects of the form (eventA,eventB,traceID), i.e. which traces + * contain at least one occurrence of the pair (eventA, eventB) + * @param uniqueSingleDf A Dataset that contains a map of the form (event type) -> # traces + * * that contain this event type + * @param support the user-defined support + * @param totalTraces Total traces in this log database */ - private void respondedExistence(JavaRDD joined, Broadcast> bUniqueSingle, - Broadcast bSupport, Broadcast bTotalTraces) { - List responseExistence = joined - .filter(x->!x.getEventA().equals(x.getEventB())) //remove duplicates, i.e. (eventA,eventA) pairs - .flatMap((FlatMapFunction)x->{ - List eps = new ArrayList<>(); - //check support for the constraint responded-existence(a,b) - double sup = ((double) x.getNumberOfTraces() + bTotalTraces.getValue() - - bUniqueSingle.getValue().get(x.getEventA())) / bTotalTraces.getValue(); - eps.add(new EventPairSupport(x.getEventA(), x.getEventB(), sup)); - //check support for the constraint responded-existence(b,a) - sup = ((double) x.getNumberOfTraces() + bTotalTraces.getValue() - - bUniqueSingle.getValue().get(x.getEventB())) / bTotalTraces.getValue(); - eps.add(new EventPairSupport(x.getEventB(), x.getEventA(), sup)); - return eps.iterator(); - } ) - .distinct() - .filter(x -> x.getSupport() >= bSupport.getValue()) - .collect(); + + private void respondedExistence(Dataset joinedUnion, Dataset uniqueSingleDf, + double support, long totalTraces) { + Dataset initialFiltered = joinedUnion + .filter(functions.col("eventA").notEqual(functions.col("eventB"))); // Remove self pairs + + Dataset joinedWithUnique = initialFiltered + .join(uniqueSingleDf.withColumnRenamed("eventName", "eventA") + .withColumnRenamed("numberOfTraces", "uniqueA"), "eventA", "left") + .join(uniqueSingleDf.withColumnRenamed("eventName", "eventB") + .withColumnRenamed("numberOfTraces", "uniqueB"), "eventB", "left"); + + // detects exclusive choice in pairs that appear at least once in the log database + Dataset extractedBothSupports = joinedWithUnique + .withColumn("supportA", functions.expr( + String.format("(numberOfTraces + %s - uniqueA)/%s",totalTraces,totalTraces) + )) + .withColumn("supportB", functions.expr( + String.format("(numberOfTraces + %s - uniqueB)/%s",totalTraces,totalTraces) + )); + + Dataset eventsForward = extractedBothSupports + .filter(functions.col("supportA").geq(support)) + .selectExpr("eventA","eventB","supportA as support") + .as(Encoders.bean(EventPairSupport.class)); + + Dataset eventsBackwards = extractedBothSupports + .filter(functions.col("supportB").geq(support)) + .selectExpr("eventB as eventA","eventA as eventB","supportB as support") + .as(Encoders.bean(EventPairSupport.class)); + + + List responseExistence = eventsForward.union(eventsBackwards) + .distinct().collectAsList(); //pass to the response this.queryResponseExistence.setRespondedExistence(responseExistence); diff --git a/src/main/java/com/datalab/siesta/queryprocessor/declare/queryPlans/orderedRelations/OrderedRelationsUtilityFunctions.java b/src/main/java/com/datalab/siesta/queryprocessor/declare/queryPlans/orderedRelations/OrderedRelationsUtilityFunctions.java deleted file mode 100644 index a06e41a2..00000000 --- a/src/main/java/com/datalab/siesta/queryprocessor/declare/queryPlans/orderedRelations/OrderedRelationsUtilityFunctions.java +++ /dev/null @@ -1,105 +0,0 @@ -package com.datalab.siesta.queryprocessor.declare.queryPlans.orderedRelations; - -import com.datalab.siesta.queryprocessor.declare.model.Abstract2OrderConstraint; -import com.datalab.siesta.queryprocessor.declare.model.EventPairTraceOccurrences; -import org.springframework.stereotype.Service; - - -import java.io.Serializable; -import java.util.List; -import java.util.stream.Collectors; - -/** - * Contain functions that are utilized to count the occurrences of order-relation constraints. - * It is required two list of integers (one for each event type) which contains the positions of - * the occurrences within this trace - */ -@Service -public class OrderedRelationsUtilityFunctions implements Serializable { - - /** - * Counts the occurrences for the 'response(eventA,eventB)' constraint - * @param line a {@link EventPairTraceOccurrences} object - * @return # of occurrences for the 'response(eventA,eventB)' constraint - */ - public Abstract2OrderConstraint countResponse(EventPairTraceOccurrences line) { - int s = 0; - for (int a : line.getOccurrencesA()) { - if (line.getOccurrencesB().stream().anyMatch(y -> y > a)) s += 1; - } - return new Abstract2OrderConstraint(line.getEventA(), line.getEventB(), "r", s); - } - - /** - * Counts the occurrences for the 'precedence(eventA,eventB)' constraint - * @param line a {@link EventPairTraceOccurrences} object - * @return # of occurrences for the 'response(eventA,eventB)' constraint - */ - public Abstract2OrderConstraint countPrecedence(EventPairTraceOccurrences line) { - int s = 0; - for (int a : line.getOccurrencesB()) { - if (line.getOccurrencesA().stream().anyMatch(y -> y < a)) s += 1; - } - return new Abstract2OrderConstraint(line.getEventA(), line.getEventB(), "p", s); - } - - /** - * Counts the occurrences for the 'alternate response(eventA,eventB)' constraint - * @param line a {@link EventPairTraceOccurrences} object - * @return # of occurrences for the 'response(eventA,eventB)' constraint - */ - public Abstract2OrderConstraint countResponseAlternate(EventPairTraceOccurrences line) { - int s = 0; - List aList = line.getOccurrencesA().stream().sorted().collect(Collectors.toList()); - for (int i = 0; i < aList.size() - 1; i++) { - int finalI = i; - if(line.getOccurrencesB().stream().anyMatch(y-> y>aList.get(finalI) && y y>aList.get(aList.size()-1))) s+=1; - return new Abstract2OrderConstraint(line.getEventA(), line.getEventB(), "r", s); - } - - /** - * Counts the occurrences for the 'alternate precedence(eventA,eventB)' constraint - * @param line a {@link EventPairTraceOccurrences} object - * @return # of occurrences for the 'response(eventA,eventB)' constraint - */ - public Abstract2OrderConstraint countPrecedenceAlternate(EventPairTraceOccurrences line) { - int s = 0; - List bList = line.getOccurrencesB().stream().sorted().collect(Collectors.toList()); - for (int i = 1; i < bList.size() ; i++) { - int finalI = i; - if(line.getOccurrencesA().stream().anyMatch(y-> ybList.get(finalI -1))) s+=1; - } - if(line.getOccurrencesA().stream().anyMatch(y-> y y == a+1)) s += 1; - } - return new Abstract2OrderConstraint(line.getEventA(), line.getEventB(), "r", s); - } - - /** - * Counts the occurrences for the 'chain precedence(eventA,eventB)' constraint - * @param line a {@link EventPairTraceOccurrences} object - * @return # of occurrences for the 'response(eventA,eventB)' constraint - */ - public Abstract2OrderConstraint countPrecedenceChain(EventPairTraceOccurrences line) { - int s = 0; - for (int a : line.getOccurrencesB()) { - if (line.getOccurrencesA().stream().anyMatch(y -> y == a-1)) s += 1; - } - return new Abstract2OrderConstraint(line.getEventA(), line.getEventB(), "p", s); - } - - -} diff --git a/src/main/java/com/datalab/siesta/queryprocessor/declare/queryPlans/orderedRelations/QueryPlanOrderRelationsState.java b/src/main/java/com/datalab/siesta/queryprocessor/declare/queryPlans/orderedRelations/QueryPlanOrderRelationsState.java index 631ba981..81c914dc 100644 --- a/src/main/java/com/datalab/siesta/queryprocessor/declare/queryPlans/orderedRelations/QueryPlanOrderRelationsState.java +++ b/src/main/java/com/datalab/siesta/queryprocessor/declare/queryPlans/orderedRelations/QueryPlanOrderRelationsState.java @@ -3,12 +3,10 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; -import org.apache.spark.api.java.JavaPairRDD; -import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; -import org.apache.spark.api.java.function.FlatMapFunction; -import org.apache.spark.broadcast.Broadcast; import com.datalab.siesta.queryprocessor.declare.DeclareDBConnector; import com.datalab.siesta.queryprocessor.declare.model.EventPairSupport; @@ -23,49 +21,50 @@ import com.datalab.siesta.queryprocessor.model.Queries.QueryResponses.QueryResponse; import com.datalab.siesta.queryprocessor.model.Queries.Wrapper.QueryWrapper; import com.datalab.siesta.queryprocessor.storage.DBConnector; +import org.apache.spark.sql.*; import org.springframework.stereotype.Component; import org.springframework.web.context.annotation.RequestScope; -import scala.Tuple2; import scala.Tuple3; @Component @RequestScope public class QueryPlanOrderRelationsState extends QueryPlanState { - private DBConnector dbConnector; + private DBConnector dbConnector; + private SparkSession sparkSession; public QueryPlanOrderRelationsState(DeclareDBConnector declareDBConnector, JavaSparkContext javaSparkContext, - DBConnector dbConnector) { - super(declareDBConnector,javaSparkContext); + DBConnector dbConnector, SparkSession sparkSession) { + super(declareDBConnector, javaSparkContext); this.dbConnector = dbConnector; + this.sparkSession = sparkSession; } @Override public QueryResponse execute(QueryWrapper qw) { QueryOrderRelationWrapper qow = (QueryOrderRelationWrapper) qw; - //create activity matrix from the event names + //get all possible activities from database to create activity matrix List activities = dbConnector.getEventNames(metadata.getLogname()); - JavaRDD activityRDD = javaSparkContext.parallelize(activities); - JavaPairRDD activityMatrix = activityRDD.cartesian(activityRDD); + Dataset activityDF = sparkSession.createDataset(activities, Encoders.STRING()); + Dataset activityMatrix = activityDF.crossJoin(activityDF).toDF("activityA", "activityB"); //Extract All constraints - Broadcast bSupport = this.javaSparkContext.broadcast(qow.getSupport()); - List constraints = this.extractAll(bSupport, activityMatrix); + List constraints = this.extractAll(qow.getSupport(), activityMatrix); //filter the constraints to create the response QueryResponseOrderedRelationsState response = new QueryResponseOrderedRelationsState(); this.setSpecificConstraints(qow, constraints, response); - + this.extractStatistics(qow); response.setUpToDate(qow.isStateUpToDate()); - if(!qow.isStateUpToDate()){ - response.setEventsPercentage((qow.getIndexedEvents()/metadata.getEvents())*100); - response.setTracesPercentage((qow.getIndexedTraces()/metadata.getTraces())*100); + if (!qow.isStateUpToDate()) { + response.setEventsPercentage(((double) qow.getIndexedEvents() / metadata.getEvents()) * 100); + response.setTracesPercentage(((double) qow.getIndexedTraces() / metadata.getTraces()) * 100); response.setMessage("State is not fully updated. Consider re-running the preprocess to get 100% accurate constraints"); - }else{ + } else { response.setEventsPercentage(100); response.setTracesPercentage(100); } @@ -73,159 +72,158 @@ public QueryResponse execute(QueryWrapper qw) { return response; } - public QueryResponseOrderedRelationsState extractConstraintFunction(List constraints, String mode){ + public QueryResponseOrderedRelationsState extractConstraintFunction(List constraints, String mode) { QueryResponseOrderedRelationsState response = new QueryResponseOrderedRelationsState(); response.setMode(mode); - for(PairConstraint uc:constraints){ - if (mode.equals("simple")){ - if(uc.getRule().equals("response")){ + for (PairConstraint uc : constraints) { + if (mode.equals("simple")) { + if (uc.getRule().equals("response")) { response.getResponse().add(uc.getEventPairSupport()); - }else if(uc.getRule().equals("precedence") ){ + } else if (uc.getRule().equals("precedence")) { response.getPrecedence().add(uc.getEventPairSupport()); - }else if(uc.getRule().equals("succession")){ + } else if (uc.getRule().equals("succession")) { response.getSuccession().add(uc.getEventPairSupport()); - }else if(uc.getRule().equals("not-succession")){ + } else if (uc.getRule().equals("not-succession")) { response.getNotSuccession().add(uc.getEventPairSupport()); } - }else{ - if(uc.getRule().contains("response") && uc.getRule().contains(mode)){ + } else { + if (uc.getRule().contains("response") && uc.getRule().contains(mode)) { response.getResponse().add(uc.getEventPairSupport()); - }else if(uc.getRule().contains("precedence") && uc.getRule().contains(mode) ){ + } else if (uc.getRule().contains("precedence") && uc.getRule().contains(mode)) { response.getPrecedence().add(uc.getEventPairSupport()); - }else if(uc.getRule().contains("succession") && uc.getRule().contains(mode)){ + } else if (uc.getRule().contains("succession") && uc.getRule().contains(mode)) { response.getSuccession().add(uc.getEventPairSupport()); } - if(mode.equals("alternate") && uc.getRule().equals("not-succession")){ + if (mode.equals("alternate") && uc.getRule().equals("not-succession")) { response.getNotSuccession().add(uc.getEventPairSupport()); - }else if(mode.equals("chain") && uc.getRule().equals("not-chain-succession")){ + } else if (mode.equals("chain") && uc.getRule().equals("not-chain-succession")) { response.getNotSuccession().add(uc.getEventPairSupport()); } } - + } return response; } - public List extractAll(Broadcast bSupport, JavaPairRDD activityMatrix){ + public List extractAll(double support, Dataset activityMatrix) { //load order constraints from the database - JavaRDD orderStateRDD = this.declareDBConnector.queryOrderState(this.metadata.getLogname()); - - List constraints = new ArrayList<>(); - List constraints0 = activityMatrix.filter(x->{ - return !x._1().equals(x._2()); - }).keyBy(x->{ - return new Tuple2<>(x._1(),x._2()); - }).subtractByKey( - orderStateRDD.filter(y->{ - return y.getRule().contains("chain"); - }) - .distinct() - .keyBy(x->{ - return new Tuple2<>(x.getEventA(),x.getEventB()); - })).map(x->{ - EventPairSupport eps = new EventPairSupport(x._1()._1(),x._1()._2(),1.0); - return new PairConstraint(eps,"not-chain-succession"); - }).collect(); - constraints.addAll(constraints0); + Dataset orderStateDF = this.declareDBConnector.queryOrderState(this.metadata.getLogname()); + // activityMatrix has format + Dataset filteredActivities = activityMatrix + .filter(functions.col("activityA").notEqual(functions.col("activityB"))); + + Dataset onlyChains = orderStateDF + .filter(functions.col("rule").contains("chain")) + .select("eventA", "eventB") + .distinct(); + + Dataset remainingPairs = filteredActivities + .join(onlyChains, functions.col("activityA").equalTo(functions.col("eventA")) + .and(functions.col("activityB").equalTo(functions.col("eventB"))), "left_anti"); + + List constraints0 = remainingPairs.collectAsList() + .stream().map(row -> new PairConstraint( + new EventPairSupport(row.getString(0), row.getString(1), 1.0), + "not-chain-succession")).toList(); + + List constraints = new ArrayList<>(constraints0); //get unique traces per event type - JavaRDD eventOccurrencesRDD = this.declareDBConnector.querySingleTable(this.metadata.getLogname()); - Map eventOccurrences = eventOccurrencesRDD.mapToPair(x->{ - return new Tuple2<>(x.getEvent(),x.getSupport()); - }).collectAsMap(); - Broadcast> bEventOccurrences = javaSparkContext.broadcast(eventOccurrences); - - List constraints2 = orderStateRDD.flatMap((FlatMapFunction)x->{ - List l = new ArrayList<>(); - double sup = x.getOccurrences()/bEventOccurrences.getValue().get(x.getEventB()); - if(x.getRule().contains("response")){ - sup = x.getOccurrences()/bEventOccurrences.getValue().get(x.getEventA()); - } - l.add(new PairConstraint(new EventPairSupport(x.getEventA(),x.getEventB(),sup),x.getRule())); - if(x.getRule().contains("chain")){ - l.add(new PairConstraint(new EventPairSupport(x.getEventA(),x.getEventB(),sup),"chain-succession")); - l.add(new PairConstraint(new EventPairSupport(x.getEventA(),x.getEventB(),1-sup),"not-chain-succession")); - }else if(x.getRule().contains("alternate")){ - l.add(new PairConstraint(new EventPairSupport(x.getEventA(),x.getEventB(),sup),"alternate-succession")); - }else{ - l.add(new PairConstraint(new EventPairSupport(x.getEventA(),x.getEventB(),sup),"succession")); - l.add(new PairConstraint(new EventPairSupport(x.getEventA(),x.getEventB(),1-sup),"not-succession")); - } - return l.iterator(); - }) - .keyBy(x->{ - return new Tuple3<>(x.getRule(),x.getEventPairSupport().getEventA(),x.getEventPairSupport().getEventB()); - }) - .reduceByKey((x,y)->{ - EventPairSupport eps = x.getEventPairSupport(); - eps.setEventA(x.getEventPairSupport().getEventA()); - eps.setEventB(x.getEventPairSupport().getEventB()); - eps.setSupport(x.getEventPairSupport().getSupport()*y.getEventPairSupport().getSupport()); - return new PairConstraint(eps,x.getRule()); - }) - .filter(x->{ - return x._2().getEventPairSupport().getSupport()>=bSupport.getValue(); - }) - .map(x->{ - return x._2(); - }) - .collect(); - + Dataset eventOccurrencesDF = this.declareDBConnector.querySingleTable(this.metadata.getLogname()); + Map eventOccurrences = eventOccurrencesDF.collectAsList().stream() + .collect(Collectors.toMap( + EventSupport::getEvent, + EventSupport::getSupport + )); + List orderStateList = orderStateDF.collectAsList(); + List allConstraints = orderStateList.parallelStream(). + flatMap(x -> { + List l = new ArrayList<>(); + double sup = x.getOccurrences() / eventOccurrences.get(x.getEventB()); + if (x.getRule().contains("response")) { + sup = x.getOccurrences() / eventOccurrences.get(x.getEventA()); + } + l.add(new PairConstraint(new EventPairSupport(x.getEventA(), x.getEventB(), sup), x.getRule())); + if (x.getRule().contains("chain")) { + l.add(new PairConstraint(new EventPairSupport(x.getEventA(), x.getEventB(), sup), "chain-succession")); + l.add(new PairConstraint(new EventPairSupport(x.getEventA(), x.getEventB(), 1 - sup), "not-chain-succession")); + } else if (x.getRule().contains("alternate")) { + l.add(new PairConstraint(new EventPairSupport(x.getEventA(), x.getEventB(), sup), "alternate-succession")); + } else { + l.add(new PairConstraint(new EventPairSupport(x.getEventA(), x.getEventB(), sup), "succession")); + l.add(new PairConstraint(new EventPairSupport(x.getEventA(), x.getEventB(), 1 - sup), "not-succession")); + } + return l.parallelStream(); + }) + .toList(); + + Map, PairConstraint> mergedConstraints = new ConcurrentHashMap<>(); + + allConstraints.parallelStream().forEach(pc -> { + Tuple3 key = new Tuple3<>(pc.getRule(), pc.getEventPairSupport().getEventA(), pc.getEventPairSupport().getEventB()); + + mergedConstraints.merge(key, pc, (existing, newValue) -> { + EventPairSupport eps = existing.getEventPairSupport(); + eps.setSupport(eps.getSupport() * newValue.getEventPairSupport().getSupport()); // Multiply support values + return new PairConstraint(eps, existing.getRule()); + }); + }); + + List constraints2 = mergedConstraints.values().parallelStream() + .filter(pc -> pc.getEventPairSupport().getSupport() >= support) + .toList(); constraints.addAll(constraints2); //handle negatives - JavaRDD negativeStateRDD = this.declareDBConnector.queryNegativeState(this.metadata.getLogname()); - List constraints3 = negativeStateRDD - .map(x->{ - EventPairSupport eps = new EventPairSupport(x.get_1(),x.get_2(),1.0); - return new PairConstraint(eps,"not-succession"); - }) - .collect(); - constraints.addAll(constraints3); + Dataset negativeStateDF = this.declareDBConnector.queryNegativeState(this.metadata.getLogname()); + List constraints3 = negativeStateDF.collectAsList() + .parallelStream().map(row -> { + EventPairSupport eps = new EventPairSupport(row.get_1(), row.get_2(), 1.0); + return new PairConstraint(eps, "not-succession"); + }).toList(); + constraints.addAll(constraints3); //return all the constraints to be filtered return constraints; - } private void setSpecificConstraints(QueryOrderRelationWrapper qow, List constraints, - QueryResponseOrderedRelationsState response){ + QueryResponseOrderedRelationsState response) { response.setMode(qow.getMode()); // Wrapper has 3 modes simple, alternate and chain. // Based on these values this methods keeps the correct constraints. - for(PairConstraint uc:constraints){ - if(qow.getMode().equals("simple")){ - if(uc.getRule().equals("response") && uc.getRule().equals(qow.getConstraint())){ + for (PairConstraint uc : constraints) { + if (qow.getMode().equals("simple")) { + if (uc.getRule().equals("response") && qow.getConstraint().contains(uc.getRule())) { response.getResponse().add(uc.getEventPairSupport()); - }else if(uc.getRule().equals("precedence") && uc.getRule().equals(qow.getConstraint())){ + } else if (uc.getRule().equals("precedence") && qow.getConstraint().contains(uc.getRule())) { response.getPrecedence().add(uc.getEventPairSupport()); - }else if(uc.getRule().equals("succession") && uc.getRule().equals(qow.getConstraint())){ + } else if (uc.getRule().equals("succession") && qow.getConstraint().contains(uc.getRule())) { response.getSuccession().add(uc.getEventPairSupport()); - }else if(uc.getRule().equals("not-succession")){ + } else if (uc.getRule().contains("not-succession")) { response.getNotSuccession().add(uc.getEventPairSupport()); } - }else if(qow.getMode().equals("alternate")){ - if(uc.getRule().equals("alternate-succession") && qow.getConstraint().equals("alternate-succession")){ + } else if (qow.getMode().equals("alternate")) { + if (uc.getRule().equals("alternate-succession") && qow.getConstraint().contains("alternate-succession")) { response.getSuccession().add(uc.getEventPairSupport()); - }else if(uc.getRule().equals("not-succession")){ + } else if (uc.getRule().equals("not-succession")) { response.getNotSuccession().add(uc.getEventPairSupport()); - }else if(uc.getRule().equals("alternate-response") && qow.getConstraint().equals("response")){ + } else if (uc.getRule().equals("alternate-response") && qow.getConstraint().contains("response")) { response.getResponse().add(uc.getEventPairSupport()); - }else if(uc.getRule().equals("alternate-precedence") && qow.getConstraint().equals("precedence")){ + } else if (uc.getRule().equals("alternate-precedence") && qow.getConstraint().contains("precedence")) { response.getPrecedence().add(uc.getEventPairSupport()); } - } - else if(qow.getMode().equals("chain")){ - if(uc.getRule().equals("chain-succession") && qow.getConstraint().equals("succession")){ + } else if (qow.getMode().equals("chain")) { + if (uc.getRule().equals("chain-succession") && qow.getConstraint().contains("succession")) { response.getSuccession().add(uc.getEventPairSupport()); - }else if(uc.getRule().equals("not-chain-succession")){ + } else if (uc.getRule().equals("not-chain-succession")) { response.getNotSuccession().add(uc.getEventPairSupport()); - }else if(uc.getRule().equals("chain-response") && qow.getConstraint().equals("response")){ + } else if (uc.getRule().equals("chain-response") && qow.getConstraint().contains("response")) { response.getResponse().add(uc.getEventPairSupport()); - }else if(uc.getRule().equals("chain-precedence") && qow.getConstraint().equals("precedence")){ + } else if (uc.getRule().equals("chain-precedence") && qow.getConstraint().contains("precedence")) { response.getPrecedence().add(uc.getEventPairSupport()); } } - + } } diff --git a/src/main/java/com/datalab/siesta/queryprocessor/declare/queryPlans/orderedRelations/QueryPlanOrderedRelations.java b/src/main/java/com/datalab/siesta/queryprocessor/declare/queryPlans/orderedRelations/QueryPlanOrderedRelations.java index 0519d228..edd2096c 100644 --- a/src/main/java/com/datalab/siesta/queryprocessor/declare/queryPlans/orderedRelations/QueryPlanOrderedRelations.java +++ b/src/main/java/com/datalab/siesta/queryprocessor/declare/queryPlans/orderedRelations/QueryPlanOrderedRelations.java @@ -5,31 +5,30 @@ import com.datalab.siesta.queryprocessor.declare.model.*; import com.datalab.siesta.queryprocessor.declare.queryResponses.QueryResponseOrderedRelations; import com.datalab.siesta.queryprocessor.declare.queryWrappers.QueryOrderRelationWrapper; +import com.datalab.siesta.queryprocessor.model.DBModel.EventTypes; import com.datalab.siesta.queryprocessor.model.DBModel.Metadata; -import com.datalab.siesta.queryprocessor.model.Events.EventPair; import com.datalab.siesta.queryprocessor.model.Queries.QueryPlans.QueryPlan; import com.datalab.siesta.queryprocessor.model.Queries.QueryResponses.QueryResponse; import com.datalab.siesta.queryprocessor.model.Queries.Wrapper.QueryWrapper; +import com.datalab.siesta.queryprocessor.storage.model.EventTypeTracePositions; import lombok.Getter; import lombok.Setter; -import org.apache.spark.api.java.JavaPairRDD; -import org.apache.spark.api.java.JavaRDD; + import org.apache.spark.api.java.JavaSparkContext; -import org.apache.spark.broadcast.Broadcast; +import org.apache.spark.sql.*; import org.apache.spark.storage.StorageLevel; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.springframework.web.context.annotation.RequestScope; -import scala.Tuple2; -import scala.Tuple3; + import java.util.*; -import java.util.stream.Collectors; + @Component @RequestScope -public class QueryPlanOrderedRelations implements QueryPlan{ +public class QueryPlanOrderedRelations implements QueryPlan { @Setter protected Metadata metadata; @@ -37,16 +36,14 @@ public class QueryPlanOrderedRelations implements QueryPlan{ protected JavaSparkContext javaSparkContext; @Getter protected QueryResponseOrderedRelations queryResponseOrderedRelations; - protected OrderedRelationsUtilityFunctions utils; protected DeclareUtilities declareUtilities; @Autowired public QueryPlanOrderedRelations(DeclareDBConnector declareDBConnector, JavaSparkContext javaSparkContext, - OrderedRelationsUtilityFunctions utils, DeclareUtilities declareUtilities) { + DeclareUtilities declareUtilities) { this.declareDBConnector = declareDBConnector; this.javaSparkContext = javaSparkContext; - this.utils = utils; - this.declareUtilities=declareUtilities; + this.declareUtilities = declareUtilities; } public void initQueryResponse() { @@ -57,31 +54,26 @@ public void initQueryResponse() { @Override public QueryResponse execute(QueryWrapper qw) { QueryOrderRelationWrapper qpw = (QueryOrderRelationWrapper) qw; - //query IndexTable - JavaRDD indexRDD = declareDBConnector.queryIndexOriginalDeclare(this.metadata.getLogname()) - .filter(x -> !x.getEventA().equals(x.getEventB())); + //query IndexTable + Dataset indexRDD = declareDBConnector.queryIndexOriginalDeclare(this.metadata.getLogname()) + .filter(functions.col("eventA").notEqual("eventB")); //query SingleTable - JavaPairRDD, List> singleRDD = declareDBConnector + Dataset singleRDD = declareDBConnector .querySingleTableAllDeclare(this.metadata.getLogname()); //join tables using joinTables and flat map to get the single events - JavaRDD joined = joinTables(indexRDD, singleRDD); + Dataset joined = joinTables(indexRDD, singleRDD); joined.persist(StorageLevel.MEMORY_AND_DISK()); //count the occurrences using the evaluate constraints - JavaRDD c = evaluateConstraint(joined, qpw.getConstraint()); + Dataset c = evaluateConstraint(joined); //filter based on the values of the SingleTable and the provided support and write to the response - Map uEventType = declareDBConnector.querySingleTableDeclare(this.metadata.getLogname()) - .map(x -> { - long all = x.getOccurrences().stream().mapToLong(OccurrencesPerTrace::getOccurrences).sum(); - return new Tuple2<>(x.getEventType(), all); - }).keyBy(x -> x._1).mapValues(x -> x._2).collectAsMap(); - Broadcast> bUEventTypes = javaSparkContext.broadcast(uEventType); - + Dataset eventTypeOccurrencesDataset = declareDBConnector + .extractTotalOccurrencesPerEventType(metadata.getLogname()); //add the not-succession constraints detected from the event pairs that did not occur in the database log - extendNotSuccession(bUEventTypes.getValue(), this.metadata.getLogname(), c); + extendNotSuccession(eventTypeOccurrencesDataset, this.metadata.getLogname(), c); //filter based on the user defined support - filterBasedOnSupport(c, bUEventTypes, qpw.getSupport()); + filterBasedOnSupport(c, eventTypeOccurrencesDataset, qpw.getSupport()); joined.unpersist(); return this.queryResponseOrderedRelations; } @@ -94,25 +86,29 @@ public QueryResponse execute(QueryWrapper qw) { * @param singleRDD a rdd of records that have the format (event * @ a rdd of {@link EventPairTraceOccurrences} */ - public JavaRDD joinTables(JavaRDD indexRDD, - JavaPairRDD, List> singleRDD) { + public Dataset joinTables(Dataset indexRDD, + Dataset singleRDD) { //for the traces that contain an occurrence of the event pair (a,b), joins their occurrences of these //event types (extracted from the Single Table) - return indexRDD - .keyBy(r -> new Tuple2<>(r.getEventA(), r.getTrace_id())) - .join(singleRDD) - .map(x -> x._2) - //join based on the second event - .keyBy(x -> new Tuple2<>(x._1.getEventB(), x._1.getTrace_id())) - .join(singleRDD) - .map(x -> { - String eventA = x._2._1._1.getEventA();//event a - String eventB = x._2._1._1.getEventB();//event b - List f = x._2._1._2; // occurrences of the first event - List s = x._2._2;//occurrences of the second event - String tid = x._1._2; //trace id - return new EventPairTraceOccurrences(eventA, eventB, tid, f, s); - }); + + Dataset singleTransformed = singleRDD. + selectExpr("eventName", "traceId as trace_id", "positions");// Alias for first join + + Dataset joined = indexRDD.as("primary") + .join(singleTransformed.as("singleA"), functions.col("primary.eventA") + .equalTo(functions.col("singleA.eventName")) + .and(functions.col("primary.trace_id") + .equalTo(functions.col("singleA.trace_id"))), "inner") + .selectExpr("eventA", "eventB", "primary.trace_id", "singleA.positions as positionsA") + .join(singleTransformed.as("singleB"), functions.col("primary.eventB") + .equalTo(functions.col("singleB.eventName")) + .and(functions.col("primary.trace_id") + .equalTo(functions.col("singleB.trace_id"))), "inner") + .selectExpr("eventA", "eventB", "primary.trace_id as traceId", + "positionsA as occurrencesA", "singleB.positions as occurrencesB") + .as(Encoders.bean(EventPairTraceOccurrences.class)); + + return joined; } /** @@ -120,69 +116,61 @@ public JavaRDD joinTables(JavaRDD i * constraint type. * * @param joined a rdd of {@link EventPairTraceOccurrences} - * @param constraint a string that describes the constraint under evaluation 'response', 'precedence' - * or 'succession' (which is the default execution) * @return a rdd of {@link Abstract2OrderConstraint} */ - public JavaRDD evaluateConstraint - (JavaRDD joined, String constraint) { + public Dataset evaluateConstraint + (Dataset joined) { + Dataset response = joined + .withColumn("s_r", functions.expr( + "size(filter(occurrencesA, a -> exists(occurrencesB, y -> y > a)))" + )) + .selectExpr("eventA", "eventB", "'r' as type", "s_r as count"); // Precedence constraint - JavaRDD tuple4JavaRDD; - switch (constraint) { - case "response": - tuple4JavaRDD = joined.map(utils::countResponse); - break; - case "precedence": - tuple4JavaRDD = joined.map(utils::countPrecedence); - break; - default: - tuple4JavaRDD = joined.map(utils::countPrecedence).union(joined.map(utils::countResponse)); - break; - } - return tuple4JavaRDD - //reduce by (eventA, eventB, mode - r/p) - .keyBy(y -> new Tuple3<>(y.getEventA(), y.getEventB(), y.getMode())) - .reduceByKey((x, y) -> { - x.setOccurrences(x.getOccurrences() + y.getOccurrences()); - return x; - }) - .map(x -> x._2); + Dataset precedence = joined + .withColumn("s_p", functions.expr( + "size(filter(occurrencesB, b -> exists(occurrencesA, y -> y < b)))" + )) + .selectExpr("eventA", "eventB", "'p' as type", "s_p as count"); + Dataset unioned = response.union(precedence) + .groupBy("eventA", "eventB", "type") + .agg(functions.sum("count").alias("occurrences")) + .withColumn("occurrences", functions.col("occurrences").cast("int")) + .selectExpr("eventA", "eventB", "type as mode", "occurrences") + .as(Encoders.bean(Abstract2OrderConstraint.class)); + return unioned; } /** * filters based on support and constraint required and write them to the response * - * @param c the occurrences of different templates detected - * @param bUEventType a spark broadcast map fo the form (event type) -> total occurrences in the log database - * @param support the user-defined support + * @param c the occurrences of different templates detected + * @param eventTypeOccurrences a spark broadcast map fo the form (event type) -> total occurrences in the log database + * @param support the user-defined support */ - public void filterBasedOnSupport(JavaRDD c, - Broadcast> bUEventType, double support) { - Broadcast bSupport = javaSparkContext.broadcast(support); + public void filterBasedOnSupport(Dataset c, + Dataset eventTypeOccurrences, double support) { + //calculates the support based on either the total occurrences of the first event (response) //or the occurrences of the second event (precedence) - JavaRDD> intermediate = c.map(x -> { - long total; - if (x.getMode().equals("r")) { //response - total = bUEventType.getValue().get(x.getEventA()); - } else { - total = bUEventType.getValue().get(x.getEventB()); - } - long found = x.getOccurrences(); - return new Tuple2<>(x.getMode(), new EventPairSupport(x.getEventA(), x.getEventB(), (double) found / total)); - }); + Dataset intermediate = getIntermediate(eventTypeOccurrences,c); + intermediate.persist(StorageLevel.MEMORY_AND_DISK()); - //filters based on the user-defined support and collects the result - List> detected = intermediate - .filter(x -> x._2.getSupport() >= bSupport.getValue()) - .collect(); - //splits the patterns that correspond to response and precedence into two separete lists - List responses = detected.stream().filter(x -> x._1.equals("r")) - .map(x -> x._2).collect(Collectors.toList()); - List precedence = detected.stream().filter(x -> x._1.equals("p")) - .map(x -> x._2).collect(Collectors.toList()); + + // filters based on the user-defined support and collect the results + Dataset detected = intermediate.filter(functions.col("support").geq(support)); + List responses = detected + .filter(functions.col("mode").equalTo("r")) + .selectExpr("eventA", "eventB", "support") + .as(Encoders.bean(EventPairSupport.class)) + .collectAsList(); + + List precedence = detected + .filter(functions.col("mode").equalTo("p")) + .selectExpr("eventA", "eventB", "support") + .as(Encoders.bean(EventPairSupport.class)) + .collectAsList(); if (!precedence.isEmpty() && !responses.isEmpty()) { //we are looking for succession and no succession setResults(responses, "response"); @@ -203,13 +191,22 @@ public void filterBasedOnSupport(JavaRDD c, //handle no succession //event pairs where both "r" and "p" have support less than the user-defined //for the pairs that do not appear here the function extendNotSuccession() is executed - check below - List> notSuccession = intermediate - .filter(x -> x._2.getSupport() <= (1 - bSupport.getValue())) - .collect(); - List notSuccessionR = notSuccession.stream().filter(x -> x._1.equals("r")) - .map(x -> x._2).collect(Collectors.toList()); - List notSuccessionP = notSuccession.stream().filter(x -> x._1.equals("p")) - .map(x -> x._2).collect(Collectors.toList()); + + Dataset notSuccession = intermediate + .filter(functions.col("support").leq(1-support)); + + List notSuccessionR = notSuccession + .filter(functions.col("mode").equalTo("r")) + .selectExpr("eventA", "eventB", "support") + .as(Encoders.bean(EventPairSupport.class)) + .collectAsList(); + + List notSuccessionP = notSuccession + .filter(functions.col("mode").equalTo("p")) + .selectExpr("eventA", "eventB", "support") + .as(Encoders.bean(EventPairSupport.class)) + .collectAsList(); + //create the event pair support based on the above event pairs List notSuccessionList = new ArrayList<>(); for (EventPairSupport r1 : notSuccessionR) { @@ -232,6 +229,39 @@ public void filterBasedOnSupport(JavaRDD c, intermediate.unpersist(); } + /** + * This method joins the two datasets and calculates the support, which is based + * either on the total occurrences of the first event (response) or the occurrences of the + * second event (precedence). This is handled by the case statement + * @param eventTypeOccurrences occurrences of each event type + * @param c dataset contains the pattern occurrences + * @return + */ + protected Dataset getIntermediate(Dataset eventTypeOccurrences, + Dataset c){ + Dataset firstEvent = eventTypeOccurrences.as("et") + .withColumnRenamed("eventName", "eventA") + .withColumnRenamed("numberOfTraces", "totalA"); + + Dataset secondEvent = eventTypeOccurrences.as("et2") + .withColumnRenamed("eventName", "eventB") + .withColumnRenamed("numberOfTraces", "totalB"); + + Dataset joinedDf = c.as("primary") + .join(firstEvent.as("et"), + functions.col("primary.eventA").equalTo(functions.col("et.eventA")), "left") + .join(secondEvent.as("et2"), + functions.col("primary.eventB").equalTo(functions.col("et2.eventB")), "left"); + + Dataset intermediate = joinedDf + .withColumn("total", functions.expr( + "CASE WHEN mode = 'r' THEN totalA ELSE totalB END" + )) + .withColumn("support", functions.col("occurrences").cast("double").divide(functions.col("total"))) + .select("mode", "primary.eventA", "primary.eventB", "support"); + return intermediate; + } + /** * Sets the different list of constraints to the appropriate fields in the response @@ -263,22 +293,24 @@ protected void setResults(List results, String constraint) { * @param logname the name of the log database (used to load information from the database) * @param cSimple the extracted constraints, a rdd of {@link Abstract2OrderConstraint} */ - public void extendNotSuccession(Map uEventType, String logname, - JavaRDD cSimple) { + public void extendNotSuccession(Dataset uEventType, String logname, + Dataset cSimple) { // since the first argument may not be available it will be loaded and calculated from the database if (uEventType == null) { uEventType = declareDBConnector.extractTotalOccurrencesPerEventType(logname); } //transform rdd to a compatible version to be used by the extractNotFoundPairs - JavaRDD mappedRdd = cSimple - .map(x->new EventPairToNumberOfTrace(x.getEventA(),x.getEventB(),1)); - Set notFound = declareUtilities.extractNotFoundPairs(uEventType.keySet(),mappedRdd); + Dataset mappedRdd = cSimple + .selectExpr("eventA", "eventB", "1 as numberOfTraces") + .as(Encoders.bean(EventPairToNumberOfTrace.class)); + + Set keys = new HashSet<>(uEventType.select("eventName").as(Encoders.STRING()).collectAsList()); + Set notFound = declareUtilities.extractNotFoundPairs(keys, mappedRdd); //transform event pairs to event pairs with support (which is 100% by definition) - List result = notFound.stream().map(x -> new EventPairSupport(x.getEventA().getName(), - x.getEventB().getName(), 1)).collect(Collectors.toList()); + List result = notFound.stream().map(x -> new EventPairSupport(x.getEventA(), + x.getEventB(), 1)).toList(); this.queryResponseOrderedRelations.setNotSuccession(result); } - } diff --git a/src/main/java/com/datalab/siesta/queryprocessor/declare/queryPlans/orderedRelations/QueryPlanOrderedRelationsAlternate.java b/src/main/java/com/datalab/siesta/queryprocessor/declare/queryPlans/orderedRelations/QueryPlanOrderedRelationsAlternate.java index d2ed0870..3de201b6 100644 --- a/src/main/java/com/datalab/siesta/queryprocessor/declare/queryPlans/orderedRelations/QueryPlanOrderedRelationsAlternate.java +++ b/src/main/java/com/datalab/siesta/queryprocessor/declare/queryPlans/orderedRelations/QueryPlanOrderedRelationsAlternate.java @@ -5,26 +5,25 @@ import com.datalab.siesta.queryprocessor.declare.model.Abstract2OrderConstraint; import com.datalab.siesta.queryprocessor.declare.model.EventPairSupport; import com.datalab.siesta.queryprocessor.declare.model.EventPairTraceOccurrences; +import com.datalab.siesta.queryprocessor.declare.model.EventTypeOccurrences; import com.datalab.siesta.queryprocessor.declare.queryResponses.QueryResponseOrderedRelations; -import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; -import org.apache.spark.broadcast.Broadcast; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.functions; import org.apache.spark.storage.StorageLevel; import org.springframework.stereotype.Component; import org.springframework.web.context.annotation.RequestScope; -import scala.Tuple2; -import scala.Tuple3; import java.util.ArrayList; import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; @Component @RequestScope public class QueryPlanOrderedRelationsAlternate extends QueryPlanOrderedRelations { public QueryPlanOrderedRelationsAlternate(DeclareDBConnector declareDBConnector, JavaSparkContext javaSparkContext, - OrderedRelationsUtilityFunctions utils, DeclareUtilities declareUtilities) { - super(declareDBConnector, javaSparkContext, utils, declareUtilities); + DeclareUtilities declareUtilities) { + super(declareDBConnector, javaSparkContext, declareUtilities); } @Override @@ -33,57 +32,85 @@ public void initQueryResponse() { } @Override - public JavaRDD evaluateConstraint - (JavaRDD joined, String constraint) { - JavaRDD tuple4JavaRDD; - switch (constraint) { - case "response": - tuple4JavaRDD = joined.map(utils::countResponseAlternate); - break; - case "precedence": - tuple4JavaRDD = joined.map(utils::countPrecedenceAlternate); - break; - default: - tuple4JavaRDD = joined.map(utils::countPrecedenceAlternate).union(joined.map(utils::countResponseAlternate)); - break; - } - return tuple4JavaRDD - //reduce by (eventA, eventB, mode - r/p) - .keyBy(y -> new Tuple3<>(y.getEventA(), y.getEventB(), y.getMode())) - .reduceByKey((x, y) -> { - x.setOccurrences(x.getOccurrences() + y.getOccurrences()); - return x; - }) - .map(x -> x._2); + public Dataset evaluateConstraint(Dataset joined) { + Dataset response = joined + // sort the list of occurrences A + .withColumn("sortedA", functions.expr("array_sort(occurrencesA)")) + // split list into 2 consecutive pairs + .withColumn("consecutivePairs", + functions.expr("arrays_zip(sortedA, slice(sortedA, 2, size(sortedA)-1))")) + // count elements in occurrencesB that fall between consecutive pairs + .withColumn("s_r1", functions.expr( + "size(filter(consecutivePairs, pair -> exists(occurrencesB, y -> y > pair.sortedA AND y < pair.`1`)))" + )) + // finally check if there is any event greater than the last one + .withColumn("lastA", functions.expr("element_at(sortedA, -1)")) + .withColumn("s_r2", functions.expr( + "CASE WHEN exists(occurrencesB, y-> y>lastA) THEN 1 ELSE 0 END" + )) + // determine the final occurrences + .withColumn("summary", functions.expr("s_r1 + s_r2")) + // select the appropriate fields for Abstract2OrderConstraint + .selectExpr("eventA", "eventB", "'r' as type", "summary"); + + Dataset precedence = joined + // sort the list of occurrences A + .withColumn("sortedB", functions.expr("array_sort(occurrencesB)")) + // split list into 2 consecutive pairs + .withColumn("consecutivePairs", + functions.expr("arrays_zip(sortedB, slice(sortedB, 2, size(sortedB)-1))")) + // count elements in occurrencesB that fall between consecutive pairs + .withColumn("s_r1", functions.expr( + "size(filter(consecutivePairs, pair -> exists(occurrencesA, y -> y > pair.sortedB AND y < pair.`1`)))" + )) + // finally check if there is any event smaller than the first one + .withColumn("firstA", functions.expr("element_at(sortedB, 1)")) + .withColumn("s_r2", functions.expr( + "CASE WHEN exists(occurrencesA, y-> y unioned = response.union(precedence) + .groupBy("eventA", "eventB", "type") + .agg(functions.sum("summary").alias("occurrences")) + .withColumn("occurrences", functions.col("occurrences").cast("int")) + .selectExpr("eventA", "eventB", "type as mode", "occurrences") + .as(Encoders.bean(Abstract2OrderConstraint.class)); + + return unioned; } + /** + * This method is similar to the one in the super class, with the only difference that it does not + * calculate notSuccession relations + * @param c the occurrences of different templates detected + * @param eventTypeOccurrences a spark broadcast map fo the form (event type) -> total occurrences in the log database + * @param support the user-defined support + */ @Override - public void filterBasedOnSupport(JavaRDD c, - Broadcast> bUEventType, double support) { - Broadcast bSupport = javaSparkContext.broadcast(support); - //calculates the support based on either the total occurrences of the first event (response) - //or the occurrences of the second event (precedence) - JavaRDD> intermediate = c.map(x -> { - long total; - if (x.getMode().equals("r")) { //response - total = bUEventType.getValue().get(x.getEventA()); - } else { - total = bUEventType.getValue().get(x.getEventB()); - } - long found = x.getOccurrences(); - return new Tuple2<>(x.getMode(), new EventPairSupport(x.getEventA(), x.getEventB(), - (double) found / total)); - }); + public void filterBasedOnSupport(Dataset c, + Dataset eventTypeOccurrences, double support) { + Dataset intermediate = super.getIntermediate(eventTypeOccurrences,c); + intermediate.persist(StorageLevel.MEMORY_AND_DISK()); - //filters based on the user-defined support and collects the result - List> detected = intermediate - .filter(x -> x._2.getSupport() >= bSupport.getValue()) - .collect(); - //splits the patterns that correspond to response and precedence into two separete lists - List responses = detected.stream().filter(x -> x._1.equals("r")) - .map(x -> x._2).collect(Collectors.toList()); - List precedence = detected.stream().filter(x -> x._1.equals("p")) - .map(x -> x._2).collect(Collectors.toList()); + + // filters based on the user-defined support and collect the results + Dataset detected = intermediate.filter(functions.col("support").geq(support)); + List responses = detected + .filter(functions.col("mode").equalTo("r")) + .selectExpr("eventA", "eventB", "support") + .as(Encoders.bean(EventPairSupport.class)) + .collectAsList(); + + List precedence = detected + .filter(functions.col("mode").equalTo("p")) + .selectExpr("eventA", "eventB", "support") + .as(Encoders.bean(EventPairSupport.class)) + .collectAsList(); + if (!precedence.isEmpty() && !responses.isEmpty()) { //we are looking for succession and no succession setResults(responses, "response"); setResults(precedence, "precedence"); @@ -100,7 +127,7 @@ public void filterBasedOnSupport(JavaRDD c, } } setResults(succession, "succession"); - } else if (precedence.isEmpty()) { + }else if (precedence.isEmpty()) { setResults(responses, "response"); } else { setResults(precedence, "precedence"); diff --git a/src/main/java/com/datalab/siesta/queryprocessor/declare/queryPlans/orderedRelations/QueryPlanOrderedRelationsChain.java b/src/main/java/com/datalab/siesta/queryprocessor/declare/queryPlans/orderedRelations/QueryPlanOrderedRelationsChain.java index cf106577..05a20d2c 100644 --- a/src/main/java/com/datalab/siesta/queryprocessor/declare/queryPlans/orderedRelations/QueryPlanOrderedRelationsChain.java +++ b/src/main/java/com/datalab/siesta/queryprocessor/declare/queryPlans/orderedRelations/QueryPlanOrderedRelationsChain.java @@ -5,20 +5,18 @@ import com.datalab.siesta.queryprocessor.declare.model.Abstract2OrderConstraint; import com.datalab.siesta.queryprocessor.declare.model.EventPairTraceOccurrences; import com.datalab.siesta.queryprocessor.declare.queryResponses.QueryResponseOrderedRelations; -import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.sql.*; import org.springframework.stereotype.Component; import org.springframework.web.context.annotation.RequestScope; -import scala.Tuple3; - @Component @RequestScope public class QueryPlanOrderedRelationsChain extends QueryPlanOrderedRelations{ public QueryPlanOrderedRelationsChain(DeclareDBConnector declareDBConnector, JavaSparkContext javaSparkContext, - OrderedRelationsUtilityFunctions utils, DeclareUtilities declareUtilities) { - super(declareDBConnector, javaSparkContext, utils,declareUtilities); + DeclareUtilities declareUtilities) { + super(declareDBConnector, javaSparkContext, declareUtilities); } @Override @@ -27,26 +25,25 @@ public void initQueryResponse() { } @Override - public JavaRDD evaluateConstraint(JavaRDD joined, - String constraint) { - JavaRDD tuple4JavaRDD; - switch (constraint) { - case "response": - tuple4JavaRDD = joined.map(utils::countResponseChain); - break; - case "precedence": - tuple4JavaRDD = joined.map(utils::countPrecedenceChain); - break; - default: - tuple4JavaRDD =joined.map(utils::countPrecedenceChain).union(joined.map(utils::countResponseChain)); - break; - } - return tuple4JavaRDD - .keyBy(y -> new Tuple3<>(y.getEventA(), y.getEventB(), y.getMode())) - .reduceByKey((x, y) -> { - x.setOccurrences(x.getOccurrences() + y.getOccurrences()); - return x; - }) - .map(x -> x._2); + public Dataset evaluateConstraint(Dataset joined) { + Dataset response = joined + .withColumn("s_r", functions.expr( + "size(filter(occurrencesA, a -> exists(occurrencesB, y -> y = a+1)))" + )) + .selectExpr("eventA", "eventB", "'r' as type", "s_r as count"); // Precedence constraint + + Dataset precedence = joined + .withColumn("s_p", functions.expr( + "size(filter(occurrencesB, b -> exists(occurrencesA, y -> y = b-1)))" + )) + .selectExpr("eventA", "eventB", "'p' as type", "s_p as count"); + Dataset unioned = response.union(precedence) + .groupBy("eventA", "eventB", "type") + .agg(functions.sum("count").alias("occurrences")) + .withColumn("occurrences", functions.col("occurrences").cast("int")) + .selectExpr("eventA", "eventB", "type as mode", "occurrences") + .as(Encoders.bean(Abstract2OrderConstraint.class)); + + return unioned; } } diff --git a/src/main/java/com/datalab/siesta/queryprocessor/declare/queryPlans/position/QueryPlanPositions.java b/src/main/java/com/datalab/siesta/queryprocessor/declare/queryPlans/position/QueryPlanPositions.java index e5b8359f..cfdcf1e0 100644 --- a/src/main/java/com/datalab/siesta/queryprocessor/declare/queryPlans/position/QueryPlanPositions.java +++ b/src/main/java/com/datalab/siesta/queryprocessor/declare/queryPlans/position/QueryPlanPositions.java @@ -2,10 +2,12 @@ import java.util.List; -import org.apache.spark.api.java.JavaPairRDD; -import org.apache.spark.api.java.JavaRDD; +import com.datalab.siesta.queryprocessor.declare.model.EventSupport; import org.apache.spark.api.java.JavaSparkContext; -import org.apache.spark.broadcast.Broadcast; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.functions; import org.apache.spark.storage.StorageLevel; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -15,11 +17,10 @@ import com.datalab.siesta.queryprocessor.declare.queryResponses.QueryResponsePosition; import com.datalab.siesta.queryprocessor.declare.queryWrappers.QueryPositionWrapper; import com.datalab.siesta.queryprocessor.model.DBModel.Metadata; -import com.datalab.siesta.queryprocessor.model.DBModel.Trace; +import com.datalab.siesta.queryprocessor.storage.model.Trace; import com.datalab.siesta.queryprocessor.model.Queries.QueryPlans.QueryPlan; import com.datalab.siesta.queryprocessor.model.Queries.QueryResponses.QueryResponse; import com.datalab.siesta.queryprocessor.model.Queries.Wrapper.QueryWrapper; -import scala.Tuple2; @Component @RequestScope @@ -52,39 +53,44 @@ public QueryResponse execute(QueryWrapper qw) { QueryPositionWrapper qpw = (QueryPositionWrapper) qw; //get number of total traces in database Long totalTraces = metadata.getTraces(); - //broadcast support and traces size - Broadcast bSupport = this.javaSparkContext.broadcast(qpw.getSupport()); - Broadcast bTraces = this.javaSparkContext.broadcast(totalTraces); //get all sequences from the sequence table - JavaRDD traces = declareDBConnector.querySequenceTableDeclare(this.metadata.getLogname()); - - JavaRDD> events = traces.map(x -> new Tuple2<>(x.getEvents().get(0).getName(), x.getEvents().get(x.getEvents().size() - 1).getName())); - events.persist(StorageLevel.MEMORY_AND_DISK()); - List> firsts = filterThem(events.map(x -> new Tuple2<>(x._1, 1)), - bSupport,bTraces); - List> lasts = filterThem(events.map(x -> new Tuple2<>(x._2, 1)), - bSupport,bTraces); - + Dataset traces = declareDBConnector.querySequenceTableDeclare(this.metadata.getLogname()); + + Dataset firstLast = traces.withColumn("firstEvent", functions.expr("events[0].eventName")) // First event name + .withColumn("lastEvent", functions.expr("events[size(events) - 1].eventName")) // Last event name + .select("firstEvent", "lastEvent"); + firstLast.persist(StorageLevel.MEMORY_AND_DISK()); + + List first = firstLast + .groupBy("firstEvent") + .agg(functions.count("*").alias("count")) + .selectExpr("firstEvent as event", String.format("count/%s as support",totalTraces)) + .filter(functions.col("support").geq((qpw.getSupport()))) + .as(Encoders.bean(EventSupport.class)) + .collectAsList(); + + List last = firstLast + .groupBy("lastEvent") + .agg(functions.count("*").alias("count")) + .selectExpr("lastEvent as event", String.format("count/%s as support",totalTraces)) + .filter(functions.col("support").geq((qpw.getSupport()))) + .as(Encoders.bean(EventSupport.class)) + .collectAsList(); + + //set up the response QueryResponsePosition response = new QueryResponsePosition(); if(qpw.getMode().equals("first")){ - response.setFirstTuple(firsts); + response.setFirst(first); }else if(qpw.getMode().equals("last")){ - response.setLastTuple(lasts); + response.setLast(last); }else{ - response.setFirstTuple(firsts); - response.setLastTuple(lasts); + response.setFirst(first); + response.setLast(last); } return response; } - private List> filterThem(JavaRDD> traces, Broadcast bSupport, Broadcast bTraces) { - JavaPairRDD> one = traces.keyBy(x -> x._1) - .reduceByKey((x, y) -> new Tuple2<>(x._1, x._2 + y._2)); - JavaRDD> two = one.map(x -> new Tuple2<>(x._1, x._2._2.doubleValue() / bTraces.getValue())); - //filter counts based on the support - return two.filter(x -> x._2 >= bSupport.getValue()).collect(); - } @Override public void setMetadata(Metadata metadata) { diff --git a/src/main/java/com/datalab/siesta/queryprocessor/declare/queryPlans/position/QueryPlanPositionsState.java b/src/main/java/com/datalab/siesta/queryprocessor/declare/queryPlans/position/QueryPlanPositionsState.java index 47e6eed7..0a12eba8 100644 --- a/src/main/java/com/datalab/siesta/queryprocessor/declare/queryPlans/position/QueryPlanPositionsState.java +++ b/src/main/java/com/datalab/siesta/queryprocessor/declare/queryPlans/position/QueryPlanPositionsState.java @@ -2,10 +2,12 @@ import java.util.List; -import org.apache.spark.api.java.JavaRDD; +import com.datalab.siesta.queryprocessor.declare.model.EventSupport; import org.apache.spark.api.java.JavaSparkContext; -import org.apache.spark.api.java.function.Function; -import org.apache.spark.broadcast.Broadcast; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.functions; import org.springframework.stereotype.Component; import org.springframework.web.context.annotation.RequestScope; @@ -17,8 +19,7 @@ import com.datalab.siesta.queryprocessor.model.Queries.QueryResponses.QueryResponse; import com.datalab.siesta.queryprocessor.model.Queries.Wrapper.QueryWrapper; -import scala.Tuple2; -import scala.Tuple3; + @Component @RequestScope @@ -32,19 +33,16 @@ public QueryPlanPositionsState(DeclareDBConnector declareDBConnector, JavaSparkC @Override public QueryResponse execute(QueryWrapper qw) { QueryPositionWrapper qpw = (QueryPositionWrapper) qw; - - - Broadcast bSupport = this.javaSparkContext.broadcast(qpw.getSupport()); - Broadcast bTraces = this.javaSparkContext.broadcast(metadata.getTraces()); - QueryResponsePositionState response = this.extractConstraintsFunction(bSupport,bTraces,qpw); + QueryResponsePositionState response = this.extractConstraintsFunction(qpw.getSupport(), metadata.getTraces(), qpw); this.extractStatistics(qpw); response.setUpToDate(qpw.isStateUpToDate()); if(!qpw.isStateUpToDate()){ - response.setEventsPercentage((qpw.getIndexedEvents()/metadata.getEvents())*100); - response.setTracesPercentage((qpw.getIndexedTraces()/metadata.getTraces())*100); - response.setMessage("State is not fully updated. Consider re-running the preprocess to get 100% accurate constraints"); + response.setEventsPercentage(((double) qpw.getIndexedEvents() /metadata.getEvents())*100); + response.setTracesPercentage(((double) qpw.getIndexedTraces() /metadata.getTraces())*100); + response.setMessage("State is not fully updated. " + + "Consider re-running the preprocess to get 100% accurate constraints"); }else{ response.setEventsPercentage(100); response.setTracesPercentage(100); @@ -53,36 +51,32 @@ public QueryResponse execute(QueryWrapper qw) { return response; } - public QueryResponsePositionState extractConstraintsFunction(Broadcast bSupport, Broadcast bTraces, QueryPositionWrapper qpw){ - JavaRDD> data = this.declareDBConnector.queryPositionState(metadata.getLogname()) - .map((Function>)x->{ - return new Tuple3(x.getRule(),x.getEvent_type(),x.getOccurrences()/bTraces.getValue()); - }) - .filter((Function,Boolean>)x->{ - return x._3() >= bSupport.getValue(); - }); - - List> firsts = data.filter((Function,Boolean>)x->{ - return x._1().equals("first"); - }).map((Function,Tuple2>)x->{ - return new Tuple2(x._2(),x._3()); - }).collect(); - - List> lasts = data.filter((Function,Boolean>)x->{ - return x._1().equals("last"); - }).map((Function,Tuple2>)x->{ - return new Tuple2(x._2(),x._3()); - }).collect(); + public QueryResponsePositionState extractConstraintsFunction(double support, long traces, QueryPositionWrapper qpw){ + Dataset data = this.declareDBConnector.queryPositionState(metadata.getLogname()); + Dataset filtered = data.withColumn("support", functions.col("occurrences").divide(traces)) + .filter(functions.col("support").geq(support)); + + List firsts = filtered + .filter(functions.col("rule").equalTo("first")) + .selectExpr("event_type as event","support") + .as(Encoders.bean(EventSupport.class)) + .collectAsList(); + + List lasts = filtered + .filter(functions.col("rule").equalTo("last")) + .selectExpr("event_type as event","support") + .as(Encoders.bean(EventSupport.class)) + .collectAsList(); QueryResponsePositionState response = new QueryResponsePositionState(); if(qpw.getMode().equals("first")){ - response.setFirstTuple(firsts); + response.setFirst(firsts); }else if(qpw.getMode().equals("last")){ - response.setLastTuple(lasts); + response.setLast(lasts); }else{ - response.setFirstTuple(firsts); - response.setLastTuple(lasts); + response.setFirst(firsts); + response.setLast(lasts); } return response; } diff --git a/src/main/java/com/datalab/siesta/queryprocessor/model/Constraints/TimeConstraint.java b/src/main/java/com/datalab/siesta/queryprocessor/model/Constraints/TimeConstraint.java index 189c3910..36aa7ba3 100644 --- a/src/main/java/com/datalab/siesta/queryprocessor/model/Constraints/TimeConstraint.java +++ b/src/main/java/com/datalab/siesta/queryprocessor/model/Constraints/TimeConstraint.java @@ -32,9 +32,9 @@ public TimeConstraint(int posA, int posB, long constraint, String granularity) { public boolean isConstraintHolds(Count c) { if (method.equals("within")) { - return c.getMin_duration() <= this.getConstraintInSeconds(); + return c.getMinDuration() <= this.getConstraintInSeconds(); } else if (method.equals("atleast")) { - return this.getConstraintInSeconds() <= c.getMax_duration(); + return this.getConstraintInSeconds() <= c.getMaxDuration(); } else return false; } diff --git a/src/main/java/com/datalab/siesta/queryprocessor/model/DBModel/Count.java b/src/main/java/com/datalab/siesta/queryprocessor/model/DBModel/Count.java index fecae386..e075dd2e 100644 --- a/src/main/java/com/datalab/siesta/queryprocessor/model/DBModel/Count.java +++ b/src/main/java/com/datalab/siesta/queryprocessor/model/DBModel/Count.java @@ -1,5 +1,8 @@ package com.datalab.siesta.queryprocessor.model.DBModel; +import lombok.Getter; +import lombok.Setter; + import java.io.Serializable; /** @@ -7,21 +10,23 @@ * based on the time distance between every event-pair of these events, the number of event-pairs in the database and the * sum of all the durations (used to calculate mean duration) */ +@Getter +@Setter public class Count implements Serializable { private String eventA; private String eventB; - private long sum_duration; + private long sumDuration; private int count; - private long min_duration; + private long minDuration; - private long max_duration; + private long maxDuration; - private double sum_squares; + private double sumSquares; public Count() { } @@ -29,78 +34,23 @@ public Count() { public Count(String eventA, String eventB, long sum_duration, int count, long min_duration, long max_duration, double sum_squares) { this.eventA = eventA; this.eventB = eventB; - this.sum_duration = sum_duration; + this.sumDuration = sum_duration; this.count = count; - this.min_duration = min_duration; - this.max_duration = max_duration; - this.sum_squares = sum_squares; + this.minDuration = min_duration; + this.maxDuration = max_duration; + this.sumSquares = sum_squares; } public Count(String eventA, String[] record) { this.eventA = eventA; this.eventB = record[0]; - this.sum_duration = Long.parseLong(record[1]); + this.sumDuration = Long.parseLong(record[1]); this.count = Integer.parseInt(record[2]); - this.min_duration = Long.parseLong(record[3]); - this.max_duration = Long.parseLong(record[4]); - this.sum_squares = Double.parseDouble(record[5]); - } - - public String getEventA() { - return eventA; - } - - public void setEventA(String eventA) { - this.eventA = eventA; - } - - public String getEventB() { - return eventB; + this.minDuration = Long.parseLong(record[3]); + this.maxDuration = Long.parseLong(record[4]); + this.sumSquares = Double.parseDouble(record[5]); } - public void setEventB(String eventB) { - this.eventB = eventB; - } - - public long getSum_duration() { - return sum_duration; - } - - public void setSum_duration(long sum_duration) { - this.sum_duration = sum_duration; - } - - public int getCount() { - return count; - } - - public void setCount(int count) { - this.count = count; - } - - public long getMin_duration() { - return min_duration; - } - - public void setMin_duration(long min_duration) { - this.min_duration = min_duration; - } - - public long getMax_duration() { - return max_duration; - } - - public void setMax_duration(long max_duration) { - this.max_duration = max_duration; - } - - public double getSum_squares() { - return sum_squares; - } - - public void setSum_squares(double sum_squares) { - this.sum_squares = sum_squares; - } public String getPair() { return this.eventA + this.eventB; @@ -111,11 +61,11 @@ public String toString() { return "Count{" + "eventA='" + eventA + '\'' + ", eventB='" + eventB + '\'' + - ", sum_duration=" + sum_duration + + ", sum_duration=" + sumDuration + ", count=" + count + - ", min_duration=" + min_duration + - ", max_duration=" + max_duration + - ", sum_squares=" + sum_squares + + ", min_duration=" + minDuration + + ", max_duration=" + maxDuration + + ", sum_squares=" + sumSquares + '}'; } } diff --git a/src/main/java/com/datalab/siesta/queryprocessor/model/DBModel/EventTypes.java b/src/main/java/com/datalab/siesta/queryprocessor/model/DBModel/EventTypes.java index 3361f514..e136e574 100644 --- a/src/main/java/com/datalab/siesta/queryprocessor/model/DBModel/EventTypes.java +++ b/src/main/java/com/datalab/siesta/queryprocessor/model/DBModel/EventTypes.java @@ -1,25 +1,26 @@ package com.datalab.siesta.queryprocessor.model.DBModel; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +import java.io.Serializable; import java.util.Objects; /** * The class that represents an et-pair without the conditions. * @see com.datalab.siesta.queryprocessor.model.Events.EventPair (for implementation with conditions) */ -public class EventTypes{ +@Getter +@Setter +@AllArgsConstructor +@NoArgsConstructor +public class EventTypes implements Serializable{ private String eventA; private String eventB; - public EventTypes(String eventA, String eventB) { - this.eventA = eventA; - this.eventB = eventB; - } - - public String getEventA() { return eventA; } - - public String getEventB() { return eventB; } - @Override public String toString() { return eventA + eventB; @@ -37,4 +38,5 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(eventA, eventB); } + } diff --git a/src/main/java/com/datalab/siesta/queryprocessor/model/DBModel/IndexPair.java b/src/main/java/com/datalab/siesta/queryprocessor/model/DBModel/IndexPair.java index 5afa5a3f..7730f7dc 100644 --- a/src/main/java/com/datalab/siesta/queryprocessor/model/DBModel/IndexPair.java +++ b/src/main/java/com/datalab/siesta/queryprocessor/model/DBModel/IndexPair.java @@ -2,9 +2,10 @@ import com.datalab.siesta.queryprocessor.model.Events.Event; import com.datalab.siesta.queryprocessor.model.Events.EventPair; -import com.datalab.siesta.queryprocessor.model.Events.EventPos; -import com.datalab.siesta.queryprocessor.model.Events.EventTs; import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.Setter; import java.io.Serializable; @@ -23,13 +24,15 @@ * it is expected the other fields to be empty (null/-1 respectively). If both information is required to answer a * query, they can be retrieved from SequenceTable (which contains both). */ +@Getter +@Setter +@AllArgsConstructor public class IndexPair implements Serializable { - - private String traceId; + private String trace_id; private String eventA; private String eventB; - private Timestamp timestampA; - private Timestamp timestampB; + private String timestampA; + private String timestampB; private int positionA; private int positionB; @@ -42,8 +45,8 @@ public IndexPair() { this.timestampB = null; } - public IndexPair(String traceId, String eventA, String eventB, Timestamp timestampA, Timestamp timestampB) { - this.traceId = traceId; + public IndexPair(String traceId, String eventA, String eventB, String timestampA, String timestampB) { + this.trace_id = traceId; this.positionA = -1; this.positionB = -1; this.eventA = eventA; @@ -52,18 +55,8 @@ public IndexPair(String traceId, String eventA, String eventB, Timestamp timesta this.timestampB = timestampB; } - public IndexPair(String traceId, String eventA, String eventB, String timestampA, String timestampB) { - this.traceId = traceId; - this.positionA = -1; - this.positionB = -1; - this.eventA = eventA; - this.eventB = eventB; - this.timestampA = Timestamp.valueOf(timestampA); - this.timestampB = Timestamp.valueOf(timestampB); - } - public IndexPair(String traceId, String eventA, String eventB, int positionA, int positionB) { - this.traceId = traceId; + this.trace_id = traceId; this.timestampA = null; this.timestampB = null; this.eventA = eventA; @@ -72,82 +65,6 @@ public IndexPair(String traceId, String eventA, String eventB, int positionA, in this.positionB = positionB; } - public String getEventA() { - return eventA; - } - - public void setEventA(String eventA) { - this.eventA = eventA; - } - - public String getEventB() { - return eventB; - } - - public void setEventB(String eventB) { - this.eventB = eventB; - } - - public Timestamp getTimestampA() { - return timestampA; - } - - public void setTimestampA(Timestamp timestampA) { - this.timestampA = timestampA; - } - - public Timestamp getTimestampB() { - return timestampB; - } - - public void setTimestampB(Timestamp timestampB) { - this.timestampB = timestampB; - } - - public int getPositionA() { - return positionA; - } - - public void setPositionA(int positionA) { - this.positionA = positionA; - } - - public int getPositionB() { - return positionB; - } - - public void setPositionB(int positionB) { - this.positionB = positionB; - } - - public String getTraceId() { - return traceId; - } - - public void setTraceId(String traceId) { - this.traceId = traceId; - } - - @JsonIgnore - public List getEvents(){ - List e = new ArrayList<>(); - if(timestampA==null){//the events are pos - EventPos eventPos1 = new EventPos(this.eventA,this.positionA); - EventPos eventPos2 = new EventPos(this.eventB,this.positionB); - eventPos1.setTraceID(this.traceId); - eventPos2.setTraceID(this.traceId); - e.add(eventPos1); - e.add(eventPos2); - }else{//the events are ts - EventTs eventTs1 = new EventTs(this.eventA,this.timestampA); - EventTs eventTs2 = new EventTs(this.eventB,this.timestampB); - eventTs1.setTraceID(this.traceId); - eventTs2.setTraceID(this.traceId); - e.add(eventTs1); - e.add(eventTs2); - } - return e; - } @JsonIgnore public boolean validate(Set pairs){ @@ -157,5 +74,5 @@ public boolean validate(Set pairs){ return false; } - public long getDuration() { return (timestampB.getTime() - timestampA.getTime()) / 1000; } + public long getDuration() { return (Timestamp.valueOf(timestampB).getTime() - Timestamp.valueOf(timestampA).getTime()) / 1000; } } diff --git a/src/main/java/com/datalab/siesta/queryprocessor/model/DBModel/IndexRecords.java b/src/main/java/com/datalab/siesta/queryprocessor/model/DBModel/IndexRecords.java index c9c0690b..80b4aa4e 100644 --- a/src/main/java/com/datalab/siesta/queryprocessor/model/DBModel/IndexRecords.java +++ b/src/main/java/com/datalab/siesta/queryprocessor/model/DBModel/IndexRecords.java @@ -16,12 +16,8 @@ public class IndexRecords { private Map> records; - public IndexRecords(List, Iterable>> results) { - records = new HashMap<>(); - results.forEach(x -> { - EventTypes et = new EventTypes(x._1._1, x._1._2); - records.put(et, Lists.newArrayList(x._2)); - }); + public IndexRecords(Map> results) { + this.records = results; } public Map> getRecords() { diff --git a/src/main/java/com/datalab/siesta/queryprocessor/model/DBModel/Trace.java b/src/main/java/com/datalab/siesta/queryprocessor/model/DBModel/Trace.java deleted file mode 100644 index d2bbbfc2..00000000 --- a/src/main/java/com/datalab/siesta/queryprocessor/model/DBModel/Trace.java +++ /dev/null @@ -1,60 +0,0 @@ -package com.datalab.siesta.queryprocessor.model.DBModel; - -import com.datalab.siesta.queryprocessor.model.Events.EventBoth; -import lombok.Getter; - -import java.io.Serializable; -import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; - -/** - * A sequence of events. It is represented by a trace id and a list of the events it contains in the correct order. - */ - -public class Trace implements Serializable { - - private String traceID; - - private List events; - - public Trace() { - } - - public Trace(String traceID, List events) { - this.traceID = traceID; - this.events = events; - } - - public void filter(Timestamp from, Timestamp till){ - events= events.stream().filter(x-> from==null || !x.getTimestamp().before(from)) - .filter(x -> till==null || !x.getTimestamp().after(till)) - .collect(Collectors.toList()); - } - - public String getTraceID() { - return traceID; - } - - public void setTraceID(String traceID) { - this.traceID = traceID; - } - - public List getEvents() { - return events; - } - - public void setEvents(List events) { - this.events = events; - } - - public List clearTrace(Set events_types){ - List result = new ArrayList<>(); - for(EventBoth eb : this.events){ - if(events_types.contains(eb.getName())) result.add(eb); - } - return result; - } -} diff --git a/src/main/java/com/datalab/siesta/queryprocessor/model/Events/EventBoth.java b/src/main/java/com/datalab/siesta/queryprocessor/model/Events/EventBoth.java index 50444e9b..fdb4f776 100644 --- a/src/main/java/com/datalab/siesta/queryprocessor/model/Events/EventBoth.java +++ b/src/main/java/com/datalab/siesta/queryprocessor/model/Events/EventBoth.java @@ -8,6 +8,7 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import java.io.Serializable; import java.sql.Timestamp; import java.util.Objects; @@ -15,7 +16,7 @@ * A SIESTA event that contains both time and position information */ @JsonSerialize(using = EventBothSerializer.class) -public class EventBoth extends EventTs implements Comparable{ +public class EventBoth extends EventTs implements Comparable, Serializable { @JsonView(MappingJacksonViews.EventAllInfo.class) private int position; @@ -91,4 +92,4 @@ public long getPrimaryMetric() { public void setPrimaryMetric(long newPrimaryMetric) { this.timestamp= new Timestamp(newPrimaryMetric); } -} +} \ No newline at end of file diff --git a/src/main/java/com/datalab/siesta/queryprocessor/model/Occurrence.java b/src/main/java/com/datalab/siesta/queryprocessor/model/Occurrence.java index 34cb95ea..10d326ad 100644 --- a/src/main/java/com/datalab/siesta/queryprocessor/model/Occurrence.java +++ b/src/main/java/com/datalab/siesta/queryprocessor/model/Occurrence.java @@ -59,4 +59,4 @@ public double getDuration(){ } return 0; } -} +} \ No newline at end of file diff --git a/src/main/java/com/datalab/siesta/queryprocessor/model/Occurrences.java b/src/main/java/com/datalab/siesta/queryprocessor/model/Occurrences.java index 55a046a9..cace803b 100644 --- a/src/main/java/com/datalab/siesta/queryprocessor/model/Occurrences.java +++ b/src/main/java/com/datalab/siesta/queryprocessor/model/Occurrences.java @@ -58,15 +58,17 @@ public void setOccurrences(List occurrences) { public void clearOccurrences(boolean returnAll) { //here we can determine different selection policies List response = new ArrayList<>() { { - Occurrence e = occurrences.get(0); - if (occurrences.size()>1){ - for (int i=1;ie.getOccurrence().size()){ - e=occurrences.get(i); + if (!occurrences.isEmpty()) { + Occurrence e = occurrences.get(0); + if (occurrences.size() > 1) { + for (int i = 1; i < occurrences.size(); i++) { //add the occurrence with the largest size + if (occurrences.get(i).getOccurrence().size() > e.getOccurrence().size()) { + e = occurrences.get(i); + } } } + add(e); } - add(e); } }; if (!returnAll) { //return the one occurrence with the largest size (that is if Kleene* or Kleene+ was used) diff --git a/src/main/java/com/datalab/siesta/queryprocessor/model/Queries/QueryPlans/Detection/QueryPlanPatternDetection.java b/src/main/java/com/datalab/siesta/queryprocessor/model/Queries/QueryPlans/Detection/QueryPlanPatternDetection.java index 5bc5c8c8..70b385d4 100644 --- a/src/main/java/com/datalab/siesta/queryprocessor/model/Queries/QueryPlans/Detection/QueryPlanPatternDetection.java +++ b/src/main/java/com/datalab/siesta/queryprocessor/model/Queries/QueryPlans/Detection/QueryPlanPatternDetection.java @@ -255,7 +255,8 @@ protected Map> querySeqDB(List trace_ids, SIESTAPatt Set eventTypes = pattern.getEventTypes(); Map> fromDB = dbConnector.querySeqTable(logname, trace_ids, eventTypes, from, till); return fromDB.entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().stream().map(s -> (Event) s) + .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().stream() + .sorted(Comparator.comparingInt(EventBoth::getPosition)).map(s -> (Event) s) .collect(Collectors.toList()))); } diff --git a/src/main/java/com/datalab/siesta/queryprocessor/model/Queries/QueryPlans/Detection/QueryPlanPatternDetectionSingle.java b/src/main/java/com/datalab/siesta/queryprocessor/model/Queries/QueryPlans/Detection/QueryPlanPatternDetectionSingle.java index 604701ab..2ceadeaa 100644 --- a/src/main/java/com/datalab/siesta/queryprocessor/model/Queries/QueryPlans/Detection/QueryPlanPatternDetectionSingle.java +++ b/src/main/java/com/datalab/siesta/queryprocessor/model/Queries/QueryPlans/Detection/QueryPlanPatternDetectionSingle.java @@ -17,6 +17,7 @@ import java.util.List; import java.util.Map; +import java.util.stream.Collectors; /** * The query plan responsible for detecting patterns of a single event (i.e. patterns that contain 2 characters @@ -42,7 +43,8 @@ public QueryResponse execute(QueryWrapper qw) { long ts_trace = System.currentTimeMillis(); if (!firstCheck.isEmpty()) return firstCheck; //stop the process as an error was found QueryResponsePatternDetection queryResponsePatternDetection = new QueryResponsePatternDetection(); - List occurrences = saseConnector.evaluateSmallPatterns(qpdw.getPattern(), intermediateResults); + List occurrences = saseConnector.evaluateSmallPatterns(qpdw.getPattern(), intermediateResults) + .stream().filter(o->o.getOccurrences().size() > 1).collect(Collectors.toList()); occurrences.forEach(x -> x.clearOccurrences(qpdw.isReturnAll())); long ts_eval = System.currentTimeMillis(); queryResponsePatternDetection.setOccurrences(occurrences); diff --git a/src/main/java/com/datalab/siesta/queryprocessor/model/Queries/QueryPlans/Exploration/QueryPlanExplorationAccurate.java b/src/main/java/com/datalab/siesta/queryprocessor/model/Queries/QueryPlans/Exploration/QueryPlanExplorationAccurate.java index ad206304..d6d75b02 100644 --- a/src/main/java/com/datalab/siesta/queryprocessor/model/Queries/QueryPlans/Exploration/QueryPlanExplorationAccurate.java +++ b/src/main/java/com/datalab/siesta/queryprocessor/model/Queries/QueryPlans/Exploration/QueryPlanExplorationAccurate.java @@ -57,15 +57,22 @@ public QueryResponse execute(QueryWrapper qw) { String lastEvent = events.get(events.size() - 1).getName(); List freqs = dbConnector.getCountForExploration(queryExploreWrapper.getLog_name(), lastEvent); List props = new ArrayList<>(); - for (Count freq : freqs) { - try { - SimplePattern sp = (SimplePattern) queryExploreWrapper.getPattern().clone(); - Proposition p = this.patternDetection(sp, freq.getEventB(), qw.getLog_name()); - if (p != null) props.add(p); - } catch (CloneNotSupportedException e) { - throw new RuntimeException(e); - } + if (queryExploreWrapper.getPattern().getEvents().size() > 1) { + for (Count freq : freqs) { + try { + SimplePattern sp = (SimplePattern) queryExploreWrapper.getPattern().clone(); + Proposition p = this.patternDetection(sp, freq.getEventB(), qw.getLog_name()); + if (p != null) props.add(p); + } catch (CloneNotSupportedException e) { + throw new RuntimeException(e); + } + } + } else if (queryExploreWrapper.getPattern().getEvents().size() == 1) { + for (Count freq : freqs) { + props.add(new Proposition(freq.getEventB(), freq.getCount(), + (double) freq.getSumDuration() / freq.getCount())); + } } props.sort(Collections.reverseOrder()); return new QueryResponseExploration(props); diff --git a/src/main/java/com/datalab/siesta/queryprocessor/model/Queries/QueryPlans/Exploration/QueryPlanExplorationFast.java b/src/main/java/com/datalab/siesta/queryprocessor/model/Queries/QueryPlans/Exploration/QueryPlanExplorationFast.java index 8713a8c2..296c72ab 100644 --- a/src/main/java/com/datalab/siesta/queryprocessor/model/Queries/QueryPlans/Exploration/QueryPlanExplorationFast.java +++ b/src/main/java/com/datalab/siesta/queryprocessor/model/Queries/QueryPlans/Exploration/QueryPlanExplorationFast.java @@ -50,10 +50,14 @@ public QueryResponse execute(QueryWrapper qw) { QueryExploreWrapper queryExploreWrapper = (QueryExploreWrapper) qw; Set pairs = queryExploreWrapper.getPattern().extractPairsConsecutive(); //approximate the total completions of the pattern based on the occurrences of the consecutive pairs - List pairCount = dbConnector.getStats(queryExploreWrapper.getLog_name(), pairs); - int lastCompletions = this.getCompletionCountOfFullFunnel(pairCount); List events = queryExploreWrapper.getPattern().getEvents(); String lastEvent = events.get(events.size() - 1).getName(); + int lastCompletions = Integer.MAX_VALUE; + if(!pairs.isEmpty()) { + List pairCount = dbConnector.getStats(queryExploreWrapper.getLog_name(), pairs); + lastCompletions = this.getCompletionCountOfFullFunnel(pairCount); + + } List props = this.exploreFast(lastEvent, queryExploreWrapper.getLog_name(), lastCompletions); props.sort(Collections.reverseOrder()); return new QueryResponseExploration(props); @@ -76,7 +80,7 @@ protected List exploreFast(String lastEvent, String logname, int la for (Count freq : freqs) { int upper = Math.min(lastCompletions, freq.getCount()); Proposition prop = new Proposition(freq.getEventB(), upper, - (double) freq.getSum_duration() / freq.getCount()); + (double) freq.getSumDuration() / freq.getCount()); if (upper != 0) propositions.add(prop); } diff --git a/src/main/java/com/datalab/siesta/queryprocessor/model/Queries/QueryPlans/Exploration/QueryPlanExplorationHybrid.java b/src/main/java/com/datalab/siesta/queryprocessor/model/Queries/QueryPlans/Exploration/QueryPlanExplorationHybrid.java index 3ae9091b..4d1c7006 100644 --- a/src/main/java/com/datalab/siesta/queryprocessor/model/Queries/QueryPlans/Exploration/QueryPlanExplorationHybrid.java +++ b/src/main/java/com/datalab/siesta/queryprocessor/model/Queries/QueryPlans/Exploration/QueryPlanExplorationHybrid.java @@ -52,12 +52,16 @@ public QueryResponse execute(QueryWrapper qw) { List props = new ArrayList<>(); int k = Math.min(fast.size(), queryExploreWrapper.getK()); for (Proposition p : fast.subList(0, k)) { - try { - SimplePattern sp = (SimplePattern) queryExploreWrapper.getPattern().clone(); - Proposition newp = this.patternDetection(sp, p.getEvent(), qw.getLog_name()); - if (newp != null) props.add(newp); - } catch (CloneNotSupportedException e) { - throw new RuntimeException(e); + if(queryExploreWrapper.getPattern().getEvents().size() == 1) { + props.add(p); + }else { + try { + SimplePattern sp = (SimplePattern) queryExploreWrapper.getPattern().clone(); + Proposition newp = this.patternDetection(sp, p.getEvent(), qw.getLog_name()); + if (newp != null) props.add(newp); + } catch (CloneNotSupportedException e) { + throw new RuntimeException(e); + } } } props.sort(Collections.reverseOrder()); diff --git a/src/main/java/com/datalab/siesta/queryprocessor/model/Utils/Utils.java b/src/main/java/com/datalab/siesta/queryprocessor/model/Utils/Utils.java index fd0454cb..7a481686 100644 --- a/src/main/java/com/datalab/siesta/queryprocessor/model/Utils/Utils.java +++ b/src/main/java/com/datalab/siesta/queryprocessor/model/Utils/Utils.java @@ -3,6 +3,7 @@ import com.datalab.siesta.queryprocessor.SaseConnection.SaseEvent; import com.datalab.siesta.queryprocessor.model.Constraints.*; import com.datalab.siesta.queryprocessor.model.Events.Event; +import com.datalab.siesta.queryprocessor.model.Events.EventBoth; import com.datalab.siesta.queryprocessor.model.Events.EventPos; import com.datalab.siesta.queryprocessor.model.Events.EventTs; import org.apache.spark.broadcast.Broadcast; @@ -48,7 +49,12 @@ public Tuple2, List> splitConstraints(List transformToSaseEvents(List events){ List ses = new ArrayList<>(); Event fe = events.get(0); - if(fe instanceof EventTs){ // handling events ts + if(fe instanceof EventBoth){ + for(int i=0;i m = new HashMap<>(); for (String l : dbConnector.findAllLongNames()) { Metadata metadata = dbConnector.getMetadata(l); + if(metadata == null) { //might be an incomplete logdatabase + continue; + } // TODO: determine a way to find the starting ts if(metadata.getStart_ts()==null){ metadata.setStart_ts(""); diff --git a/src/main/java/com/datalab/siesta/queryprocessor/storage/DBConnector.java b/src/main/java/com/datalab/siesta/queryprocessor/storage/DBConnector.java index 72b419c4..174bda40 100644 --- a/src/main/java/com/datalab/siesta/queryprocessor/storage/DBConnector.java +++ b/src/main/java/com/datalab/siesta/queryprocessor/storage/DBConnector.java @@ -120,29 +120,6 @@ public Map> querySeqTable(String logname, List t return db.querySeqTable(logname, traceIds, eventTypes, from, till); } - /** - * Retrieves all the events from specific traces in the SequenceTable - * @param logname the log database - * @param traceIds the ids of the traces that will be retrieved - * @return a map where the key is the trace id and the value is a list of the retrieved events (with their - * * timestamps) - */ - public Map> querySeqTable(String logname, List traceIds) { - return db.querySeqTable(logname, traceIds); - } - - /** - * Retrieves the appropriate events from the SingleTable, which contains the single inverted index - * - * @param logname the log database - * @param traceIds the ids of the traces that wil be retrieved - * @param eventTypes the events that will we retrieved - * @return a list of all the retrieved events (wth their timestamps) - */ - public List querySingleTable(String logname, Set traceIds, Set eventTypes) { - return db.querySingleTable(logname, traceIds, eventTypes); - } - /** * Retrieves event from the SingleTable and group them based on the traceID * @param logname the log database diff --git a/src/main/java/com/datalab/siesta/queryprocessor/storage/DatabaseRepository.java b/src/main/java/com/datalab/siesta/queryprocessor/storage/DatabaseRepository.java index 2031ab69..9a22fa81 100644 --- a/src/main/java/com/datalab/siesta/queryprocessor/storage/DatabaseRepository.java +++ b/src/main/java/com/datalab/siesta/queryprocessor/storage/DatabaseRepository.java @@ -11,15 +11,13 @@ import com.datalab.siesta.queryprocessor.declare.model.declareState.UnorderStateI; import com.datalab.siesta.queryprocessor.declare.model.declareState.UnorderStateU; import com.datalab.siesta.queryprocessor.model.DBModel.*; -import com.datalab.siesta.queryprocessor.model.Events.Event; import com.datalab.siesta.queryprocessor.model.Events.EventBoth; import com.datalab.siesta.queryprocessor.model.Events.EventPair; -import com.datalab.siesta.queryprocessor.model.Events.EventPos; import com.datalab.siesta.queryprocessor.model.ExtractedPairsForPatternDetection; -import org.apache.spark.api.java.JavaPairRDD; -import org.apache.spark.api.java.JavaRDD; +import com.datalab.siesta.queryprocessor.storage.model.EventTypeTracePositions; +import com.datalab.siesta.queryprocessor.storage.model.Trace; +import org.apache.spark.sql.Dataset; import scala.Tuple2; -import scala.Tuple3; import java.sql.Timestamp; import java.util.List; @@ -102,14 +100,6 @@ public interface DatabaseRepository { */ IndexMiddleResult patterDetectionTraceIds(String logname, List> combined, Metadata metadata, ExtractedPairsForPatternDetection pairs, Timestamp from, Timestamp till); - /** - * Retrieves data from the primary inverted index - * @param pairs a set of the pairs that we need to retrieve information for - * @param logname the log database - * @return the corresponding records from the index - */ - IndexRecords queryIndexTable(Set pairs, String logname); - /** * Retrieves data from the primary inverted index * @param pairs a set of the pairs that we need to retrieve information for @@ -160,27 +150,25 @@ public interface DatabaseRepository { // Below are for Declare // - JavaRDD querySequenceTableDeclare(String logname); - - JavaRDD querySingleTableDeclare(String logname); + Dataset querySequenceTableDeclare(String logname); - JavaRDD querySingleTable(String logname); + Dataset querySingleTableDeclare(String logname); - JavaRDD queryIndexTableDeclare(String logname); + Dataset querySingleTable(String logname); - JavaRDD queryIndexTableAllDeclare(String logname); + Dataset queryIndexTableDeclare(String logname); - JavaPairRDD, List> querySingleTableAllDeclare(String logname); + Dataset querySingleTableAllDeclare(String logname); - JavaRDD queryIndexOriginalDeclare(String logname); + Dataset queryIndexOriginalDeclare(String logname); //Below are for the states of Declare - JavaRDD queryPositionState(String logname); - JavaRDD queryExistenceState(String logname); - JavaRDD queryUnorderStateI(String logname); - JavaRDD queryUnorderStateU(String logname); - JavaRDD queryOrderState(String logname); - JavaRDD queryNegativeState(String logname); + Dataset queryPositionState(String logname); + Dataset queryExistenceState(String logname); + Dataset queryUnorderStateI(String logname); + Dataset queryUnorderStateU(String logname); + Dataset queryOrderState(String logname); + Dataset queryNegativeState(String logname); } diff --git a/src/main/java/com/datalab/siesta/queryprocessor/storage/model/EventModel.java b/src/main/java/com/datalab/siesta/queryprocessor/storage/model/EventModel.java new file mode 100644 index 00000000..37873482 --- /dev/null +++ b/src/main/java/com/datalab/siesta/queryprocessor/storage/model/EventModel.java @@ -0,0 +1,19 @@ +package com.datalab.siesta.queryprocessor.storage.model; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +import java.io.Serializable; + +@Getter +@Setter +@AllArgsConstructor +@NoArgsConstructor +public class EventModel implements Serializable{ + private String traceId; + private String eventName; + private String timestamp; + private int position; +} diff --git a/src/main/java/com/datalab/siesta/queryprocessor/storage/model/EventTypeTracePositions.java b/src/main/java/com/datalab/siesta/queryprocessor/storage/model/EventTypeTracePositions.java new file mode 100644 index 00000000..3b91cc3b --- /dev/null +++ b/src/main/java/com/datalab/siesta/queryprocessor/storage/model/EventTypeTracePositions.java @@ -0,0 +1,19 @@ +package com.datalab.siesta.queryprocessor.storage.model; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +import java.io.Serializable; +import java.util.List; + +@Getter +@Setter +@NoArgsConstructor +@AllArgsConstructor +public class EventTypeTracePositions implements Serializable { + private String EventName; + private String traceId; + private List positions; +} diff --git a/src/main/java/com/datalab/siesta/queryprocessor/storage/model/GroupEvents.java b/src/main/java/com/datalab/siesta/queryprocessor/storage/model/GroupEvents.java new file mode 100644 index 00000000..a20ce711 --- /dev/null +++ b/src/main/java/com/datalab/siesta/queryprocessor/storage/model/GroupEvents.java @@ -0,0 +1,17 @@ +package com.datalab.siesta.queryprocessor.storage.model; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +import java.util.List; + +@Getter +@Setter +@NoArgsConstructor +@AllArgsConstructor +public class GroupEvents { + private int group_id; + private List events; +} diff --git a/src/main/java/com/datalab/siesta/queryprocessor/storage/model/Trace.java b/src/main/java/com/datalab/siesta/queryprocessor/storage/model/Trace.java new file mode 100644 index 00000000..a8932baf --- /dev/null +++ b/src/main/java/com/datalab/siesta/queryprocessor/storage/model/Trace.java @@ -0,0 +1,57 @@ +package com.datalab.siesta.queryprocessor.storage.model; + +import lombok.Getter; +import lombok.Setter; + +import java.io.Serializable; +import java.util.List; + +/** + * A sequence of events. It is represented by a trace id and a list of the events it contains in the correct order. + */ +@Getter +@Setter +public class Trace implements Serializable { + + private String traceId; + + private List events; + + public Trace() { + } + + public Trace(String traceId, List events) { + this.traceId = traceId; + this.events = events; + } + +// public void filter(Timestamp from, Timestamp till){ +// events= events.stream().filter(x-> from==null || !x.getTimestamp().before(from)) +// .filter(x -> till==null || !x.getTimestamp().after(till)) +// .collect(Collectors.toList()); +// } + +// public String getTraceID() { +// return traceID; +// } +// +// public void setTraceID(String traceID) { +// this.traceID = traceID; +// } +// +// public List getEvents() { +// return events; +// } +// +// public void setEvents(List events) { +// this.events = events; +// } +// +// public List clearTrace(Set events_types){ +// List result = new ArrayList<>(); +// for(EventBoth eb : this.events){ +// if(events_types.contains(eb.getName())) result.add(eb); +// } +// return result; +// } +} diff --git a/src/main/java/com/datalab/siesta/queryprocessor/storage/repositories/Cassandra/CassConnector.java b/src/main/java/com/datalab/siesta/queryprocessor/storage/repositories/Cassandra/CassConnector.java new file mode 100644 index 00000000..6bd796b4 --- /dev/null +++ b/src/main/java/com/datalab/siesta/queryprocessor/storage/repositories/Cassandra/CassConnector.java @@ -0,0 +1,294 @@ +package com.datalab.siesta.queryprocessor.storage.repositories.Cassandra; + +import com.datalab.siesta.queryprocessor.declare.model.declareState.*; +import com.datalab.siesta.queryprocessor.model.DBModel.Count; +import com.datalab.siesta.queryprocessor.model.DBModel.IndexPair; +import com.datalab.siesta.queryprocessor.model.DBModel.Metadata; +import com.datalab.siesta.queryprocessor.model.Events.EventBoth; +import com.datalab.siesta.queryprocessor.model.Utils.Utils; +import com.datalab.siesta.queryprocessor.storage.model.EventModel; +import com.datalab.siesta.queryprocessor.storage.model.Trace; +import com.datalab.siesta.queryprocessor.storage.repositories.SparkDatabaseRepository; +import com.datastax.spark.connector.cql.CassandraConnector; +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.api.java.function.Function; +import org.apache.spark.sql.*; +import org.apache.spark.sql.expressions.Window; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.PropertySource; +import scala.Tuple2; + +import org.apache.spark.sql.*; +import org.apache.spark.sql.functions; +import org.apache.spark.sql.types.*; +import static org.apache.spark.sql.functions.*; +import scala.collection.JavaConverters; + + +import java.sql.Timestamp; +import java.util.*; +import java.util.stream.Collectors; + +@Configuration +@PropertySource("classpath:application.properties") +@ConditionalOnExpression("'${database}' == 'cassandra'") +public class CassConnector extends SparkDatabaseRepository { + + + protected String bucket = "s3a://siesta/"; + + @Autowired + public CassConnector(SparkSession sparkSession, JavaSparkContext javaSparkContext, Utils utils) { + super(sparkSession, javaSparkContext, utils); + } + + @Override + public Metadata getMetadata(String logname) { + Dataset df = sparkSession.read() + .format("org.apache.spark.sql.cassandra") + .options(Map.of("table", logname + "_meta", "keyspace", "siesta")) + .load(); + try { + Map m = new HashMap<>(); + df.toJavaRDD().map((Function>) row -> + new Tuple2<>(row.getString(0), row.getString(1))) + .collect().forEach(t -> { + m.put(t._1, t._2); + }); + return new Metadata(m); + + } catch (Exception e){ //handle metadata from delta + Map metadataMap = new HashMap<>(); + List rows = df.collectAsList(); // Collect rows as a list + for (Row row : rows) { + String key = row.getAs("key"); + String value = row.getAs("value"); + metadataMap.put(key, value); + } + return new Metadata(metadataMap, "delta"); + } + } + + @Override + public Set findAllLongNames() { + CassandraConnector connector = CassandraConnector.apply(sparkSession.sparkContext().getConf()); + List keywords = new ArrayList<>() {{ + add("set"); + add("sign"); + add("meta"); + add("idx"); + add("count"); + add("index"); + add("seq"); + add("lastchecked"); + add("single"); + }}; + return connector.withSessionDo(session -> session.execute("SELECT table_name FROM system_schema.tables WHERE keyspace_name = '" + + "siesta" + "';").all()) + .stream().map(x -> x.get("table_name", String.class)).filter(Objects::nonNull) + .map(x -> + Arrays.stream(x.split("_")). + filter(y -> !keywords.contains(y)).collect(Collectors.joining("_")) + ).collect(Collectors.toSet()); + } + + @Override + protected Dataset readSequenceTable(String logname) { + String path = String.format("%s_seq", logname); + Dataset df = sparkSession.read() + .format("org.apache.spark.sql.cassandra") + .options(Map.of("table", path, "keyspace", "siesta")) + .load(); + + // Explode the sequence data and transform to EventModel format + Dataset explodedDF = df + .withColumn("event_data", functions.explode(functions.col("events"))) + .withColumn("event_parts", functions.split(functions.col("event_data"), ",")) + .withColumn("event_name", functions.element_at(functions.col("event_parts"), 1)) + .withColumn("timestamp", functions.element_at(functions.col("event_parts"), 2)) + .withColumn("position", functions.row_number().over( + Window.partitionBy("sequence_id").orderBy(functions.monotonically_increasing_id()) + ).minus(1)) + .select( + functions.col("sequence_id").alias("traceId"), + functions.col("event_name").alias("eventName"), + functions.col("timestamp"), + functions.col("position") + ); + return explodedDF.as(Encoders.bean(EventModel.class)); + } + + @Override + protected Dataset readSingleTable(String logname){ + String tableName = String.format("%s_single", logname); + Dataset df = sparkSession.read() + .format("org.apache.spark.sql.cassandra") + .options(Map.of("table", tableName, "keyspace", "siesta")) + .load(); + + // Explode the occurrences list and parse each occurrence + Dataset explodedDF = df + .withColumn("occurrence", functions.explode(functions.col("occurrences"))) + .withColumn("occurrence_parts", functions.split(functions.col("occurrence"), ",")) + .withColumn("timestamp", functions.element_at(functions.col("occurrence_parts"), 2)) + .withColumn("position", functions.element_at(functions.col("occurrence_parts"), 1).cast("int")) + .select( + functions.col("trace_id").alias("traceId"), + functions.col("event_type").alias("eventName"), + functions.col("timestamp"), + functions.col("position") + ); + + return explodedDF.as(Encoders.bean(EventModel.class)); + } + + @Override + protected Dataset readCountTable(String logname){ + String tableName = String.format("%s_count", logname); + Dataset df = sparkSession.read() + .format("org.apache.spark.sql.cassandra") + .options(Map.of("table", tableName, "keyspace", "siesta")) + .load(); + + // Explode the times list and parse each time record + Dataset explodedDF = df + .withColumn("time_record", functions.explode(functions.col("times"))) + .withColumn("time_parts", functions.split(functions.col("time_record"), ",")) + .select( + functions.col("event_a").alias("eventA"), + functions.element_at(functions.col("time_parts"), 1).alias("eventB"), + functions.element_at(functions.col("time_parts"), 2).cast("long").alias("sumDuration"), + functions.element_at(functions.col("time_parts"), 3).cast("int").alias("count"), + functions.element_at(functions.col("time_parts"), 4).cast("long").alias("minDuration"), + functions.element_at(functions.col("time_parts"), 5).cast("long").alias("maxDuration") + ,functions.element_at(functions.col("time_parts"), 6).cast("double").alias("sumSquares") + ); + return explodedDF.as(Encoders.bean(Count.class)); + } + + protected Dataset readIndexTable(String logname) { + boolean positions = sparkSession.read() + .format("org.apache.spark.sql.cassandra") + .options(Map.of("table", logname + "_meta", "keyspace", "siesta")) + .load() + .filter(col("key").equalTo("mode")) + .select("value") + .first() + .getString(0) + .equals("positions"); + + String tableName = String.format("%s_index", logname); + Dataset df = sparkSession.read() + .format("org.apache.spark.sql.cassandra") + .options(Map.of("table", tableName, "keyspace", "siesta")) + .load(); + + Dataset explodedDF = df + // explode each occurrence entry + .withColumn("occurrence", functions.explode(col("occurrences"))) + + // split into trace_id and values string + .withColumn("parts", split(col("occurrence"), "\\|\\|")) + .withColumn("trace_id", trim(element_at(col("parts"), 1))) + .withColumn("values_raw", trim(element_at(col("parts"), 2))) + + // split raw values into individual pairs (valueA|valueB) + .withColumn("pair", explode(split(col("values_raw"), ","))) + + // split each pair into valueA and valueB + .withColumn("values", split(col("pair"), "\\|")) + .withColumn("valueA", trim(element_at(col("values"), 1))) + .withColumn("valueB", trim(element_at(col("values"), 2))) + + .withColumnRenamed("event_a", "eventA") + .withColumnRenamed("event_b", "eventB") + ; + +// explodedDF.filter(functions.not(functions.col("valueB").rlike("^[+-]?[0-9]+$"))).show(false); + + if (positions) { + explodedDF = explodedDF + .withColumn("positionA", col("valueA").cast(DataTypes.IntegerType)) + .withColumn("positionB", col("valueB").cast(DataTypes.IntegerType)) + .withColumn("timestampA", functions.lit(null)) + .withColumn("timestampB", functions.lit(null)) +// .filter("positionA >= 0 AND positionB >= 0") + .select("trace_id", "eventA", "eventB", "timestampA", "timestampB", "positionA", "positionB"); +// explodedDF.filter(col("positionB").isNull()).show(false); + + } else { + explodedDF = explodedDF + .withColumn("timestampA", col("valueA")) + .withColumn("timestampB", col("valueB")) + .withColumn("positionA", functions.lit(-1)) + .withColumn("positionB", functions.lit(-1)) +// .filter("timestampA IS NOT NULL AND timestampB IS NOT NULL") + .select("trace_id", "eventA", "eventB", "timestampA", "timestampB", "positionA", "positionB"); + } + + return explodedDF.as(Encoders.bean(IndexPair.class)); + } + + //Below are for declare// + @Override + public Dataset queryPositionState(String logname) { + String path = String.format("%s%s%s", bucket, logname, "/declare/position.parquet/"); + + return sparkSession.read() + .parquet(path) + .as(Encoders.bean(PositionState.class)); + } + + @Override + public Dataset queryExistenceState(String logname) { + String path = String.format("%s%s%s", bucket, logname, "/declare/existence.parquet/"); + + return sparkSession.read() + .parquet(path) + .as(Encoders.bean(ExistenceState.class)); + } + + + @Override + public Dataset queryUnorderStateI(String logname) { + String path = String.format("%s%s%s", bucket, logname, "/declare/unorder/i.parquet/"); + + return sparkSession.read() + .parquet(path) + .as(Encoders.bean(UnorderStateI.class)); + } + + + @Override + public Dataset queryUnorderStateU(String logname) { + String path = String.format("%s%s%s", bucket, logname, "/declare/unorder/u.parquet/"); + + return sparkSession.read() + .parquet(path) + .as(Encoders.bean(UnorderStateU.class)); + } + + + @Override + public Dataset queryOrderState(String logname) { + String path = String.format("%s%s%s", bucket, logname, "/declare/order.parquet"); + + return sparkSession.read() + .parquet(path) + .as(Encoders.bean(OrderState.class)); + } + + + @Override + public Dataset queryNegativeState(String logname) { + String path = String.format("%s%s%s", bucket, logname, "/declare/negatives.parquet"); + + return sparkSession.read() + .parquet(path) + .as(Encoders.bean(NegativeState.class)); + } + + +} diff --git a/src/main/java/com/datalab/siesta/queryprocessor/storage/repositories/Cassandra/SparkConfiguration.java b/src/main/java/com/datalab/siesta/queryprocessor/storage/repositories/Cassandra/SparkConfiguration.java new file mode 100644 index 00000000..33de63e6 --- /dev/null +++ b/src/main/java/com/datalab/siesta/queryprocessor/storage/repositories/Cassandra/SparkConfiguration.java @@ -0,0 +1,88 @@ +package com.datalab.siesta.queryprocessor.storage.repositories.Cassandra; + +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.sql.SparkSession; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.PropertySource; +import org.springframework.context.support.PropertySourcesPlaceholderConfigurer; + +/** + * Contains the configuration of Spark in order to connect to ScyllaDB database + */ +@Configuration +@PropertySource("classpath:application.properties") +@ConditionalOnExpression("'${database}' == 'cassandra' and '${delta}' == 'false'") +public class SparkConfiguration { + + @Value("${app.name:siesta2}") + private String appName; + + @Value("${master.uri:local[*]}") + private String masterUri; + + @Value("${cassandra.contact.points:cassandra}") + private String cassContactPoints; + + @Value("${scylla.port:9042}") + private String cassPort; + + @Value("${scylla.keyspace:siesta}") + private String cassKeyspace; + + @Value("${spring.data.cassandra.user:cassandra}") + private String cassUser; + + @Value("${spring.data.cassandra.password:cassandra}") + private String cassPassword; + + @Bean + public SparkConf sparkConf() { + return new SparkConf() + .setAppName(appName) + .setMaster(masterUri) + .set("spark.driver.extraJavaOptions", "--add-opens java.base/sun.security.action=ALL-UNNAMED") + .set("spark.executor.extraJavaOptions", "--add-opens java.base/sun.security.action=ALL-UNNAMED") + .set("spark.driver.maxResultSize", "5g") + // ScyllaDB/Cassandra specific configurations + .set("spark.cassandra.connection.host", cassContactPoints) + .set("spark.cassandra.connection.port", cassPort) + .set("spark.cassandra.connection.keep_alive_ms", "60000") + .set("spark.cassandra.connection.timeout_ms", "30000") + .set("spark.cassandra.read.timeout_ms", "30000") +// .set("spark.cassandra.connection.reconnection_delay_ms.base", "1000") + .set("spark.cassandra.connection.reconnection_delay_ms.max", "60000") + .set("spark.cassandra.auth.username", cassUser) + .set("spark.cassandra.auth.password", cassPassword) + .set("spark.sql.extensions", "com.datastax.spark.connector.CassandraSparkExtensions"); + } + + @Bean + public JavaSparkContext javaSparkContext() { + return new JavaSparkContext(this.sparkConf()); + } + + @Bean + public SparkSession sparkSession() { + SparkSession spark = SparkSession + .builder() + .sparkContext(this.javaSparkContext().sc()) + .appName("siesta 2") + .getOrCreate(); + + // Configure Spark SQL for Cassandra/ScyllaDB + spark.conf().set("spark.sql.sources.partitionOverwriteMode", "dynamic"); + spark.conf().set("spark.sql.adaptive.enabled", "true"); + spark.conf().set("spark.sql.adaptive.coalescePartitions.enabled", "true"); + + return spark; + } + + @Bean + public static PropertySourcesPlaceholderConfigurer propertySourcesPlaceholderConfigurer() { + return new PropertySourcesPlaceholderConfigurer(); + } +} diff --git a/src/main/java/com/datalab/siesta/queryprocessor/storage/repositories/DeltaLakes/DeltaConnector.java b/src/main/java/com/datalab/siesta/queryprocessor/storage/repositories/DeltaLakes/DeltaConnector.java deleted file mode 100644 index 46047d1d..00000000 --- a/src/main/java/com/datalab/siesta/queryprocessor/storage/repositories/DeltaLakes/DeltaConnector.java +++ /dev/null @@ -1,507 +0,0 @@ -package com.datalab.siesta.queryprocessor.storage.repositories.DeltaLakes; - -import com.datalab.siesta.queryprocessor.declare.model.*; -import com.datalab.siesta.queryprocessor.declare.model.declareState.*; -import com.datalab.siesta.queryprocessor.model.DBModel.Count; -import com.datalab.siesta.queryprocessor.model.DBModel.IndexPair; -import com.datalab.siesta.queryprocessor.model.DBModel.Metadata; -import com.datalab.siesta.queryprocessor.model.DBModel.Trace; -import com.datalab.siesta.queryprocessor.model.Events.EventBoth; -import com.datalab.siesta.queryprocessor.model.Events.EventPair; -import com.datalab.siesta.queryprocessor.model.Utils.Utils; -import com.datalab.siesta.queryprocessor.storage.repositories.SparkDatabaseRepository; -import org.apache.commons.collections4.IteratorUtils; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.LocatedFileStatus; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.RemoteIterator; -import org.apache.hadoop.fs.s3a.S3AFileStatus; -import org.apache.spark.api.java.JavaPairRDD; -import org.apache.spark.api.java.JavaRDD; -import org.apache.spark.api.java.JavaSparkContext; -import org.apache.spark.api.java.function.FlatMapFunction; -import org.apache.spark.api.java.function.Function; -import org.apache.spark.broadcast.Broadcast; -import org.apache.spark.sql.*; -import org.apache.spark.sql.execution.datasources.*; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.PropertySource; -import scala.Tuple2; -import scala.Tuple3; -import scala.collection.JavaConverters; - -import java.io.IOException; -import java.io.File; -import java.net.URI; -import java.net.URISyntaxException; -import java.sql.Timestamp; -import java.util.*; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -@Configuration -@PropertySource("classpath:application.properties") -@ConditionalOnExpression("'${database}' == 's3' and '${delta}' == 'true'") -public class DeltaConnector extends SparkDatabaseRepository { - private String bucket = "s3a://siesta/"; - - @Autowired - public DeltaConnector(SparkSession sparkSession, JavaSparkContext javaSparkContext, Utils utils) { - super(sparkSession, javaSparkContext, utils); - } - - - @Override - public Metadata getMetadata(String logname) { - String path = String.format(String.format("%s%s%s", bucket, logname, "/meta")); - Dataset df = sparkSession.read().format("delta").load(path); - Map metadataMap = new HashMap<>(); - List rows = df.collectAsList(); // Collect rows as a list - for (Row row : rows) { - String key = row.getAs("key"); - String value = row.getAs("value"); - metadataMap.put(key, value); - } - return new Metadata(metadataMap, "delta"); - } - - @Override - public Set findAllLongNames() { - try { - FileSystem fs = FileSystem.get(new URI(this.bucket), sparkSession.sparkContext().hadoopConfiguration()); - RemoteIterator f = fs.listFiles(new Path(this.bucket), true); - Pattern pattern = Pattern.compile(String.format("%s[^/]*/", this.bucket)); - Set files = new HashSet<>(); - while (f.hasNext()) { - LocatedFileStatus fin = f.next(); - Matcher matcher = pattern.matcher(fin.getPath().toString()); - if (matcher.find()) { - String logname = matcher.group(0).replace(this.bucket, "").replace("/", ""); - files.add(logname); - } - } - return files; - - } catch (IOException | URISyntaxException e) { - throw new RuntimeException(e); - } - } - - @Override - public List getCountForExploration(String logname, String event) { - String path = String.format("%s%s%s", bucket, logname, "/count/"); - List counts = sparkSession.read() - .format("delta") - .load(path) - .where(String.format("eventA = '%s'", event)) - .toJavaRDD() - .flatMap((FlatMapFunction) row -> { - List c = new ArrayList<>(); - String eventA = row.getString(0); - String eventB = row.getString(1); - long sum_duration = row.getLong(2); - int count = row.getInt(3); - long min_duration = row.getLong(4); - long max_duration = row.getLong(5); - double sum_squared = row.getDouble(6); - c.add(new Count(eventA, eventB, sum_duration, count, min_duration, max_duration, sum_squared)); - return c.iterator(); - }).collect(); - return new ArrayList<>(counts); - } - - @Override - public List getCounts(String logname, Set pairs) { - String path = String.format("%s%s%s", bucket, logname, "/count/"); - String firstFilter = pairs.stream().map(x -> x.getEventA().getName()).collect(Collectors.toSet()) - .stream().map(x -> String.format("eventA = '%s'", x)).collect(Collectors.joining(" or ")); - Broadcast> b_pairs = javaSparkContext.broadcast(pairs); - - Dataset df = sparkSession.read() - .format("delta") - .load(path) - .where(firstFilter); - - // Print the schema of the DataFrame - System.out.println("Schema of the DataFrame:"); - df.printSchema(); - System.out.println("Seires: " + df.count()); - System.out.println("Sthles: " + df.columns().length); - List counts = sparkSession.read() - .format("delta") - .load(path) - .where(firstFilter) - .toJavaRDD() - .flatMap((FlatMapFunction) row -> { - List c = new ArrayList<>(); - String eventA = row.getString(0); - String eventB = row.getString(1); - long sum_duration = row.getLong(2); - int count = row.getInt(3); - long min_duration = row.getLong(4); - long max_duration = row.getLong(5); - double sum_squared = row.getDouble(6); - c.add(new Count(eventA, eventB, sum_duration, count, min_duration, max_duration, sum_squared)); - return c.iterator(); - }) - .filter((Function) c -> { - for (EventPair p : b_pairs.getValue()) { - if (c.getEventA().equals(p.getEventA().getName()) && c.getEventB().equals(p.getEventB().getName())) { - return true; - } - } - return false; - }) - .collect(); - List response = new ArrayList<>(); - pairs.forEach(p -> { - for (Count c : counts) { - if (c.getEventA().equals(p.getEventA().getName()) && c.getEventB().equals(p.getEventB().getName())) { - response.add(c); - break; - } - } - }); - - return response; - } - - @Override - public List getEventPairs(String logname) { - String path = String.format("%s%s%s", bucket, logname, "/count/"); - List counts = sparkSession.read() - .format("delta") - .load(path) - .toJavaRDD() - .flatMap((FlatMapFunction) row -> { - List c = new ArrayList<>(); - String eventA = row.getString(0); - String eventB = row.getString(1); - long sum_duration = row.getLong(2); - int count = row.getInt(3); - long min_duration = row.getLong(4); - long max_duration = row.getLong(5); - double sum_squared = row.getDouble(6); - c.add(new Count(eventA, eventB, sum_duration, count, min_duration, max_duration, sum_squared)); - return c.iterator(); - }) - .collect(); - return counts; - } - - @Override - public List getEventNames(String logname) { - String path = String.format("%s%s%s", bucket, logname, "/single/"); - - return sparkSession.read().format("delta") - .load(path) - .select("event_type") - .distinct() - .toJavaRDD() - .map((Function) row -> row.getString(0)) - .collect(); - } - - - @Override - protected JavaRDD querySequenceTablePrivate(String logname, Broadcast> bTraceIds) { - return querySequenceTableDeclare(logname) - .filter((Function) trace -> bTraceIds.getValue().contains(trace.getTraceID())); - } - - - @Override - protected JavaRDD getFromSingle(String logname, Set traceIds, Set eventTypes) { - String path = String.format("%s%s%s", bucket, logname, "/single/"); - Broadcast> bTraceIds = javaSparkContext.broadcast(traceIds); - Broadcast> bEventTypes = javaSparkContext.broadcast(eventTypes); - return sparkSession.read() - .format("delta") - .load(path) - .toJavaRDD() - .filter((Function) x -> bEventTypes.value().contains((String)x.getAs("event_type"))) - .filter((Function) x -> bTraceIds.value().contains((String)x.getAs("trace"))) - .map((Function) row->{ - String trace_id = row.getAs("trace"); - String event_type = row.getAs("event_type"); - String ts = row.getAs("timestamp"); - Integer position = row.getAs("position"); - return new EventBoth(event_type,trace_id, Timestamp.valueOf(ts),position); - }); - } - - - @Override - protected JavaPairRDD, Iterable> getAllEventPairs(Set pairs, - String logname, - Metadata metadata, - Timestamp from, - Timestamp till) { - String path = String.format("%s%s%s", bucket, logname, "/index/"); - Broadcast> bPairs = javaSparkContext.broadcast(pairs); - Broadcast mode = javaSparkContext.broadcast(metadata.getMode()); - Broadcast bFrom = javaSparkContext.broadcast(from); - Broadcast bTill = javaSparkContext.broadcast(till); - - List whereStatements = new ArrayList<>(); - whereStatements.add( - pairs.stream().map(x -> x.getEventA().getName()).distinct() - .map(p -> String.format("eventA = '%s'", p)) - .collect(Collectors.joining(" or "))); - - for (int i = 0; i < whereStatements.size(); i++) { - whereStatements.set(i, String.format("( %s )", whereStatements.get(i))); - } - String whereStatement = String.join(" and ", whereStatements); - - JavaPairRDD, java.lang.Iterable> rows = sparkSession.read() - .format("delta") - .load(path) - .where(whereStatement) - .toJavaRDD() - .flatMap((FlatMapFunction) row -> { - String eventA = row.getAs("eventA"); - String eventB = row.getAs("eventB"); - boolean checkContained = false; - for (EventPair ep : bPairs.getValue()) { - if (eventA.equals(ep.getEventA().getName()) && eventB.equals(ep.getEventB().getName())) { - checkContained = true; - break; - } - } - List response = new ArrayList<>(); - if (checkContained) { - String tid = row.getAs("id"); - if (mode.getValue().equals("positions")) { - int posA = row.getAs("positionA"); - int posB = row.getAs("positionB"); - response.add(new IndexPair(tid, eventA, eventB, posA, posB)); - } else { - Timestamp tsA = row.getAs("timestampA"); - Timestamp tsB = row.getAs("timestampB"); - if (!(bTill.value() != null && tsA.after(bTill.value()) || - bFrom.value() != null && tsB.before(bFrom.value()))) { - response.add(new IndexPair(tid, eventA, eventB, tsA, tsB)); - } - } - } - return response.iterator(); - }) - .groupBy((Function>) indexPair -> new Tuple2<>(indexPair.getEventA(), indexPair.getEventB())); - return rows; - } - - - //Below are for declare// - - @Override - public JavaRDD querySequenceTableDeclare(String logname) { - String path = String.format("%s%s%s", bucket, logname, "/seq"); - return sparkSession.read() - .format("delta") - .load(path) - .toJavaRDD() - .map((Function) row -> { - String trace_id = row.getAs("trace"); - String event_name = row.getAs("event_type"); - Timestamp ts = row.getAs("timestamp"); - Integer pos = row.getAs("position"); - return new EventBoth(event_name, trace_id, ts, pos); - }) - .groupBy((Function) EventBoth::getTraceID) - .map((Function>, Trace>) t -> { - String traceID = t._1(); - List sortedEvents = IteratorUtils.toList(t._2().iterator()); - sortedEvents.sort(Comparator.comparingInt(EventBoth::getPosition)); - return new Trace(traceID, sortedEvents); - }); - } - - @Override - public JavaRDD querySingleTable(String logname){ - String path = String.format("%s%s%s", bucket, logname, "/single/"); - - return sparkSession.read() - .format("delta") - .load(path) - .select("event_type","trace") - .groupBy("event_type") - .agg(functions.size(functions.collect_list("event_type")).alias("unique")) - .toJavaRDD() - .map((Function) row -> { - String event = row.getAs("event_type"); - int s = row.getAs("unique"); - return new EventSupport(event,s); - }); - } - - @Override - public JavaRDD querySingleTableDeclare(String logname) { - String path = String.format("%s%s%s", bucket, logname, "/single"); - - return sparkSession.read() - .format("delta") - .load(path) - .select("event_type","trace") - .groupBy("event_type","trace") - .agg(functions.size(functions.collect_list("event_type")).alias("unique")) - .toJavaRDD() - .groupBy((Function) ev->ev.getAs("event_type")) - .map((Function>,UniqueTracesPerEventType>) ev->{ - String event_type = ev._1(); - List opt = new ArrayList<>(); - for(Row r: ev._2()){ - opt.add(new OccurrencesPerTrace(r.getAs("trace"),r.getAs("unique"))); - } - return new UniqueTracesPerEventType(event_type,opt); - }); - } - - @Override - public JavaPairRDD, List> querySingleTableAllDeclare(String logname) { - String path = String.format("%s%s%s", bucket, logname, "/single"); - JavaPairRDD, List> rdd = sparkSession.read() - .format("delta") - .load(path) - .select("event_type","trace","position") - .groupBy("event_type","trace") - .agg(functions.collect_list("position").alias("positions")) - .toJavaRDD() - .map(row->{ - String eventType = row.getAs("event_type"); - String trace_id = row.getAs("trace"); - List positions = JavaConverters.seqAsJavaList(row.getSeq(2)); - return new Tuple3<>(eventType,trace_id,positions); - }) - .keyBy(r -> new Tuple2<>(r._1(), r._2())) - .mapValues(Tuple3::_3); - - return rdd; - - } - - @Override - public JavaRDD queryIndexOriginalDeclare(String logname) { - String path = String.format("%s%s%s", bucket, logname, "/index"); - - return sparkSession.read() - .format("delta") - .load(path) - .withColumnRenamed("id", "trace_id") - .select("eventA","eventB","trace_id") - .distinct() - .as(Encoders.bean(EventPairToTrace.class)) - .toJavaRDD(); - } - - @Override - public JavaRDD queryIndexTableDeclare(String logname) { - String path = String.format("%s%s%s", bucket, logname, "/index"); - - return sparkSession.read().format("delta") - .load(path) - .select("eventA","eventB","id") - .distinct() - .toJavaRDD() - .groupBy((Function>)row->new Tuple2<>(row.getAs("eventA"),row.getAs("eventB"))) - .map((Function, Iterable>, UniqueTracesPerEventPair>)row->{ - List uniqueTraces = new ArrayList<>(); - for(Row r: row._2()){ - uniqueTraces.add(r.getAs("id")); - } - return new UniqueTracesPerEventPair(row._1()._1(),row._1()._2,uniqueTraces); - } ); - } - - @Override - public JavaRDD queryIndexTableAllDeclare(String logname) { - String path = String.format("%s%s%s", bucket, logname, "/index"); - - return sparkSession.read() - .format("delta") - .load(path) - .toJavaRDD() - .map((Function) row -> { - String eventA = row.getAs("eventA"); - String eventB = row.getAs("eventB"); - String trace_id = row.getAs("id"); - int positionA = row.getAs("positionA"); - int positionB = row.getAs("positionB"); - return new IndexPair(trace_id,eventA,eventB,positionA,positionB); - }); - } - - @Override - public JavaRDD queryPositionState(String logname) { - String path = String.format("%s%s%s", bucket, logname, "/declare/position/"); - - return sparkSession.read() - .format("delta") - .load(path) - .as(Encoders.bean(PositionState.class)) - .toJavaRDD(); - } - - @Override - public JavaRDD queryExistenceState(String logname) { - String path = String.format("%s%s%s", bucket, logname, "/declare/existence/"); - - return sparkSession.read() - .format("delta") - .load(path) - .as(Encoders.bean(ExistenceState.class)) - .toJavaRDD(); - } - - - @Override - public JavaRDD queryUnorderStateI(String logname) { - String path = String.format("%s%s%s", bucket, logname, "/declare/unorder/i/"); - - return sparkSession.read() - .format("delta") - .load(path) - .as(Encoders.bean(UnorderStateI.class)) - .toJavaRDD(); - } - - - @Override - public JavaRDD queryUnorderStateU(String logname) { - String path = String.format("%s%s%s", bucket, logname, "/declare/unorder/u/"); - - return sparkSession.read() - .format("delta") - .load(path) - .as(Encoders.bean(UnorderStateU.class)) - .toJavaRDD(); - } - - - @Override - public JavaRDD queryOrderState(String logname) { - String path = String.format("%s%s%s", bucket, logname, "/declare/order"); - - return sparkSession.read() - .format("delta") - .load(path) - .as(Encoders.bean(OrderState.class)) - .toJavaRDD(); - } - - - @Override - public JavaRDD queryNegativeState(String logname) { - String path = String.format("%s%s%s", bucket, logname, "/declare/negatives"); - - return sparkSession.read() - .format("delta") - .load(path) - .as(Encoders.bean(NegativeState.class)) - .toJavaRDD(); - } -} diff --git a/src/main/java/com/datalab/siesta/queryprocessor/storage/repositories/DeltaLakes/SparkConfiguration.java b/src/main/java/com/datalab/siesta/queryprocessor/storage/repositories/DeltaLakes/SparkConfiguration.java deleted file mode 100644 index d1f802fe..00000000 --- a/src/main/java/com/datalab/siesta/queryprocessor/storage/repositories/DeltaLakes/SparkConfiguration.java +++ /dev/null @@ -1,90 +0,0 @@ -package com.datalab.siesta.queryprocessor.storage.repositories.DeltaLakes; - -import org.apache.spark.SparkConf; -import org.apache.spark.api.java.JavaSparkContext; -import org.apache.spark.sql.SparkSession; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.PropertySource; -import org.springframework.context.support.PropertySourcesPlaceholderConfigurer; - -/** - * Contains the configuration of spark in he.maven.plugins:maven-compiler-plugin:3.13.0:compile (default-compile) on project siesta-query-processor: Fatal error compiling: error: release version 17 not supported -> [Help 1] - order to connect to s3 database - */ -@Configuration -@PropertySource("classpath:application.properties") -@ConditionalOnExpression("'${database}' == 's3' and '${delta}' == 'true'") -public class SparkConfiguration { - - @Value("${app.name:siesta2}") - private String appName; - - @Value("${master.uri:local[*]}") - private String masterUri; - - @Value("${s3.user:minioadmin}") - private String s3user; - - @Value("${s3.key:minioadmin}") - private String s3key; - - @Value("${s3.timeout:600000}") - private String s3timeout; - - @Value("${s3.endpoint:http://127.0.0.1:9000}") - private String s3endpoint; - - public SparkConf sparkConf() { - return new SparkConf() - .setAppName(appName) - .setMaster(masterUri) - .set("spark.driver.extraJavaOptions", "--add-opens java.base/sun.security.action=ALL-UNNAMED") - .set("spark.executor.extraJavaOptions", "--add-opens java.base/sun.security.action=ALL-UNNAMED") - .set("spark.sql.extensions","io.delta.sql.DeltaSparkSessionExtension") - .set("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") - .set("spark.sql.streaming.statefulOperator.checkCorrectness.enabled", "false") -// .set("spark.driver.memory","25g") -// .set("spark.driver.memoryOverhead","2g") -// .set("spark.memory.fraction","0.8") -// .set("spark.memory.storageFraction","0.5") - .set("spark.driver.maxResultSize","5g"); - } - - @Bean - public JavaSparkContext javaSparkContext() { - return new JavaSparkContext(this.sparkConf()); - } - - @Bean - public SparkSession sparkSession() { - SparkSession spark= SparkSession - .builder() - .sparkContext(this.javaSparkContext().sc()) - .appName("siesta 2") - .getOrCreate(); - spark.sparkContext().hadoopConfiguration().set("fs.s3a.endpoint", s3endpoint); - spark.sparkContext().hadoopConfiguration().set("fs.s3a.access.key", s3user); - spark.sparkContext().hadoopConfiguration().set("fs.s3a.secret.key", s3key); - spark.sparkContext().hadoopConfiguration().set("fs.s3a.connection.timeout", s3timeout); - spark.sparkContext().hadoopConfiguration().set("fs.s3a.path.style.access", "true"); - spark.sparkContext().hadoopConfiguration().set("fs.s3a.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem"); - spark.sparkContext().hadoopConfiguration().set("fs.s3a.connection.ssl.enabled", "true"); - spark.sparkContext().hadoopConfiguration().set("fs.s3a.bucket.create.enabled", "true"); - spark.conf().set("spark.sql.sources.partitionOverwriteMode", "dynamic"); - spark.conf().set("spark.sql.files.metadata.log.parsing.enabled", "true"); - spark.conf().set("spark.sql.sources.useV1SourceList", "delta"); - spark.conf().set("spark.delta.logStore.class", "org.apache.spark.sql.delta.storage.S3SingleDriverLogStore"); - System.out.println("Spark version: ".concat(spark.version())); -// spark.conf().set("spark.executor.memory", "30g"); - return spark; - } - - @Bean - public static PropertySourcesPlaceholderConfigurer propertySourcesPlaceholderConfigurer() { - return new PropertySourcesPlaceholderConfigurer(); - } -} diff --git a/src/main/java/com/datalab/siesta/queryprocessor/storage/repositories/S3/S3Connector.java b/src/main/java/com/datalab/siesta/queryprocessor/storage/repositories/S3/S3Connector.java index a2e8c30a..b3e2bf3c 100644 --- a/src/main/java/com/datalab/siesta/queryprocessor/storage/repositories/S3/S3Connector.java +++ b/src/main/java/com/datalab/siesta/queryprocessor/storage/repositories/S3/S3Connector.java @@ -1,66 +1,41 @@ package com.datalab.siesta.queryprocessor.storage.repositories.S3; -import com.datalab.siesta.queryprocessor.declare.model.EventPairToTrace; -import com.datalab.siesta.queryprocessor.declare.model.EventSupport; -import com.datalab.siesta.queryprocessor.declare.model.OccurrencesPerTrace; -import com.datalab.siesta.queryprocessor.declare.model.UniqueTracesPerEventPair; -import com.datalab.siesta.queryprocessor.declare.model.UniqueTracesPerEventType; import com.datalab.siesta.queryprocessor.declare.model.declareState.ExistenceState; import com.datalab.siesta.queryprocessor.declare.model.declareState.NegativeState; import com.datalab.siesta.queryprocessor.declare.model.declareState.OrderState; import com.datalab.siesta.queryprocessor.declare.model.declareState.PositionState; import com.datalab.siesta.queryprocessor.declare.model.declareState.UnorderStateI; import com.datalab.siesta.queryprocessor.declare.model.declareState.UnorderStateU; -import com.datalab.siesta.queryprocessor.model.DBModel.Count; -import com.datalab.siesta.queryprocessor.model.DBModel.IndexPair; -import com.datalab.siesta.queryprocessor.model.DBModel.Metadata; -import com.datalab.siesta.queryprocessor.model.DBModel.Trace; -import com.datalab.siesta.queryprocessor.model.Events.EventBoth; -import com.datalab.siesta.queryprocessor.model.Events.EventPair; +import com.datalab.siesta.queryprocessor.model.DBModel.*; import com.datalab.siesta.queryprocessor.model.Utils.Utils; +import com.datalab.siesta.queryprocessor.storage.model.EventModel; import com.datalab.siesta.queryprocessor.storage.repositories.SparkDatabaseRepository; -import org.apache.commons.collections4.IteratorUtils; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; -import org.apache.spark.api.java.JavaPairRDD; -import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; -import org.apache.spark.api.java.function.FlatMapFunction; -import org.apache.spark.api.java.function.Function; -import org.apache.spark.broadcast.Broadcast; import org.apache.spark.sql.*; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.PropertySource; import org.springframework.stereotype.Service; -import scala.Tuple2; -import scala.Tuple3; -import scala.collection.JavaConverters; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; -import java.sql.Timestamp; import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; -import java.util.stream.Collectors; @Configuration -//@ConditionalOnProperty( -// value = "database", -// havingValue = "s3", -// matchIfMissing = true -//) -@ConditionalOnExpression("'${database}' == 's3' and '${delta}' == 'false'") -@Service +@PropertySource("classpath:application.properties") +@ConditionalOnExpression("'${database}' == 's3'") public class S3Connector extends SparkDatabaseRepository { - private String bucket = "s3a://siesta/"; + protected String bucket = "s3a://siesta/"; @Autowired public S3Connector(SparkSession sparkSession, JavaSparkContext javaSparkContext, Utils utils) { @@ -70,8 +45,31 @@ public S3Connector(SparkSession sparkSession, JavaSparkContext javaSparkContext, @Override public Metadata getMetadata(String logname) { - Dataset df = sparkSession.read().parquet(String.format("%s%s%s", bucket, logname, "/meta.parquet/")); - return new Metadata(df.toJavaRDD().collect().get(0)); + Dataset df = null; + boolean parquet = true; + try{ + df = sparkSession.read().parquet(String.format("%s%s%s", bucket, logname, "/meta.parquet/")); + }catch (Exception e){ + try { + String path = String.format(String.format("%s%s%s", bucket, logname, "/meta/")); + df = sparkSession.read().format("delta").load(path); + parquet = false; + }catch (Exception e2){ + return null; + } + } + if(parquet){ //handle metadata from parquets + return new Metadata(df.toJavaRDD().collect().get(0)); + }else{ //handle metadata from delta + Map metadataMap = new HashMap<>(); + List rows = df.collectAsList(); // Collect rows as a list + for (Row row : rows) { + String key = row.getAs("key"); + String value = row.getAs("value"); + metadataMap.put(key, value); + } + return new Metadata(metadataMap, "delta"); + } } @Override @@ -97,392 +95,153 @@ public Set findAllLongNames() { } @Override - public List getCountForExploration(String logname, String event) { - String path = String.format("%s%s%s", bucket, logname, "/count.parquet/"); - List counts = sparkSession.read() - .parquet(path) - .where(String.format("eventA = '%s'", event)) - .toJavaRDD() - .flatMap((FlatMapFunction) row -> { - String eventA = row.getString(1); - List countRecords = JavaConverters.seqAsJavaList(row.getSeq(0)); - List c = new ArrayList<>(); - for (Row v1 : countRecords) { - String eventB = v1.getString(0); - long sum_duration = v1.getLong(1); - int count = v1.getInt(2); - long min_duration = v1.getLong(3); - long max_duration = v1.getLong(4); - double sum_squared = v1.getDouble(5); - c.add(new Count(eventA, eventB, sum_duration, count, min_duration, max_duration, sum_squared)); - } - return c.iterator(); - }).collect(); - return new ArrayList<>(counts); - } - - @Override - public List getCounts(String logname, Set pairs) { - String path = String.format("%s%s%s", bucket, logname, "/count.parquet/"); - String firstFilter = pairs.stream().map(x -> x.getEventA().getName()).collect(Collectors.toSet()) - .stream().map(x -> String.format("eventA = '%s'", x)).collect(Collectors.joining(" or ")); - Broadcast> b_pairs = javaSparkContext.broadcast(pairs); - List counts = sparkSession.read() - .parquet(path) - .where(firstFilter) - .toJavaRDD() - .flatMap((FlatMapFunction) row -> { - String eventA = row.getString(1); - List countRecords = JavaConverters.seqAsJavaList(row.getSeq(0)); - List c = new ArrayList<>(); - for (Row v1 : countRecords) { - String eventB = v1.getString(0); - long sum_duration = v1.getLong(1); - int count = v1.getInt(2); - long min_duration = v1.getLong(3); - long max_duration = v1.getLong(4); - double sum_squares = v1.getDouble(5); - c.add(new Count(eventA, eventB, sum_duration, count, min_duration, max_duration, sum_squares)); - } - return c.iterator(); - }) - .filter((Function) c -> { - for (EventPair p : b_pairs.getValue()) { - if (c.getEventA().equals(p.getEventA().getName()) && c.getEventB().equals(p.getEventB().getName())) { - return true; - } - } - return false; - }) - .collect(); - List response = new ArrayList<>(); - pairs.forEach(p -> { - for (Count c : counts) { - if (c.getEventA().equals(p.getEventA().getName()) && c.getEventB().equals(p.getEventB().getName())) { - response.add(c); - break; - } - } - }); - - return response; - } - - @Override - public List getEventPairs(String logname) { - String path = String.format("%s%s%s", bucket, logname, "/count.parquet/"); - List counts = sparkSession.read() - .parquet(path) - .toJavaRDD() - .flatMap((FlatMapFunction) row -> { - String eventA = row.getString(1); - List countRecords = JavaConverters.seqAsJavaList(row.getSeq(0)); - List c = new ArrayList<>(); - for (Row v1 : countRecords) { - String eventB = v1.getString(0); - long sum_duration = v1.getLong(1); - int count = v1.getInt(2); - long min_duration = v1.getLong(3); - long max_duration = v1.getLong(4); - double sum_squares = v1.getDouble(5); - c.add(new Count(eventA, eventB, sum_duration, count, min_duration, max_duration, sum_squares)); - } - return c.iterator(); - }) - .collect(); + protected Dataset readSequenceTable(String logname){ + Dataset df; + try{ + String path = String.format("%s%s%s", bucket, logname, "/seq.parquet/"); + df = sparkSession.read().parquet(path); + }catch (Exception e){ + String path = String.format("%s%s%s", bucket, logname, "/seq/"); + df = sparkSession.read().format("delta").load(path) + .withColumnRenamed("trace","trace_id"); + } + Dataset eventsDF = df + .selectExpr( + "trace_id as traceId", + "event_type as eventName", + "CAST(timestamp AS STRING) as timestamp", // Ensure timestamp is correctly formatted + "position" + ) + .as(Encoders.bean(EventModel.class)); + return eventsDF; + } + + @Override + protected Dataset readSingleTable(String logname){ + Dataset df; + try{ + String path = String.format("%s%s%s", bucket, logname, "/single.parquet/"); + df = sparkSession.read().parquet(path); + }catch (Exception e){ + String path = String.format("%s%s%s", bucket, logname, "/single/"); + df = sparkSession.read().format("delta").load(path) + .withColumnRenamed("trace","trace_id"); + } + Dataset eventsDF = df + .selectExpr( + "trace_id as traceId", + "event_type as eventName", + "CAST(timestamp AS STRING) as timestamp", // Ensure timestamp is correctly formatted + "position" + ) + .as(Encoders.bean(EventModel.class)); + return eventsDF; + } + + @Override + protected Dataset readCountTable(String logname){ + Dataset df; + + try{ + String path = String.format("%s%s%s", bucket, logname, "/count.parquet/"); + df = sparkSession.read().parquet(path) + .withColumn("countRecord", functions.explode( + functions.col("times"))) + .select( + functions.col("eventA"), + functions.col("countRecord._1").alias("eventB"), + functions.col("countRecord._2").alias("sumDuration"), + functions.col("countRecord._3").alias("count"), + functions.col("countRecord._4").alias("minDuration"), + functions.col("countRecord._5").alias("maxDuration"), + functions.col("countRecord._6").alias("sumSquares") + ); + }catch (Exception e){ + String path = String.format("%s%s%s", bucket, logname, "/count/"); + df = sparkSession.read().format("delta").load(path) + .withColumnRenamed("sum_duration","sumDuration") + .withColumnRenamed("min_duration","minDuration") + .withColumnRenamed("max_duration","maxDuration") + .withColumnRenamed("sum_squares","sumSquares"); + } + Dataset counts = df.as(Encoders.bean(Count.class)); return counts; } - @Override - public List getEventNames(String logname) { - String path = String.format("%s%s%s", bucket, logname, "/single.parquet/"); - return sparkSession.read().parquet(path) - .select("event_type") - .distinct() - .toJavaRDD() - .map((Function) row -> row.getString(0)) - .collect(); - } - - - @Override - protected JavaRDD querySequenceTablePrivate(String logname, Broadcast> bTraceIds) { - return querySequenceTableDeclare(logname) - .filter((Function) trace -> bTraceIds.getValue().contains(trace.getTraceID())); - } - - - @Override - protected JavaRDD getFromSingle(String logname, Set traceIds, Set eventTypes) { - String path = String.format("%s%s%s", bucket, logname, "/single.parquet/"); - Broadcast> bTraceIds = javaSparkContext.broadcast(traceIds); - Broadcast> bEventTypes = javaSparkContext.broadcast(eventTypes); - return sparkSession.read() - .parquet(path) - .toJavaRDD() - .filter((Function) x -> bEventTypes.value().contains((String)x.getAs("event_type"))) - .filter((Function) x -> bTraceIds.value().contains((String)x.getAs("trace_id"))) - .map((Function) row->{ - String trace_id = row.getAs("trace_id"); - String event_type = row.getAs("event_type"); - String ts = row.getAs("timestamp"); - Integer position = row.getAs("position"); - return new EventBoth(event_type,trace_id,Timestamp.valueOf(ts),position); - }); - } - - - @Override - protected JavaPairRDD, java.lang.Iterable> getAllEventPairs(Set pairs, - String logname, - Metadata metadata, - Timestamp from, - Timestamp till) { - String path = String.format("%s%s%s", bucket, logname, "/index.parquet/"); - Broadcast> bPairs = javaSparkContext.broadcast(pairs); - Broadcast mode = javaSparkContext.broadcast(metadata.getMode()); - Broadcast bFrom = javaSparkContext.broadcast(from); - Broadcast bTill = javaSparkContext.broadcast(till); - - List whereStatements = new ArrayList<>(); - whereStatements.add( - pairs.stream().map(x -> x.getEventA().getName()).distinct() - .map(p -> String.format("eventA = '%s'", p)) - .collect(Collectors.joining(" or "))); - - for (int i = 0; i < whereStatements.size(); i++) { - whereStatements.set(i, String.format("( %s )", whereStatements.get(i))); + protected Dataset readIndexTable(String logname) { + Dataset df; + try{ + String path = String.format("%s%s%s", bucket, logname, "/index.parquet/"); + df = sparkSession.read().parquet(path); + }catch (Exception e){ + String path = String.format("%s%s%s", bucket, logname, "/index/"); + df = sparkSession.read().format("delta").load(path) + .withColumnRenamed("id","trace_id") + .withColumn("timestampA",functions.col("timeA").cast("string")) + .withColumn("timestampB",functions.col("timeB").cast("string")); } - String whereStatement = String.join(" and ", whereStatements); - - JavaPairRDD, java.lang.Iterable> rows = sparkSession.read() - .parquet(path) - .where(whereStatement) - .toJavaRDD() - .flatMap((FlatMapFunction) row -> { - String eventA = row.getAs("eventA"); - String eventB = row.getAs("eventB"); - boolean checkContained = false; - for (EventPair ep : bPairs.getValue()) { - if (eventA.equals(ep.getEventA().getName()) && eventB.equals(ep.getEventB().getName())) { - checkContained = true; - break; - } - } - List response = new ArrayList<>(); - if (checkContained) { - String tid = row.getAs("trace_id"); - if (mode.getValue().equals("positions")) { - int posA = row.getAs("positionA"); - int posB = row.getAs("positionB"); - response.add(new IndexPair(tid, eventA, eventB, posA, posB)); - } else { - Timestamp tsA = row.getAs("timestampA"); - Timestamp tsB = row.getAs("timestampB"); - if (!(bTill.value() != null && tsA.after(bTill.value()) || - bFrom.value() != null && tsB.before(bFrom.value()))) { - response.add(new IndexPair(tid, eventA, eventB, tsA, tsB)); - } - } - } - return response.iterator(); - }) - .groupBy((Function>) indexPair -> new Tuple2<>(indexPair.getEventA(), indexPair.getEventB())); - return rows; + Dataset fixMissingFields = super.transformToIndexPairSet(df) + .as(Encoders.bean(IndexPair.class)); + return fixMissingFields; } - //Below are for declare// - - @Override - public JavaRDD querySequenceTableDeclare(String logname) { - String path = String.format("%s%s%s", bucket, logname, "/seq.parquet/"); - return sparkSession.read() - .parquet(path) - .toJavaRDD() - .map((Function) row -> { - String trace_id = row.getAs("trace_id"); - String event_name = row.getAs("event_type"); - Timestamp ts = Timestamp.valueOf((String) row.getAs("timestamp")); - Integer pos = row.getAs("position"); - return new EventBoth(event_name, trace_id, ts, pos); - }) - .groupBy((Function) EventBoth::getTraceID) - .map((Function>, Trace>) t -> - new Trace(t._1(), IteratorUtils.toList(t._2().iterator())) - ); - } - - @Override - public JavaRDD querySingleTableDeclare(String logname) { - String path = String.format("%s%s%s", bucket, logname, "/single.parquet/"); - - return sparkSession.read() - .parquet(path) - .select("event_type","trace_id") - .groupBy("event_type","trace_id") - .agg(functions.size(functions.collect_list("event_type")).alias("unique")) - .toJavaRDD() - .groupBy((Function) ev->ev.getAs("event_type")) - .map((Function>,UniqueTracesPerEventType>) ev->{ - String event_type = ev._1(); - List opt = new ArrayList<>(); - for(Row r: ev._2()){ - opt.add(new OccurrencesPerTrace(r.getAs("trace_id"),r.getAs("unique"))); - } - return new UniqueTracesPerEventType(event_type,opt); - }); - } - - @Override - public JavaRDD querySingleTable(String logname){ - String path = String.format("%s%s%s", bucket, logname, "/single.parquet/"); - - return sparkSession.read() - .parquet(path) - .select("event_type","trace_id") - .groupBy("event_type") - .agg(functions.size(functions.collect_list("event_type")).alias("unique")) - .toJavaRDD() - .map((Function) row -> { - String event = row.getAs("event_type"); - int s = row.getAs("unique"); - return new EventSupport(event,s); - }); - } - - @Override - public JavaPairRDD, List> querySingleTableAllDeclare(String logname) { - String path = String.format("%s%s%s", bucket, logname, "/single.parquet/"); - JavaPairRDD, List> rdd = sparkSession.read() - .parquet(path) - .select("event_type","trace_id","position") - .groupBy("event_type","trace_id") - .agg(functions.collect_list("position").alias("positions")) - .toJavaRDD() - .map(row->{ - String eventType = row.getAs("event_type"); - String trace_id = row.getAs("trace_id"); - List positions = JavaConverters.seqAsJavaList(row.getSeq(2)); - return new Tuple3<>(eventType,trace_id,positions); - }) - .keyBy(r -> new Tuple2<>(r._1(), r._2())) - .mapValues(Tuple3::_3); - - return rdd; - - } - - @Override - public JavaRDD queryIndexOriginalDeclare(String logname) { - String path = String.format("%s%s%s", bucket, logname, "/index.parquet/"); - - return sparkSession.read() - .parquet(path) - .select("eventA","eventB","trace_id") - .distinct() - .as(Encoders.bean(EventPairToTrace.class)) - .toJavaRDD(); - } - - @Override - public JavaRDD queryIndexTableDeclare(String logname) { - String path = String.format("%s%s%s", bucket, logname, "/index.parquet/"); - - return sparkSession.read().parquet(path) - .select("eventA","eventB","trace_id") - .distinct() - .toJavaRDD() - .groupBy((Function>)row->new Tuple2<>(row.getAs("eventA"),row.getAs("eventB"))) - .map((Function, Iterable>, UniqueTracesPerEventPair>)row->{ - List uniqueTraces = new ArrayList<>(); - for(Row r: row._2()){ - uniqueTraces.add(r.getAs("trace_id")); - } - return new UniqueTracesPerEventPair(row._1()._1(),row._1()._2,uniqueTraces); - } ); - } - - @Override - public JavaRDD queryIndexTableAllDeclare(String logname) { - String path = String.format("%s%s%s", bucket, logname, "/index.parquet/"); - - return sparkSession.read() - .parquet(path) - .toJavaRDD() - .map((Function) row -> { - String eventA = row.getAs("eventA"); - String eventB = row.getAs("eventB"); - String trace_id = row.getAs("trace_id"); - int positionA = row.getAs("positionA"); - int positionB = row.getAs("positionB"); - return new IndexPair(trace_id,eventA,eventB,positionA,positionB); - }); - } - - @Override - public JavaRDD queryPositionState(String logname) { + public Dataset queryPositionState(String logname) { String path = String.format("%s%s%s", bucket, logname, "/declare/position.parquet/"); return sparkSession.read() .parquet(path) - .as(Encoders.bean(PositionState.class)) - .toJavaRDD(); + .as(Encoders.bean(PositionState.class)); } @Override - public JavaRDD queryExistenceState(String logname) { + public Dataset queryExistenceState(String logname) { String path = String.format("%s%s%s", bucket, logname, "/declare/existence.parquet/"); return sparkSession.read() .parquet(path) - .as(Encoders.bean(ExistenceState.class)) - .toJavaRDD(); + .as(Encoders.bean(ExistenceState.class)); } @Override - public JavaRDD queryUnorderStateI(String logname) { + public Dataset queryUnorderStateI(String logname) { String path = String.format("%s%s%s", bucket, logname, "/declare/unorder/i.parquet/"); return sparkSession.read() .parquet(path) - .as(Encoders.bean(UnorderStateI.class)) - .toJavaRDD(); + .as(Encoders.bean(UnorderStateI.class)); } @Override - public JavaRDD queryUnorderStateU(String logname) { + public Dataset queryUnorderStateU(String logname) { String path = String.format("%s%s%s", bucket, logname, "/declare/unorder/u.parquet/"); return sparkSession.read() .parquet(path) - .as(Encoders.bean(UnorderStateU.class)) - .toJavaRDD(); + .as(Encoders.bean(UnorderStateU.class)); } @Override - public JavaRDD queryOrderState(String logname) { + public Dataset queryOrderState(String logname) { String path = String.format("%s%s%s", bucket, logname, "/declare/order.parquet"); return sparkSession.read() .parquet(path) - .as(Encoders.bean(OrderState.class)) - .toJavaRDD(); + .as(Encoders.bean(OrderState.class)); } @Override - public JavaRDD queryNegativeState(String logname) { + public Dataset queryNegativeState(String logname) { String path = String.format("%s%s%s", bucket, logname, "/declare/negatives.parquet"); return sparkSession.read() .parquet(path) - .as(Encoders.bean(NegativeState.class)) - .toJavaRDD(); + .as(Encoders.bean(NegativeState.class)); } diff --git a/src/main/java/com/datalab/siesta/queryprocessor/storage/repositories/S3/SparkConfiguration.java b/src/main/java/com/datalab/siesta/queryprocessor/storage/repositories/S3/SparkConfiguration.java index 190b12ce..0602fe62 100644 --- a/src/main/java/com/datalab/siesta/queryprocessor/storage/repositories/S3/SparkConfiguration.java +++ b/src/main/java/com/datalab/siesta/queryprocessor/storage/repositories/S3/SparkConfiguration.java @@ -5,7 +5,6 @@ import org.apache.spark.sql.SparkSession; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.PropertySource; @@ -13,16 +12,11 @@ /** * Contains the configuration of spark in he.maven.plugins:maven-compiler-plugin:3.13.0:compile (default-compile) on project siesta-query-processor: Fatal error compiling: error: release version 17 not supported -> [Help 1] -order to connect to s3 database + * order to connect to s3 database */ @Configuration @PropertySource("classpath:application.properties") -//@ConditionalOnProperty( -// value = "database", -// havingValue = "s3", -// matchIfMissing = true -//) -@ConditionalOnExpression("'${database}' == 's3' and '${delta}' == 'false'") +@ConditionalOnExpression("'${database}' == 's3'") public class SparkConfiguration { @Value("${app.name:siesta2}") @@ -40,21 +34,23 @@ public class SparkConfiguration { @Value("${s3.timeout:600000}") private String s3timeout; - @Value("${s3.endpoint:http://127.0.0.1:9000}") + @Value("${s3.endpoint:http://localhost:9000}") private String s3endpoint; @Bean public SparkConf sparkConf() { + // Get the directory where JARs are located + String jarDir = "/code/src/main/resources/jars"; + return new SparkConf() .setAppName(appName) .setMaster(masterUri) .set("spark.driver.extraJavaOptions", "--add-opens java.base/sun.security.action=ALL-UNNAMED") .set("spark.executor.extraJavaOptions", "--add-opens java.base/sun.security.action=ALL-UNNAMED") -// .set("spark.driver.memory","25g") -// .set("spark.driver.memoryOverhead","2g") -// .set("spark.memory.fraction","0.8") -// .set("spark.memory.storageFraction","0.5") - .set("spark.driver.maxResultSize","5g"); + .set("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") + .set("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") + .set("spark.deploy.mode", "client") + .set("spark.sql.streaming.statefulOperator.checkCorrectness.enabled", "false"); } @Bean @@ -64,10 +60,10 @@ public JavaSparkContext javaSparkContext() { @Bean public SparkSession sparkSession() { - SparkSession spark= SparkSession + SparkSession spark = SparkSession .builder() .sparkContext(this.javaSparkContext().sc()) - .appName("siesta 2") + .appName("SIESTA Query") .getOrCreate(); spark.sparkContext().hadoopConfiguration().set("fs.s3a.endpoint", s3endpoint); spark.sparkContext().hadoopConfiguration().set("fs.s3a.access.key", s3user); @@ -78,7 +74,9 @@ public SparkSession sparkSession() { spark.sparkContext().hadoopConfiguration().set("fs.s3a.connection.ssl.enabled", "true"); spark.sparkContext().hadoopConfiguration().set("fs.s3a.bucket.create.enabled", "true"); spark.conf().set("spark.sql.sources.partitionOverwriteMode", "dynamic"); -// spark.conf().set("spark.executor.memory", "30g"); + spark.conf().set("spark.sql.files.metadata.log.parsing.enabled", "true"); + spark.conf().set("spark.sql.sources.useV1SourceList", "delta"); + spark.conf().set("spark.delta.logStore.class", "org.apache.spark.sql.delta.storage.S3SingleDriverLogStore"); return spark; } diff --git a/src/main/java/com/datalab/siesta/queryprocessor/storage/repositories/SparkDatabaseRepository.java b/src/main/java/com/datalab/siesta/queryprocessor/storage/repositories/SparkDatabaseRepository.java index 89e20ab6..f0046293 100644 --- a/src/main/java/com/datalab/siesta/queryprocessor/storage/repositories/SparkDatabaseRepository.java +++ b/src/main/java/com/datalab/siesta/queryprocessor/storage/repositories/SparkDatabaseRepository.java @@ -1,25 +1,25 @@ package com.datalab.siesta.queryprocessor.storage.repositories; +import com.datalab.siesta.queryprocessor.declare.model.*; import com.datalab.siesta.queryprocessor.model.DBModel.*; import com.datalab.siesta.queryprocessor.model.Events.*; import com.datalab.siesta.queryprocessor.model.ExtractedPairsForPatternDetection; import com.datalab.siesta.queryprocessor.model.Utils.Utils; import com.datalab.siesta.queryprocessor.storage.DatabaseRepository; -import org.apache.spark.api.java.JavaPairRDD; -import org.apache.spark.api.java.JavaRDD; +import com.datalab.siesta.queryprocessor.storage.model.EventModel; +import com.datalab.siesta.queryprocessor.storage.model.EventTypeTracePositions; +import com.datalab.siesta.queryprocessor.storage.model.GroupEvents; +import com.datalab.siesta.queryprocessor.storage.model.Trace; import org.apache.spark.api.java.JavaSparkContext; -import org.apache.spark.api.java.function.FlatMapFunction; -import org.apache.spark.api.java.function.Function; -import org.apache.spark.broadcast.Broadcast; -import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.*; +import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.StructType; import org.apache.spark.storage.StorageLevel; import org.springframework.beans.factory.annotation.Autowired; import scala.Tuple2; -import scala.Tuple3; import java.sql.Timestamp; import java.util.*; -import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -43,199 +43,326 @@ public SparkDatabaseRepository(SparkSession sparkSession, JavaSparkContext javaS } /** - * return all the IndexPairs grouped by the eventA and eventB + * Return all events in the SequenceTable as a dataset in order to be filtered latter * needs to be implemented by each different connector - * @param pairs set of the pairs + * * @param logname the log database - * @return extract the pairs + * @return all events in the SequenceTable + */ + protected Dataset readSequenceTable(String logname) { + return null; + } + + /** + * Return all events in the SequenceTable as a dataset in order to be filtered latter + * needs to be implemented by each different connector + * + * @param logname the log database + * @return all events in the SequenceTable + */ + protected Dataset readSingleTable(String logname) { + return null; + } + + /** + * Return all events in the CountTable as a dataset in order to be filtered latter + * needs to be implemented by each different connector + * + * @param logname the log database + * @return all events in the CountTable */ - protected JavaPairRDD, java.lang.Iterable> getAllEventPairs(Set pairs, String logname, Metadata metadata, Timestamp from, Timestamp till) { + protected Dataset readCountTable(String logname) { return null; } + /** + * Return all events in the IndexTable as a dataset in order to be filtered latter + * needs to be implemented by each different connector + * + * @param logname the log database + * @return all events in the IndexTable + */ + protected Dataset readIndexTable(String logname) { + return null; + } + + /** * return all the IndexPairs grouped by the eventA and eventB * needs to be implemented by each different connector - * @param pairs set of the pairs + * + * @param pairs set of the pairs * @param logname the log database * @return extract the pairs */ - protected JavaPairRDD, java.lang.Iterable> getAllEventPairs(Set pairs, String logname) { - return null; + protected Dataset getAllEventPairs(Set pairs, + String logname, Metadata metadata, Timestamp from, Timestamp till) { + String filter = pairs.stream().map(x -> new Tuple2<>(x.getEventA().getName(), x.getEventB().getName())) + .collect(Collectors.toSet()) + .stream().map(x -> String.format("(eventA = '%s' and eventB = '%s')", x._1(), x._2())) + .collect(Collectors.joining(" or ")); + Dataset indexDataset = readIndexTable(logname) + .where(filter); //filter based on events + + if (!metadata.getMode().equals("position")) { // we can filter based on the timestamp also + Dataset indexRows = indexDataset + .withColumn("timestampA-2", functions.to_timestamp(functions.col("timestampA"), "yyyy-MM-dd HH:mm:ss")) + .withColumn("timestampB-2", functions.to_timestamp(functions.col("timestampB"), "yyyy-MM-dd HH:mm:ss")); + + if (from != null) { + indexRows = indexRows.filter(functions.col("timestampA-2").isNull() + .or(functions.col("timestampA-2").geq(from))); + } + if (till != null) { + indexRows = indexRows.filter(functions.col("timestampB-2").isNull() + .or(functions.col("timestampB-2").leq(till))); + } + indexDataset = indexRows.select("trace_id", "eventA", "eventB", "timestampA", "timestampB", "positionA", "positionB") + .as(Encoders.bean(IndexPair.class)); + } + + return indexDataset; } + /** * Retrieves the appropriate events from the SequenceTable, which contains the original traces - * @param logname the log database + * + * @param logname the log database * @param traceIds the ids of the traces that will be retrieved * @return a map where the key is the trace id and the value is a list of the retrieved events (with their - * * timestamps) + * * timestamps) */ @Override public Map> querySeqTable(String logname, List traceIds) { - Broadcast> bTraceIds = javaSparkContext.broadcast(new HashSet<>(traceIds)); - return this.querySequenceTablePrivate(logname, bTraceIds) - .keyBy((Function) Trace::getTraceID) - .mapValues((Function>) Trace::getEvents) - .collectAsMap(); + Dataset eventsDF = this.readSequenceTable(logname) + .filter(functions.col("traceId").isin(traceIds.toArray())); + return this.transformEventModelToMap(eventsDF.toDF()); } /** * Retrieves the appropriate events from the SequenceTable, which contains the original traces - * @param logname the log database - * @param traceIds the ids of the traces that will be retrieved + * + * @param logname the log database + * @param traceIds the ids of the traces that will be retrieved * @param eventTypes the events that will be retrieved - * @param from the starting timestamp, set to null if not used - * @param till the ending timestamp, set to null if not used + * @param from the starting timestamp, set to null if not used + * @param till the ending timestamp, set to null if not used * @return a map where the key is the trace id and the value is a list of the retrieved events (with their * timestamps) */ @Override - public Map> querySeqTable(String logname, List traceIds, Set eventTypes, Timestamp from, Timestamp till) { - Broadcast> bTraceIds = javaSparkContext.broadcast(new HashSet<>(traceIds)); - Broadcast> bevents = javaSparkContext.broadcast(new HashSet<>(eventTypes)); - Broadcast bFrom = javaSparkContext.broadcast(from); - Broadcast bTill = javaSparkContext.broadcast(till); - JavaRDD df = this.querySequenceTablePrivate(logname, bTraceIds) - .map((Function) trace -> { - trace.filter(bFrom.getValue(), bTill.getValue()); - return trace; - }); - return df.keyBy((Function) Trace::getTraceID) - .mapValues((Function>) trace -> trace.clearTrace(bevents.getValue())) - .collectAsMap(); + public Map> querySeqTable(String logname, List traceIds, Set eventTypes, + Timestamp from, Timestamp till) { + Dataset eventsDF = this.readSequenceTable(logname); + //filter based on id and based on eventType + eventsDF = eventsDF.filter(functions.col("traceId").isin(traceIds.toArray())) + .filter(functions.col("eventName").isin(eventTypes.toArray())); + //filter based on the timestamps and the parameters from and till + Dataset filteredTimestamps = eventsDF + .withColumn("timestamp-true", functions.col("timestamp").cast("timestamp")); + if (from != null) { + filteredTimestamps = filteredTimestamps.filter(functions.col("timestamp-true").geq(from)); + } + if (till != null) { + filteredTimestamps = filteredTimestamps.filter(functions.col("timestamp-true").leq(till)); + } + + return this.transformEventModelToMap(filteredTimestamps); } /** - * This function reads data from the Sequence table into a JavaRDD, any database that utilizes spark should - * override it + * Utility class that is used from querySequenceTable methods to transform the Dataset of + * EventModel to Map> * - * @param logname Name of the log - * @param bTraceIds broadcasted the values of the trace ids we are interested in - * @return a JavaRDD + * @param events a dataset of augmented EventModel (can include more than the standard fields of EventModel) + * @return a Map of the traceIds to the corresponding events */ - protected JavaRDD querySequenceTablePrivate(String logname, Broadcast> bTraceIds) { - return null; - } + private Map> transformEventModelToMap(Dataset events) { + List eventsList = events + .select("traceId", "eventName", "timestamp", "position") + .as(Encoders.bean(EventModel.class)) + .collectAsList(); - /** - * Retrieves data from the primary inverted index - * @param pairs a set of the pairs that we need to retrieve information for - * @param logname the log database - * @return the corresponding records from the index - */ - @Override - public IndexRecords queryIndexTable(Set pairs, String logname) { - List, Iterable>> results = this.getAllEventPairs(pairs, logname) - .collect(); - return new IndexRecords(results); + Map> response = eventsList.parallelStream() + .map(x -> + new EventBoth(x.getEventName(), x.getTraceId(), Timestamp.valueOf(x.getTimestamp()), x.getPosition())) + .collect(Collectors.groupingByConcurrent(EventBoth::getTraceID)); + return response; } /** * Retrieves data from the primary inverted index - * @param pairs a set of the pairs that we need to retrieve information for - * @param logname the log database + * + * @param pairs a set of the pairs that we need to retrieve information for + * @param logname the log database * @param metadata the metadata for this log database - * @param from the starting timestamp, set to null if not used - * @param till the ending timestamp, set to null if not used + * @param from the starting timestamp, set to null if not used + * @param till the ending timestamp, set to null if not used * @return the corresponding records from the index */ @Override public IndexRecords queryIndexTable(Set pairs, String logname, Metadata metadata, Timestamp from, Timestamp till) { - List, Iterable>> results = this.getAllEventPairs(pairs, logname, metadata, from, till) - .collect(); - return new IndexRecords(results); + Dataset results = this.getAllEventPairs(pairs, logname, metadata, from, till); + return this.transformToIndexRecords(results); } - protected JavaRDD getPairs(JavaPairRDD, java.lang.Iterable> pairs) { - return pairs.flatMap((FlatMapFunction, Iterable>, IndexPair>) g -> g._2.iterator()); + /** + * Transform the Dataset of IndexPair (which is a utility class in storage package) to IndexRecords + * which are objects handled by the remaining program + * + * @param indexPairs records from the IndexTable + * @return an IndexRecords object + */ + private IndexRecords transformToIndexRecords(Dataset indexPairs) { + Dataset groupedDf = indexPairs.withColumn("indexPair", + functions.struct("trace_id", "eventA", "eventB", "timestampA", + "timestampB", "positionA", "positionB")) + .groupBy("eventA", "eventB") + .agg(functions.collect_list("indexPair").alias("indexPairs")) + .select("traceId", "events"); + + // Convert DataFrame to Map> + Map> eventsMap = groupedDf + .collectAsList() + .stream() + .collect(Collectors.toMap( + row -> new EventTypes(row.getString(0), row.getString(1)), + row -> { + List eventRows = row.getList(2); + return eventRows.stream() + .map(eventRow -> new IndexPair( + eventRow.getString(0), + eventRow.getString(1), + eventRow.getString(2), + eventRow.getString(3), + eventRow.getString(4), + eventRow.getInt(5), + eventRow.getInt(6) + )) + .collect(Collectors.toList()); + } + )); + return new IndexRecords(eventsMap); } /** * Extract the ids of the traces that contains all the provided pairs - * @param pairs pairs retrieved from the storage + * + * @param pairs pairs retrieved from the storage * @param trueEventPairs the pairs that required to appear in a trace in order to be a candidate * @return the candidate trace ids */ - protected List getCommonIds(JavaRDD pairs, Set trueEventPairs) { - Broadcast> truePairs = javaSparkContext.broadcast(trueEventPairs); - return pairs.map((Function>) pair -> - new Tuple3<>(pair.getEventA(), pair.getEventB(), pair.getTraceId())) - .distinct() //remove all the duplicate event pairs that refer to the same trace - .groupBy((Function, String>) Tuple3::_3) - .map((Function>>, Tuple2>)x->{ - AtomicInteger acc = new AtomicInteger(0); - x._2.forEach(pair-> { - Optional op =truePairs.getValue().stream().filter(y->y.getEventA().getName().equals(pair._1())&& - y.getEventB().getName().equals(pair._2())).findFirst(); - if(op.isPresent()) acc.incrementAndGet(); - }); - return new Tuple2<>(x._1, acc.get()); - } ) - .filter((Function, Boolean>) p -> p._2 == truePairs.getValue().size()) - .map((Function, String>) p -> p._1) - .collect(); + protected List getCommonIds(Dataset pairs, Set trueEventPairs) { + Set truePairs = trueEventPairs.stream() + .map(x -> new EventTypes(x.getEventA().getName(), x.getEventB().getName())) + .collect(Collectors.toSet()); + String eventFilter = truePairs.stream() + .map(x -> String.format("(eventA = '%s' AND eventB = '%s')", x.getEventA(), x.getEventB())) + .collect(Collectors.joining(" OR ")); + + // Next sequence extract the ids of the traces that contains all the required et-pairs (iun the truePairs list) + List listOfTraces = pairs + .select("eventA", "eventB", "trace_id") // Maintain the required fields + .distinct() // Remove duplicates + .where(eventFilter) // Maintain only the et-pairs in the true pairs + .groupBy("trace_id") // Group based on the trace id + .agg(functions.count("*").alias("valid_et_pairs")) // Count how many of the valid pairs each trace has + .filter(functions.col("valid_et_pairs").equalTo(truePairs.size()))// Keeps thr traces that have all valid et-pairs + .select("trace_id") // Return the trace_ids + .as(Encoders.STRING()) //Transform to string + .collectAsList(); //Collect as List + return listOfTraces; } /** * Filters te results based on the starting and ending timestamp - * @param pairs the retrieved pairs from the IndexTable + * + * @param pairs the retrieved pairs from the IndexTable * @param traceIds the trace ids that contains all the required pairs - * @param from the starting timestamp, set to null if not used - * @param till the ending timestamp, set to null if not used + * @param from the starting timestamp, set to null if not used + * @param till the ending timestamp, set to null if not used * @return the intermediate results, i.e. the candidate traces before remove false positives */ - protected IndexMiddleResult addFilterIds(JavaRDD pairs, List traceIds, Timestamp from, Timestamp till) { - Broadcast> bTraces = javaSparkContext.broadcast(new HashSet<>(traceIds)); - Broadcast bFrom = javaSparkContext.broadcast(from); - Broadcast bTill = javaSparkContext.broadcast(till); - JavaRDD filtered = pairs.filter((Function) pair -> - bTraces.getValue().contains(pair.getTraceId())); + protected IndexMiddleResult addFilterIds(Dataset pairs, List traceIds, Timestamp from, Timestamp till) { IndexMiddleResult imr = new IndexMiddleResult(); imr.setTrace_ids(traceIds); - Map> events = filtered.flatMap((FlatMapFunction) indexPair -> indexPair.getEvents().iterator()) - .groupBy((Function) Event::getTraceID) - .mapValues((Function, List>) p -> { - Set eventSet = new HashSet<>(); - for (Event ev : p) { - if (ev instanceof EventPos) eventSet.add(ev); - else { - EventTs et = (EventTs) ev; - if (bFrom.value() != null & bTill.value() != null) { - if (!et.getTimestamp().before(bFrom.value()) && !et.getTimestamp().after(bTill.value())) { - eventSet.add(ev); - } - } else if (bFrom.value() != null) { - if (!et.getTimestamp().before(bFrom.value())) { - eventSet.add(ev); - } - } else if (bTill.value() != null) { - if (!et.getTimestamp().after(bTill.value())) { - eventSet.add(ev); - } - } else { - eventSet.add(ev); - } + // Filter to maintain only the pruned traces + Dataset filteredDf = pairs.filter(functions.col("trace_id").isin(traceIds.toArray())); + // Extract EventBoth from the IndexPair + Dataset eventsDf = this.getEventsFromIndexRecords(filteredDf); + + // Check if timestamp filtering is needed + boolean filterByTime = from != null || till != null; + + if (filterByTime) { + Dataset rows = eventsDf.withColumn("timestamp-2", + functions.to_timestamp(functions.col("timestamp"), "yyyy-MM-dd HH:mm:ss")); + if (from != null) { + rows = rows.filter( + functions.col("timestamp-2").isNull().or( + functions.col("timestamp-2").geq(from))); + } + if (till != null) { + rows = rows.filter( + functions.col("timestamp-2").isNull().or( + functions.col("timestamp-2").leq(till))); + } + eventsDf = rows.select("eventName", "traceId", "position", "timestamp") + .as(Encoders.bean(EventModel.class)); + } + + Dataset groupedDf = eventsDf + .withColumn("event", functions.struct("traceId", "eventName", "position", "timestamp")) + .groupBy("traceId") + .agg(functions.collect_list("event").alias("events")) + .select("traceId", "events"); + + // Convert DataFrame to Map> + Map> eventsMap = groupedDf + .collectAsList() + .stream() + .collect(Collectors.toMap( + row -> row.getString(0), + row -> { + List eventRows = row.getList(1); + return eventRows.stream() + .map(eventRow -> { + if (eventRow.getString(3) == null) { + return new EventPos( + eventRow.getString(1), // eventName + eventRow.getString(0), // traceId + eventRow.getInt(2) // position + ); + } else { + return new EventTs( + eventRow.getString(1), // eventName + eventRow.getString(0), // traceId + Timestamp.valueOf(eventRow.getString(3))// timestamp + ); + } + }) + .collect(Collectors.toList()); } - } - List eventsList = new ArrayList<>(eventSet); - Collections.sort(eventsList); - return eventsList; - }) - .collectAsMap(); - imr.setEvents(events); + )); + + eventsMap.forEach((key, eventList) -> Collections.sort(eventList)); + imr.setEvents(eventsMap); + return imr; + } /** * Detects the traces that contain all the given event pairs - * @param logname the log database + * + * @param logname the log database * @param combined a list where each event pair is combined with the according stats from the CountTable * @param metadata the log database metadata - * @param expairs the event pairs extracted from the query - * @param from the starting timestamp, set to null if not used - * @param till the ending timestamp, set to null if not used + * @param expairs the event pairs extracted from the query + * @param from the starting timestamp, set to null if not used + * @param till the ending timestamp, set to null if not used * @return the traces that contain all the pairs. It will be then processed by SASE in order to remove false * positives. */ @@ -243,8 +370,7 @@ protected IndexMiddleResult addFilterIds(JavaRDD pairs, List public IndexMiddleResult patterDetectionTraceIds(String logname, List> combined, Metadata metadata, ExtractedPairsForPatternDetection expairs, Timestamp from, Timestamp till) { Set pairs = combined.stream().map(x -> x._1).collect(Collectors.toSet()); - JavaPairRDD, java.lang.Iterable> gpairs = this.getAllEventPairs(pairs, logname, metadata, from, till); - JavaRDD indexPairs = this.getPairs(gpairs); + Dataset indexPairs = this.getAllEventPairs(pairs, logname, metadata, from, till); indexPairs.persist(StorageLevel.MEMORY_AND_DISK()); List traces = this.getCommonIds(indexPairs, expairs.getTruePairs()); IndexMiddleResult imr = this.addFilterIds(indexPairs, traces, from, till); @@ -253,91 +379,327 @@ public IndexMiddleResult patterDetectionTraceIds(String logname, List that will be used in querySingleTable and querySingleTableGroups + * @param logname the log database + * @param eventTypes the events that will we retrieved + * @return */ - protected JavaRDD getFromSingle(String logname, Set traceIds, Set eventTypes) { - Broadcast> bTraces = javaSparkContext.broadcast(traceIds); - return queryFromSingle(logname,eventTypes).filter((Function) event-> - bTraces.getValue().contains(event.getTraceID())); - } - - protected JavaRDD queryFromSingle(String logname, Set eventTypes){ - return null; - } - @Override public Map> querySingleTable(String logname, Set eventTypes) { - JavaRDD events = queryFromSingle(logname, eventTypes); - JavaPairRDD> pairs = events.groupBy((Function) Event::getName); - return pairs.mapValues((Function, List>) e -> { - List tempList = new ArrayList<>(); - for (EventBoth ev : e) { - tempList.add(ev); - } - return tempList; - }).collectAsMap(); + Dataset events = this.readSingleTable(logname) + .filter(functions.col("eventName").isin(eventTypes.toArray())) + .withColumn("event", functions.struct("traceId", "eventName", "timestamp", "position")) + .groupBy("traceId") + .agg(functions.collect_list("event").alias("events")) + .as(Encoders.bean(Trace.class)); + + Map> response = events.collectAsList().stream() + .collect(Collectors.toMap( + Trace::getTraceId, + trace -> trace.getEvents().stream().map(event -> new EventBoth( + event.getEventName(), + event.getTraceId(), + Timestamp.valueOf(event.getTimestamp()), + event.getPosition())).collect(Collectors.toList()) + ) + ); + return response; } /** * Retrieves the appropriate events from the SingleTable, which contains the single inverted index - * @param logname the log database - * @param traceIds the ids of the traces that wil be retrieved + * + * @param logname the log database + * @param traceIds the ids of the traces that wil be retrieved * @param eventTypes the events that will we retrieved * @return a list of all the retrieved events (wth their timestamps) */ @Override public List querySingleTable(String logname, Set traceIds, Set eventTypes) { - return this.getFromSingle(logname, traceIds, eventTypes).collect(); + Dataset data = this.readSingleTable(logname) + .filter(functions.col("eventName").isin(eventTypes.toArray()))//filter based on type + .filter(functions.col("traceId").isin(traceIds.toArray()));//filter based on trace + List response = data.collectAsList() + .parallelStream().map(x -> new EventBoth(x.getEventName(), + x.getTraceId(), Timestamp.valueOf(x.getTimestamp()), x.getPosition())) + .collect(Collectors.toList()); + return response; } /** * Retrieves the appropriate events from the SingleTable, which contains the single inverted index - * @param logname the log database - * @param groups a list of the groups as defined in the query + * + * @param logname the log database + * @param groups a list of the groups as defined in the query * @param eventTypes the events that will we retrieved * @return a map where the key is the group id and the value is a list of the retrieved events (with their t * imestamps) */ @Override - public Map> querySingleTableGroups(String logname, List> groups, Set eventTypes) { + public Map> querySingleTableGroups(String logname, List> groups, + Set eventTypes) { + // extract all traces that appear in groups Set allTraces = groups.stream() .flatMap((java.util.function.Function, Stream>) Collection::stream) .collect(Collectors.toSet()); - Broadcast>> bgroups = javaSparkContext.broadcast(groups); - Broadcast bEventTypesSize = javaSparkContext.broadcast(eventTypes.size()); - JavaRDD eventRDD = this.getFromSingle(logname, allTraces, eventTypes); - Map> response = eventRDD.map((Function>) event -> { - for (int g = 0; g < bgroups.value().size(); g++) { - if (bgroups.value().get(g).contains(event.getTraceID())) return new Tuple2<>(g + 1, event); - } - return new Tuple2<>(-1, event); - }) - .filter((Function, Boolean>) event -> event._1 != -1) - .groupBy((Function, Integer>) event -> event._1) - //maintain only these groups that contain all the event types in the query - .filter((Function>>, Boolean>) group -> { - Set events = new HashSet<>(); - group._2.forEach(x -> events.add(x._2.getName())); - return events.size() == bEventTypesSize.value(); - }) - .mapValues((Function>, List>) group -> { - List eventBoth = new ArrayList<>(); - for (Tuple2 e : group) { - eventBoth.add(e._2); - } - return eventBoth.stream().sorted().collect(Collectors.toList()); - }).collectAsMap(); + //create dataframe with groups + List groupRows = new ArrayList<>(); + for (int i = 0; i < groups.size(); i++) { + groupRows.add(RowFactory.create(i + 1, new ArrayList<>(groups.get(i)))); // (group index, traceIds) + } + Dataset groupDF = sparkSession.createDataFrame(groupRows, new StructType() + .add("group_id", DataTypes.IntegerType) + .add("traceIds", DataTypes.createArrayType(DataTypes.StringType))); + + + //extract events from the single table + Dataset data = this.readSingleTable(logname) + .filter(functions.col("eventName").isin(eventTypes.toArray()))//filter based on type + .filter(functions.col("traceId").isin(allTraces.toArray()));//filter based on trace + // Assign group IDs to events + Dataset eventsWithGroup = data.crossJoin(groupDF) + .filter(functions.array_contains(functions.col("traceIds"), functions.col("traceId"))) + .select("group_id", "traceId", "eventName", "position", "timestamp"); // Keep relevant fields + + + // Group by group_id and collect event details + Dataset groupedEvents = eventsWithGroup.groupBy("group_id") + .agg(functions.collect_list(functions.struct("eventName", "traceId", "position", "timestamp")) + .alias("events")); + // Maintains only the groups that contain all the required event types + Dataset validGroups = groupedEvents + .withColumn("eventTypes", functions.expr("transform(events, x -> x.eventName)")) // Extract event names + .filter(functions.size(functions.array_distinct(functions.col("eventTypes"))).equalTo(eventTypes.size())); + + //Extract from the dataframe the response that follows the format Map + List eventsList = validGroups + .as(Encoders.bean(GroupEvents.class)) + .collectAsList(); + Map> response = eventsList.parallelStream() + .collect(Collectors.toMap( + GroupEvents::getGroup_id, + group -> group.getEvents().stream() + .map(event -> new EventBoth( + event.getEventName(), + event.getTraceId(), + Timestamp.valueOf(event.getTimestamp()), + event.getPosition() + )) + .sorted() + .collect(Collectors.toList()) + )); + return response; } + /** + * For a given log database, returns all the event pairs found in the log + * + * @param logname the log database + * @return all event pairs found in the log + */ + @Override + public List getEventPairs(String logname) { + List counts = readCountTable(logname) + .collectAsList(); + return counts; + } + + /** + * @param logname the log database + * @return a list with all the event types stored in it + */ + @Override + public List getEventNames(String logname) { + Dataset events = readSingleTable(logname) + .select("eventName") + .distinct() + .as(Encoders.STRING()); + return events.collectAsList(); + } + + + /** + * Retrieves the corresponding stats (min, max duration and so on) from the CountTable, for a given set of event + * pairs + * + * @param logname the log database + * @param pairs a set with the event pairs + * @return a list of the stats for the set of event pairs + */ + @Override + public List getCounts(String logname, Set pairs) { + String firstFilter = pairs.stream().map(x -> x.getEventA().getName()).collect(Collectors.toSet()) + .stream().map(x -> String.format("eventA = '%s'", x)) + .collect(Collectors.joining(" or ")); + String secondFilter = pairs.stream().map(x -> new Tuple2<>(x.getEventA().getName(), x.getEventB().getName())) + .collect(Collectors.toSet()) + .stream().map(x -> String.format("(eventA = '%s' and eventB = '%s')", x._1(), x._2())) + .collect(Collectors.joining(" or ")); + Dataset counts = readCountTable(logname); + //Spark should be able to run this query efficiently and push the first filter before explosion + List countList = counts + .filter(firstFilter) //filter only based on the first event + .filter(secondFilter) //filter based on the et-pair + .collectAsList(); + return countList; + } + + /** + * For a given event type inside a log database, returns all the possible next events. That is, since Count + * contains for each pair the stats, return all the events that have at least one pair with the given event + * + * @param logname the log database + * @param event the event type + * @return the possible next events + */ + @Override + public List getCountForExploration(String logname, String event) { + Dataset counts = readCountTable(logname); + List countList = counts + .filter(String.format("eventA = '%s'", event)) + .collectAsList(); + return countList; + } + + /** + * This method transforms the rows read from the Database to IndexPair. Since SIESTA supports both timestamp + * and positions, this method is responsible to extract the schema and make the corresponding changes. Finally, + * since this is the place that identifies if there are timestamps in the index, we have also included the + * from/till filtering + * + * @param indexRows + * @return + */ + protected Dataset transformToIndexPairSet(Dataset indexRows) { + StructType schema = indexRows.schema(); + // Check if each column exists before selecting it + boolean hasTimestampA = Arrays.asList(schema.fieldNames()).contains("timestampA"); + boolean hasTimestampB = Arrays.asList(schema.fieldNames()).contains("timestampB"); + boolean hasPositionA = Arrays.asList(schema.fieldNames()).contains("positionA"); + boolean hasPositionB = Arrays.asList(schema.fieldNames()).contains("positionB"); + + Column traceId = functions.col("trace_id"); + Column eventA = functions.col("eventA"); + Column eventB = functions.col("eventB"); + + //here is the filtering for the till and from if the indexing has been done using timestamp + if (hasTimestampA && hasTimestampB) { + } + Column timestampA = hasTimestampA ? functions.col("timestampA") : functions.lit(null).cast("string"); + Column timestampB = hasTimestampB ? functions.col("timestampB") : functions.lit(null).cast("string"); + Column positionA = hasPositionA ? functions.col("positionA") : functions.lit(null).cast("int"); + Column positionB = hasPositionB ? functions.col("positionB") : functions.lit(null).cast("int"); + + Dataset indexPairDataset = indexRows.select(traceId, eventA, eventB, timestampA.alias("timestampA"), + timestampB.alias("timestampB"), positionA.alias("positionA"), + positionB.alias("positionB")) + .as(Encoders.bean(IndexPair.class)); + return indexPairDataset; + } + + /** + * Extract all events that appear in IndexTable records. Essentially, split event pairs into two + * + * @param indexPairs records from IndexTable in the form of a Dataset + * @return a dataset of the unique events (i.e, uses distinct since one event might appear in multiple pairs) + */ + private Dataset getEventsFromIndexRecords(Dataset indexPairs) { + Dataset eventA_DF = indexPairs + .selectExpr( + "eventA as eventName", + "timestampA as timestamp", + "positionA as position", + "trace_id as traceId" + ); + Dataset eventB_DF = indexPairs + .selectExpr( + "eventB as eventName", + "timestampB as timestamp", + "positionB as position", + "trace_id as traceId" + ); + + Dataset eventsDF = eventA_DF.union(eventB_DF) + .distinct() + .as(Encoders.bean(EventModel.class)); + return eventsDF; + } + + //Below are for Declare// + @Override + public Dataset querySequenceTableDeclare(String logname) { + Dataset eventDF = this.readSequenceTable(logname); + Dataset groupedDF = eventDF + .groupBy("traceId")// Group by trace_id and collect events into a list + .agg(functions.collect_list(functions + .struct("eventName", "traceID", "timestamp", "position")) + .alias("events")) + .as(Encoders.bean(Trace.class)); + return groupedDF; + } + + @Override + public Dataset querySingleTableDeclare(String logname) { + Dataset eventDF = this.readSingleTable(logname); + Dataset uniqueTracesPerEventTypeDataset = eventDF + .selectExpr("eventName", "traceId") + .groupBy("eventName", "traceId") + .agg(functions.count("*").alias("occs")) + .withColumn("occs", functions.col("occs").cast("int")) + .withColumn("occurrence", functions.struct("traceId", "occs")) + .groupBy("eventName") + .agg(functions.collect_list("occurrence").alias("occurrences")) + .as(Encoders.bean(UniqueTracesPerEventType.class)); + return uniqueTracesPerEventTypeDataset; + } + @Override + public Dataset querySingleTable(String logname) { + Dataset eventDF = this.readSingleTable(logname); + Dataset supportDF = eventDF.select("eventName", "traceId") + .groupBy("eventName") + .agg(functions.count("traceId").alias("support")) + .selectExpr("eventName as event", "support") + .as(Encoders.bean(EventSupport.class)); + return supportDF; + } + + @Override + public Dataset queryIndexTableDeclare(String logname) { + Dataset indexRecords = readIndexTable(logname); + Dataset uniqueTracesPerEventPairDataset = indexRecords + .select("eventA", "eventB", "trace_id") + .distinct() + .groupBy("eventA", "eventB") + .agg(functions.collect_list("trace_id").alias("uniqueTraces")) + .as(Encoders.bean(UniqueTracesPerEventPair.class)); + return uniqueTracesPerEventPairDataset; + } + + @Override + public Dataset queryIndexOriginalDeclare(String logname) { + Dataset indexPairDataset = readIndexTable(logname); + Dataset response = indexPairDataset + .select("eventA", "eventB", "trace_id") + .distinct() + .as(Encoders.bean(EventPairToTrace.class)); + return response; + } + + @Override + public Dataset querySingleTableAllDeclare(String logname) { + Dataset eventDF = this.readSingleTable(logname); + Dataset response = eventDF + .select("eventName","traceId","position") + .groupBy("eventName","traceId") + .agg(functions.collect_list("position").alias("positions")) + .as(Encoders.bean(EventTypeTracePositions.class)); + return response; + } -} +} \ No newline at end of file diff --git a/src/main/java/edu/umass/cs/sase/engine/Engine.java b/src/main/java/edu/umass/cs/sase/engine/Engine.java index 148eda40..54d08c3c 100755 --- a/src/main/java/edu/umass/cs/sase/engine/Engine.java +++ b/src/main/java/edu/umass/cs/sase/engine/Engine.java @@ -982,7 +982,7 @@ public void createNewRun(Event e) throws EvaluationException { } //check if next state is Kleene*. In that case two runs needs to be added, one that will wait for it to //have a next event and the other that will consider it finished - if (this.nfa.getStates(1).getStateType().equalsIgnoreCase("kleeneClosure*")) { + if (this.nfa.getSize()>1 && this.nfa.getStates(1).getStateType().equalsIgnoreCase("kleeneClosure*")) { Run newRun2 = this.engineRunController.getRun(); newRun2.initializeRun(this.nfa); newRun2.addEvent(e); //may need to add to buffer diff --git a/src/main/java/edu/umass/cs/sase/query/NFA.java b/src/main/java/edu/umass/cs/sase/query/NFA.java index aa9c966b..01a12cb8 100755 --- a/src/main/java/edu/umass/cs/sase/query/NFA.java +++ b/src/main/java/edu/umass/cs/sase/query/NFA.java @@ -503,12 +503,6 @@ public State[] getStates() { } public State getStates(int order) { - // for debug - /* - if(order == 3){ - System.out.println(); - } - */ return states[order]; } diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index f2f3a322..12ebb916 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -1,8 +1,34 @@ +# Application configuration app.name=siesta-query-processor -master.uri=local[*] - -s3.endpoint=http://localhost:9000 database=s3 delta=false server.port=8090 -spring.mvc.pathmatch.matching-strategy = ANT_PATH_MATCHER \ No newline at end of file +spring.mvc.pathmatch.matching-strategy = ANT_PATH_MATCHER + +# Spark configuration +# for running in spark cluster mode with docker-compose +master.uri=spark://:7077 +# spark.driver.host=spark-worker-1 +# spark.driver.port=42315 + +# for running in local mode +# master.uri=local[*] +# spark.driver.host=localhost +# spark.driver.port=42315 + +# for running in spark cluster mode without docker-compose (using IDE) +# master.uri=spark://localhost:7077 +# spark.driver.host=localhost +# spark.driver.port=42315 + +# S3 configuration +s3.endpoint=http://:9000 +s3.user=minioadmin +s3.key=minioadmin +s3.timeout=600000 + +# Scylla/Cassandra configuration +cassandra.contact.points=localhost +cassandra.port=9042 +cassandra.keyspace=siesta + diff --git a/src/main/resources/jars/.gitkeep b/src/main/resources/jars/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/src/test/java/com/datalab/siesta/queryprocessor/storage/repositories/S3/S3ConnectorTest.java b/src/test/java/com/datalab/siesta/queryprocessor/storage/repositories/S3/S3ConnectorTest.java index 3754b05d..3c1fd037 100644 --- a/src/test/java/com/datalab/siesta/queryprocessor/storage/repositories/S3/S3ConnectorTest.java +++ b/src/test/java/com/datalab/siesta/queryprocessor/storage/repositories/S3/S3ConnectorTest.java @@ -56,24 +56,24 @@ void queryIndexTable(){ Map> r = ir.getRecords(); // Assertions.assertEquals(2,r.get(new EventTypes("B","A")).size()); - List contained = r.get(new EventTypes("C","B")).stream().map(IndexPair::getTraceId).collect(Collectors.toList()); + List contained = r.get(new EventTypes("C","B")).stream().map(IndexPair::getTrace_id).collect(Collectors.toList()); Assertions.assertTrue(contained.contains(2L)); // Assertions.assertEquals(2,r.get(new EventTypes("C","B")).size()); contained.clear(); - contained = r.get(new EventTypes("C","B")).stream().map(IndexPair::getTraceId).collect(Collectors.toList()); + contained = r.get(new EventTypes("C","B")).stream().map(IndexPair::getTrace_id).collect(Collectors.toList()); Assertions.assertTrue(contained.contains(2L)); Assertions.assertTrue(contained.contains(3L)); // Assertions.assertEquals(2,r.get(new EventTypes("C","C")).size()); contained.clear(); - contained = r.get(new EventTypes("C","C")).stream().map(IndexPair::getTraceId).collect(Collectors.toList()); + contained = r.get(new EventTypes("C","C")).stream().map(IndexPair::getTrace_id).collect(Collectors.toList()); Assertions.assertTrue(contained.contains(2L)); Assertions.assertTrue(contained.contains(3L)); // Assertions.assertEquals(6,r.get(new EventTypes("A","A")).size()); contained.clear(); - contained = r.get(new EventTypes("A","A")).stream().map(IndexPair::getTraceId).collect(Collectors.toList()); + contained = r.get(new EventTypes("A","A")).stream().map(IndexPair::getTrace_id).collect(Collectors.toList()); Assertions.assertTrue(contained.contains(2L)); Assertions.assertTrue(contained.contains(3L)); Assertions.assertTrue(contained.contains(1L)); @@ -81,21 +81,21 @@ void queryIndexTable(){ // Assertions.assertEquals(5,r.get(new EventTypes("A","B")).size()); contained.clear(); - contained = r.get(new EventTypes("A","B")).stream().map(IndexPair::getTraceId).collect(Collectors.toList()); + contained = r.get(new EventTypes("A","B")).stream().map(IndexPair::getTrace_id).collect(Collectors.toList()); Assertions.assertTrue(contained.contains(2L)); Assertions.assertTrue(contained.contains(3L)); Assertions.assertTrue(contained.contains(1L)); // Assertions.assertEquals(5,r.get(new EventTypes("B","C")).size()); contained.clear(); - contained = r.get(new EventTypes("B","C")).stream().map(IndexPair::getTraceId).collect(Collectors.toList()); + contained = r.get(new EventTypes("B","C")).stream().map(IndexPair::getTrace_id).collect(Collectors.toList()); Assertions.assertTrue(contained.contains(2L)); Assertions.assertTrue(contained.contains(3L)); Assertions.assertTrue(contained.contains(1L)); // Assertions.assertEquals(5,r.get(new EventTypes("A","C")).size()); contained.clear(); - contained = r.get(new EventTypes("A","C")).stream().map(IndexPair::getTraceId).collect(Collectors.toList()); + contained = r.get(new EventTypes("A","C")).stream().map(IndexPair::getTrace_id).collect(Collectors.toList()); Assertions.assertTrue(contained.contains(2L)); Assertions.assertTrue(contained.contains(3L)); Assertions.assertTrue(contained.contains(1L)); @@ -103,13 +103,13 @@ void queryIndexTable(){ // Assertions.assertEquals(2,r.get(new EventTypes("C","A")).size()); contained.clear(); - contained = r.get(new EventTypes("C","A")).stream().map(IndexPair::getTraceId).collect(Collectors.toList()); + contained = r.get(new EventTypes("C","A")).stream().map(IndexPair::getTrace_id).collect(Collectors.toList()); Assertions.assertTrue(contained.contains(2L)); Assertions.assertTrue(contained.contains(4L)); // Assertions.assertEquals(3,r.get(new EventTypes("B","B")).size()); contained.clear(); - contained = r.get(new EventTypes("B","B")).stream().map(IndexPair::getTraceId).collect(Collectors.toList()); + contained = r.get(new EventTypes("B","B")).stream().map(IndexPair::getTrace_id).collect(Collectors.toList()); Assertions.assertTrue(contained.contains(2L)); Assertions.assertTrue(contained.contains(3L)); }