Develop

.github/workflows/ci.yml

@@ -31,12 +31,14 @@ jobs:
         uses: astral-sh/ruff-action@v3
         with:
           args: check --output-format=github
-      - name: Check dependencies with import-linter
+      - name: Check import dependencies with import-linter
         run: |
-          python -m venv venv
-          source venv/bin/activate
           pip install import-linter
           PYTHONPATH=source lint-imports
+      - name: Looking for dead code with vulture
+        run: |
+          pip install vulture
+          vulture
 
   build-docker-db:
     name: Build docker db
@@ -131,10 +133,8 @@ jobs:
         uses: actions/checkout@v4
       - name: Start development server
         run: |
-          # Even though, we use --env-file option when running docker compose, this is still necessary, because the compose has a env_file attribute :(
-          # TODO should move basic.env file, which is in directory tests, up. It's used in several places. Maybe, rename it into dev.env
-          cp tests/data/basic.env .env
-          docker compose --file docker-compose.dev.yml --env-file tests/data/basic.env up --detach --wait
+          cp .env.tests.model .env
+          docker compose --file docker-compose.dev.yml up --detach --wait
       - name: Generate GraphQL documentation
         run: |
           npx spectaql@^3.0.2 source/spectaql/config.yml
@@ -168,11 +168,19 @@ jobs:
           docker load --input ${{ runner.temp }}/iriswebapp_app.tar
       - name: Check out iris
         uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: npm
+          cache-dependency-path: ui/package-lock.json
+      - name: Build ui to be mounted in development docker
+        working-directory: ui
+        run: |
+          npm ci
+          npm run build
       - name: Start development server
         run: |
-          # Even though, we use --env-file option when running docker compose, this is still necessary, because the compose has a env_file attribute :(
-          # TODO should move basic.env file, which is in directory tests, up. It's used in several places. Maybe, rename it into dev.env
-          cp tests/data/basic.env .env
+          cp .env.tests.model .env
           docker compose --file docker-compose.dev.yml up --detach --wait
       - name: Inspect development server start failure
         if: ${{ failure() || cancelled() }}
@@ -226,8 +234,7 @@ jobs:
       - name: Check out iris
         uses: actions/checkout@v4
       - name: Set up .env file
-        # TODO should move basic.env file, which is in directory tests, up. It's used in several places. Maybe, rename it into dev.env?
-        run: cp tests/data/basic.env .env
+        run: cp .env.tests.model .env
       - name: Run tests
         working-directory: tests_database_migration
         run: |
@@ -277,8 +284,7 @@ jobs:
         run: npx playwright install chromium firefox
       - name: Start development server
         run: |
-          # TODO should move basic.env file, which is in directory tests, up. It's used in several places. Maybe, rename it into dev.env
-          cp tests/data/basic.env .env
+          cp .env.tests.model .env
           docker compose --file docker-compose.dev.yml up --detach --wait
       - name: Run end to end tests
         working-directory: e2e

deploy/kubernetes/charts/Chart.yaml

@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.2
+version: 0.2.1
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

deploy/kubernetes/charts/templates/ingress.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.ingress.enabled }}
+{{- if and .Values.ingress.enabled (not .Values.ingress.traefik.enabled) }}
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:

deploy/kubernetes/charts/templates/ingress_route.yaml

@@ -0,0 +1,28 @@
+{{- if .Values.ingress.traefik.enabled }}
+apiVersion: traefik.io/v1alpha1
+kind: IngressRoute
+metadata:
+  name: {{ .Values.ingress.name }}
+  {{- with .Values.ingress.customAnnotations }}
+  annotations:
+  {{- range $k, $v := . }}
+    {{- if not (hasPrefix "nginx.ingress.kubernetes.io" $k) }}
+    {{ $k }}: {{ $v }}
+    {{- end }}
+  {{- end }}
+  {{- end }}
+spec:
+  entryPoints:
+    - {{ .Values.ingress.traefik.entryPoint | default "web" }}
+  routes:
+  {{- range $host := .Values.ingress.hosts }}
+    {{- range $path := $host.paths }}
+    - kind: Rule
+      match: >
+        Host(`{{ $host.host }}`) && PathPrefix(`{{ $path.path }}`)
+      services:
+        - name: {{ $.Values.irisapp.name }}
+          port: {{ $.Values.irisapp.service.port }}
+    {{- end }}
+  {{- end }}
+{{- end }}

deploy/kubernetes/charts/templates/iris_app.yaml

@@ -58,10 +58,10 @@ spec:
             {{- end }}
           {{- end }}
 
-          - name:  IRIS_SECRET_KEY 
+          - name:  IRIS_SECRET_KEY
             value: {{ .Values.irisapp.IRIS_SECRET_KEY | quote }}
 
-          - name:  IRIS_SECURITY_PASSWORD_SALT 
+          - name:  IRIS_SECURITY_PASSWORD_SALT
             value: {{ .Values.irisapp.IRIS_SECURITY_PASSWORD_SALT | quote }}
 
           - name:  DB_RETRY_COUNT
@@ -78,8 +78,12 @@ spec:
 
           - name:  IRIS_ADM_PASSWORD
             value: {{ .Values.irisapp.IRIS_ADM_PASSWORD | quote }}
-          
+
           {{- if eq .Values.irisapp.IRIS_AUTHENTICATION_TYPE "oidc" }}
+
+          - name: IRIS_AUTHENTICATION_TYPE:
+            value: {{ .Values.irisapp.IRIS_AUTHENTICATION_TYPE | quote }}
+
           - name: OIDC_ISSUER_URL
             value: {{ .Values.irisapp.OIDC_ISSUER_URL | quote }}
 
@@ -104,13 +108,13 @@ spec:
           - name: OIDC_MAPPING_ROLES
             value: {{ .Values.irisapp.OIDC_MAPPING_ROLES | quote }}
           {{- end }}
-          
+
           ports:
             - containerPort: 8000
 
           volumeMounts:
             - mountPath: /home/iris/downloads
-              name: iris-downloads  
+              name: iris-downloads
             - mountPath: /home/iris/user_templates
               name: user-templates
             - mountPath: /home/iris/server_data

deploy/kubernetes/charts/templates/postgres.yaml

@@ -1,5 +1,5 @@
 {{- if .Values.postgres.enabled }}
----    
+---
 # Here I have used a hostpath
 # Local volumes can only be used as a statically created PersistentVolume. Dynamic provisioning is not supported.
 # If you need to go with Dynamic volumes you may choose AWS EBS or EFS
@@ -68,13 +68,13 @@ spec:
           - name: POSTGRES_USER  # Setting Database username
             value: {{ .Values.postgres.POSTGRES_USER | quote }}
 
-          - name:  POSTGRES_PASSWORDD # Setting Database password
+          - name:  POSTGRES_PASSWORD # Setting Database password
             value: {{ .Values.postgres.POSTGRES_PASSWORD | quote }}
 
-          - name:  POSTGRES_ADMIN_USER # Setting Database admin user 
+          - name:  POSTGRES_ADMIN_USER # Setting Database admin user
             value: {{ .Values.postgres.POSTGRES_ADMIN_USER | quote }}
 
-          - name:  POSTGRES_ADMIN_PASSWORD # Setting Database admin password 
+          - name:  POSTGRES_ADMIN_PASSWORD # Setting Database admin password
             value: {{ .Values.postgres.POSTGRES_ADMIN_PASSWORD | quote }}
 
           - name:  POSTGRES_PORT # Setting Database port
@@ -108,5 +108,5 @@ spec:
    - port: {{ .Values.postgres.service.port }}
   selector:
    app: {{ .Values.postgres.app }}
----  
-{{- end }}
+---
+{{- end }}

deploy/kubernetes/charts/values.yaml

@@ -224,6 +224,9 @@ ingress:
   enabled: true
   name: iris-ingress
   className: nginx
+  traefik:
+    enabled: false
+    entryPoint: web
   enableTls: false
   sslRedirect: false
   customAnnotations:

pyproject.toml

@@ -1,8 +1,12 @@
 [tool.ruff.lint]
 preview = true
-select = ["E101", "E225", "E23", "E24", "E3", "E4", "E7", "E9", "F", "PLR0402", "RET506", "TID252", "UP032", "W29"]
+select = ["ARG003", "ARG005", "B00", "E101", "E20", "E211", "E221", "E225", "E226", "E272", "E23", "E24", "E3", "E4", "E7", "E9", "F", "FURB142", "FURB145", "FURB148", "PLR0402", "RET501", "RET505", "RET506", "RUF029", "RUF100", "TID252", "UP032", "W29", "W391"]
 ignore = ["E402", "E711", "E712", "E721", "E722"]
 
+[tool.vulture]
+paths = ["source/app", ".vulture.ignore"]
+ignore_decorators = ["@*.route", "@app.*", "@*.post", "@*.get", "@*.put", "@*.delete", "@pre_load", "@post_load"]
+
 [tool.importlinter]
 root_package = "app"
 include_external_packages = true
@@ -24,8 +28,8 @@ allow_indirect_imports = true
 [[tool.importlinter.contracts]]
 name = "Do not import API layer from the business layer"
 type = "forbidden"
-source_modules = ["app.business.access_controls", "app.business.assets"]
-forbidden_modules = "app.blueprints.iris_user"
+source_modules = ["app.business.access_controls", "app.business.assets", "app.business.cases", "app.business.alerts"]
+forbidden_modules = "app.blueprints"
 allow_indirect_imports = true
 
 [[tool.importlinter.contracts]]
@@ -36,9 +40,44 @@ forbidden_modules = "sqlalchemy"
 allow_indirect_imports = true
 
 [[tool.importlinter.contracts]]
-name = "Do not import API layer from the persistence layer"
+name = "Do not import API layer from the persistence layer (access_controls)"
+type = "forbidden"
+source_modules = "app.datamgmt"
+forbidden_modules = "app.blueprints.access_controls"
+allow_indirect_imports = true
+
+[[tool.importlinter.contracts]]
+name = "Do not import API layer from the persistence layer (iris_user)"
 type = "forbidden"
-source_modules = "app.datamgmt.dashboard"
+source_modules = ["app.datamgmt.case", "app.datamgmt.datastore", "app.datamgmt.filters"]
 forbidden_modules = "app.blueprints.iris_user"
 allow_indirect_imports = true
 
+[[tool.importlinter.contracts]]
+name = "Do not import marshables from the persistence layer"
+type = "forbidden"
+source_modules = ["app.datamgmt.manage.manage_case_state_db", "app.datamgmt.manage.manage_groups_db"]
+forbidden_modules = "app.schema.marshables"
+allow_indirect_imports = true
+
+[[tool.importlinter.contracts]]
+name = "Do not import marshmallow from the persistence layer"
+type = "forbidden"
+source_modules = "app.datamgmt.client"
+forbidden_modules = "marshmallow"
+allow_indirect_imports = true
+
+[[tool.importlinter.contracts]]
+name = "Do not import the engine from the persistence layer"
+type = "forbidden"
+source_modules = "app.datamgmt.case"
+forbidden_modules = "app.iris_engine"
+allow_indirect_imports = true
+
+[[tool.importlinter.contracts]]
+name = "Do not import the persistence layer from the models"
+type = "forbidden"
+source_modules = "app.models.cases"
+forbidden_modules = "app.datamgmt"
+allow_indirect_imports = true
+

scripts/gunicorn-cfg.py

@@ -27,4 +27,3 @@
 
 def worker_exit(server, worker):
     sys.exit(4)
-

source/app/__init__.py

@@ -16,20 +16,18 @@
 #  along with this program; if not, write to the Free Software Foundation,
 #  Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 
-import collections
-import json
 import os
-from flask import Flask, g
+from flask import Flask
+from flask import g
 from flask import session
 from flask_bcrypt import Bcrypt
 from flask_caching import Cache
 from flask_cors import CORS
 
 from flask_login import LoginManager
 from flask_marshmallow import Marshmallow
-from flask_socketio import SocketIO, Namespace
-from flask_sqlalchemy import SQLAlchemy
-from functools import partial
+from flask_socketio import SocketIO
+from flask_socketio import Namespace
 
 from werkzeug.middleware.proxy_fix import ProxyFix
 
@@ -39,6 +37,8 @@
 from app.iris_engine.tasker.celery import set_celery_flask_context
 from app.iris_engine.access_control.oidc_handler import get_oidc_client
 from app.jinja_filters import register_jinja_filters
+from app.models.authorization import ac_flag_match_mask
+from app.db import db
 
 
 class ReverseProxied(object):
@@ -59,12 +59,6 @@ class AlertsNamespace(Namespace):
 APP_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 TEMPLATE_PATH = os.path.join(APP_PATH, 'templates/')
 
-SQLALCHEMY_ENGINE_OPTIONS = {
-    "json_deserializer": partial(json.loads, object_pairs_hook=collections.OrderedDict),
-    "pool_pre_ping": True
-}
-
-db = SQLAlchemy(engine_options=SQLALCHEMY_ENGINE_OPTIONS)  # flask-sqlalchemy
 bc = Bcrypt()  # flask-bcrypt
 ma = Marshmallow()
 celery = make_celery(__name__)
@@ -75,10 +69,13 @@ def ac_current_user_has_permission(*permissions):
     """
     Return True if current user has permission
     """
+    if 'permissions' not in session:
+        return False
+
+    current_user_permissions = session['permissions']
     for permission in permissions:
 
-        if ('permissions' in session and
-                session['permissions'] & permission.value == permission.value):
+        if ac_flag_match_mask(current_user_permissions, permission.value):
             return True
 
     return False
@@ -129,7 +126,7 @@ def ac_current_user_has_manage_perms():
      ]}})
 
 
-app.wsgi_app = ProxyFix(app.wsgi_app, x_for=1, x_proto=1)
+app.wsgi_app = ProxyFix(app.wsgi_app, x_for=1, x_proto=1, x_host=1, x_port=1)
 #app.wsgi_app = store.wsgi_middleware(app.wsgi_app)
 
 socket_io = SocketIO(app, cors_allowed_origins="*")

source/app/alembic/versions/10a7616f3cc7_add_module_types.py

@@ -42,4 +42,3 @@ def upgrade():
 
 def downgrade():
     pass
-

source/app/alembic/versions/4ecdfcb34f7c_add_compromise_status_to_assets.py

@@ -10,7 +10,7 @@
 
 # revision identifiers, used by Alembic.
 from app.alembic.alembic_utils import _table_has_column
-from app.models.models import CompromiseStatus
+from app.models.assets import CompromiseStatus
 
 revision = '4ecdfcb34f7c'
 down_revision = 'a929ef458490'

source/app/alembic/versions/ad4e0cd17597_add_ioctype_validation.py

@@ -93,4 +93,3 @@ def upgrade():
 def downgrade():
     op.drop_column('ioc_type', 'type_validation_regex')
     op.drop_column('ioc_type', 'type_validation_expect')
-

source/app/alembic/versions/cd519d2d24df_password_policy_edition.py

@@ -57,4 +57,3 @@ def upgrade():
 
 def downgrade():
     pass
-

source/app/alembic/versions/d5a720d1b99b_add_alerts_indexes.py

@@ -61,4 +61,3 @@ def downgrade():
 
     # Drop AlertSimilarity table
     op.drop_table('alert_similarity')
-