Connector Tests #561
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Connector Tests | |
| concurrency: | |
| group: Connector-Tests-${{ github.workflow }}-${{ github.head_ref || github.event.workflow_run.head_branch || github.run_id }} | |
| cancel-in-progress: true | |
| on: | |
| merge_group: | |
| pull_request: | |
| branches: [main] | |
| schedule: | |
| # This cron expression runs the job daily at 16:00 UTC (9am PT) | |
| - cron: "0 16 * * *" | |
| permissions: | |
| contents: read | |
| env: | |
| # AWS | |
| AWS_ACCESS_KEY_ID_DAILY_CONNECTOR_TESTS: ${{ secrets.AWS_ACCESS_KEY_ID_DAILY_CONNECTOR_TESTS }} | |
| AWS_SECRET_ACCESS_KEY_DAILY_CONNECTOR_TESTS: ${{ secrets.AWS_SECRET_ACCESS_KEY_DAILY_CONNECTOR_TESTS }} | |
| # Cloudflare R2 | |
| R2_ACCOUNT_ID_DAILY_CONNECTOR_TESTS: ${{ vars.R2_ACCOUNT_ID_DAILY_CONNECTOR_TESTS }} | |
| R2_ACCESS_KEY_ID_DAILY_CONNECTOR_TESTS: ${{ secrets.R2_ACCESS_KEY_ID_DAILY_CONNECTOR_TESTS }} | |
| R2_SECRET_ACCESS_KEY_DAILY_CONNECTOR_TESTS: ${{ secrets.R2_SECRET_ACCESS_KEY_DAILY_CONNECTOR_TESTS }} | |
| # Google Cloud Storage | |
| GCS_ACCESS_KEY_ID_DAILY_CONNECTOR_TESTS: ${{ secrets.GCS_ACCESS_KEY_ID_DAILY_CONNECTOR_TESTS }} | |
| GCS_SECRET_ACCESS_KEY_DAILY_CONNECTOR_TESTS: ${{ secrets.GCS_SECRET_ACCESS_KEY_DAILY_CONNECTOR_TESTS }} | |
| # Confluence | |
| CONFLUENCE_TEST_SPACE_URL: ${{ vars.CONFLUENCE_TEST_SPACE_URL }} | |
| CONFLUENCE_TEST_SPACE: ${{ vars.CONFLUENCE_TEST_SPACE }} | |
| CONFLUENCE_TEST_PAGE_ID: ${{ secrets.CONFLUENCE_TEST_PAGE_ID }} | |
| CONFLUENCE_USER_NAME: ${{ vars.CONFLUENCE_USER_NAME }} | |
| CONFLUENCE_ACCESS_TOKEN: ${{ secrets.CONFLUENCE_ACCESS_TOKEN }} | |
| CONFLUENCE_ACCESS_TOKEN_SCOPED: ${{ secrets.CONFLUENCE_ACCESS_TOKEN_SCOPED }} | |
| # Jira | |
| JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }} | |
| JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }} | |
| JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }} | |
| JIRA_API_TOKEN_SCOPED: ${{ secrets.JIRA_API_TOKEN_SCOPED }} | |
| # Gong | |
| GONG_ACCESS_KEY: ${{ secrets.GONG_ACCESS_KEY }} | |
| GONG_ACCESS_KEY_SECRET: ${{ secrets.GONG_ACCESS_KEY_SECRET }} | |
| GOOGLE_DRIVE_SERVICE_ACCOUNT_JSON_STR: ${{ secrets.GOOGLE_DRIVE_SERVICE_ACCOUNT_JSON_STR }} | |
| GOOGLE_DRIVE_OAUTH_CREDENTIALS_JSON_STR_TEST_USER_1: ${{ secrets.GOOGLE_DRIVE_OAUTH_CREDENTIALS_JSON_STR_TEST_USER_1 }} | |
| GOOGLE_DRIVE_OAUTH_CREDENTIALS_JSON_STR: ${{ secrets.GOOGLE_DRIVE_OAUTH_CREDENTIALS_JSON_STR }} | |
| GOOGLE_GMAIL_SERVICE_ACCOUNT_JSON_STR: ${{ secrets.GOOGLE_GMAIL_SERVICE_ACCOUNT_JSON_STR }} | |
| GOOGLE_GMAIL_OAUTH_CREDENTIALS_JSON_STR: ${{ secrets.GOOGLE_GMAIL_OAUTH_CREDENTIALS_JSON_STR }} | |
| # Slab | |
| SLAB_BOT_TOKEN: ${{ secrets.SLAB_BOT_TOKEN }} | |
| # Zendesk | |
| ZENDESK_SUBDOMAIN: ${{ secrets.ZENDESK_SUBDOMAIN }} | |
| ZENDESK_EMAIL: ${{ secrets.ZENDESK_EMAIL }} | |
| ZENDESK_TOKEN: ${{ secrets.ZENDESK_TOKEN }} | |
| # Salesforce | |
| SF_USERNAME: ${{ secrets.SF_USERNAME }} | |
| SF_PASSWORD: ${{ secrets.SF_PASSWORD }} | |
| SF_SECURITY_TOKEN: ${{ secrets.SF_SECURITY_TOKEN }} | |
| # Hubspot | |
| HUBSPOT_ACCESS_TOKEN: ${{ secrets.HUBSPOT_ACCESS_TOKEN }} | |
| # IMAP | |
| IMAP_HOST: ${{ vars.IMAP_HOST }} | |
| IMAP_USERNAME: ${{ vars.IMAP_USERNAME }} | |
| IMAP_PASSWORD: ${{ secrets.IMAP_PASSWORD }} | |
| IMAP_MAILBOXES: ${{ vars.IMAP_MAILBOXES }} | |
| # Airtable | |
| AIRTABLE_TEST_BASE_ID: ${{ vars.AIRTABLE_TEST_BASE_ID }} | |
| AIRTABLE_TEST_TABLE_ID: ${{ vars.AIRTABLE_TEST_TABLE_ID }} | |
| AIRTABLE_TEST_TABLE_NAME: ${{ vars.AIRTABLE_TEST_TABLE_NAME }} | |
| AIRTABLE_ACCESS_TOKEN: ${{ secrets.AIRTABLE_ACCESS_TOKEN }} | |
| # Sharepoint | |
| SHAREPOINT_CLIENT_ID: ${{ vars.SHAREPOINT_CLIENT_ID }} | |
| SHAREPOINT_CLIENT_SECRET: ${{ secrets.SHAREPOINT_CLIENT_SECRET }} | |
| SHAREPOINT_CLIENT_DIRECTORY_ID: ${{ vars.SHAREPOINT_CLIENT_DIRECTORY_ID }} | |
| SHAREPOINT_SITE: ${{ vars.SHAREPOINT_SITE }} | |
| # Github | |
| ACCESS_TOKEN_GITHUB: ${{ secrets.ACCESS_TOKEN_GITHUB }} | |
| # Gitlab | |
| GITLAB_ACCESS_TOKEN: ${{ secrets.GITLAB_ACCESS_TOKEN }} | |
| # Gitbook | |
| GITBOOK_SPACE_ID: ${{ secrets.GITBOOK_SPACE_ID }} | |
| GITBOOK_API_KEY: ${{ secrets.GITBOOK_API_KEY }} | |
| # Notion | |
| NOTION_INTEGRATION_TOKEN: ${{ secrets.NOTION_INTEGRATION_TOKEN }} | |
| # Highspot | |
| HIGHSPOT_KEY: ${{ secrets.HIGHSPOT_KEY }} | |
| HIGHSPOT_SECRET: ${{ secrets.HIGHSPOT_SECRET }} | |
| # Slack | |
| SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} | |
| # Teams | |
| TEAMS_APPLICATION_ID: ${{ secrets.TEAMS_APPLICATION_ID }} | |
| TEAMS_DIRECTORY_ID: ${{ secrets.TEAMS_DIRECTORY_ID }} | |
| TEAMS_SECRET: ${{ secrets.TEAMS_SECRET }} | |
| # Bitbucket | |
| BITBUCKET_WORKSPACE: ${{ secrets.BITBUCKET_WORKSPACE }} | |
| BITBUCKET_REPOSITORIES: ${{ secrets.BITBUCKET_REPOSITORIES }} | |
| BITBUCKET_PROJECTS: ${{ secrets.BITBUCKET_PROJECTS }} | |
| BITBUCKET_EMAIL: ${{ vars.BITBUCKET_EMAIL }} | |
| BITBUCKET_API_TOKEN: ${{ secrets.BITBUCKET_API_TOKEN }} | |
| # Fireflies | |
| FIREFLIES_API_KEY: ${{ secrets.FIREFLIES_API_KEY }} | |
| jobs: | |
| connectors-check: | |
| # See https://runs-on.com/runners/linux/ | |
| runs-on: [runs-on, runner=8cpu-linux-x64, "run-id=${{ github.run_id }}-connectors-check", "extras=s3-cache"] | |
| env: | |
| PYTHONPATH: ./backend | |
| steps: | |
| - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # ratchet:runs-on/action@v2 | |
| - name: Checkout code | |
| uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # ratchet:actions/checkout@v6 | |
| with: | |
| persist-credentials: false | |
| - name: Setup Python and Install Dependencies | |
| uses: ./.github/actions/setup-python-and-install-dependencies | |
| - name: Setup Playwright | |
| uses: ./.github/actions/setup-playwright | |
| - name: Detect Connector changes | |
| id: changes | |
| uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # ratchet:dorny/paths-filter@v3 | |
| with: | |
| filters: | | |
| hubspot: | |
| - 'backend/onyx/connectors/hubspot/**' | |
| - 'backend/tests/daily/connectors/hubspot/**' | |
| salesforce: | |
| - 'backend/onyx/connectors/salesforce/**' | |
| - 'backend/tests/daily/connectors/salesforce/**' | |
| github: | |
| - 'backend/onyx/connectors/github/**' | |
| - 'backend/tests/daily/connectors/github/**' | |
| file_processing: | |
| - 'backend/onyx/file_processing/**' | |
| - name: Run Tests (excluding HubSpot, Salesforce, and GitHub) | |
| shell: script -q -e -c "bash --noprofile --norc -eo pipefail {0}" | |
| run: | | |
| py.test \ | |
| -n 8 \ | |
| --dist loadfile \ | |
| --durations=8 \ | |
| -o junit_family=xunit2 \ | |
| -xv \ | |
| --ff \ | |
| backend/tests/daily/connectors \ | |
| --ignore backend/tests/daily/connectors/hubspot \ | |
| --ignore backend/tests/daily/connectors/salesforce \ | |
| --ignore backend/tests/daily/connectors/github | |
| - name: Run HubSpot Connector Tests | |
| if: ${{ github.event_name == 'schedule' || steps.changes.outputs.hubspot == 'true' || steps.changes.outputs.file_processing == 'true' }} | |
| shell: script -q -e -c "bash --noprofile --norc -eo pipefail {0}" | |
| run: | | |
| py.test \ | |
| -n 8 \ | |
| --dist loadfile \ | |
| --durations=8 \ | |
| -o junit_family=xunit2 \ | |
| -xv \ | |
| --ff \ | |
| backend/tests/daily/connectors/hubspot | |
| - name: Run Salesforce Connector Tests | |
| if: ${{ github.event_name == 'schedule' || steps.changes.outputs.salesforce == 'true' || steps.changes.outputs.file_processing == 'true' }} | |
| shell: script -q -e -c "bash --noprofile --norc -eo pipefail {0}" | |
| run: | | |
| py.test \ | |
| -n 8 \ | |
| --dist loadfile \ | |
| --durations=8 \ | |
| -o junit_family=xunit2 \ | |
| -xv \ | |
| --ff \ | |
| backend/tests/daily/connectors/salesforce | |
| - name: Run GitHub Connector Tests | |
| if: ${{ github.event_name == 'schedule' || steps.changes.outputs.github == 'true' || steps.changes.outputs.file_processing == 'true' }} | |
| shell: script -q -e -c "bash --noprofile --norc -eo pipefail {0}" | |
| run: | | |
| py.test \ | |
| -n 8 \ | |
| --dist loadfile \ | |
| --durations=8 \ | |
| -o junit_family=xunit2 \ | |
| -xv \ | |
| --ff \ | |
| backend/tests/daily/connectors/github | |
| - name: Alert on Failure | |
| if: failure() && github.event_name == 'schedule' | |
| env: | |
| SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} | |
| REPO: ${{ github.repository }} | |
| RUN_ID: ${{ github.run_id }} | |
| run: | | |
| curl -X POST \ | |
| -H 'Content-type: application/json' \ | |
| --data "{\"text\":\"Scheduled Connector Tests failed! Check the run at: https://github.com/${REPO}/actions/runs/${RUN_ID}\"}" \ | |
| $SLACK_WEBHOOK |