Merge branch 'master' into ADO-4387-move-workflows-from-overview

This commit is contained in:
Ricardo Espinoza 2025-11-20 09:05:04 -05:00 committed by GitHub
commit 5369c5c4ea
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
249 changed files with 14667 additions and 3024 deletions

3
.github/CODEOWNERS vendored
View File

@ -1 +1,4 @@
packages/@n8n/db/src/migrations/ @n8n-io/migrations-review
# Node popularity data updates
packages/frontend/editor-ui/data/node-popularity.json @n8n-io/catalysts

View File

@ -1,119 +0,0 @@
#!/bin/bash
set -euo pipefail
# Script to determine Docker tags for runners images (Alpine and distroless variants)
#
# Usage: determine-runners-tags.sh RELEASE_TYPE N8N_VERSION_TAG GHCR_BASE DOCKER_BASE PLATFORM GITHUB_OUTPUT
#
# Example:
# determine-runners-tags.sh \
# "stable" \
# "1.123.0" \
# "ghcr.io/n8n-io/runners" \
# "n8nio/runners" \
# "amd64" \
# "$GITHUB_OUTPUT"
#
# Output (written to GITHUB_OUTPUT):
# Alpine variant:
# tags=ghcr.io/n8n-io/runners:1.123.0-amd64, n8nio/runners:1.123.0-amd64
# ghcr_platform_tag=ghcr.io/n8n-io/runners:1.123.0-amd64
# dockerhub_platform_tag=n8nio/runners:1.123.0-amd64
# primary_ghcr_manifest_tag=ghcr.io/n8n-io/runners:1.123.0
#
# Distroless variant:
# tags_distroless=ghcr.io/n8n-io/runners:1.123.0-distroless-amd64, n8nio/runners:1.123.0-distroless-amd64
# ghcr_platform_tag_distroless=ghcr.io/n8n-io/runners:1.123.0-distroless-amd64
# dockerhub_platform_tag_distroless=n8nio/runners:1.123.0-distroless-amd64
# primary_ghcr_manifest_tag_distroless=ghcr.io/n8n-io/runners:1.123.0-distroless
RELEASE_TYPE="${1:?Missing RELEASE_TYPE argument}"
N8N_VERSION_TAG="${2:?Missing N8N_VERSION_TAG argument}"
GHCR_BASE="${3:?Missing GHCR_BASE argument}"
DOCKER_BASE="${4:?Missing DOCKER_BASE argument}"
PLATFORM="${5:?Missing PLATFORM argument}"
GITHUB_OUTPUT="${6:?Missing GITHUB_OUTPUT argument}"
generate_tags() {
local VARIANT_SUFFIX="$1"
local OUTPUT_FILE="$2"
local GHCR_TAGS_FOR_PUSH=""
local DOCKER_TAGS_FOR_PUSH=""
local PRIMARY_GHCR_MANIFEST_TAG_VALUE=""
case "$RELEASE_TYPE" in
"stable")
PRIMARY_GHCR_MANIFEST_TAG_VALUE="${GHCR_BASE}:${N8N_VERSION_TAG}${VARIANT_SUFFIX}"
GHCR_TAGS_FOR_PUSH="${PRIMARY_GHCR_MANIFEST_TAG_VALUE}-${PLATFORM}"
DOCKER_TAGS_FOR_PUSH="${DOCKER_BASE}:${N8N_VERSION_TAG}${VARIANT_SUFFIX}-${PLATFORM}"
;;
"nightly")
PRIMARY_GHCR_MANIFEST_TAG_VALUE="${GHCR_BASE}:nightly${VARIANT_SUFFIX}"
GHCR_TAGS_FOR_PUSH="${PRIMARY_GHCR_MANIFEST_TAG_VALUE}-${PLATFORM}"
DOCKER_TAGS_FOR_PUSH="${DOCKER_BASE}:nightly${VARIANT_SUFFIX}-${PLATFORM}"
;;
"branch")
PRIMARY_GHCR_MANIFEST_TAG_VALUE="${GHCR_BASE}:${N8N_VERSION_TAG}${VARIANT_SUFFIX}"
GHCR_TAGS_FOR_PUSH="${PRIMARY_GHCR_MANIFEST_TAG_VALUE}-${PLATFORM}"
DOCKER_TAGS_FOR_PUSH=""
;;
"dev"|*)
if [[ "$N8N_VERSION_TAG" == pr-* ]]; then
PRIMARY_GHCR_MANIFEST_TAG_VALUE="${GHCR_BASE}:${N8N_VERSION_TAG}${VARIANT_SUFFIX}"
GHCR_TAGS_FOR_PUSH="${PRIMARY_GHCR_MANIFEST_TAG_VALUE}-${PLATFORM}"
DOCKER_TAGS_FOR_PUSH=""
else
PRIMARY_GHCR_MANIFEST_TAG_VALUE="${GHCR_BASE}:dev${VARIANT_SUFFIX}"
GHCR_TAGS_FOR_PUSH="${PRIMARY_GHCR_MANIFEST_TAG_VALUE}-${PLATFORM}"
DOCKER_TAGS_FOR_PUSH="${DOCKER_BASE}:dev${VARIANT_SUFFIX}-${PLATFORM}"
fi
;;
esac
local ALL_TAGS="${GHCR_TAGS_FOR_PUSH}"
if [[ -n "$DOCKER_TAGS_FOR_PUSH" ]]; then
ALL_TAGS="${ALL_TAGS}\n${DOCKER_TAGS_FOR_PUSH}"
fi
{
echo "tags<<EOF"
echo -e "$ALL_TAGS"
echo "EOF"
echo "ghcr_platform_tag=${GHCR_TAGS_FOR_PUSH}"
echo "dockerhub_platform_tag=${DOCKER_TAGS_FOR_PUSH}"
} >> "$OUTPUT_FILE"
# Only output manifest tags from the first platform to avoid duplicates
if [[ "$PLATFORM" == "amd64" ]]; then
echo "primary_ghcr_manifest_tag=${PRIMARY_GHCR_MANIFEST_TAG_VALUE}" >> "$OUTPUT_FILE"
fi
}
# Reads outputs from a temp file and appends them to GITHUB_OUTPUT with _distroless suffix
# Transforms variable names: tags -> tags_distroless, ghcr_platform_tag -> ghcr_platform_tag_distroless
transform_and_append_distroless_outputs() {
local TEMP_FILE="$1"
while IFS= read -r line; do
if [[ "$line" == "tags<<EOF" ]]; then
echo "tags_distroless<<EOF" >> "$GITHUB_OUTPUT"
elif [[ "$line" =~ ^(ghcr_platform_tag|dockerhub_platform_tag|primary_ghcr_manifest_tag)= ]]; then
key="${line%%=*}"
value="${line#*=}"
echo "${key}_distroless=${value}" >> "$GITHUB_OUTPUT"
else
# Pass through EOF markers and tag content
echo "$line" >> "$GITHUB_OUTPUT"
fi
done < "$TEMP_FILE"
}
# Generate tags for Alpine variant (no suffix)
generate_tags "" "$GITHUB_OUTPUT"
# Generate tags for distroless variant
DISTROLESS_OUTPUT=$(mktemp)
generate_tags "-distroless" "$DISTROLESS_OUTPUT"
transform_and_append_distroless_outputs "$DISTROLESS_OUTPUT"
rm "$DISTROLESS_OUTPUT"

171
.github/scripts/docker/docker-config.mjs vendored Normal file
View File

@ -0,0 +1,171 @@
#!/usr/bin/env node
import { appendFileSync } from 'node:fs';
class BuildContext {
constructor() {
this.githubOutput = process.env.GITHUB_OUTPUT || null;
}
determine({ event, pr, branch, version, releaseType, pushEnabled }) {
let context = {
version: '',
release_type: '',
platforms: ['linux/amd64', 'linux/arm64'],
push_to_ghcr: true,
push_to_docker: false,
};
// Determine version and release type based on event
switch (event) {
case 'schedule':
context.version = 'nightly';
context.release_type = 'nightly';
context.push_to_docker = true;
break;
case 'pull_request':
context.version = `pr-${pr}`;
context.release_type = 'dev';
context.push_to_ghcr = false;
break;
case 'workflow_dispatch':
context.version = `branch-${this.sanitizeBranch(branch)}`;
context.release_type = 'branch';
context.platforms = ['linux/amd64'];
break;
case 'push':
if (branch === 'master') {
context.version = 'dev';
context.release_type = 'dev';
context.push_to_docker = true;
} else {
context.version = `branch-${this.sanitizeBranch(branch)}`;
context.release_type = 'branch';
context.platforms = ['linux/amd64'];
}
break;
case 'workflow_call':
case 'release':
if (!version) throw new Error('Version required for release');
context.version = version;
context.release_type = releaseType || 'stable';
context.push_to_docker = true;
break;
default:
throw new Error(`Unknown event: ${event}`);
}
// Handle push_enabled override
if (pushEnabled !== undefined) {
context.push_enabled = pushEnabled;
} else {
context.push_enabled = context.push_to_ghcr;
}
return context;
}
sanitizeBranch(branch) {
if (!branch) return 'unknown';
return branch
.toLowerCase()
.replace(/[^a-z0-9._-]/g, '-')
.replace(/^[.-]/, '')
.replace(/[.-]$/, '')
.substring(0, 128);
}
buildMatrix(platforms) {
const runners = {
'linux/amd64': 'blacksmith-4vcpu-ubuntu-2204',
'linux/arm64': 'blacksmith-4vcpu-ubuntu-2204-arm',
};
const matrix = {
platform: [],
include: [],
};
for (const platform of platforms) {
const shortName = platform.split('/').pop(); // amd64 or arm64
matrix.platform.push(shortName);
matrix.include.push({
platform: shortName,
runner: runners[platform],
docker_platform: platform,
});
}
return matrix;
}
output(context, matrix = null) {
const buildMatrix = matrix || this.buildMatrix(context.platforms);
if (this.githubOutput) {
const outputs = [
`version=${context.version}`,
`release_type=${context.release_type}`,
`platforms=${JSON.stringify(context.platforms)}`,
`push_to_ghcr=${context.push_to_ghcr}`,
`push_to_docker=${context.push_to_docker}`,
`push_enabled=${context.push_enabled}`,
`build_matrix=${JSON.stringify(buildMatrix)}`,
];
appendFileSync(this.githubOutput, outputs.join('\n') + '\n');
} else {
console.log(JSON.stringify({ ...context, build_matrix: buildMatrix }, null, 2));
}
}
}
// CLI - Simple argument parsing
if (import.meta.url === `file://${process.argv[1]}`) {
const args = process.argv.slice(2);
const getArg = (name) => {
const index = args.indexOf(`--${name}`);
if (index === -1 || !args[index + 1]) return undefined;
const value = args[index + 1];
// Handle empty strings and 'null' as undefined
return value === '' || value === 'null' ? undefined : value;
};
try {
const context = new BuildContext();
const pushEnabledArg = getArg('push-enabled');
const result = context.determine({
event: getArg('event') || process.env.GITHUB_EVENT_NAME,
pr: getArg('pr') || process.env.GITHUB_PR_NUMBER,
branch: getArg('branch') || process.env.GITHUB_REF_NAME,
version: getArg('version'),
releaseType: getArg('release-type'),
pushEnabled: pushEnabledArg === 'true' ? true : pushEnabledArg === 'false' ? false : undefined,
});
const matrix = context.buildMatrix(result.platforms);
// Debug output when GITHUB_OUTPUT is set (running in Actions)
if (context.githubOutput) {
console.log('=== Build Context ===');
console.log(`version: ${result.version}`);
console.log(`release_type: ${result.release_type}`);
console.log(`platforms: ${JSON.stringify(result.platforms, null, 2)}`);
console.log(`push_to_ghcr: ${result.push_to_ghcr}`);
console.log(`push_to_docker: ${result.push_to_docker}`);
console.log(`push_enabled: ${result.push_enabled}`);
console.log('build_matrix:', JSON.stringify(matrix, null, 2));
}
context.output(result, matrix);
} catch (error) {
console.error(`Error: ${error.message}`);
process.exit(1);
}
}
export default BuildContext;

113
.github/scripts/docker/docker-tags.mjs vendored Normal file
View File

@ -0,0 +1,113 @@
#!/usr/bin/env node
import { appendFileSync } from 'node:fs';
class TagGenerator {
constructor() {
this.githubOwner = process.env.GITHUB_REPOSITORY_OWNER || 'n8n-io';
this.dockerUsername = process.env.DOCKER_USERNAME || 'n8nio';
this.githubOutput = process.env.GITHUB_OUTPUT || null;
}
generate({ image, version, platform, includeDockerHub = false }) {
let imageName = image;
let versionSuffix = '';
if (image === 'runners-distroless') {
imageName = 'runners';
versionSuffix = '-distroless';
}
const platformSuffix = platform ? `-${platform.split('/').pop()}` : '';
const fullVersion = `${version}${versionSuffix}${platformSuffix}`;
const tags = {
ghcr: [`ghcr.io/${this.githubOwner}/${imageName}:${fullVersion}`],
docker: includeDockerHub ? [`${this.dockerUsername}/${imageName}:${fullVersion}`] : [],
};
tags.all = [...tags.ghcr, ...tags.docker];
return tags;
}
output(tags, prefix = '') {
if (this.githubOutput) {
const prefixStr = prefix ? `${prefix}_` : '';
const primaryTag = tags.ghcr[0] ? tags.ghcr[0].replace(/-amd64$|-arm64$/, '') : '';
const outputs = [
`${prefixStr}tags=${tags.all.join(',')}`,
`${prefixStr}ghcr_tag=${tags.ghcr[0] || ''}`,
`${prefixStr}docker_tag=${tags.docker[0] || ''}`,
`${prefixStr}primary_tag=${primaryTag}`,
];
appendFileSync(this.githubOutput, outputs.join('\n') + '\n');
} else {
console.log(JSON.stringify(tags, null, 2));
}
}
generateAll({ version, platform, includeDockerHub = false }) {
const images = ['n8n', 'runners', 'runners-distroless'];
const results = {};
for (const image of images) {
const tags = this.generate({ image, version, platform, includeDockerHub });
const prefix = image.replace('-distroless', '_distroless');
results[prefix] = tags;
if (this.githubOutput) {
this.output(tags, prefix);
}
}
return results;
}
}
if (import.meta.url === `file://${process.argv[1]}`) {
const args = process.argv.slice(2);
const getArg = (name) => {
const index = args.indexOf(`--${name}`);
return index !== -1 && args[index + 1] ? args[index + 1] : undefined;
};
const hasFlag = (name) => args.includes(`--${name}`);
try {
const generator = new TagGenerator();
const version = getArg('version');
if (!version) {
console.error('Error: --version is required');
process.exit(1);
}
if (hasFlag('all')) {
const results = generator.generateAll({
version,
platform: getArg('platform'),
includeDockerHub: hasFlag('include-docker'),
});
if (!generator.githubOutput) {
console.log(JSON.stringify(results, null, 2));
}
} else {
const image = getArg('image');
if (!image) {
console.error('Error: Either --image or --all is required');
process.exit(1);
}
const tags = generator.generate({
image,
version,
platform: getArg('platform'),
includeDockerHub: hasFlag('include-docker'),
});
generator.output(tags);
}
} catch (error) {
console.error(`Error: ${error.message}`);
process.exit(1);
}
}
export default TagGenerator;

View File

@ -59,7 +59,7 @@ jobs:
name: MariaDB
needs: build
runs-on: blacksmith-4vcpu-ubuntu-2204
timeout-minutes: 20
timeout-minutes: 30
env:
DB_MYSQLDB_PASSWORD: password
DB_MYSQLDB_POOL_SIZE: 1

View File

@ -6,6 +6,9 @@ on:
- '**'
- '!release/*'
env:
COVERAGE_ENABLED: 'true' # Set globally for all jobs - ensures Turbo cache consistency
jobs:
install-and-build:
name: Install & Build
@ -16,7 +19,7 @@ jobs:
frontend_changed: ${{ steps.paths-filter.outputs.frontend == 'true' }}
non_python_changed: ${{ steps.paths-filter.outputs.non-python == 'true' }}
steps:
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
ref: refs/pull/${{ github.event.pull_request.number }}/merge
@ -61,7 +64,7 @@ jobs:
if: needs.install-and-build.outputs.frontend_changed == 'true'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
ref: refs/pull/${{ github.event.pull_request.number }}/merge

View File

@ -1,9 +1,7 @@
# This workflow is used to build and push the Docker image for n8nio/n8n and n8nio/runners
# - determine-build-context: Determines what needs to be built based on the trigger
# - build-and-push-docker: This builds on both an ARM64 and AMD64 runner so the builds are native to the platform. Uses blacksmith native runners and build-push-action
# - create_multi_arch_manifest: This creates the multi-arch manifest for the Docker image. Needed to recombine the images from the build-and-push-docker job since they are separate runners.
# - security-scan: This scans the n8nio/n8n Docker image for security vulnerabilities using Trivy.
# - security-scan-runners: This scans the n8nio/runners Docker image for security vulnerabilities using Trivy.
#
# - Uses docker-config.mjs for context determination, this determines what needs to be built based on the trigger
# - Uses docker-tags.mjs for tag generation, this generates the tags for the images
name: 'Docker: Build and Push'
@ -50,6 +48,8 @@ on:
- ready_for_review
paths:
- '.github/workflows/docker-build-push.yml'
- '.github/scripts/docker/docker-config.mjs'
- '.github/scripts/docker/docker-tags.mjs'
- 'docker/images/n8n/Dockerfile'
- 'docker/images/runners/Dockerfile'
- 'docker/images/runners/Dockerfile.distroless'
@ -60,107 +60,24 @@ jobs:
runs-on: ubuntu-latest
outputs:
release_type: ${{ steps.context.outputs.release_type }}
n8n_version: ${{ steps.context.outputs.n8n_version }}
n8n_version: ${{ steps.context.outputs.version }}
push_enabled: ${{ steps.context.outputs.push_enabled }}
build_matrix: ${{ steps.matrix.outputs.matrix }}
push_to_docker: ${{ steps.context.outputs.push_to_docker }}
build_matrix: ${{ steps.context.outputs.build_matrix }}
steps:
- name: Determine build context values
- name: Checkout code
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Determine build context
id: context
run: |
# Debug info
echo "Event: ${{ github.event_name }}"
echo "Ref: ${{ github.ref }}"
echo "Ref Name: ${{ github.ref_name }}"
# Check if called by another workflow (has n8n_version input)
if [[ -n "${{ inputs.n8n_version }}" ]]; then
# workflow_call - used for releases
{
echo "release_type=${{ inputs.release_type }}"
echo "n8n_version=${{ inputs.n8n_version }}"
echo "push_enabled=${{ inputs.push_enabled }}"
} >> "$GITHUB_OUTPUT"
elif [[ "${{ github.event_name }}" == "schedule" ]]; then
# Nightly builds
{
echo "release_type=nightly"
echo "n8n_version=snapshot"
echo "push_enabled=true"
} >> "$GITHUB_OUTPUT"
elif [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
# Build branches for Nathan deploy
BRANCH_NAME="${{ github.ref_name }}"
# Fallback to parsing ref if ref_name is empty
if [[ -z "$BRANCH_NAME" ]] && [[ "${{ github.ref }}" =~ ^refs/heads/(.+)$ ]]; then
BRANCH_NAME="${BASH_REMATCH[1]}"
fi
# Sanitize branch name for Docker tag
SAFE_BRANCH_NAME=$(echo "$BRANCH_NAME" | tr '/' '-' | tr -cd '[:alnum:]-_')
if [[ -z "$SAFE_BRANCH_NAME" ]]; then
echo "Error: Could not determine valid branch name"
exit 1
fi
{
echo "release_type=branch"
echo "n8n_version=branch-${SAFE_BRANCH_NAME}"
echo "push_enabled=${{ inputs.push_enabled }}"
} >> "$GITHUB_OUTPUT"
elif [[ "${{ github.event_name }}" == "pull_request" ]]; then
# Direct PR triggers for testing Dockerfile changes
{
echo "release_type=dev"
echo "n8n_version=pr-${{ github.event.pull_request.number }}"
echo "push_enabled=false"
} >> "$GITHUB_OUTPUT"
fi
# Output summary for logs
echo "=== Build Context Summary ==="
echo "Release type: $(grep release_type "$GITHUB_OUTPUT" | cut -d= -f2)"
echo "N8N version: $(grep n8n_version "$GITHUB_OUTPUT" | cut -d= -f2)"
echo "Push enabled: $(grep push_enabled "$GITHUB_OUTPUT" | cut -d= -f2)"
- name: Determine build matrix
id: matrix
run: |
RELEASE_TYPE="${{ steps.context.outputs.release_type }}"
# Branch builds only need AMD64, everything else needs both platforms
if [[ "$RELEASE_TYPE" == "branch" ]]; then
MATRIX='{
"platform": ["amd64"],
"include": [{
"platform": "amd64",
"runner": "blacksmith-4vcpu-ubuntu-2204",
"docker_platform": "linux/amd64"
}]
}'
else
# All other builds (stable, nightly, dev, PR) need both platforms
MATRIX='{
"platform": ["amd64", "arm64"],
"include": [{
"platform": "amd64",
"runner": "blacksmith-4vcpu-ubuntu-2204",
"docker_platform": "linux/amd64"
}, {
"platform": "arm64",
"runner": "blacksmith-4vcpu-ubuntu-2204-arm",
"docker_platform": "linux/arm64"
}]
}'
fi
# Output matrix as single line for GITHUB_OUTPUT
echo "matrix=$(echo "$MATRIX" | jq -c .)" >> "$GITHUB_OUTPUT"
echo "Build matrix: $(echo "$MATRIX" | jq .)"
node .github/scripts/docker/docker-config.mjs \
--event "${{ github.event_name }}" \
--pr "${{ github.event.pull_request.number }}" \
--branch "${{ github.ref_name }}" \
--version "${{ inputs.n8n_version }}" \
--release-type "${{ inputs.release_type }}" \
--push-enabled "${{ inputs.push_enabled }}"
build-and-push-docker:
name: Build App, then Build and Push Docker Image (${{ matrix.platform }})
@ -170,10 +87,10 @@ jobs:
strategy:
matrix: ${{ fromJSON(needs.determine-build-context.outputs.build_matrix) }}
outputs:
image_ref: ${{ steps.determine-tags.outputs.primary_ghcr_manifest_tag }}
primary_ghcr_manifest_tag: ${{ steps.determine-tags.outputs.primary_ghcr_manifest_tag }}
runners_primary_ghcr_manifest_tag: ${{ steps.determine-runners-tags.outputs.primary_ghcr_manifest_tag }}
runners_distroless_primary_ghcr_manifest_tag: ${{ steps.determine-runners-tags.outputs.primary_ghcr_manifest_tag_distroless }}
image_ref: ${{ steps.determine-tags.outputs.n8n_primary_tag }}
primary_ghcr_manifest_tag: ${{ steps.determine-tags.outputs.n8n_primary_tag }}
runners_primary_ghcr_manifest_tag: ${{ steps.determine-tags.outputs.runners_primary_tag }}
runners_distroless_primary_ghcr_manifest_tag: ${{ steps.determine-tags.outputs.runners_distroless_primary_tag }}
steps:
- name: Checkout code
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
@ -181,101 +98,23 @@ jobs:
fetch-depth: 0
- name: Setup and Build
uses: n8n-io/n8n/.github/actions/setup-nodejs-blacksmith@f5fbbbe0a28a886451c886cac6b49192a39b0eea # v1.104.1
uses: ./.github/actions/setup-nodejs-blacksmith
with:
build-command: pnpm build:n8n
- name: Determine Docker tags
- name: Determine Docker tags for all images
id: determine-tags
run: |
RELEASE_TYPE="${{ needs.determine-build-context.outputs.release_type }}"
N8N_VERSION_TAG="${{ needs.determine-build-context.outputs.n8n_version }}"
GHCR_BASE="ghcr.io/${{ github.repository_owner }}/n8n"
DOCKER_BASE="${{ secrets.DOCKER_USERNAME }}/n8n"
PLATFORM="${{ matrix.platform }}"
node .github/scripts/docker/docker-tags.mjs \
--all \
--version "${{ needs.determine-build-context.outputs.n8n_version }}" \
--platform "${{ matrix.docker_platform }}" \
${{ needs.determine-build-context.outputs.push_to_docker == 'true' && '--include-docker' || '' }}
GHCR_TAGS_FOR_PUSH=""
DOCKER_TAGS_FOR_PUSH=""
PRIMARY_GHCR_MANIFEST_TAG_VALUE=""
# Validate inputs
if [[ "$RELEASE_TYPE" == "stable" && -z "$N8N_VERSION_TAG" ]]; then
echo "Error: N8N_VERSION_TAG is empty for a stable release."
exit 1
fi
if [[ "$RELEASE_TYPE" == "branch" && -z "$N8N_VERSION_TAG" ]]; then
echo "Error: N8N_VERSION_TAG is empty for a branch release."
exit 1
fi
# Determine tags based on release type
case "$RELEASE_TYPE" in
"stable")
PRIMARY_GHCR_MANIFEST_TAG_VALUE="${GHCR_BASE}:${N8N_VERSION_TAG}"
GHCR_TAGS_FOR_PUSH="${PRIMARY_GHCR_MANIFEST_TAG_VALUE}-${PLATFORM}"
DOCKER_TAGS_FOR_PUSH="${DOCKER_BASE}:${N8N_VERSION_TAG}-${PLATFORM}"
;;
"nightly")
PRIMARY_GHCR_MANIFEST_TAG_VALUE="${GHCR_BASE}:nightly"
GHCR_TAGS_FOR_PUSH="${PRIMARY_GHCR_MANIFEST_TAG_VALUE}-${PLATFORM}"
DOCKER_TAGS_FOR_PUSH="${DOCKER_BASE}:nightly-${PLATFORM}"
;;
"branch")
PRIMARY_GHCR_MANIFEST_TAG_VALUE="${GHCR_BASE}:${N8N_VERSION_TAG}"
GHCR_TAGS_FOR_PUSH="${PRIMARY_GHCR_MANIFEST_TAG_VALUE}-${PLATFORM}"
# No Docker Hub tags for branch builds
DOCKER_TAGS_FOR_PUSH=""
;;
"dev"|*)
if [[ "$N8N_VERSION_TAG" == pr-* ]]; then
# PR builds only go to GHCR
PRIMARY_GHCR_MANIFEST_TAG_VALUE="${GHCR_BASE}:${N8N_VERSION_TAG}"
GHCR_TAGS_FOR_PUSH="${PRIMARY_GHCR_MANIFEST_TAG_VALUE}-${PLATFORM}"
DOCKER_TAGS_FOR_PUSH=""
else
# Regular dev builds go to both registries
PRIMARY_GHCR_MANIFEST_TAG_VALUE="${GHCR_BASE}:dev"
GHCR_TAGS_FOR_PUSH="${PRIMARY_GHCR_MANIFEST_TAG_VALUE}-${PLATFORM}"
DOCKER_TAGS_FOR_PUSH="${DOCKER_BASE}:dev-${PLATFORM}"
fi
;;
esac
# Combine all tags
ALL_TAGS="${GHCR_TAGS_FOR_PUSH}"
if [[ -n "$DOCKER_TAGS_FOR_PUSH" ]]; then
ALL_TAGS="${ALL_TAGS}\n${DOCKER_TAGS_FOR_PUSH}"
fi
echo "Generated Tags for push: $ALL_TAGS"
{
echo "tags<<EOF"
echo -e "$ALL_TAGS"
echo "EOF"
} >> "$GITHUB_OUTPUT"
{
echo "ghcr_platform_tag=${GHCR_TAGS_FOR_PUSH}"
echo "dockerhub_platform_tag=${DOCKER_TAGS_FOR_PUSH}"
} >> "$GITHUB_OUTPUT"
# Only output manifest tags from the first platform to avoid duplicates
if [[ "$PLATFORM" == "amd64" ]]; then
echo "primary_ghcr_manifest_tag=${PRIMARY_GHCR_MANIFEST_TAG_VALUE}" >> "$GITHUB_OUTPUT"
fi
- name: Determine Docker tags (runners variants)
id: determine-runners-tags
run: |
.github/scripts/determine-runners-tags.sh \
"${{ needs.determine-build-context.outputs.release_type }}" \
"${{ needs.determine-build-context.outputs.n8n_version }}" \
"ghcr.io/${{ github.repository_owner }}/runners" \
"${{ secrets.DOCKER_USERNAME }}/runners" \
"${{ matrix.platform }}" \
"$GITHUB_OUTPUT"
echo "=== Generated Docker Tags ==="
cat "$GITHUB_OUTPUT" | grep "_tags=" | while IFS='=' read -r key value; do
echo "${key}: ${value%%,*}..." # Show first tag for brevity
done
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0
@ -289,10 +128,9 @@ jobs:
password: ${{ secrets.GITHUB_TOKEN }}
- name: Login to Docker Hub
if: needs.determine-build-context.outputs.push_enabled == 'true' && (
steps.determine-tags.outputs.dockerhub_platform_tag != '' ||
steps.determine-runners-tags.outputs.dockerhub_platform_tag != '' ||
steps.determine-runners-tags.outputs.dockerhub_platform_tag_distroless != '')
if: |
needs.determine-build-context.outputs.push_enabled == 'true' &&
needs.determine-build-context.outputs.push_to_docker == 'true'
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
with:
username: ${{ secrets.DOCKER_USERNAME }}
@ -311,7 +149,7 @@ jobs:
provenance: true
sbom: true
push: ${{ needs.determine-build-context.outputs.push_enabled == 'true' }}
tags: ${{ steps.determine-tags.outputs.tags }}
tags: ${{ steps.determine-tags.outputs.n8n_tags }}
- name: Build and push task runners Docker image (Alpine)
uses: useblacksmith/build-push-action@574eb0ee0b59c6a687ace24192f0727dfb65d6d7 # v1.2
@ -326,7 +164,7 @@ jobs:
provenance: true
sbom: true
push: ${{ needs.determine-build-context.outputs.push_enabled == 'true' }}
tags: ${{ steps.determine-runners-tags.outputs.tags }}
tags: ${{ steps.determine-tags.outputs.runners_tags }}
- name: Build and push task runners Docker image (distroless)
uses: useblacksmith/build-push-action@574eb0ee0b59c6a687ace24192f0727dfb65d6d7 # v1.2
@ -341,7 +179,7 @@ jobs:
provenance: true
sbom: true
push: ${{ needs.determine-build-context.outputs.push_enabled == 'true' }}
tags: ${{ steps.determine-runners-tags.outputs.tags_distroless }}
tags: ${{ steps.determine-tags.outputs.runners_distroless_tags }}
create_multi_arch_manifest:
name: Create Multi-Arch Manifest
@ -361,181 +199,70 @@ jobs:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Determine Docker Hub manifest tag
id: dockerhub_check
run: |
RELEASE_TYPE="${{ needs.determine-build-context.outputs.release_type }}"
N8N_VERSION="${{ needs.determine-build-context.outputs.n8n_version }}"
DOCKER_BASE="${{ secrets.DOCKER_USERNAME }}/n8n"
# Determine if Docker Hub manifest is needed and construct the tag
case "$RELEASE_TYPE" in
"stable")
{
echo "DOCKER_MANIFEST_TAG=${DOCKER_BASE}:${N8N_VERSION}"
echo "CREATE_DOCKERHUB_MANIFEST=true"
} >> "$GITHUB_OUTPUT"
;;
"nightly")
{
echo "DOCKER_MANIFEST_TAG=${DOCKER_BASE}:nightly"
echo "CREATE_DOCKERHUB_MANIFEST=true"
} >> "$GITHUB_OUTPUT"
;;
"dev")
if [[ "$N8N_VERSION" != pr-* ]]; then
{
echo "DOCKER_MANIFEST_TAG=${DOCKER_BASE}:dev"
echo "CREATE_DOCKERHUB_MANIFEST=true"
} >> "$GITHUB_OUTPUT"
else
echo "CREATE_DOCKERHUB_MANIFEST=false" >> "$GITHUB_OUTPUT"
fi
;;
*)
echo "CREATE_DOCKERHUB_MANIFEST=false" >> "$GITHUB_OUTPUT"
;;
esac
- name: Determine Docker Hub manifest tags (runners variants)
id: dockerhub_runners_check
run: |
RELEASE_TYPE="${{ needs.determine-build-context.outputs.release_type }}"
N8N_VERSION="${{ needs.determine-build-context.outputs.n8n_version }}"
DOCKER_BASE="${{ secrets.DOCKER_USERNAME }}/runners"
# Generate manifest tags for both Alpine and distroless variants
for VARIANT in "" "_distroless"; do
SUFFIX="${VARIANT//_/-}" # Convert _distroless to -distroless for tag
OUTPUT_SUFFIX="${VARIANT}" # Keep underscore for output var names
case "$RELEASE_TYPE" in
"stable")
echo "DOCKER_MANIFEST_TAG${OUTPUT_SUFFIX}=${DOCKER_BASE}:${N8N_VERSION}${SUFFIX}" >> "$GITHUB_OUTPUT"
echo "CREATE_DOCKERHUB_MANIFEST${OUTPUT_SUFFIX}=true" >> "$GITHUB_OUTPUT"
;;
"nightly")
echo "DOCKER_MANIFEST_TAG${OUTPUT_SUFFIX}=${DOCKER_BASE}:nightly${SUFFIX}" >> "$GITHUB_OUTPUT"
echo "CREATE_DOCKERHUB_MANIFEST${OUTPUT_SUFFIX}=true" >> "$GITHUB_OUTPUT"
;;
"dev")
if [[ "$N8N_VERSION" != pr-* ]]; then
echo "DOCKER_MANIFEST_TAG${OUTPUT_SUFFIX}=${DOCKER_BASE}:dev${SUFFIX}" >> "$GITHUB_OUTPUT"
echo "CREATE_DOCKERHUB_MANIFEST${OUTPUT_SUFFIX}=true" >> "$GITHUB_OUTPUT"
else
echo "CREATE_DOCKERHUB_MANIFEST${OUTPUT_SUFFIX}=false" >> "$GITHUB_OUTPUT"
fi
;;
*)
echo "CREATE_DOCKERHUB_MANIFEST${OUTPUT_SUFFIX}=false" >> "$GITHUB_OUTPUT"
;;
esac
done
- name: Login to Docker Hub
if: steps.dockerhub_check.outputs.CREATE_DOCKERHUB_MANIFEST == 'true' ||
steps.dockerhub_runners_check.outputs.CREATE_DOCKERHUB_MANIFEST == 'true' ||
steps.dockerhub_runners_check.outputs.CREATE_DOCKERHUB_MANIFEST_distroless == 'true'
if: needs.determine-build-context.outputs.push_to_docker == 'true'
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Create GHCR multi-arch manifest
if: needs.build-and-push-docker.outputs.primary_ghcr_manifest_tag != ''
- name: Create GHCR multi-arch manifests
run: |
MANIFEST_TAG="${{ needs.build-and-push-docker.outputs.primary_ghcr_manifest_tag }}"
RELEASE_TYPE="${{ needs.determine-build-context.outputs.release_type }}"
echo "Creating GHCR manifest: $MANIFEST_TAG"
# Function to create manifest for an image
create_manifest() {
local IMAGE_NAME=$1
local MANIFEST_TAG=$2
# For branch builds, only AMD64 is built
if [[ "$RELEASE_TYPE" == "branch" ]]; then
docker buildx imagetools create \
--tag $MANIFEST_TAG \
${MANIFEST_TAG}-amd64
else
docker buildx imagetools create \
--tag $MANIFEST_TAG \
${MANIFEST_TAG}-amd64 \
${MANIFEST_TAG}-arm64
fi
if [[ -z "$MANIFEST_TAG" ]]; then
echo "Skipping $IMAGE_NAME - no manifest tag"
return
fi
- name: Create GHCR multi-arch manifest for runners (Alpine)
if: needs.build-and-push-docker.outputs.runners_primary_ghcr_manifest_tag != ''
echo "Creating GHCR manifest for $IMAGE_NAME: $MANIFEST_TAG"
# For branch builds, only AMD64 is built
if [[ "$RELEASE_TYPE" == "branch" ]]; then
docker buildx imagetools create \
--tag "$MANIFEST_TAG" \
"${MANIFEST_TAG}-amd64"
else
docker buildx imagetools create \
--tag "$MANIFEST_TAG" \
"${MANIFEST_TAG}-amd64" \
"${MANIFEST_TAG}-arm64"
fi
}
# Create manifests for all images
create_manifest "n8n" "${{ needs.build-and-push-docker.outputs.primary_ghcr_manifest_tag }}"
create_manifest "runners" "${{ needs.build-and-push-docker.outputs.runners_primary_ghcr_manifest_tag }}"
create_manifest "runners-distroless" "${{ needs.build-and-push-docker.outputs.runners_distroless_primary_ghcr_manifest_tag }}"
- name: Create Docker Hub manifests
if: needs.determine-build-context.outputs.push_to_docker == 'true'
run: |
MANIFEST_TAG="${{ needs.build-and-push-docker.outputs.runners_primary_ghcr_manifest_tag }}"
RELEASE_TYPE="${{ needs.determine-build-context.outputs.release_type }}"
VERSION="${{ needs.determine-build-context.outputs.n8n_version }}"
DOCKER_BASE="${{ secrets.DOCKER_USERNAME }}"
echo "Creating GHCR runners manifest: $MANIFEST_TAG"
# Create manifests for each image type
declare -A images=(
["n8n"]="${VERSION}"
["runners"]="${VERSION}"
["runners-distroless"]="${VERSION}-distroless"
)
# For branch builds, only AMD64 is built
if [[ "$RELEASE_TYPE" == "branch" ]]; then
for image in "${!images[@]}"; do
TAG_SUFFIX="${images[$image]}"
IMAGE_NAME="${image//-distroless/}" # Remove -distroless from image name
echo "Creating Docker Hub manifest for $image"
docker buildx imagetools create \
--tag $MANIFEST_TAG \
${MANIFEST_TAG}-amd64
else
docker buildx imagetools create \
--tag $MANIFEST_TAG \
${MANIFEST_TAG}-amd64 \
${MANIFEST_TAG}-arm64
fi
- name: Create GHCR multi-arch manifest for runners (distroless)
if: needs.build-and-push-docker.outputs.runners_distroless_primary_ghcr_manifest_tag != ''
run: |
MANIFEST_TAG="${{ needs.build-and-push-docker.outputs.runners_distroless_primary_ghcr_manifest_tag }}"
RELEASE_TYPE="${{ needs.determine-build-context.outputs.release_type }}"
echo "Creating GHCR runners distroless manifest: $MANIFEST_TAG"
# For branch builds, only AMD64 is built
if [[ "$RELEASE_TYPE" == "branch" ]]; then
docker buildx imagetools create \
--tag $MANIFEST_TAG \
${MANIFEST_TAG}-amd64
else
docker buildx imagetools create \
--tag $MANIFEST_TAG \
${MANIFEST_TAG}-amd64 \
${MANIFEST_TAG}-arm64
fi
- name: Create Docker Hub multi-arch manifest
if: steps.dockerhub_check.outputs.CREATE_DOCKERHUB_MANIFEST == 'true'
run: |
MANIFEST_TAG="${{ steps.dockerhub_check.outputs.DOCKER_MANIFEST_TAG }}"
echo "Creating Docker Hub manifest: $MANIFEST_TAG"
docker buildx imagetools create \
--tag $MANIFEST_TAG \
${MANIFEST_TAG}-amd64 \
${MANIFEST_TAG}-arm64
- name: Create Docker Hub multi-arch manifest for runners (Alpine)
if: steps.dockerhub_runners_check.outputs.CREATE_DOCKERHUB_MANIFEST == 'true'
run: |
MANIFEST_TAG="${{ steps.dockerhub_runners_check.outputs.DOCKER_MANIFEST_TAG }}"
echo "Creating Docker Hub manifest: $MANIFEST_TAG"
docker buildx imagetools create \
--tag $MANIFEST_TAG \
${MANIFEST_TAG}-amd64 \
${MANIFEST_TAG}-arm64
- name: Create Docker Hub multi-arch manifest for runners (distroless)
if: steps.dockerhub_runners_check.outputs.CREATE_DOCKERHUB_MANIFEST_distroless == 'true'
run: |
MANIFEST_TAG="${{ steps.dockerhub_runners_check.outputs.DOCKER_MANIFEST_TAG_distroless }}"
echo "Creating Docker Hub distroless manifest: $MANIFEST_TAG"
docker buildx imagetools create \
--tag $MANIFEST_TAG \
${MANIFEST_TAG}-amd64 \
${MANIFEST_TAG}-arm64
--tag "${DOCKER_BASE}/${IMAGE_NAME}:${TAG_SUFFIX}" \
"${DOCKER_BASE}/${IMAGE_NAME}:${TAG_SUFFIX}-amd64" \
"${DOCKER_BASE}/${IMAGE_NAME}:${TAG_SUFFIX}-arm64"
done
call-success-url:
name: Call Success URL
@ -566,7 +293,7 @@ jobs:
security-scan-runners:
name: Security Scan (runners)
needs: [determine-build-context, build-and-push-docker]
needs: [determine-build-context, build-and-push-docker, create_multi_arch_manifest]
if: |
success() &&
(needs.determine-build-context.outputs.release_type == 'stable' ||

View File

@ -7,7 +7,7 @@ on:
description: GitHub ref to lint.
required: false
type: string
default: master
default: ''
nodeVersion:
description: Version of node to use.
required: false
@ -22,7 +22,7 @@ jobs:
name: Lint
runs-on: blacksmith-4vcpu-ubuntu-2204
steps:
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
ref: ${{ inputs.ref }}

View File

@ -7,7 +7,7 @@ on:
description: GitHub ref to test.
required: false
type: string
default: master
default: ''
nodeVersion:
description: Version of node to use.
required: false
@ -30,7 +30,7 @@ jobs:
name: Backend Unit Tests
runs-on: blacksmith-4vcpu-ubuntu-2204
env:
COVERAGE_ENABLED: ${{ inputs.collectCoverage }} # Coverage collected when true
COVERAGE_ENABLED: ${{ inputs.collectCoverage }} # Coverage collected when true
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
@ -52,6 +52,7 @@ jobs:
name: backend-unit
- name: Upload coverage to Codecov
if: env.COVERAGE_ENABLED == 'true'
uses: codecov/codecov-action@18283e04ce6e62d37312384ff67231eb8fd56d24 # v5.4.3
with:
token: ${{ secrets.CODECOV_TOKEN }}
@ -62,7 +63,7 @@ jobs:
name: Backend Integration Tests
runs-on: blacksmith-4vcpu-ubuntu-2204
env:
COVERAGE_ENABLED: ${{ inputs.collectCoverage }} # Coverage collected when true
COVERAGE_ENABLED: ${{ inputs.collectCoverage }} # Coverage collected when true
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
@ -84,12 +85,46 @@ jobs:
name: backend-integration
- name: Upload coverage to Codecov
if: env.COVERAGE_ENABLED == 'true'
uses: codecov/codecov-action@18283e04ce6e62d37312384ff67231eb8fd56d24 # v5.4.3
with:
token: ${{ secrets.CODECOV_TOKEN }}
flags: backend-integration
name: backend-integration
unit-test-nodes:
name: Nodes Unit Tests
runs-on: blacksmith-4vcpu-ubuntu-2204
env:
COVERAGE_ENABLED: ${{ inputs.collectCoverage }} # Coverage collected when true
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
ref: ${{ inputs.ref }}
- name: Build
uses: n8n-io/n8n/.github/actions/setup-nodejs-blacksmith@f5fbbbe0a28a886451c886cac6b49192a39b0eea # v1.104.1
with:
node-version: ${{ inputs.nodeVersion }}
- name: Test Nodes
run: pnpm --filter=n8n-nodes-base test
- name: Upload test results to Codecov
if: ${{ !cancelled() }}
uses: codecov/test-results-action@47f89e9acb64b76debcd5ea40642d25a4adced9f # v1.1.1
with:
token: ${{ secrets.CODECOV_TOKEN }}
name: nodes-unit
- name: Upload coverage to Codecov
if: env.COVERAGE_ENABLED == 'true'
uses: codecov/codecov-action@18283e04ce6e62d37312384ff67231eb8fd56d24 # v5.4.3
with:
token: ${{ secrets.CODECOV_TOKEN }}
flags: nodes-unit
name: nodes-unit
unit-test-frontend:
name: Frontend (${{ matrix.shard }}/2)
runs-on: blacksmith-4vcpu-ubuntu-2204
@ -122,6 +157,7 @@ jobs:
name: frontend-shard-${{ matrix.shard }}
- name: Upload coverage to Codecov
if: env.COVERAGE_ENABLED == 'true'
uses: codecov/codecov-action@18283e04ce6e62d37312384ff67231eb8fd56d24 # v5.4.3
with:
token: ${{ secrets.CODECOV_TOKEN }}
@ -131,9 +167,9 @@ jobs:
unit-test:
name: Unit tests
runs-on: ubuntu-latest
needs: [unit-test-backend, integration-test-backend, unit-test-frontend]
needs: [unit-test-backend, integration-test-backend, unit-test-nodes, unit-test-frontend]
if: always()
steps:
- name: Fail if tests failed
if: needs.unit-test-backend.result == 'failure' || needs.integration-test-backend.result == 'failure' || needs.unit-test-frontend.result == 'failure'
if: needs.unit-test-backend.result == 'failure' || needs.integration-test-backend.result == 'failure' || needs.unit-test-nodes.result == 'failure' || needs.unit-test-frontend.result == 'failure'
run: exit 1

View File

@ -0,0 +1,60 @@
name: Update Node Popularity Data
on:
schedule:
# Run every Monday at 00:00 UTC
- cron: '0 0 * * 1'
workflow_dispatch: # Allow manual trigger for testing
permissions:
contents: write
pull-requests: write
jobs:
update-popularity:
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Setup Node.js and Dependencies
uses: ./.github/actions/setup-nodejs-github
with:
build-command: '' # Skip build, we only need to fetch data
- name: Fetch node popularity data
run: |
cd packages/frontend/editor-ui
node scripts/fetch-node-popularity.mjs
env:
N8N_FAIL_ON_POPULARITY_FETCH_ERROR: 'false' # Don't fail if API is down
- name: Check for changes
id: check-changes
run: |
if git diff --quiet packages/frontend/editor-ui/data/node-popularity.json; then
echo "No changes to popularity data"
echo "has_changes=false" >> "$GITHUB_OUTPUT"
else
echo "Popularity data has changed"
echo "has_changes=true" >> "$GITHUB_OUTPUT"
fi
- name: Create Pull Request
if: steps.check-changes.outputs.has_changes == 'true'
uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8
with:
branch-token: ${{ secrets.GITHUB_TOKEN }}
commit-message: 'chore: Update node popularity data'
title: 'chore: Update node popularity data'
body: |
This automated PR updates the node popularity data used for sorting nodes in the node creator panel.
The data is fetched weekly from the n8n telemetry endpoint to reflect current usage patterns.
_Generated by the weekly node popularity update workflow._
branch: update-node-popularity
base: master
delete-branch: true
author: n8n Bot <18347049+n8n-bot@users.noreply.github.com>
committer: n8n Bot <18347049+n8n-bot@users.noreply.github.com>

View File

@ -11,15 +11,6 @@ WORKDIR /app/task-runner-javascript
RUN corepack enable pnpm
# Install extra runtime-only npm packages. Allow usage in the Code node via
# 'NODE_FUNCTION_ALLOW_EXTERNAL' env variable on n8n-task-runners.json.
RUN rm -f node_modules/.modules.yaml
RUN mv package.json package.json.bak
COPY docker/images/runners/package.json /app/task-runner-javascript/package.json
RUN pnpm install --prod --no-lockfile --silent
RUN mv package.json extras.json
RUN mv package.json.bak package.json
# Remove `catalog` and `workspace` references from package.json to allow `pnpm add` in extended images
RUN node -e "const pkg = require('./package.json'); \
Object.keys(pkg.dependencies || {}).forEach(k => { \
@ -79,11 +70,6 @@ RUN uv sync \
--all-extras \
--no-editable
# Install extra runtime-only Python packages. Allow usage in the Code node via
# 'N8N_RUNNERS_EXTERNAL_ALLOW' env variable on n8n-task-runners.json.
COPY docker/images/runners/extras.txt /app/task-runner-python/extras.txt
RUN uv pip install -r /app/task-runner-python/extras.txt
# ==============================================================================
# STAGE 3: Task Runner Launcher download
# ==============================================================================

View File

@ -26,14 +26,24 @@ WORKDIR /app/task-runner-javascript
RUN corepack enable pnpm
# Install extra runtime-only npm packages. Allow usage in the Code node via
# 'NODE_FUNCTION_ALLOW_EXTERNAL' env variable on n8n-task-runners.json.
RUN rm -f node_modules/.modules.yaml
RUN mv package.json package.json.bak
COPY docker/images/runners/package.json /app/task-runner-javascript/package.json
RUN pnpm install --prod --no-lockfile --silent
RUN mv package.json extras.json
RUN mv package.json.bak package.json
# Remove `catalog` and `workspace` references from package.json to allow `pnpm add`
RUN node -e "const pkg = require('./package.json'); \
Object.keys(pkg.dependencies || {}).forEach(k => { \
const val = pkg.dependencies[k]; \
if (val === 'catalog:' || val.startsWith('catalog:') || val.startsWith('workspace:')) \
delete pkg.dependencies[k]; \
}); \
Object.keys(pkg.devDependencies || {}).forEach(k => { \
const val = pkg.devDependencies[k]; \
if (val === 'catalog:' || val.startsWith('catalog:') || val.startsWith('workspace:')) \
delete pkg.devDependencies[k]; \
}); \
delete pkg.devDependencies; \
require('fs').writeFileSync('./package.json', JSON.stringify(pkg, null, 2));"
# Install moment by default (special case for n8n cloud)
RUN rm -f node_modules/.modules.yaml && \
pnpm add moment@2.30.1 --prod --no-lockfile
# ==============================================================================
# STAGE 2: Python runner build (@n8n/task-runner-python) with uv
@ -81,11 +91,6 @@ RUN uv sync \
--all-extras \
--no-editable
# Install extra runtime-only Python packages. Allow usage in the Code node via
# 'N8N_RUNNERS_EXTERNAL_ALLOW' env variable on n8n-task-runners.json.
COPY docker/images/runners/extras.txt /app/task-runner-python/extras.txt
RUN uv pip install -r /app/task-runner-python/extras.txt
# ==============================================================================
# STAGE 3: Task Runner Launcher download
# ==============================================================================

View File

@ -1,5 +0,0 @@
# Runtime-only extra Requirements File for installing dependencies in the Python task runner image.
# Installed at Docker image build time. Allow usage in the Code node
# via 'N8N_RUNNERS_EXTERNAL_ALLOW' env variable on n8n-task-runners.json.
# numpy==2.3.2

View File

@ -1,8 +0,0 @@
{
"name": "task-runner-runtime-extras",
"description": "Runtime-only extra dependencies for installing packages in the JavaScript task runner image. Installed at Docker image build time. Allow usage in the Code node via 'NODE_FUNCTION_ALLOW_EXTERNAL' env variable on n8n-task-runners.json.",
"private": true,
"dependencies": {
"moment": "2.30.1"
}
}

View File

@ -4,9 +4,9 @@
"private": true,
"engines": {
"node": ">=22.16",
"pnpm": ">=10.18.3"
"pnpm": ">=10.22.0"
},
"packageManager": "pnpm@10.18.3",
"packageManager": "pnpm@10.22.0",
"scripts": {
"prepare": "node scripts/prepare.mjs",
"preinstall": "node scripts/block-npm-install.js",
@ -88,6 +88,7 @@
"sqlite3"
],
"overrides": {
"ast-types": "0.16.1",
"@azure/identity": "^4.3.0",
"@lezer/common": "^1.2.0",
"@mistralai/mistralai": "^1.10.0",

View File

@ -98,37 +98,37 @@ export const basicTestCases: TestCase[] = [
id: 'multi-agent-research',
name: 'Multi-agent research workflow',
prompt:
'Create a multi-agent AI workflow using GPT-4.1-mini where several agents work together to research a topic, fact-check the findings, and write a report that\'s sent as an HTML email. One agent should gather recent, credible information about the topic. Another agent should verify the facts and only mark something as "verified" if it appears in at least two independent sources. A third agent should combine the verified information into a clear, well-written report under 1,000 words. A final agent should edit and format the report to make it look clean and professional in the body of the email. Use Gmail to send the report.',
'Create a multi-agent AI workflow using `gpt-4.1-mini` where several agents work together to research a topic, fact-check the findings, and write a report that\'s sent as an HTML email. One agent should gather recent, credible information about the topic. Another agent should verify the facts and only mark something as "verified" if it appears in at least two independent sources. A third agent should combine the verified information into a clear, well-written report under 1,000 words. A final agent should edit and format the report to make it look clean and professional in the body of the email. Use Gmail to send the report.',
},
{
id: 'email-summary',
name: 'Summarize emails with AI',
prompt:
'Create an automation that runs on Monday mornings. It reads my Gmail inbox from the weekend, analyzes them with GPT-4.1-mini to find action items and priorities, and emails me a structured email using Gmail.',
'Create an automation that runs on Monday mornings. It reads my Gmail inbox from the weekend, analyzes them with `gpt-4.1-mini` to find action items and priorities, and emails me a structured email using Gmail.',
},
{
id: 'ai-news-digest',
name: 'Daily AI news digest',
prompt:
'Build an automation that runs every night 8pm. Use the NewsAPI "/everything" endpoint to search for AI-related news from the day. Pick the top 5 articles and use OpenAI GPT-4.1-mini to summarize each in two sentences. Generate an image using OpenAI based on the top article\'s summary. Send a structured Telegram message.',
'Build an automation that runs every night 8pm. Use the NewsAPI "/everything" endpoint to search for AI-related news from the day. Pick the top 5 articles and use OpenAI `gpt-4.1-mini` to summarize each in two sentences. Generate an image using OpenAI based on the top article\'s summary. Send a structured Telegram message.',
},
{
id: 'daily-weather-report',
name: 'Daily weather report',
prompt:
'Create an automation that checks the weather for my location every morning at 5 a.m using OpenWeather. Send me a short weather report by email using Gmail. Use OpenAI GPT-4.1-mini to write a short, fun formatted email body by adding personality when describing the weather and how the day might feel. Include all details relevant to decide on my plans and clothes for the day.',
'Create an automation that checks the weather for my location every morning at 5 a.m using OpenWeather. Send me a short weather report by email using Gmail. Use OpenAI `gpt-4.1-mini` to write a short, fun formatted email body by adding personality when describing the weather and how the day might feel. Include all details relevant to decide on my plans and clothes for the day.',
},
{
id: 'invoice-pipeline',
name: 'Invoice processing pipeline',
prompt:
'Create an invoice processing workflow using an n8n Form. When a user submits an invoice file (PDF or image) with their email address, use OpenAI GPT-4.1-mini to extract invoice data. Then, validate the date format is correct, the currency is valid, and the total amount is greater than zero. If validation fails, email the user a clear error message that explains which check failed from my Gmail. If the data passes validation, store the structured result in a datatable plus email the user. Every Monday morning, generate a weekly spending report using GPT-4.1-mini based on stored invoices and send a clean email using Gmail.',
'Create an invoice processing workflow using an n8n Form. When a user submits an invoice file (PDF or image) with their email address, use OpenAI `gpt-4.1-mini` to extract invoice data. Then, validate the date format is correct, the currency is valid, and the total amount is greater than zero. If validation fails, email the user a clear error message that explains which check failed from my Gmail. If the data passes validation, store the structured result in a datatable plus email the user. Every Monday morning, generate a weekly spending report using `gpt-4.1-mini` based on stored invoices and send a clean email using Gmail.',
},
{
id: 'rag-assistant',
name: 'RAG knowledge assistant',
prompt:
'Build an automation that creates a document-to-chat RAG pipeline. The workflow starts with an n8n Form where a user uploads one or more files (PDF, CSV, or JSON). Each upload should trigger a process that reads the file, splits it into chunks, and generates embeddings using OpenAI GPT-4.1-mini model, saved in one Pinecone table. Add a second part of the workflow for querying: use a Chat Message Trigger to act as a chatbot interface. When a user sends a question, retrieve the top 5 most relevant chunks from Pinecone, pass them into GPT-4.1-mini as context, and have it answer naturally using only the retrieved information. If a question can\'t be answered confidently, the bot should respond with: "I couldn\'t find that in the uploaded documents." Log each chat interaction in a Data Table with the user query, matched file(s), and timestamp. Send a daily summary email through Gmail showing total questions asked, top files referenced, and any failed lookups.',
'Build an automation that creates a document-to-chat RAG pipeline. The workflow starts with an n8n Form where a user uploads one or more files (PDF, CSV, or JSON). Each upload should trigger a process that reads the file, splits it into chunks, and generates embeddings using OpenAI `gpt-4.1-mini` model, saved in one Pinecone table. Add a second part of the workflow for querying: use a Chat Message Trigger to act as a chatbot interface. When a user sends a question, retrieve the top 5 most relevant chunks from Pinecone, pass them into `gpt-4.1-mini` as context, and have it answer naturally using only the retrieved information. If a question can\'t be answered confidently, the bot should respond with: "I couldn\'t find that in the uploaded documents." Log each chat interaction in a Data Table with the user query, matched file(s), and timestamp. Send a daily summary email through Gmail showing total questions asked, top files referenced, and any failed lookups.',
},
{
id: 'lead-qualification',

View File

@ -69,6 +69,7 @@ describe('evaluateAgentPrompt', () => {
expect(result.violations).toHaveLength(1);
expect(result.violations[0]).toEqual({
type: 'major',
name: expect.any(String),
description:
'Agent node "AI Agent" has no expression in its prompt field. This likely means it failed to use chatInput or dynamic context',
pointsDeducted: 20,

View File

@ -1,11 +1,13 @@
import { ChatAnthropic } from '@langchain/anthropic';
import { AIMessage, ToolMessage } from '@langchain/core/messages';
import type { BaseMessage } from '@langchain/core/messages';
import { LangChainTracer } from '@langchain/core/tracers/tracer_langchain';
import { Logger } from '@n8n/backend-common';
import { Service } from '@n8n/di';
import { AiAssistantClient, AiAssistantSDK } from '@n8n_io/ai-assistant-sdk';
import assert from 'assert';
import { Client as TracingClient } from 'langsmith';
import type { IUser, INodeTypeDescription } from 'n8n-workflow';
import type { IUser, INodeTypeDescription, ITelemetryTrackProperties } from 'n8n-workflow';
import { LLMServiceError } from '@/errors';
import { anthropicClaudeSonnet45 } from '@/llm-config';
@ -14,6 +16,8 @@ import { WorkflowBuilderAgent, type ChatPayload } from '@/workflow-builder-agent
type OnCreditsUpdated = (userId: string, creditsQuota: number, creditsClaimed: number) => void;
type OnTelemetryEvent = (event: string, properties: ITelemetryTrackProperties) => void;
@Service()
export class AiWorkflowBuilderService {
private readonly parsedNodeTypes: INodeTypeDescription[];
@ -23,8 +27,10 @@ export class AiWorkflowBuilderService {
parsedNodeTypes: INodeTypeDescription[],
private readonly client?: AiAssistantClient,
private readonly logger?: Logger,
private readonly instanceId?: string,
private readonly instanceUrl?: string,
private readonly onCreditsUpdated?: OnCreditsUpdated,
private readonly onTelemetryEvent?: OnTelemetryEvent,
) {
this.parsedNodeTypes = this.filterNodeTypes(parsedNodeTypes);
this.sessionManager = new SessionManagerService(this.parsedNodeTypes, logger);
@ -44,6 +50,7 @@ export class AiWorkflowBuilderService {
apiKey,
headers: {
...authHeaders,
// eslint-disable-next-line @typescript-eslint/naming-convention
'anthropic-beta': 'prompt-caching-2024-07-31',
},
});
@ -54,6 +61,7 @@ export class AiWorkflowBuilderService {
const authResponse = await this.client.getBuilderApiProxyToken(user);
const authHeaders = {
// eslint-disable-next-line @typescript-eslint/naming-convention
Authorization: `${authResponse.tokenType} ${authResponse.accessToken}`,
};
@ -63,6 +71,7 @@ export class AiWorkflowBuilderService {
private async setupModels(user: IUser): Promise<{
anthropicClaude: ChatAnthropic;
tracingClient?: TracingClient;
// eslint-disable-next-line @typescript-eslint/naming-convention
authHeaders?: { Authorization: string };
}> {
try {
@ -168,6 +177,7 @@ export class AiWorkflowBuilderService {
private async onGenerationSuccess(
user?: IUser,
// eslint-disable-next-line @typescript-eslint/naming-convention
authHeaders?: { Authorization: string },
): Promise<void> {
try {
@ -190,10 +200,66 @@ export class AiWorkflowBuilderService {
async *chat(payload: ChatPayload, user: IUser, abortSignal?: AbortSignal) {
const agent = await this.getAgent(user);
const userId = user?.id?.toString();
const workflowId = payload.workflowContext?.currentWorkflow?.id;
for await (const output of agent.chat(payload, user?.id?.toString(), abortSignal)) {
for await (const output of agent.chat(payload, userId, abortSignal)) {
yield output;
}
// After the stream completes, track telemetry
if (this.onTelemetryEvent && userId) {
try {
await this.trackBuilderReplyTelemetry(agent, workflowId, userId);
} catch (error) {
this.logger?.error('Failed to track builder reply telemetry', { error });
}
}
}
private async trackBuilderReplyTelemetry(
agent: WorkflowBuilderAgent,
workflowId: string | undefined,
userId: string,
): Promise<void> {
if (!this.onTelemetryEvent) return;
const state = await agent.getState(workflowId, userId);
const threadId = SessionManagerService.generateThreadId(workflowId, userId);
// extract the last message that was sent to the user for telemetry
const lastAiMessage = state.values.messages.findLast(
(m: BaseMessage): m is AIMessage => m instanceof AIMessage,
);
const messageAi =
typeof lastAiMessage?.content === 'string'
? lastAiMessage.content
: JSON.stringify(lastAiMessage?.content ?? '');
const toolMessages = state.values.messages.filter(
(m: BaseMessage): m is ToolMessage => m instanceof ToolMessage,
);
const toolsCalled = [
...new Set(
toolMessages
.map((m: ToolMessage) => m.name)
.filter((name: string | undefined): name is string => name !== undefined),
),
];
// Build telemetry properties
const properties: ITelemetryTrackProperties = {
user_id: userId,
instance_id: this.instanceId,
workflow_id: workflowId,
sequence_id: threadId,
message_ai: messageAi,
tools_called: toolsCalled,
techniques_categories: state.values.techniqueCategories,
validations: state.values.validationHistory,
};
this.onTelemetryEvent('Builder replied to user message', properties);
}
async getSessions(workflowId: string | undefined, user?: IUser) {

View File

@ -188,6 +188,7 @@ describe('AiWorkflowBuilderService', () => {
mockNodeTypeDescriptions,
mockClient,
mockLogger,
'test-instance-id',
'https://n8n.example.com',
mockOnCreditsUpdated,
);
@ -199,6 +200,7 @@ describe('AiWorkflowBuilderService', () => {
mockNodeTypeDescriptions,
mockClient,
mockLogger,
'test-instance-id',
'https://test.com',
mockOnCreditsUpdated,
);
@ -224,6 +226,7 @@ describe('AiWorkflowBuilderService', () => {
mockNodeTypeDescriptions,
mockClient,
mockLogger,
'test-instance-id',
'https://test.com',
mockOnCreditsUpdated,
);
@ -247,6 +250,7 @@ describe('AiWorkflowBuilderService', () => {
mockNodeTypeDescriptions,
mockClient,
mockLogger,
'test-instance-id',
'https://test.com',
mockOnCreditsUpdated,
);

View File

@ -286,6 +286,8 @@ describe('WorkflowBuilderAgent', () => {
},
},
workflowValidation: null,
validationHistory: [],
techniqueCategories: [],
previousSummary: 'EMPTY',
};
};

View File

@ -1,7 +1,8 @@
import { HumanMessage, AIMessage as AssistantMessage, ToolMessage } from '@langchain/core/messages';
import type { BaseMessage } from '@langchain/core/messages';
import { createTrimMessagesReducer } from '../workflow-state';
import type { TelemetryValidationStatus } from '../validation/types';
import { createTrimMessagesReducer, WorkflowState } from '../workflow-state';
describe('createTrimMessagesReducer', () => {
it('should return messages unchanged when human messages are within limit', () => {
@ -152,3 +153,139 @@ describe('createTrimMessagesReducer', () => {
expect(result.length).toBe(2);
});
});
describe('WorkflowState.validationHistory reducer', () => {
// Helper to create TelemetryValidationStatus avoiding ESLint naming-convention warnings
const createValidationStatus = (
violations: Array<{ name: string; result: 'pass' | 'fail' }>,
): TelemetryValidationStatus => {
const status: Record<string, 'pass' | 'fail'> = {};
for (const violation of violations) {
status[violation.name] = violation.result;
}
return status as TelemetryValidationStatus;
};
it('should append new validation history to existing history', () => {
const reducer = WorkflowState.spec.validationHistory.operator;
const existingHistory: TelemetryValidationStatus[] = [
createValidationStatus([
{ name: 'tool-node-has-no-parameters', result: 'pass' },
{ name: 'agent-static-prompt', result: 'fail' },
{ name: 'workflow-has-no-nodes', result: 'pass' },
]),
createValidationStatus([
{ name: 'tool-node-has-no-parameters', result: 'fail' },
{ name: 'agent-static-prompt', result: 'pass' },
{ name: 'workflow-has-no-nodes', result: 'pass' },
]),
];
const newHistory: TelemetryValidationStatus[] = [
createValidationStatus([
{ name: 'tool-node-has-no-parameters', result: 'pass' },
{ name: 'agent-static-prompt', result: 'pass' },
{ name: 'workflow-has-no-nodes', result: 'pass' },
]),
];
const result = reducer(existingHistory, newHistory);
expect(result).toHaveLength(3);
expect(result[0]).toBe(existingHistory[0]);
expect(result[1]).toBe(existingHistory[1]);
expect(result[2]).toBe(newHistory[0]);
});
it('should handle empty existing history with new updates', () => {
const reducer = WorkflowState.spec.validationHistory.operator;
const newHistory: TelemetryValidationStatus[] = [
createValidationStatus([
{ name: 'node-missing-required-input', result: 'pass' },
{ name: 'node-merge-single-input', result: 'fail' },
]),
];
const result = reducer([], newHistory);
expect(result).toEqual(newHistory);
expect(result[0]).toBe(newHistory[0]);
});
it('should handle multiple updates sequentially', () => {
type ReducerFn = (
x: TelemetryValidationStatus[],
y: TelemetryValidationStatus[] | undefined | null,
) => TelemetryValidationStatus[];
const reducer = WorkflowState.spec.validationHistory.operator as ReducerFn;
let history: TelemetryValidationStatus[] = [];
// First update
const update1: TelemetryValidationStatus[] = [
createValidationStatus([{ name: 'tool-node-has-no-parameters', result: 'pass' }]),
];
history = reducer(history, update1);
expect(history).toHaveLength(1);
expect(history[0]).toBe(update1[0]);
// Second update (undefined - should not change)
const prevHistory = history;
history = reducer(history, undefined);
expect(history).toBe(prevHistory);
expect(history).toHaveLength(1);
// Third update
const update2: TelemetryValidationStatus[] = [
createValidationStatus([{ name: 'agent-static-prompt', result: 'fail' }]),
];
history = reducer(history, update2);
expect(history).toHaveLength(2);
expect(history[0]).toBe(update1[0]);
expect(history[1]).toBe(update2[0]);
// Fourth update (empty array - should not change)
const prevHistory2 = history;
history = reducer(history, []);
expect(history).toBe(prevHistory2);
expect(history).toHaveLength(2);
});
});
describe('WorkflowState.techniqueCategories reducer', () => {
it('should append new technique categories to existing categories', () => {
const reducer = WorkflowState.spec.techniqueCategories.operator;
const existingCategories = ['scraping', 'data-transformation'];
const newCategories = ['notifications', 'scheduling'];
const result = reducer(existingCategories, newCategories);
expect(result).toHaveLength(4);
expect(result).toEqual(['scraping', 'data-transformation', 'notifications', 'scheduling']);
});
it('should return existing categories when update is undefined', () => {
type ReducerFn = (x: string[], y: string[] | undefined | null) => string[];
const reducer = WorkflowState.spec.techniqueCategories.operator as ReducerFn;
const existingCategories = ['api-integration', 'webhook'];
const result = reducer(existingCategories, undefined);
expect(result).toEqual(existingCategories);
expect(result).toBe(existingCategories);
});
it('should handle empty existing categories with new updates', () => {
const reducer = WorkflowState.spec.techniqueCategories.operator;
const newCategories = ['email-automation', 'file-processing'];
const result = reducer([], newCategories);
expect(result).toEqual(newCategories);
expect(result[0]).toBe(newCategories[0]);
});
});

View File

@ -68,6 +68,7 @@ export function createCategorizePromptTool(llm: BaseChatModel, logger?: Logger):
return createSuccessResponse(config, buildCategorizationMessage(categorization), {
categorization,
techniqueCategories: categorization.techniques,
});
} catch (error) {
if (error instanceof z.ZodError) {

View File

@ -20,6 +20,7 @@ export function createSuccessResponse<TState = typeof WorkflowState.State>(
new ToolMessage({
content: message,
tool_call_id: toolCallId,
name: config.toolCall?.name,
}),
];
@ -42,6 +43,7 @@ export function createErrorResponse(config: ToolRunnableConfig, error: ToolError
new ToolMessage({
content: `Error: ${error.message}`,
tool_call_id: toolCallId,
name: config.toolCall?.name,
}),
];

View File

@ -37,7 +37,7 @@ Follow this proven sequence for creating robust workflows:
- Why: Best practices help to inform which nodes to search for and use to build the workflow plus mistakes to avoid
2. **Discovery Phase** (parallel execution)
- Search for all required node types simultaneously
- Search for all required node types simultaneously, review the <node_selection> section for tips and best practices
- Why: Ensures you work with actual available nodes, not assumptions
3. **Analysis Phase** (parallel execution)
@ -66,6 +66,14 @@ Follow this proven sequence for creating robust workflows:
- Review <workflow_validation_report> and resolve any violations before finalizing
- Why: Ensures structural issues are surfaced early; rerun validation after major updates
<node_selection>
When building AI workflows prefer the AI agent node to other text LLM nodes, unless the user specifies them by name. Summarization, analysis, information
extraction and classification can all be carried out by an AI agent node, correct system prompt, and structured output parser.
For the purposes of this section provider specific nodes can be described as nodes like @n8n/n8n-nodes-langchain.openAi.
Do not use provider specific nodes for text operations - instead use an AI agent node.
For generation/analysis of content other than text (images, video, audio) provider specific nodes should be used.
</node_selection>
<best_practices_compliance>
Enforcing best practice compliance is MANDATORY

View File

@ -49,6 +49,7 @@ describe('validateWorkflow tool', () => {
trigger: [],
agentPrompt: [
{
name: 'agent-static-prompt',
type: 'minor',
description: 'Agent prompt is missing required expression.',
pointsDeducted: 5,
@ -118,6 +119,7 @@ describe('validateWorkflow tool', () => {
...sampleValidationResult,
connections: [
{
name: 'node-missing-required-input',
type: 'critical',
description: 'Node HTTP Request is missing required main input.',
pointsDeducted: 50,

View File

@ -5,6 +5,12 @@ import { z } from 'zod';
import type { BuilderTool, BuilderToolBase } from '@/utils/stream-processor';
import { programmaticValidation } from '@/validation/programmatic';
import type {
ProgrammaticViolation,
ProgrammaticChecksResult,
TelemetryValidationStatus,
} from '@/validation/types';
import { PROGRAMMATIC_VIOLATION_NAMES } from '@/validation/types';
import { ToolExecutionError, ValidationError } from '../errors';
import { formatWorkflowValidation } from '../utils/workflow-validation';
@ -19,6 +25,26 @@ export const VALIDATE_WORKFLOW_TOOL: BuilderToolBase = {
displayTitle: 'Validating workflow',
};
/**
* Creates a compacted validation result for use in telemetry
* @returns `{ X: 'pass' | 'fail', Y: 'pass' | 'fail', ... }`
*/
function collectValidationResultForTelemetry(
results: ProgrammaticChecksResult,
): TelemetryValidationStatus {
const status = Object.fromEntries(
PROGRAMMATIC_VIOLATION_NAMES.map((name) => [name, 'pass' as const]),
) as TelemetryValidationStatus;
Object.values(results).forEach((violations: ProgrammaticViolation[]) => {
violations?.forEach((violation) => {
status[violation.name] = 'fail';
});
});
return status;
}
export function createValidateWorkflowTool(
parsedNodeTypes: INodeTypeDescription[],
logger?: Logger,
@ -45,12 +71,15 @@ export function createValidateWorkflowTool(
parsedNodeTypes,
);
const validationResultForTelemetry = collectValidationResultForTelemetry(violations);
const message = formatWorkflowValidation(violations);
reporter.complete({ message });
return createSuccessResponse(config, message, {
workflowValidation: violations,
validationHistory: [validationResultForTelemetry],
});
} catch (error) {
if (error instanceof z.ZodError) {

View File

@ -593,6 +593,8 @@ describe('operations-processor', () => {
messages: [],
workflowContext: {},
workflowValidation: null,
validationHistory: [],
techniqueCategories: [],
previousSummary: 'EMPTY',
});

View File

@ -49,6 +49,8 @@ describe('tool-executor', () => {
messages,
workflowContext: {},
workflowValidation: null,
validationHistory: [],
techniqueCategories: [],
previousSummary: 'EMPTY',
});
@ -705,5 +707,114 @@ describe('tool-executor', () => {
expect(result.messages).toContain(toolResultMessage);
expect(result.workflowOperations).toHaveLength(1);
});
it('should collect validationHistory from tool state updates', async () => {
const validation1 = {
result: 'success',
checks: { total: 5, passed: 5, failed: 0 },
};
const validation2 = {
result: 'warning',
checks: { total: 3, passed: 2, failed: 1 },
};
const command1 = new MockCommand({
update: {
messages: [new ToolMessage({ content: 'Validation 1', tool_call_id: 'call-1' })],
validationHistory: [validation1],
},
});
const command2 = new MockCommand({
update: {
messages: [new ToolMessage({ content: 'Validation 2', tool_call_id: 'call-2' })],
validationHistory: [validation2],
},
});
const mockTool1 = createMockTool(command1);
const mockTool2 = createMockTool(command2);
const aiMessage = new AIMessage('');
aiMessage.tool_calls = [
{
id: 'call-1',
name: 'validate_tool_1',
args: {},
type: 'tool_call',
},
{
id: 'call-2',
name: 'validate_tool_2',
args: {},
type: 'tool_call',
},
];
const state = createState([aiMessage]);
const toolMap = new Map<string, DynamicStructuredTool>([
['validate_tool_1', mockTool1],
['validate_tool_2', mockTool2],
]);
const options: ToolExecutorOptions = { state, toolMap };
const result = await executeToolsInParallel(options);
expect(result.validationHistory).toBeDefined();
expect(result.validationHistory).toHaveLength(2);
expect(result.validationHistory).toContain(validation1);
expect(result.validationHistory).toContain(validation2);
});
it('should collect techniqueCategories from tool state updates', async () => {
const categories1 = ['scraping', 'data-transformation'];
const categories2 = ['notifications', 'scheduling'];
const command1 = new MockCommand({
update: {
messages: [new ToolMessage({ content: 'Categorized', tool_call_id: 'call-1' })],
techniqueCategories: categories1,
},
});
const command2 = new MockCommand({
update: {
messages: [new ToolMessage({ content: 'Categorized', tool_call_id: 'call-2' })],
techniqueCategories: categories2,
},
});
const mockTool1 = createMockTool(command1);
const mockTool2 = createMockTool(command2);
const aiMessage = new AIMessage('');
aiMessage.tool_calls = [
{
id: 'call-1',
name: 'categorize_tool_1',
args: {},
type: 'tool_call',
},
{
id: 'call-2',
name: 'categorize_tool_2',
args: {},
type: 'tool_call',
},
];
const state = createState([aiMessage]);
const toolMap = new Map<string, DynamicStructuredTool>([
['categorize_tool_1', mockTool1],
['categorize_tool_2', mockTool2],
]);
const options: ToolExecutorOptions = { state, toolMap };
const result = await executeToolsInParallel(options);
expect(result.techniqueCategories).toBeDefined();
expect(result.techniqueCategories).toHaveLength(4);
expect(result.techniqueCategories).toEqual([...categories1, ...categories2]);
});
});
});

View File

@ -88,6 +88,7 @@ export async function executeToolsInParallel(
return new ToolMessage({
content: errorContent,
tool_call_id: toolCall.id ?? '',
name: toolCall.name,
// Include error flag so tools can handle errors appropriately
additional_kwargs: { error: true },
});
@ -128,6 +129,24 @@ export async function executeToolsInParallel(
}
}
// Collect all technique categories
const allTechniqueCategories: string[] = [];
for (const update of stateUpdates) {
if (update.techniqueCategories && Array.isArray(update.techniqueCategories)) {
allTechniqueCategories.push(...update.techniqueCategories);
}
}
// Collect all validation history
const allValidationHistory: Array<(typeof WorkflowState.State.validationHistory)[number]> = [];
for (const update of stateUpdates) {
if (update.validationHistory && Array.isArray(update.validationHistory)) {
allValidationHistory.push(...update.validationHistory);
}
}
// Return the combined update
const finalUpdate: Partial<typeof WorkflowState.State> = {
messages: allMessages,
@ -137,5 +156,13 @@ export async function executeToolsInParallel(
finalUpdate.workflowOperations = allOperations;
}
if (allTechniqueCategories.length > 0) {
finalUpdate.techniqueCategories = allTechniqueCategories;
}
if (allValidationHistory.length > 0) {
finalUpdate.validationHistory = allValidationHistory;
}
return finalUpdate;
}

View File

@ -55,6 +55,7 @@ export function validateAgentPrompt(workflow: SimpleWorkflow): ProgrammaticViola
// Check 1: Text parameter should contain expressions for dynamic context
if (!textParam || !containsExpression(textParam)) {
violations.push({
name: 'agent-static-prompt',
type: 'major',
description: `Agent node "${node.name}" has no expression in its prompt field. This likely means it failed to use chatInput or dynamic context`,
pointsDeducted: 20,
@ -65,6 +66,7 @@ export function validateAgentPrompt(workflow: SimpleWorkflow): ProgrammaticViola
// If systemMessage is missing, it likely means all instructions are in the text field
if (!systemMessage || systemMessage.trim().length === 0) {
violations.push({
name: 'agent-no-system-prompt',
type: 'major',
description: `Agent node "${node.name}" has no system message. System-level instructions (role, tasks, behavior) should be in the system message field, not the text field`,
pointsDeducted: 25,

View File

@ -45,6 +45,7 @@ function checkMissingRequiredInputs(
if (input.required && providedCount === 0) {
issues.push({
name: 'node-missing-required-input',
type: 'critical',
description: `Node ${nodeInfo.node.name} (${nodeInfo.node.type}) is missing required input of type ${input.type}`,
pointsDeducted: 50,
@ -67,6 +68,7 @@ function checkUnsupportedConnections(
for (const [type] of providedInputTypes) {
if (!supportedTypes.has(type)) {
issues.push({
name: 'node-unsupported-connection-input',
type: 'critical',
description: `Node ${nodeInfo.node.name} (${nodeInfo.node.type}) received unsupported connection type ${type}`,
pointsDeducted: 50,
@ -90,6 +92,7 @@ function checkMergeNodeConnections(
if (totalInputConnections < 2) {
issues.push({
name: 'node-merge-single-input',
type: 'major',
description: `Merge node ${nodeInfo.node.name} has only ${totalInputConnections} input connection(s). Merge nodes require at least 2 inputs to function properly.`,
pointsDeducted: 20,
@ -101,6 +104,7 @@ function checkMergeNodeConnections(
if (totalInputConnections !== expectedInputs) {
issues.push({
name: 'node-merge-incorrect-num-inputs',
type: 'minor',
description: `Merge node ${nodeInfo.node.name} has ${totalInputConnections} input connections but is configured to accept ${expectedInputs}.`,
pointsDeducted: 10,
@ -121,6 +125,7 @@ function checkMergeNodeConnections(
if (missingIndexes.length > 0) {
issues.push({
name: 'node-merge-missing-input',
type: 'major',
description: `Merge node ${nodeInfo.node.name} is missing connections for input(s) ${missingIndexes.join(', ')}.`,
pointsDeducted: 20,
@ -165,6 +170,7 @@ function checkSubNodeRootConnections(
if (!hasRootConnection) {
issues.push({
name: 'sub-node-not-connected',
type: 'critical',
description: `Sub-node ${node.name} (${node.type}) provides ${outputType} but is not connected to a root node.`,
pointsDeducted: 50,
@ -193,6 +199,7 @@ export function validateConnections(
const nodeType = nodeTypeMap.get(node.type);
if (!nodeType) {
violations.push({
name: 'node-type-not-found',
type: 'critical',
description: `Node type ${node.type} not found for node ${node.name}`,
pointsDeducted: 50,
@ -207,6 +214,7 @@ export function validateConnections(
nodeInfo.resolvedOutputs = resolveNodeOutputs(nodeInfo);
} catch (error) {
violations.push({
name: 'failed-to-resolve-connections',
type: 'critical',
description: `Failed to resolve connections for node ${node.name} (${node.type}): ${
error instanceof Error ? error.message : String(error)

View File

@ -65,6 +65,7 @@ export function validateFromAi(
if (node.parameters && parametersContainFromAi(node.parameters)) {
violations.push({
name: 'non-tool-node-uses-fromai',
type: 'major',
description: `Non-tool node "${node.name}" (${node.type}) uses $fromAI in its parameters. $fromAI is only for tool nodes connected to AI agents.`,
pointsDeducted: 20,

View File

@ -34,6 +34,7 @@ export function validateTools(
if (isTool(nodeType) && !toolsWithoutParameters.includes(node.type)) {
if (!node.parameters || Object.keys(node.parameters).length === 0) {
violations.push({
name: 'tool-node-has-no-parameters',
type: 'major',
description: `Tool node "${node.name}" has no parameters set.`,
pointsDeducted: 20,
@ -43,6 +44,7 @@ export function validateTools(
if (!nodeParametersContainExpression(node.parameters)) {
violations.push({
name: 'tool-node-static-parameters',
type: 'major',
description: `Tool node "${node.name}" has no expressions in its parameters. This likely means it is not using dynamic input.`,
pointsDeducted: 20,

View File

@ -20,6 +20,7 @@ export function validateTrigger(
if (!workflow.nodes || workflow.nodes.length === 0) {
violations.push({
name: 'workflow-has-no-nodes',
type: 'critical',
description: 'Workflow has no nodes',
pointsDeducted: 50,
@ -44,6 +45,7 @@ export function validateTrigger(
if (!hasTrigger) {
violations.push({
name: 'workflow-has-no-trigger',
type: 'critical',
description: 'Workflow must have at least one trigger node to start execution',
pointsDeducted: 50,

View File

@ -4,7 +4,30 @@ import type { SimpleWorkflow } from '@/types';
export type ProgrammaticViolationType = 'critical' | 'major' | 'minor';
export const PROGRAMMATIC_VIOLATION_NAMES = [
'tool-node-has-no-parameters',
'tool-node-static-parameters',
'agent-static-prompt',
'agent-no-system-prompt',
'non-tool-node-uses-fromai',
'workflow-has-no-nodes',
'workflow-has-no-trigger',
'node-missing-required-input',
'node-unsupported-connection-input',
'node-merge-single-input',
'node-merge-incorrect-num-inputs',
'node-merge-missing-input',
'sub-node-not-connected',
'node-type-not-found',
'failed-to-resolve-connections',
] as const;
export type ProgrammaticViolationName = (typeof PROGRAMMATIC_VIOLATION_NAMES)[number];
export type TelemetryValidationStatus = Record<ProgrammaticViolationName, 'pass' | 'fail'>;
export interface ProgrammaticViolation {
name: ProgrammaticViolationName;
type: ProgrammaticViolationType;
description: string;
pointsDeducted: number;

View File

@ -3,7 +3,7 @@ import { AIMessage, HumanMessage, RemoveMessage } from '@langchain/core/messages
import type { ToolMessage } from '@langchain/core/messages';
import type { RunnableConfig } from '@langchain/core/runnables';
import type { LangChainTracer } from '@langchain/core/tracers/tracer_langchain';
import type { MemorySaver } from '@langchain/langgraph';
import type { MemorySaver, StateSnapshot } from '@langchain/langgraph';
import { StateGraph, END, GraphRecursionError } from '@langchain/langgraph';
import type { Logger } from '@n8n/backend-common';
import {
@ -40,6 +40,13 @@ import { estimateTokenCountFromMessages } from './utils/token-usage';
import { executeToolsInParallel } from './utils/tool-executor';
import { WorkflowState } from './workflow-state';
/**
* Type for the state snapshot with properly typed values
*/
export type TypedStateSnapshot = Omit<StateSnapshot, 'values'> & {
values: typeof WorkflowState.State;
};
/**
* Determines which node to execute next based on the current state.
* This function decides if the workflow should:
@ -385,12 +392,13 @@ export class WorkflowBuilderAgent {
return workflow;
}
async getState(workflowId: string, userId?: string) {
async getState(workflowId?: string, userId?: string): Promise<TypedStateSnapshot> {
const workflow = this.createWorkflow();
const agent = workflow.compile({ checkpointer: this.checkpointer });
return await agent.getState({
configurable: { thread_id: `workflow-${workflowId}-user-${userId ?? new Date().getTime()}` },
});
const threadId = SessionManagerService.generateThreadId(workflowId, userId);
return (await agent.getState({
configurable: { thread_id: threadId },
})) as TypedStateSnapshot;
}
private getDefaultWorkflowJSON(payload: ChatPayload): SimpleWorkflow {

View File

@ -3,7 +3,7 @@ import { HumanMessage } from '@langchain/core/messages';
import { Annotation, messagesStateReducer } from '@langchain/langgraph';
import type { SimpleWorkflow, WorkflowOperation } from './types';
import type { ProgrammaticEvaluationResult } from './validation/types';
import type { ProgrammaticEvaluationResult, TelemetryValidationStatus } from './validation/types';
import type { ChatPayload } from './workflow-builder-agent';
/**
@ -80,10 +80,21 @@ export const WorkflowState = Annotation.Root({
workflowContext: Annotation<ChatPayload['workflowContext'] | undefined>({
reducer: (x, y) => y ?? x,
}),
// Results of last workflow validation
workflowValidation: Annotation<ProgrammaticEvaluationResult | null>({
reducer: (x, y) => (y === undefined ? x : y),
default: () => null,
}),
// Compacted programmatic validations history for telemetry
validationHistory: Annotation<TelemetryValidationStatus[]>({
reducer: (x, y) => (y && y.length > 0 ? [...x, ...y] : x),
default: () => [],
}),
// Technique categories identified from categorize_prompt tool for telemetry
techniqueCategories: Annotation<string[]>({
reducer: (x, y) => (y && y.length > 0 ? [...x, ...y] : x),
default: () => [],
}),
// Previous conversation summary (used for compressing long conversations)
previousSummary: Annotation<string>({

View File

@ -17,6 +17,8 @@ export const chatHubLLMProviderSchema = z.enum([
'google',
'azureOpenAi',
'ollama',
'awsBedrock',
'mistralCloud',
]);
export type ChatHubLLMProvider = z.infer<typeof chatHubLLMProviderSchema>;
@ -40,6 +42,8 @@ export const PROVIDER_CREDENTIAL_TYPE_MAP: Record<
google: 'googlePalmApi',
ollama: 'ollamaApi',
azureOpenAi: 'azureOpenAiApi',
awsBedrock: 'aws',
mistralCloud: 'mistralCloudApi',
};
export type ChatHubAgentTool = typeof JINA_AI_TOOL_NODE_TYPE | typeof SEAR_XNG_TOOL_NODE_TYPE;
@ -72,6 +76,16 @@ const ollamaModelSchema = z.object({
model: z.string(),
});
const awsBedrockModelSchema = z.object({
provider: z.literal('awsBedrock'),
model: z.string(),
});
const mistralCloudModelSchema = z.object({
provider: z.literal('mistralCloud'),
model: z.string(),
});
const n8nModelSchema = z.object({
provider: z.literal('n8n'),
workflowId: z.string(),
@ -88,6 +102,8 @@ export const chatHubConversationModelSchema = z.discriminatedUnion('provider', [
googleModelSchema,
azureOpenAIModelSchema,
ollamaModelSchema,
awsBedrockModelSchema,
mistralCloudModelSchema,
n8nModelSchema,
chatAgentSchema,
]);
@ -97,12 +113,16 @@ export type ChatHubAnthropicModel = z.infer<typeof anthropicModelSchema>;
export type ChatHubGoogleModel = z.infer<typeof googleModelSchema>;
export type ChatHubAzureOpenAIModel = z.infer<typeof azureOpenAIModelSchema>;
export type ChatHubOllamaModel = z.infer<typeof ollamaModelSchema>;
export type ChatHubAwsBedrockModel = z.infer<typeof awsBedrockModelSchema>;
export type ChatHubMistralCloudModel = z.infer<typeof mistralCloudModelSchema>;
export type ChatHubBaseLLMModel =
| ChatHubOpenAIModel
| ChatHubAnthropicModel
| ChatHubGoogleModel
| ChatHubAzureOpenAIModel
| ChatHubOllamaModel;
| ChatHubOllamaModel
| ChatHubAwsBedrockModel
| ChatHubMistralCloudModel;
export type ChatHubN8nModel = z.infer<typeof n8nModelSchema>;
export type ChatHubCustomAgentModel = z.infer<typeof chatAgentSchema>;
@ -124,6 +144,7 @@ export interface ChatModelDto {
description: string | null;
updatedAt: string | null;
createdAt: string | null;
allowFileUploads?: boolean;
}
/**
@ -143,11 +164,25 @@ export const emptyChatModelsResponse: ChatModelsResponse = {
google: { models: [] },
azureOpenAi: { models: [] },
ollama: { models: [] },
awsBedrock: { models: [] },
mistralCloud: { models: [] },
n8n: { models: [] },
// eslint-disable-next-line @typescript-eslint/naming-convention
'custom-agent': { models: [] },
};
/**
* Chat attachment schema for incoming requests.
* Requires base64 data and fileName.
* MimeType, fileType, fileExtension, and fileSize are populated server-side.
*/
export const chatAttachmentSchema = z.object({
data: z.string(),
fileName: z.string(),
});
export type ChatAttachment = z.infer<typeof chatAttachmentSchema>;
export class ChatHubSendMessageRequest extends Z.class({
messageId: z.string().uuid(),
sessionId: z.string().uuid(),
@ -161,6 +196,7 @@ export class ChatHubSendMessageRequest extends Z.class({
}),
),
tools: z.array(INodeSchema),
attachments: z.array(chatAttachmentSchema),
}) {}
export class ChatHubRegenerateMessageRequest extends Z.class({
@ -235,9 +271,20 @@ export interface ChatHubMessageDto {
previousMessageId: ChatMessageId | null;
retryOfMessageId: ChatMessageId | null;
revisionOfMessageId: ChatMessageId | null;
attachments: Array<{ fileName?: string; mimeType?: string }>;
}
export type ChatHubConversationsResponse = ChatHubSessionDto[];
export class ChatHubConversationsRequest extends Z.class({
limit: z.coerce.number().int().min(1).max(100),
cursor: z.string().uuid().optional(),
}) {}
export interface ChatHubConversationsResponse {
data: ChatHubSessionDto[];
nextCursor: string | null;
hasMore: boolean;
}
export interface ChatHubConversationDto {
messages: Record<ChatMessageId, ChatHubMessageDto>;

View File

@ -7,4 +7,6 @@ import { dataTableNameSchema } from '../../schemas/data-table.schema';
export class CreateDataTableDto extends Z.class({
name: dataTableNameSchema,
columns: z.array(CreateDataTableColumnDto),
fileId: z.string().optional(),
hasHeaders: z.boolean().optional(),
}) {}

View File

@ -3,4 +3,5 @@ import { Z } from 'zod-class';
export class ImportWorkflowFromUrlDto extends Z.class({
url: z.string().url(),
projectId: z.string(),
}) {}

View File

@ -27,10 +27,13 @@ export {
emptyChatModelsResponse,
type ChatModelsRequest,
type ChatModelsResponse,
chatAttachmentSchema,
type ChatAttachment,
ChatHubSendMessageRequest,
ChatHubRegenerateMessageRequest,
ChatHubEditMessageRequest,
ChatHubUpdateConversationRequest,
ChatHubConversationsRequest,
type ChatMessageId,
type ChatSessionId,
type ChatHubMessageDto,

View File

@ -15,8 +15,8 @@ beforeEach(() => {
describe('eligibleModules', () => {
it('should consider all default modules eligible', () => {
// 'mcp' and 'chat-hub' aren't (yet) eligible modules by default
const NON_DEFAULT_MODULES = ['mcp', 'chat-hub'];
// 'chat-hub' isn't (yet) an eligible module by default
const NON_DEFAULT_MODULES = ['chat-hub'];
const expectedModules = MODULE_NAMES.filter((name) => !NON_DEFAULT_MODULES.includes(name));
expect(Container.get(ModuleRegistry).eligibleModules).toEqual(expectedModules);
});
@ -27,6 +27,7 @@ describe('eligibleModules', () => {
'external-secrets',
'community-packages',
'data-table',
'mcp',
'provisioning',
'breaking-changes',
]);
@ -39,6 +40,7 @@ describe('eligibleModules', () => {
'external-secrets',
'community-packages',
'data-table',
'mcp',
'provisioning',
'breaking-changes',
]);
@ -98,7 +100,7 @@ describe('initModules', () => {
const moduleRegistry = new ModuleRegistry(moduleMetadata, mock(), mock(), mock());
await moduleRegistry.initModules();
await moduleRegistry.initModules('main');
expect(ModuleClass.init).toHaveBeenCalled();
});
@ -117,7 +119,7 @@ describe('initModules', () => {
const moduleRegistry = new ModuleRegistry(moduleMetadata, licenseState, mock(), mock());
await moduleRegistry.initModules();
await moduleRegistry.initModules('main');
expect(ModuleClass.init).toHaveBeenCalled();
});
@ -136,7 +138,7 @@ describe('initModules', () => {
const moduleRegistry = new ModuleRegistry(moduleMetadata, licenseState, mock(), mock());
await moduleRegistry.initModules();
await moduleRegistry.initModules('main');
expect(ModuleClass.init).not.toHaveBeenCalled();
});
@ -153,9 +155,9 @@ describe('initModules', () => {
const moduleRegistry = new ModuleRegistry(moduleMetadata, mock(), mock(), mock());
await moduleRegistry.initModules();
await moduleRegistry.initModules('main');
await expect(moduleRegistry.initModules()).resolves.not.toThrow();
await expect(moduleRegistry.initModules('main')).resolves.not.toThrow();
});
it('registers settings', async () => {
@ -174,7 +176,7 @@ describe('initModules', () => {
const moduleRegistry = new ModuleRegistry(moduleMetadata, mock(), mock(), mock());
// ACT
await moduleRegistry.initModules();
await moduleRegistry.initModules('main');
// ASSERT
expect(ModuleClass.settings).toHaveBeenCalled();
@ -198,7 +200,7 @@ describe('initModules', () => {
const moduleRegistry = new ModuleRegistry(moduleMetadata, mock(), mock(), mock());
// ACT
await moduleRegistry.initModules();
await moduleRegistry.initModules('main');
// ASSERT
// eslint-disable-next-line @typescript-eslint/no-explicit-any
@ -220,7 +222,7 @@ describe('initModules', () => {
const moduleRegistry = new ModuleRegistry(moduleMetadata, mock(), mock(), mock());
// ACT
await moduleRegistry.initModules();
await moduleRegistry.initModules('main');
// ASSERT
// eslint-disable-next-line @typescript-eslint/no-explicit-any
@ -244,7 +246,7 @@ describe('initModules', () => {
const moduleRegistry = new ModuleRegistry(moduleMetadata, mock(), mock(), mock());
// ACT
await moduleRegistry.initModules();
await moduleRegistry.initModules('main');
// ASSERT
expect(ModuleClass.context).toHaveBeenCalled();
@ -264,11 +266,45 @@ describe('initModules', () => {
const moduleRegistry = new ModuleRegistry(moduleMetadata, mock(), mock(), mock());
// ACT
await moduleRegistry.initModules();
await moduleRegistry.initModules('main');
// ASSERT
expect(moduleRegistry.context.has(moduleName)).toBe(false);
});
it('should init module with matching instance type', async () => {
const ModuleClass = { init: jest.fn() };
const moduleMetadata = mock<ModuleMetadata>({
getEntries: jest
.fn()
.mockReturnValue([
['test-module', { instanceTypes: ['main', 'worker'], class: ModuleClass }],
]),
});
Container.get = jest.fn().mockReturnValue(ModuleClass);
const moduleRegistry = new ModuleRegistry(moduleMetadata, mock(), mock(), mock());
await moduleRegistry.initModules('main');
expect(ModuleClass.init).toHaveBeenCalled();
});
it('should skip init for module with non-matching instance type', async () => {
const ModuleClass = { init: jest.fn() };
const moduleMetadata = mock<ModuleMetadata>({
getEntries: jest
.fn()
.mockReturnValue([['test-module', { instanceTypes: ['worker'], class: ModuleClass }]]),
});
Container.get = jest.fn().mockReturnValue(ModuleClass);
const moduleRegistry = new ModuleRegistry(moduleMetadata, mock(), mock(), mock());
await moduleRegistry.initModules('main');
expect(ModuleClass.init).not.toHaveBeenCalled();
});
});
describe('loadDir', () => {

View File

@ -1,3 +1,4 @@
import type { InstanceType } from '@n8n/constants';
import { ModuleMetadata } from '@n8n/decorators';
import type { EntityClass, ModuleContext, ModuleSettings } from '@n8n/decorators';
import { Container, Service } from '@n8n/di';
@ -33,9 +34,9 @@ export class ModuleRegistry {
'external-secrets',
'community-packages',
'data-table',
'mcp',
'provisioning',
'breaking-changes',
'mcp',
];
private readonly activeModules: string[] = [];
@ -107,15 +108,22 @@ export class ModuleRegistry {
*
* `ModuleRegistry.loadModules` must have been called before.
*/
async initModules() {
async initModules(instanceType: InstanceType) {
for (const [moduleName, moduleEntry] of this.moduleMetadata.getEntries()) {
const { licenseFlag, class: ModuleClass } = moduleEntry;
const { licenseFlag, instanceTypes, class: ModuleClass } = moduleEntry;
if (licenseFlag !== undefined && !this.licenseState.isLicensed(licenseFlag)) {
this.logger.debug(`Skipped init for unlicensed module "${moduleName}"`);
continue;
}
if (instanceTypes !== undefined && !instanceTypes.includes(instanceType)) {
this.logger.debug(
`Skipped init for module "${moduleName}" (instance type "${instanceType}" not in: ${instanceTypes.join(', ')})`,
);
continue;
}
await Container.get(ModuleClass).init?.();
const moduleSettings = await Container.get(ModuleClass).settings?.();

View File

@ -1,3 +1,6 @@
import { tmpdir } from 'node:os';
import path from 'node:path';
import { Config, Env } from '../decorators';
@Config
@ -20,4 +23,39 @@ export class DataTableConfig {
*/
@Env('N8N_DATA_TABLES_SIZE_CHECK_CACHE_DURATION_MS')
sizeCheckCacheDuration: number = 60 * 1000;
/**
* The maximum allowed file size (in bytes) for CSV uploads to data tables.
* If set, this is the hard limit for file uploads.
* If not set, the upload limit will be the remaining available storage space.
*/
@Env('N8N_DATA_TABLES_UPLOAD_MAX_FILE_SIZE_BYTES')
uploadMaxFileSize?: number;
/**
* The interval in milliseconds at which orphaned uploaded files are cleaned up.
* Defaults to 60 seconds if not explicitly set via environment variable.
*/
@Env('N8N_DATA_TABLES_CLEANUP_INTERVAL_MS')
cleanupIntervalMs: number = 60 * 1000;
/**
* The maximum age in milliseconds for uploaded files before they are considered orphaned and deleted.
* Files older than this threshold are removed during cleanup.
* Defaults to 2 minutes if not explicitly set via environment variable.
*/
@Env('N8N_DATA_TABLES_FILE_MAX_AGE_MS')
fileMaxAgeMs: number = 2 * 60 * 1000;
/**
* The directory path where uploaded CSV files are temporarily stored before being imported.
* Files in this directory are automatically cleaned up after a configurable period (fileMaxAgeMs).
* Computed as: <system-tmp-dir>/n8nDataTableUploads
* Example: /tmp/n8nDataTableUploads
*/
readonly uploadDir: string;
constructor() {
this.uploadDir = path.join(tmpdir(), 'n8nDataTableUploads');
}
}

View File

@ -1,6 +1,7 @@
import path from 'node:path';
import { Config, Env } from '../decorators';
import { getN8nFolder } from '../utils/utils';
@Config
export class InstanceSettingsConfig {
@ -29,9 +30,7 @@ export class InstanceSettingsConfig {
readonly n8nFolder: string;
constructor() {
const homeVarName = process.platform === 'win32' ? 'USERPROFILE' : 'HOME';
this.userHome = process.env.N8N_USER_FOLDER ?? process.env[homeVarName] ?? process.cwd();
this.n8nFolder = path.join(this.userHome, '.n8n');
this.n8nFolder = getN8nFolder();
this.userHome = path.dirname(this.n8nFolder);
}
}

View File

@ -0,0 +1,11 @@
import path from 'node:path';
/**
* Computes the n8n folder path based on environment variables.
* This is used by various configs that need to know the n8n installation directory.
*/
export function getN8nFolder(): string {
const homeVarName = process.platform === 'win32' ? 'USERPROFILE' : 'HOME';
const userHome = process.env.N8N_USER_FOLDER ?? process.env[homeVarName] ?? process.cwd();
return path.join(userHome, '.n8n');
}

View File

@ -1,6 +1,8 @@
import { Container } from '@n8n/di';
import fs from 'fs';
import { mock } from 'jest-mock-extended';
import { tmpdir } from 'node:os';
import path from 'node:path';
import type { UserManagementConfig } from '../src/configs/user-management.config';
import { GlobalConfig } from '../src/index';
@ -55,6 +57,9 @@ describe('GlobalConfig', () => {
dataTable: {
maxSize: 50 * 1024 * 1024,
sizeCheckCacheDuration: 60000,
cleanupIntervalMs: 60 * 1000,
fileMaxAgeMs: 2 * 60 * 1000,
uploadDir: path.join(tmpdir(), 'n8nDataTableUploads'),
},
database: {
logging: {

View File

@ -0,0 +1,19 @@
import type { MigrationContext, ReversibleMigration } from '../migration-types';
const table = {
messages: 'chat_hub_messages',
} as const;
export class AddAttachmentsToChatHubMessages1761773155024 implements ReversibleMigration {
async up({ schemaBuilder: { addColumns, column } }: MigrationContext) {
await addColumns(table.messages, [
column('attachments').json.comment(
'File attachments for the message (if any), stored as JSON. Files are stored as base64-encoded data URLs.',
),
]);
}
async down({ schemaBuilder: { dropColumns } }: MigrationContext) {
await dropColumns(table.messages, ['attachments']);
}
}

View File

@ -0,0 +1,69 @@
import type { IrreversibleMigration, MigrationContext } from '../migration-types';
const TABLE_NAME = 'oauth_authorization_codes';
const TEMP_TABLE_NAME = 'temp_oauth_authorization_codes';
export class ChangeOAuthStateColumnToUnboundedVarchar1763572724000
implements IrreversibleMigration
{
async up({
isSqlite,
isMysql,
isPostgres,
escape,
copyTable,
queryRunner,
schemaBuilder: { createTable, column, dropTable },
}: MigrationContext) {
const tableName = escape.tableName(TABLE_NAME);
if (isSqlite) {
const tempTableName = escape.tableName(TEMP_TABLE_NAME);
await createTable(TEMP_TABLE_NAME)
.withColumns(
column('code').varchar(255).primary.notNull,
column('clientId').varchar().notNull,
column('userId').uuid.notNull,
column('redirectUri').varchar().notNull,
column('codeChallenge').varchar().notNull,
column('codeChallengeMethod').varchar(255).notNull,
column('expiresAt').bigint.notNull.comment('Unix timestamp in milliseconds'),
column('state').varchar(),
column('used').bool.notNull.default(false),
)
.withForeignKey('clientId', {
tableName: 'oauth_clients',
columnName: 'id',
onDelete: 'CASCADE',
})
.withForeignKey('userId', {
tableName: 'user',
columnName: 'id',
onDelete: 'CASCADE',
}).withTimestamps;
await copyTable(TABLE_NAME, TEMP_TABLE_NAME);
await dropTable(TABLE_NAME);
await queryRunner.query(`ALTER TABLE ${tempTableName} RENAME TO ${tableName};`);
} else if (isMysql) {
await queryRunner.query(
`ALTER TABLE ${tableName} MODIFY COLUMN ${escape.columnName('state')} TEXT;`,
);
await queryRunner.query(
`ALTER TABLE ${tableName} MODIFY COLUMN ${escape.columnName('codeChallenge')} TEXT NOT NULL;`,
);
await queryRunner.query(
`ALTER TABLE ${tableName} MODIFY COLUMN ${escape.columnName('redirectUri')} TEXT NOT NULL;`,
);
} else if (isPostgres) {
await queryRunner.query(
`ALTER TABLE ${tableName} ALTER COLUMN ${escape.columnName('state')} TYPE VARCHAR,` +
` ALTER COLUMN ${escape.columnName('codeChallenge')} TYPE VARCHAR,` +
` ALTER COLUMN ${escape.columnName('redirectUri')} TYPE VARCHAR;`,
);
}
}
}

View File

@ -111,8 +111,10 @@ import { CreateChatHubAgentTable1760020000000 } from '../common/1760020000000-Cr
import { UniqueRoleNames1760020838000 } from '../common/1760020838000-UniqueRoleNames';
import { CreateOAuthEntities1760116750277 } from '../common/1760116750277-CreateOAuthEntities';
import { CreateWorkflowDependencyTable1760314000000 } from '../common/1760314000000-CreateWorkflowDependencyTable';
import { AddAttachmentsToChatHubMessages1761773155024 } from '../common/1761773155024-AddAttachmentsToChatHubMessages';
import { AddWorkflowDescriptionColumn1762177736257 } from '../common/1762177736257-AddWorkflowDescriptionColumn';
import { BackfillMissingWorkflowHistoryRecords1762763704614 } from '../common/1762763704614-BackfillMissingWorkflowHistoryRecords';
import { ChangeOAuthStateColumnToUnboundedVarchar1763572724000 } from '../common/1763572724000-ChangeOAuthStateColumnToUnboundedVarchar';
import type { Migration } from '../migration-types';
export const mysqlMigrations: Migration[] = [
@ -231,4 +233,6 @@ export const mysqlMigrations: Migration[] = [
BackfillMissingWorkflowHistoryRecords1762763704614,
AddWorkflowHistoryAutoSaveFields1762847206508,
AddToolsColumnToChatHubTables1761830340990,
ChangeOAuthStateColumnToUnboundedVarchar1763572724000,
AddAttachmentsToChatHubMessages1761773155024,
];

View File

@ -109,10 +109,12 @@ import { UniqueRoleNames1760020838000 } from '../common/1760020838000-UniqueRole
import { CreateOAuthEntities1760116750277 } from '../common/1760116750277-CreateOAuthEntities';
import { CreateWorkflowDependencyTable1760314000000 } from '../common/1760314000000-CreateWorkflowDependencyTable';
import { DropUnusedChatHubColumns1760965142113 } from '../common/1760965142113-DropUnusedChatHubColumns';
import { AddAttachmentsToChatHubMessages1761773155024 } from '../common/1761773155024-AddAttachmentsToChatHubMessages';
import { AddToolsColumnToChatHubTables1761830340990 } from '../common/1761830340990-AddToolsColumnToChatHubTables';
import { AddWorkflowDescriptionColumn1762177736257 } from '../common/1762177736257-AddWorkflowDescriptionColumn';
import { BackfillMissingWorkflowHistoryRecords1762763704614 } from '../common/1762763704614-BackfillMissingWorkflowHistoryRecords';
import { AddWorkflowHistoryAutoSaveFields1762847206508 } from '../common/1762847206508-AddWorkflowHistoryAutoSaveFields';
import { ChangeOAuthStateColumnToUnboundedVarchar1763572724000 } from '../common/1763572724000-ChangeOAuthStateColumnToUnboundedVarchar';
import type { Migration } from '../migration-types';
export const postgresMigrations: Migration[] = [
@ -231,4 +233,6 @@ export const postgresMigrations: Migration[] = [
ChangeDefaultForIdInUserTable1762771264000,
AddWorkflowHistoryAutoSaveFields1762847206508,
AddToolsColumnToChatHubTables1761830340990,
ChangeOAuthStateColumnToUnboundedVarchar1763572724000,
AddAttachmentsToChatHubMessages1761773155024,
];

View File

@ -105,10 +105,12 @@ import { UniqueRoleNames1760020838000 } from '../common/1760020838000-UniqueRole
import { CreateOAuthEntities1760116750277 } from '../common/1760116750277-CreateOAuthEntities';
import { CreateWorkflowDependencyTable1760314000000 } from '../common/1760314000000-CreateWorkflowDependencyTable';
import { DropUnusedChatHubColumns1760965142113 } from '../common/1760965142113-DropUnusedChatHubColumns';
import { AddAttachmentsToChatHubMessages1761773155024 } from '../common/1761773155024-AddAttachmentsToChatHubMessages';
import { AddToolsColumnToChatHubTables1761830340990 } from '../common/1761830340990-AddToolsColumnToChatHubTables';
import { AddWorkflowDescriptionColumn1762177736257 } from '../common/1762177736257-AddWorkflowDescriptionColumn';
import { BackfillMissingWorkflowHistoryRecords1762763704614 } from '../common/1762763704614-BackfillMissingWorkflowHistoryRecords';
import { AddWorkflowHistoryAutoSaveFields1762847206508 } from '../common/1762847206508-AddWorkflowHistoryAutoSaveFields';
import { ChangeOAuthStateColumnToUnboundedVarchar1763572724000 } from '../common/1763572724000-ChangeOAuthStateColumnToUnboundedVarchar';
import type { Migration } from '../migration-types';
const sqliteMigrations: Migration[] = [
@ -223,6 +225,8 @@ const sqliteMigrations: Migration[] = [
BackfillMissingWorkflowHistoryRecords1762763704614,
AddWorkflowHistoryAutoSaveFields1762847206508,
AddToolsColumnToChatHubTables1761830340990,
ChangeOAuthStateColumnToUnboundedVarchar1763572724000,
AddAttachmentsToChatHubMessages1761773155024,
];
export { sqliteMigrations };

View File

@ -408,7 +408,7 @@ export class ExecutionRepository extends Repository<ExecutionEntity> {
async hardDelete(ids: { workflowId: string; executionId: string }) {
return await Promise.all([
this.delete(ids.executionId),
this.binaryDataService.deleteMany([ids]),
this.binaryDataService.deleteMany([{ type: 'execution', ...ids }]),
]);
}
@ -509,6 +509,7 @@ export class ExecutionRepository extends Repository<ExecutionEntity> {
}
const ids = executions.map(({ id, workflowId }) => ({
type: 'execution' as const,
executionId: id,
workflowId,
}));
@ -605,7 +606,11 @@ export class ExecutionRepository extends Repository<ExecutionEntity> {
*/
withDeleted: true,
})
).map(({ id: executionId, workflowId }) => ({ workflowId, executionId }));
).map(({ id: executionId, workflowId }) => ({
type: 'execution' as const,
workflowId,
executionId,
}));
return workflowIdsAndExecutionIds;
}

View File

@ -12,6 +12,7 @@
"lint:fix": "eslint . --fix",
"watch": "tsc -p tsconfig.build.json --watch",
"test": "jest",
"test:unit": "jest",
"test:dev": "jest --watch"
},
"main": "dist/index.js",

View File

@ -0,0 +1,148 @@
import { Container } from '@n8n/di';
import type {
ContextEstablishmentOptions,
ContextEstablishmentResult,
IContextEstablishmentHook,
} from '../context-establishment-hook';
import {
ContextEstablishmentHookMetadata,
ContextEstablishmentHook,
} from '../context-establishment-hook-metadata';
describe('@ContextEstablishmentHook decorator', () => {
let hookMetadata: ContextEstablishmentHookMetadata;
beforeEach(() => {
jest.resetAllMocks();
hookMetadata = new ContextEstablishmentHookMetadata();
Container.set(ContextEstablishmentHookMetadata, hookMetadata);
});
it('should register hook in ContextEstablishmentHookMetadata', () => {
@ContextEstablishmentHook()
class TestHook implements IContextEstablishmentHook {
hookDescription = { name: 'test.hook' };
async execute(_options: ContextEstablishmentOptions): Promise<ContextEstablishmentResult> {
return {};
}
isApplicableToTriggerNode(_nodeType: string): boolean {
return true;
}
}
const registeredHooks = hookMetadata.getClasses();
expect(registeredHooks).toContain(TestHook);
expect(registeredHooks).toHaveLength(1);
});
it('should register multiple hooks', () => {
@ContextEstablishmentHook()
class FirstHook implements IContextEstablishmentHook {
hookDescription = { name: 'first.hook' };
async execute(_options: ContextEstablishmentOptions): Promise<ContextEstablishmentResult> {
return {};
}
isApplicableToTriggerNode(_nodeType: string): boolean {
return true;
}
}
@ContextEstablishmentHook()
class SecondHook implements IContextEstablishmentHook {
hookDescription = { name: 'second.hook' };
async execute(_options: ContextEstablishmentOptions): Promise<ContextEstablishmentResult> {
return {};
}
isApplicableToTriggerNode(_nodeType: string): boolean {
return true;
}
}
@ContextEstablishmentHook()
class ThirdHook implements IContextEstablishmentHook {
hookDescription = { name: 'third.hook' };
async execute(_options: ContextEstablishmentOptions): Promise<ContextEstablishmentResult> {
return {};
}
isApplicableToTriggerNode(_nodeType: string): boolean {
return true;
}
}
const registeredHooks = hookMetadata.getClasses();
expect(registeredHooks).toContain(FirstHook);
expect(registeredHooks).toContain(SecondHook);
expect(registeredHooks).toContain(ThirdHook);
expect(registeredHooks).toHaveLength(3);
});
it('should apply Service decorator', () => {
@ContextEstablishmentHook()
class TestHook implements IContextEstablishmentHook {
hookDescription = { name: 'test.hook' };
async execute(_options: ContextEstablishmentOptions): Promise<ContextEstablishmentResult> {
return {};
}
isApplicableToTriggerNode(_nodeType: string): boolean {
return true;
}
}
expect(Container.has(TestHook)).toBe(true);
});
it('should allow instantiation of registered hooks with accessible hookDescription', () => {
@ContextEstablishmentHook()
class TestHook implements IContextEstablishmentHook {
hookDescription = { name: 'credentials.bearerToken' };
async execute(_options: ContextEstablishmentOptions): Promise<ContextEstablishmentResult> {
return {};
}
isApplicableToTriggerNode(_nodeType: string): boolean {
return true;
}
}
const hookInstance = Container.get(TestHook);
expect(hookInstance).toBeInstanceOf(TestHook);
expect(hookInstance.hookDescription).toEqual({ name: 'credentials.bearerToken' });
expect(hookInstance.hookDescription.name).toBe('credentials.bearerToken');
});
it('should register hooks with different description names', () => {
@ContextEstablishmentHook()
class BearerTokenHook implements IContextEstablishmentHook {
hookDescription = { name: 'credentials.bearerToken' };
async execute(_options: ContextEstablishmentOptions): Promise<ContextEstablishmentResult> {
return {};
}
isApplicableToTriggerNode(_nodeType: string): boolean {
return true;
}
}
@ContextEstablishmentHook()
class ApiKeyHook implements IContextEstablishmentHook {
hookDescription = { name: 'credentials.apiKey' };
async execute(_options: ContextEstablishmentOptions): Promise<ContextEstablishmentResult> {
return {};
}
isApplicableToTriggerNode(_nodeType: string): boolean {
return true;
}
}
const registeredHooks = hookMetadata.getClasses();
const bearerTokenHook = Container.get(BearerTokenHook);
const apiKeyHook = Container.get(ApiKeyHook);
expect(registeredHooks).toHaveLength(2);
expect(bearerTokenHook.hookDescription.name).toBe('credentials.bearerToken');
expect(apiKeyHook.hookDescription.name).toBe('credentials.apiKey');
});
});

View File

@ -0,0 +1,200 @@
import { Container, Service } from '@n8n/di';
import { ContextEstablishmentHookClass } from './context-establishment-hook';
/**
* Registry entry for a context establishment hook.
*
* This is a lightweight wrapper around the hook class constructor that can be
* extended in the future to include additional metadata if needed (e.g., module
* source, registration timestamp, feature flags, license flags).
*
* @internal
*/
type ContextEstablishmentHookEntry = {
/** The hook class constructor for DI container instantiation */
class: ContextEstablishmentHookClass;
};
/**
* Low-level metadata registry for context establishment hooks.
*
* This service acts as a simple collection of registered hook classes that gets
* populated automatically by the @ContextEstablishmentHook decorator at module
* load time. It serves as the foundation for the higher-level Hook Registry.
*
* **Architecture:**
* ```
* Decorator ContextEstablishmentHookMetadata Hook Registry Execution Engine
* (registration) (collection) (discovery) (execution)
* ```
*
* @see ContextEstablishmentHook decorator for automatic registration
* @see IContextEstablishmentHook for hook interface
*/
@Service()
export class ContextEstablishmentHookMetadata {
/**
* Internal collection of registered hook classes.
*
* Uses Set for efficient deduplication (though duplicate registration
* should not occur with proper decorator usage).
*/
private readonly contextEstablishmentHooks: Set<ContextEstablishmentHookEntry> = new Set();
/**
* Registers a hook class in the metadata collection.
*
* Called automatically by the @ContextEstablishmentHook decorator during
* module loading. Should not be called directly by application code.
*
* **Note:** This method does not validate uniqueness or check for naming
* conflicts. Validation happens later in the Hook Registry.
*
* @param hookEntry - The hook class entry to register
*
* @internal Called by decorator only
*/
register(hookEntry: ContextEstablishmentHookEntry) {
this.contextEstablishmentHooks.add(hookEntry);
}
/**
* Retrieves all registered hook entries.
*
* Returns an array of [index, entry] tuples compatible with Set.entries().
* Primarily used for debugging or low-level iteration.
*
* **Prefer getClasses()** for most use cases as it returns just the classes.
*
* @returns Array of [index, entry] tuples from the internal Set
*
* @example
* ```typescript
* const entries = metadata.getEntries();
* for (const [index, entry] of entries) {
* console.log(`Hook ${index}:`, entry.class.name);
* }
* ```
*/
getEntries() {
return [...this.contextEstablishmentHooks.entries()];
}
/**
* Retrieves all registered hook classes.
*
* This is the primary method used by the Hook Registry to obtain hook classes
* for instantiation and indexing. Returns just the class constructors without
* the wrapper entry objects.
*
* **Usage pattern:**
* ```typescript
* const classes = metadata.getClasses();
* const hooks = classes.map(HookClass => Container.get(HookClass));
* const hooksByName = new Map(hooks.map(h => [h.hookDescription.name, h]));
* ```
*
* @returns Array of hook class constructors ready for DI instantiation
*
* @example
* ```typescript
* @Service()
* export class HookRegistry {
* constructor(
* private metadata: ContextEstablishmentHookMetadata,
* private container: Container
* ) {
* const hookClasses = metadata.getClasses();
* this.hooks = hookClasses.map(cls => container.get(cls));
* }
* }
* ```
*/
getClasses() {
return [...this.contextEstablishmentHooks.values()].map((entry) => entry.class);
}
}
/**
* Class decorator for context establishment hooks.
*
* This decorator performs two critical functions:
* 1. **Registers** the hook class in ContextEstablishmentHookMetadata for discovery
* 2. **Enables DI** by applying @Service() to make the hook injectable
*
* The decorator executes at module load time (when the class is defined), ensuring
* all hooks are registered before the application starts. This enables automatic
* discovery without manual registration code.
*
* **Registration flow:**
* ```
* @ContextEstablishmentHook() // 1. Decorator executes
* export class BearerTokenHook // 2. Class is defined
*
* ContextEstablishmentHookMetadata // 3. Hook class registered in metadata
*
* @Service() // 4. DI container registration
*
* Hook is discoverable & injectable // 5. Ready for use
* ```
*
* **Design pattern:**
* This follows the declarative registration pattern used throughout n8n for
* extensibility (similar to node registration). Hooks self-register without
* requiring central registration files or manual imports.
*
* **Requirements:**
* - Decorated class MUST implement IContextEstablishmentHook
* - Decorated class MUST have a hookDescription property with unique name
*
* **Important notes:**
* - No decorator parameters needed (hook metadata lives on hook instance)
* - Hooks are registered as singletons via @Service()
* - Registration happens eagerly at module load, not lazily
* - Duplicate decoration of the same class is safe (Set deduplicates)
*
* @see IContextEstablishmentHook for interface requirements
* @see ContextEstablishmentHookMetadata for underlying registry
* @see HookDescription for hook metadata structure
*
* @example
* ```typescript
* // Basic hook registration:
* @ContextEstablishmentHook()
* export class BearerTokenHook implements IContextEstablishmentHook {
* hookDescription = {
* name: 'credentials.bearerToken'
* };
*
* async execute(options: ContextEstablishmentOptions) {
* // Extract bearer token from Authorization header
* const token = this.extractToken(options.triggerItem);
* return {
* triggerItem: this.removeAuthHeader(options.triggerItem),
* contextUpdate: {
* credentials: { version: 1, identity: token }
* }
* };
* }
*
* isApplicableToTriggerNode(nodeType: string) {
* return nodeType === 'n8n-nodes-base.webhook';
* }
* }
* ```
*
* @returns A class decorator function that registers and enables DI for the hook
*/
export const ContextEstablishmentHook =
<T extends ContextEstablishmentHookClass>() =>
(target: T) => {
// Register hook class in metadata for discovery by Hook Registry
Container.get(ContextEstablishmentHookMetadata).register({
class: target,
});
// Enable dependency injection for the hook class
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
return Service()(target);
};

View File

@ -0,0 +1,328 @@
import type { Constructable } from '@n8n/di';
import type {
INode,
INodeExecutionData,
PlaintextExecutionContext,
IWorkflowBase,
} from 'n8n-workflow';
/**
* Input parameters passed to a context establishment hook during execution.
*
* Hooks receive the current workflow state and extract information from
* trigger items to build the execution context (e.g., credentials, environment).
* All hooks work with plaintext (decrypted) context for runtime operations.
*
* @see IContextEstablishmentHook
* @see PlaintextExecutionContext
*/
export type ContextEstablishmentOptions = {
/** The trigger node that initiated the workflow execution */
triggerNode: INode;
/** The complete workflow definition */
workflow: IWorkflowBase;
/**
* Trigger items from the workflow execution start.
* This array represents items as modified by previous hooks in the chain.
* Hooks can extract data from these items and optionally modify them
* (e.g., removing sensitive headers before storage).
*/
triggerItems: INodeExecutionData[];
/**
* The plaintext execution context built so far.
* Includes base context plus results from any previously executed hooks.
* Contains decrypted credential data for runtime operations.
*
* @see PlaintextExecutionContext for security considerations
*/
context: PlaintextExecutionContext;
/**
* Hook-specific configuration provided by the trigger node.
* Structure varies per hook type (e.g., { removeFromItem: true } for bearer token hook).
*/
options?: Record<string, unknown>;
};
/**
* Result returned by a context establishment hook after execution.
*
* Hooks can modify trigger items (e.g., remove sensitive headers) and
* contribute partial context updates that get merged into the execution context.
* All context data is in plaintext form during hook execution.
*
* @see IContextEstablishmentHook
* @see PlaintextExecutionContext
*/
export type ContextEstablishmentResult = {
/**
* The potentially modified trigger items.
* If undefined, the original trigger items are preserved unchanged.
*
* Common use case: Removing sensitive data (e.g., Authorization headers)
* before storing items in execution history.
*
* @example
* ```typescript
* // Remove Authorization header from trigger items
* const modifiedItems = options.triggerItems.map(item => ({
* ...item,
* json: {
* ...item.json,
* headers: {
* ...item.json.headers,
* authorization: undefined
* }
* }
* }));
* return { triggerItems: modifiedItems, contextUpdate: { ... } };
* ```
*/
triggerItems?: INodeExecutionData[];
/**
* Partial context update to merge into the execution context.
* If undefined, no context updates are applied.
*
* Contains only this hook's contributions (e.g., credentials data).
* Multiple hooks' updates are merged sequentially during execution.
* Context data is in plaintext form and will be encrypted before persistence.
*
* @example
* ```typescript
* // Add credential context from bearer token
* return {
* triggerItems: modifiedItems,
* contextUpdate: {
* credentials: {
* version: 1,
* identity: extractedToken,
* metadata: { source: 'bearer-token' }
* }
* }
* };
* ```
*/
contextUpdate?: Partial<PlaintextExecutionContext>;
};
/**
* Metadata describing a context establishment hook.
*
* This object carries self-describing information about the hook that enables
* runtime discovery, lookup, and instantiation. Each hook instance serves as
* the single source of truth for its own metadata.
*
* **Design rationale:**
* - Hook instances are self-describing (no external configuration files)
* - Name lookup happens at runtime via Registry, not during registration
* - Description can be extended without changing decorator or registry internals
* - Supports future features like versioning, schema validation, and categorization
*
* **Future extensions** may include:
* - `version?: string` - Semantic version of hook implementation for compatibility checks
* - `configSchema?: ZodSchema` - Validation schema for hook-specific options
* - `tags?: string[]` - Categorization tags for grouping and filtering
* - `applicableTriggers?: string[]` - Cached list of compatible trigger node types
* - `deprecated?: boolean | string` - Deprecation status and migration guidance
*
* @see IContextEstablishmentHook.hookDescription
* @see ContextEstablishmentHookMetadata for registration mechanism
*
* @example
* ```typescript
* @ContextEstablishmentHook()
* export class BearerTokenHook implements IContextEstablishmentHook {
* hookDescription = {
* name: 'credentials.bearerToken'
* };
*
* // ... hook implementation
* }
*
* ```
*/
export type HookDescription = {
/**
* Unique identifier for this hook type.
*
* Used by the Hook Registry (to be implemented) to index and retrieve
* hook instances at runtime. Must be unique across all registered hooks.
*
* **Naming convention**: Use namespaced names like 'credentials.bearerToken'
* or 'envVars.tenantConfig' to organize hooks by domain and avoid collisions.
*
* **Usage contexts:**
* - Trigger node configuration specifies hooks by name
* - Hook Registry uses name as lookup key
* - UI displays localized names via i18n (e.g., `hooks.${name}.displayName`)
* - Logging and debugging references hooks by name
* - Error messages include hook name for troubleshooting
*
* **Versioning**: Future hook versions can use naming like 'credentials.bearerToken.v2'
* if breaking changes are needed, though this is not required initially.
*
* @example 'credentials.bearerToken'
* @example 'credentials.apiKey'
* @example 'envVars.tenantConfig'
* @example 'audit.requestMetadata'
*/
name: string;
};
/**
* Interface for context establishment hooks that extract data from trigger
* items and extend the execution context during workflow initialization.
*
* @see ContextEstablishmentOptions - Input parameters
* @see ContextEstablishmentResult - Output structure
* @see PlaintextExecutionContext - Runtime context type with decrypted data
*/
export interface IContextEstablishmentHook {
/**
* Self-describing metadata for this hook instance.
*
* Provides the unique name and future metadata used by the Hook Registry
* for discovery, lookup, and validation. This property makes each hook
* instance self-contained and discoverable without external configuration.
*
* @see HookDescription for detailed metadata structure and future extensions
*
* @example
* ```typescript
* @ContextEstablishmentHook()
* export class BearerTokenHook implements IContextEstablishmentHook {
* hookDescription = {
* name: 'credentials.bearerToken'
* };
*
* async execute(options: ContextEstablishmentOptions) {
* // Hook implementation
* }
*
* isApplicableToTriggerNode(nodeType: string) {
* return nodeType === 'n8n-nodes-base.webhook';
* }
* }
* ```
*/
hookDescription: HookDescription;
/**
* Executes the hook to extract context data from trigger information.
*
* **Implementation requirements:**
* 1. Extract relevant data from trigger items (headers, body, query params, etc.)
* 2. Optionally modify trigger items to remove sensitive data
* 3. Return partial context updates to merge into execution context
* 4. Throw errors for unrecoverable failures (stops workflow execution)
*
* **Execution order:**
* Hooks execute sequentially in the order configured by the trigger node.
* Each hook receives:
* - Trigger items as modified by all previous hooks
* - Context with updates from all previous hooks
*
* **Context handling:**
* - Input context is plaintext (PlaintextExecutionContext) for runtime operations
* - Output updates are plaintext and will be encrypted before persistence
* - Never log or expose plaintext context outside hook execution
*
* **Error handling:**
* - Throw errors if required data is missing (e.g., expected header not found)
* - Use descriptive error messages for debugging
* - Errors stop workflow execution (fail-fast approach)
*
* @param options - Input parameters including trigger node, workflow, items, and current context
* @returns Promise resolving to modified trigger items and context updates
* @throws Error if hook execution fails (stops workflow execution)
*
* @example
* ```typescript
* async execute(options: ContextEstablishmentOptions): Promise<ContextEstablishmentResult> {
* const removeHeader = options.options?.removeFromItem ?? true;
*
* // Extract data
* const token = this.extractToken(options.triggerItems);
* if (!token) {
* throw new Error('Bearer token not found in Authorization header');
* }
*
* // Optionally modify items
* const modifiedItems = removeHeader
* ? this.removeAuthHeader(options.triggerItems)
* : undefined;
*
* // Return context update
* return {
* triggerItems: modifiedItems,
* contextUpdate: {
* credentials: { version: 1, identity: token }
* }
* };
* }
* ```
*/
execute(options: ContextEstablishmentOptions): Promise<ContextEstablishmentResult>;
/**
* Method to determine if this hook is applicable to a specific trigger node type.
*
* **Use cases:**
* - **UI filtering**: Show only relevant hooks for a trigger type in node configuration
* - **Validation**: Prevent incompatible hook configurations at save time
* - **Auto-suggestion**: Suggest applicable hooks based on trigger node selection
* - **Documentation**: Generate trigger-specific hook documentation
*
* **Implementation notes:**
* - Return true if the hook can extract meaningful data from this trigger type
* - Consider transport layer (HTTP, AMQP, manual, etc.)
* - Multiple triggers can share the same hook (e.g., webhook and form trigger both support bearer tokens)
*
* @param nodeType - The node type identifier (e.g., 'n8n-nodes-base.webhook')
* @returns true if this hook can be used with the given trigger node type
*
* @example
* ```typescript
* // Hook only works with HTTP-based triggers
* isApplicableToTriggerNode(nodeType: string): boolean {
* return [
* 'n8n-nodes-base.webhook',
* 'n8n-nodes-base.formTrigger',
* 'n8n-nodes-base.httpRequest'
* ].includes(nodeType);
* }
* ```
*
* @example
* ```typescript
* // Hook works with any trigger that has HTTP headers
* isApplicableToTriggerNode(nodeType: string): boolean {
* return nodeType.includes('webhook') || nodeType.includes('http');
* }
* ```
*/
isApplicableToTriggerNode(nodeType: string): boolean;
}
/**
* Type representing the constructor/class of a context establishment hook.
*
* Used by the dependency injection container to register and instantiate
* hook classes at runtime. Works with the @ContextEstablishmentHook decorator.
*
* @see IContextEstablishmentHook
* @see ContextEstablishmentHook decorator in './index.ts'
*
* @example
* ```typescript
* import { Container } from '@n8n/di';
* import type { ContextEstablishmentHookClass } from './context-establishment-hook';
*
* const HookClass: ContextEstablishmentHookClass = BearerTokenHook;
* const hookInstance = Container.get(HookClass);
* ```
*/
export type ContextEstablishmentHookClass = Constructable<IContextEstablishmentHook>;

View File

@ -0,0 +1,5 @@
export {
ContextEstablishmentHookMetadata,
ContextEstablishmentHook,
} from './context-establishment-hook-metadata';
export type * from './context-establishment-hook';

View File

@ -1,14 +1,16 @@
import type { InstanceType } from '@n8n/constants';
import { Service } from '@n8n/di';
import type { LicenseFlag, ModuleClass } from './module';
/**
* Internal representation of a registered module.
* For field descriptions, see {@link BackendModuleOptions}.
*/
type ModuleEntry = {
class: ModuleClass;
/*
* If singular, checks if that feature ls licensed,
* if multiple, checks that any of the features are licensed
*/
licenseFlag?: LicenseFlag | LicenseFlag[];
instanceTypes?: InstanceType[];
};
@Service()

View File

@ -1,4 +1,4 @@
import type { LICENSE_FEATURES } from '@n8n/constants';
import type { LICENSE_FEATURES, InstanceType } from '@n8n/constants';
import { Container, Service, type Constructable } from '@n8n/di';
import { ModuleMetadata } from './module-metadata';
@ -83,12 +83,27 @@ export type ModuleClass = Constructable<ModuleInterface>;
export type LicenseFlag = (typeof LICENSE_FEATURES)[keyof typeof LICENSE_FEATURES];
export type BackendModuleOptions = {
/** Canonical name of the backend module. Use kebab-case.*/
name: string;
/**
* If present, initialize the module only if the instance has access to a licensed feature.
* Multiple license flags use `OR` logic, i.e. at least one must be licensed.
*/
licenseFlag?: LicenseFlag | LicenseFlag[];
/** If present, initialize the module only if the instance type is one of the specified types. */
instanceTypes?: InstanceType[];
};
export const BackendModule =
(opts: { name: string; licenseFlag?: LicenseFlag | LicenseFlag[] }): ClassDecorator =>
(opts: BackendModuleOptions): ClassDecorator =>
(target) => {
Container.get(ModuleMetadata).register(opts.name, {
class: target as unknown as ModuleClass,
licenseFlag: opts?.licenseFlag,
instanceTypes: opts?.instanceTypes,
});
// eslint-disable-next-line @typescript-eslint/no-unsafe-return

View File

@ -1,5 +1,6 @@
import { describe, it, expect } from 'vitest';
import { sanitizeFilename } from './fileUtils';
import { sanitizeFilename } from './sanitize';
describe('sanitizeFilename', () => {
it('should return normal filenames unchanged', () => {

View File

@ -0,0 +1,87 @@
// Constants definition
/* eslint-disable no-control-regex */
const INVALID_CHARS_REGEX = /[<>:"/\\|?*\u0000-\u001F\u007F-\u009F]/g;
const ZERO_WIDTH_CHARS_REGEX = /[\u200B-\u200D\u2060\uFEFF]/g;
const UNICODE_SPACES_REGEX = /[\u00A0\u2000-\u200A]/g;
const LEADING_TRAILING_DOTS_SPACES_REGEX = /^[\s.]+|[\s.]+$/g;
/* eslint-enable no-control-regex */
const WINDOWS_RESERVED_NAMES = new Set([
'CON',
'PRN',
'AUX',
'NUL',
'COM1',
'COM2',
'COM3',
'COM4',
'COM5',
'COM6',
'COM7',
'COM8',
'COM9',
'LPT1',
'LPT2',
'LPT3',
'LPT4',
'LPT5',
'LPT6',
'LPT7',
'LPT8',
'LPT9',
]);
const DEFAULT_FALLBACK_NAME = 'untitled';
const MAX_FILENAME_LENGTH = 200;
/**
* Sanitizes a filename to be compatible with Mac, Linux, and Windows file systems
*
* Main features:
* - Replace invalid characters (e.g. ":" in hello:world)
* - Handle Windows reserved names
* - Limit filename length
* - Normalize Unicode characters
*
* @param filename - The filename to sanitize (without extension)
* @param maxLength - Maximum filename length (default: 200)
* @returns A sanitized filename (without extension)
*
* @example
* sanitizeFilename('hello:world') // returns 'hello_world'
* sanitizeFilename('CON') // returns '_CON'
* sanitizeFilename('') // returns 'untitled'
*/
export const sanitizeFilename = (
filename: string,
maxLength: number = MAX_FILENAME_LENGTH,
): string => {
// Input validation
if (!filename) {
return DEFAULT_FALLBACK_NAME;
}
let baseName = filename
.trim()
.replace(INVALID_CHARS_REGEX, '_')
.replace(ZERO_WIDTH_CHARS_REGEX, '')
.replace(UNICODE_SPACES_REGEX, ' ')
.replace(LEADING_TRAILING_DOTS_SPACES_REGEX, '');
// Handle empty or invalid filenames after cleaning
if (!baseName) {
baseName = DEFAULT_FALLBACK_NAME;
}
// Handle Windows reserved names
if (WINDOWS_RESERVED_NAMES.has(baseName.toUpperCase())) {
baseName = `_${baseName}`;
}
// Truncate if too long
if (baseName.length > maxLength) {
baseName = baseName.slice(0, maxLength);
}
return baseName;
};

View File

@ -7,3 +7,4 @@ export * from './search/reRankSearchResults';
export * from './search/sublimeSearch';
export * from './sort/sortByProperty';
export * from './string/truncate';
export * from './files/sanitize';

View File

@ -2,6 +2,16 @@
This list shows all the versions which include breaking changes and how to upgrade.
# 1.122.0
### What changed?
The way to add third-party dependencies to the `n8nio/runners` image has changed. More details [here](https://docs.n8n.io/hosting/configuration/task-runners/#adding-extra-dependencies).
### When is action necessary?
If you adding third-party dependencies to the `n8nio/runners` image using `package.json` and `extras.txt` and building the image yourself, please extend the image as instructed in the link above.
# 1.113.0
### What changed?

View File

@ -110,6 +110,7 @@
"@n8n/permissions": "workspace:*",
"@n8n/task-runner": "workspace:*",
"@n8n/typeorm": "catalog:",
"@n8n/utils": "workspace:*",
"@n8n_io/ai-assistant-sdk": "catalog:",
"@n8n_io/license-sdk": "2.24.1",
"@rudderstack/rudder-sdk-node": "2.1.4",

View File

@ -87,6 +87,9 @@ export class AuthService {
'/types/nodes.json',
'/types/credentials.json',
'/mcp-oauth/authorize/',
// Skip browser ID check for chat hub attachments
`/${restEndpoint}/chat/conversations/:sessionId/messages/:messageId/attachments/:index`,
];
}

View File

@ -251,7 +251,7 @@ export class Start extends BaseCommand<z.infer<typeof flagsSchema>> {
await this.generateStaticAssets();
}
await this.moduleRegistry.initModules();
await this.moduleRegistry.initModules(this.instanceSettings.instanceType);
if (this.instanceSettings.isMultiMain) {
// we instantiate `PrometheusMetricsService` early to register its multi-main event handlers

View File

@ -82,7 +82,7 @@ export class Webhook extends BaseCommand {
});
Container.get(LogStreamingEventRelay).init();
await this.moduleRegistry.initModules();
await this.moduleRegistry.initModules(this.instanceSettings.instanceType);
}
async run() {

View File

@ -114,7 +114,7 @@ export class Worker extends BaseCommand<z.infer<typeof flagsSchema>> {
}),
);
await this.moduleRegistry.initModules();
await this.moduleRegistry.initModules(this.instanceSettings.instanceType);
}
async initEventBus() {

View File

@ -68,7 +68,9 @@ describe('ExecutionRepository', () => {
await executionRepository.deleteExecutionsByFilter({ id: '1' }, ['1'], { ids: ['1'] });
expect(binaryDataService.deleteMany).toHaveBeenCalledWith([{ executionId: '1', workflowId }]);
expect(binaryDataService.deleteMany).toHaveBeenCalledWith([
{ type: 'execution', executionId: '1', workflowId },
]);
});
});

View File

@ -49,6 +49,12 @@ export const rawBodyReader: RequestHandler = async (req, _res, next) => {
};
export const parseBody = async (req: Request) => {
// Skip multipart requests (e.g., file uploads) - these need specialized parsing by multer.
// Reading the body stream here would consume it, making it unavailable for multer processing.
if (req.contentType?.startsWith('multipart/')) {
return;
}
await req.readRawBody();
const { rawBody, contentType, encoding } = req;
if (rawBody?.length) {

View File

@ -1,12 +1,15 @@
import { testDb, testModules } from '@n8n/backend-test-utils';
import { mockInstance, testDb, testModules } from '@n8n/backend-test-utils';
import type { User } from '@n8n/db';
import { Container } from '@n8n/di';
import { BinaryDataService } from 'n8n-core';
import { createAdmin, createMember } from '@test-integration/db/users';
import { ChatHubService } from '../chat-hub.service';
import { ChatHubMessageRepository } from '../chat-message.repository';
import { ChatHubSessionRepository } from '../chat-session.repository';
mockInstance(BinaryDataService);
beforeAll(async () => {
await testModules.loadModules(['chat-hub']);
await testDb.init();
@ -45,9 +48,9 @@ describe('chatHub', () => {
describe('getConversations', () => {
it('should list empty conversations', async () => {
const conversations = await chatHubService.getConversations(member.id);
const conversations = await chatHubService.getConversations(member.id, 20);
expect(conversations).toBeDefined();
expect(conversations).toHaveLength(0);
expect(conversations.data).toHaveLength(0);
});
it("should list user's own conversations in expected order", async () => {
@ -80,11 +83,182 @@ describe('chatHub', () => {
tools: [],
});
const conversations = await chatHubService.getConversations(member.id);
expect(conversations).toHaveLength(3);
expect(conversations[0].id).toBe(session1.id);
expect(conversations[1].id).toBe(session2.id);
expect(conversations[2].id).toBe(session3.id);
const conversations = await chatHubService.getConversations(member.id, 20);
expect(conversations.data).toHaveLength(3);
expect(conversations.data[0].id).toBe(session1.id);
expect(conversations.data[1].id).toBe(session2.id);
expect(conversations.data[2].id).toBe(session3.id);
});
describe('pagination', () => {
it('should return hasMore=false and nextCursor=null when all sessions fit in one page', async () => {
await sessionsRepository.createChatSession({
id: crypto.randomUUID(),
ownerId: member.id,
title: 'session 1',
lastMessageAt: new Date('2025-01-01T00:00:00Z'),
tools: [],
});
const conversations = await chatHubService.getConversations(member.id, 10);
expect(conversations.data).toHaveLength(1);
expect(conversations.hasMore).toBe(false);
expect(conversations.nextCursor).toBeNull();
});
it('should fetch next page using cursor', async () => {
const session1 = await sessionsRepository.createChatSession({
id: crypto.randomUUID(),
ownerId: member.id,
title: 'session 1',
lastMessageAt: new Date('2025-01-05T00:00:00Z'),
tools: [],
});
const session2 = await sessionsRepository.createChatSession({
id: crypto.randomUUID(),
ownerId: member.id,
title: 'session 2',
lastMessageAt: new Date('2025-01-04T00:00:00Z'),
tools: [],
});
const session3 = await sessionsRepository.createChatSession({
id: crypto.randomUUID(),
ownerId: member.id,
title: 'session 3',
lastMessageAt: new Date('2025-01-03T00:00:00Z'),
tools: [],
});
const session4 = await sessionsRepository.createChatSession({
id: crypto.randomUUID(),
ownerId: member.id,
title: 'session 4',
lastMessageAt: new Date('2025-01-02T00:00:00Z'),
tools: [],
});
// First page
const page1 = await chatHubService.getConversations(member.id, 2);
expect(page1.data).toHaveLength(2);
expect(page1.data[0].id).toBe(session1.id);
expect(page1.data[1].id).toBe(session2.id);
expect(page1.hasMore).toBe(true);
expect(page1.nextCursor).toBe(session2.id);
// Second page using cursor
const page2 = await chatHubService.getConversations(member.id, 2, page1.nextCursor!);
expect(page2.data).toHaveLength(2);
expect(page2.data[0].id).toBe(session3.id);
expect(page2.data[1].id).toBe(session4.id);
expect(page2.hasMore).toBe(false);
expect(page2.nextCursor).toBeNull();
});
it('should handle sessions with same lastMessageAt using id for ordering', async () => {
const sameDate = new Date('2025-01-01T00:00:00Z');
const session1 = await sessionsRepository.createChatSession({
id: '00000000-0000-0000-0000-000000000001',
ownerId: member.id,
title: 'Session 1',
lastMessageAt: sameDate,
tools: [],
});
const session2 = await sessionsRepository.createChatSession({
id: '00000000-0000-0000-0000-000000000002',
ownerId: member.id,
title: 'Session 2',
lastMessageAt: sameDate,
tools: [],
});
const session3 = await sessionsRepository.createChatSession({
id: '00000000-0000-0000-0000-000000000003',
ownerId: member.id,
title: 'Session 3',
lastMessageAt: sameDate,
tools: [],
});
// Fetch first page
const page1 = await chatHubService.getConversations(member.id, 2);
expect(page1.data).toHaveLength(2);
expect(page1.data[0].id).toBe(session1.id);
expect(page1.data[1].id).toBe(session2.id);
expect(page1.hasMore).toBe(true);
// Fetch second page
const page2 = await chatHubService.getConversations(member.id, 2, page1.nextCursor!);
expect(page2.data).toHaveLength(1);
expect(page2.data[0].id).toBe(session3.id);
expect(page2.hasMore).toBe(false);
});
it('should throw error when cursor session does not exist', async () => {
await sessionsRepository.createChatSession({
id: crypto.randomUUID(),
ownerId: member.id,
title: 'session 1',
lastMessageAt: new Date('2025-01-01T00:00:00Z'),
tools: [],
});
const nonExistentCursor = '00000000-0000-0000-0000-000000000000';
await expect(
chatHubService.getConversations(member.id, 10, nonExistentCursor),
).rejects.toThrow('Cursor session not found');
});
it('should throw error when cursor session belongs to different user', async () => {
await sessionsRepository.createChatSession({
id: crypto.randomUUID(),
ownerId: member.id,
title: 'Member Session',
lastMessageAt: new Date('2025-01-02T00:00:00Z'),
tools: [],
});
const adminSession = await sessionsRepository.createChatSession({
id: crypto.randomUUID(),
ownerId: admin.id,
title: 'Admin Session',
lastMessageAt: new Date('2025-01-01T00:00:00Z'),
tools: [],
});
await expect(
chatHubService.getConversations(member.id, 10, adminSession.id),
).rejects.toThrow('Cursor session not found');
});
it('should handle sessions with null lastMessageAt', async () => {
const session1 = await sessionsRepository.createChatSession({
id: crypto.randomUUID(),
ownerId: member.id,
title: 'Session with date',
lastMessageAt: new Date('2025-01-01T00:00:00Z'),
tools: [],
});
const session2 = await sessionsRepository.createChatSession({
id: crypto.randomUUID(),
ownerId: member.id,
title: 'Session without date',
lastMessageAt: null,
tools: [],
});
const conversations = await chatHubService.getConversations(member.id, 10);
expect(conversations.data).toHaveLength(2);
expect(conversations.data[0].id).toBe(session1.id);
expect(conversations.data[1].id).toBe(session2.id);
});
});
});

View File

@ -32,6 +32,7 @@ export class ChatHubAgentService {
},
createdAt: agent.createdAt.toISOString(),
updatedAt: agent.updatedAt.toISOString(),
allowFileUploads: true,
})),
};
}

View File

@ -11,6 +11,7 @@ import {
} from '@n8n/typeorm';
import type { ChatHubSession } from './chat-hub-session.entity';
import type { IBinaryData } from 'n8n-workflow';
@Entity({ name: 'chat_hub_messages' })
export class ChatHubMessage extends WithTimestamps {
@ -167,4 +168,13 @@ export class ChatHubMessage extends WithTimestamps {
*/
@Column({ type: 'varchar', length: 16, default: 'success' })
status: ChatHubMessageStatus;
/**
* File attachments for the message (if any), stored as JSON.
* Storage strategy depends on the binary data mode configuration:
* - When using external storage (e.g., filesystem-v2): Only metadata is stored, with 'id' referencing the external location
* - When using default mode: Base64-encoded data is stored directly in the 'data' field
*/
@Column({ type: 'json', nullable: true })
attachments: Array<IBinaryData> | null;
}

View File

@ -21,8 +21,10 @@ import {
IWorkflowBase,
MEMORY_BUFFER_WINDOW_NODE_TYPE,
MEMORY_MANAGER_NODE_TYPE,
MERGE_NODE_TYPE,
NodeConnectionTypes,
OperationalError,
type IBinaryData,
} from 'n8n-workflow';
import { v4 as uuidv4 } from 'uuid';
@ -49,6 +51,7 @@ export class ChatHubWorkflowService {
projectId: string,
history: ChatHubMessage[],
humanMessage: string,
attachments: IBinaryData[],
credentials: INodeCredentials,
model: ChatHubConversationModel,
systemMessage: string | undefined,
@ -65,6 +68,7 @@ export class ChatHubWorkflowService {
sessionId,
history,
humanMessage,
attachments,
credentials,
model,
systemMessage,
@ -72,6 +76,7 @@ export class ChatHubWorkflowService {
});
const newWorkflow = new WorkflowEntity();
newWorkflow.versionId = uuidv4();
newWorkflow.name = `Chat ${sessionId}`;
newWorkflow.active = false;
@ -147,6 +152,38 @@ export class ChatHubWorkflowService {
});
}
prepareExecutionData(
triggerNode: INode,
sessionId: string,
message: string,
attachments: IBinaryData[],
): IExecuteData[] {
// Attachments are already processed (id field populated) by the caller
return [
{
node: triggerNode,
data: {
main: [
[
{
json: {
sessionId,
action: 'sendMessage',
chatInput: message,
files: attachments.map(({ data, ...metadata }) => metadata),
},
binary: Object.fromEntries(
attachments.map((attachment, index) => [`data${index}`, attachment]),
),
},
],
],
},
source: null,
},
];
}
private getUniqueNodeName(originalName: string, existingNames: Set<string>): string {
if (!existingNames.has(originalName)) {
return originalName;
@ -168,6 +205,7 @@ export class ChatHubWorkflowService {
sessionId,
history,
humanMessage,
attachments,
credentials,
model,
systemMessage,
@ -177,6 +215,7 @@ export class ChatHubWorkflowService {
sessionId: ChatSessionId;
history: ChatHubMessage[];
humanMessage: string;
attachments: IBinaryData[];
credentials: INodeCredentials;
model: ChatHubConversationModel;
systemMessage?: string;
@ -188,6 +227,7 @@ export class ChatHubWorkflowService {
const memoryNode = this.buildMemoryNode(20);
const restoreMemoryNode = this.buildRestoreMemoryNode(history);
const clearMemoryNode = this.buildClearMemoryNode();
const mergeNode = this.buildMergeNode();
const nodes: INode[] = [
chatTriggerNode,
@ -196,6 +236,7 @@ export class ChatHubWorkflowService {
memoryNode,
restoreMemoryNode,
clearMemoryNode,
mergeNode,
];
const nodeNames = new Set(nodes.map((node) => node.name));
@ -221,10 +262,18 @@ export class ChatHubWorkflowService {
const connections: IConnections = {
[NODE_NAMES.CHAT_TRIGGER]: {
[NodeConnectionTypes.Main]: [
[{ node: NODE_NAMES.RESTORE_CHAT_MEMORY, type: NodeConnectionTypes.Main, index: 0 }],
[
{ node: NODE_NAMES.RESTORE_CHAT_MEMORY, type: NodeConnectionTypes.Main, index: 0 },
{ node: NODE_NAMES.MERGE, type: NodeConnectionTypes.Main, index: 0 },
],
],
},
[NODE_NAMES.RESTORE_CHAT_MEMORY]: {
[NodeConnectionTypes.Main]: [
[{ node: NODE_NAMES.MERGE, type: NodeConnectionTypes.Main, index: 1 }],
],
},
[NODE_NAMES.MERGE]: {
[NodeConnectionTypes.Main]: [
[{ node: NODE_NAMES.REPLY_AGENT, type: NodeConnectionTypes.Main, index: 0 }],
],
@ -271,25 +320,12 @@ export class ChatHubWorkflowService {
}, {}),
};
const nodeExecutionStack: IExecuteData[] = [
{
node: chatTriggerNode,
data: {
main: [
[
{
json: {
sessionId,
action: 'sendMessage',
chatInput: humanMessage,
},
},
],
],
},
source: null,
},
];
const nodeExecutionStack = this.prepareExecutionData(
chatTriggerNode,
sessionId,
humanMessage,
attachments,
);
const executionData = createRunExecutionData({
executionData: {
@ -452,7 +488,7 @@ export class ChatHubWorkflowService {
return {
...common,
parameters: {
model: { __rl: true, mode: 'id', value: model },
model,
options: {},
},
};
@ -465,6 +501,24 @@ export class ChatHubWorkflowService {
},
};
}
case 'awsBedrock': {
return {
...common,
parameters: {
model,
options: {},
},
};
}
case 'mistralCloud': {
return {
...common,
parameters: {
model,
options: {},
},
};
}
default:
throw new OperationalError('Unsupported model provider');
}
@ -532,6 +586,22 @@ export class ChatHubWorkflowService {
};
}
private buildMergeNode(): INode {
return {
parameters: {
mode: 'combine',
fieldsToMatchString: 'chatInput',
joinMode: 'enrichInput1',
options: {},
},
type: MERGE_NODE_TYPE,
typeVersion: 3.2,
position: [224, -100],
id: uuidv4(),
name: NODE_NAMES.MERGE,
};
}
private buildTitleGeneratorAgentNode(): INode {
return {
parameters: {

View File

@ -0,0 +1,160 @@
import { Service } from '@n8n/di';
import { BINARY_ENCODING, type IBinaryData } from 'n8n-workflow';
import { sanitizeFilename } from '@n8n/utils';
import { BinaryDataService, FileLocation } from 'n8n-core';
import { Not, IsNull } from '@n8n/typeorm';
import { ChatHubMessageRepository } from './chat-message.repository';
import type { ChatMessageId, ChatSessionId, ChatAttachment } from '@n8n/api-types';
import { NotFoundError } from '@/errors/response-errors/not-found.error';
import { BadRequestError } from '@/errors/response-errors/bad-request.error';
import type Stream from 'node:stream';
import FileType from 'file-type';
@Service()
export class ChatHubAttachmentService {
private readonly maxTotalSizeBytes = 200 * 1024 * 1024; // 200 MB
constructor(
private readonly binaryDataService: BinaryDataService,
private readonly messageRepository: ChatHubMessageRepository,
) {}
/**
* Stores attachments through BinaryDataService.
* This populates the 'id' and other metadata for attachments. When external storage is used,
* BinaryDataService replaces base64 data with the storage mode string (e.g., "filesystem-v2").
*/
async store(
sessionId: ChatSessionId,
messageId: ChatMessageId,
attachments: ChatAttachment[],
): Promise<IBinaryData[]> {
let totalSize = 0;
const storedAttachments: IBinaryData[] = [];
for (const attachment of attachments) {
const buffer = Buffer.from(attachment.data, BINARY_ENCODING);
totalSize += buffer.length;
if (totalSize > this.maxTotalSizeBytes) {
const maxSizeMB = Math.floor(this.maxTotalSizeBytes / (1024 * 1024));
throw new BadRequestError(
`Total size of attachments exceeds maximum size of ${maxSizeMB} MB`,
);
}
const stored = await this.processAttachment(sessionId, messageId, attachment, buffer);
storedAttachments.push(stored);
}
return storedAttachments;
}
/*
* Gets a specific attachment from a message by index and returns it as either buffer or stream
*/
async getAttachment(
sessionId: ChatSessionId,
messageId: ChatMessageId,
attachmentIndex: number,
): Promise<
[
IBinaryData,
(
| { type: 'buffer'; buffer: Buffer<ArrayBufferLike>; fileSize: number }
| { type: 'stream'; stream: Stream.Readable; fileSize: number }
),
]
> {
const message = await this.messageRepository.getOneById(messageId, sessionId, []);
if (!message) {
throw new NotFoundError('Message not found');
}
const attachment = message.attachments?.[attachmentIndex];
if (!attachment) {
throw new NotFoundError('Attachment not found');
}
if (attachment.id) {
const metadata = await this.binaryDataService.getMetadata(attachment.id);
const stream = await this.binaryDataService.getAsStream(attachment.id);
return [attachment, { type: 'stream', stream, fileSize: metadata.fileSize }];
}
if (attachment.data) {
const buffer = await this.binaryDataService.getAsBuffer(attachment);
return [attachment, { type: 'buffer', buffer, fileSize: buffer.length }];
}
throw new NotFoundError('Attachment has no stored file');
}
/**
* Deletes all files attached to messages in the session
*/
async deleteAllBySessionId(sessionId: string): Promise<void> {
const messages = await this.messageRepository.getManyBySessionId(sessionId);
await this.deleteAttachments(messages.flatMap((message) => message.attachments ?? []));
}
/**
* Deletes all chat attachment files.
*/
async deleteAll(): Promise<void> {
const messages = await this.messageRepository.find({
where: {
attachments: Not(IsNull()),
},
select: ['attachments'],
});
await this.deleteAttachments(messages.flatMap((message) => message.attachments ?? []));
}
/**
* Deletes attachments by their binary data directly (used for rollback when message wasn't saved)
*/
async deleteAttachments(attachments: IBinaryData[]): Promise<void> {
await this.binaryDataService.deleteManyByBinaryDataId(
attachments.flatMap((attachment) => (attachment.id ? [attachment.id] : [])),
);
}
/**
* Processes a single attachment by populating metadata and storing it.
*/
private async processAttachment(
sessionId: ChatSessionId,
messageId: ChatMessageId,
attachment: ChatAttachment,
buffer: Buffer,
): Promise<IBinaryData> {
const sanitizedFileName = sanitizeFilename(attachment.fileName);
const fileTypeData = await FileType.fromBuffer(buffer);
// Only trust content-based detection for security
const mimeType = fileTypeData?.mime ?? 'application/octet-stream';
// Construct IBinaryData with all required fields
const binaryData: IBinaryData = {
data: attachment.data,
mimeType,
fileName: sanitizedFileName,
fileSize: `${buffer.length}`,
fileExtension: fileTypeData?.ext,
};
return await this.binaryDataService.store(
FileLocation.ofChatHubMessageAttachment(sessionId, messageId),
buffer,
binaryData,
);
}
}

View File

@ -32,6 +32,14 @@ export const PROVIDER_NODE_TYPE_MAP: Record<ChatHubLLMProvider, INodeTypeNameVer
name: '@n8n/n8n-nodes-langchain.lmChatAzureOpenAi',
version: 1,
},
awsBedrock: {
name: '@n8n/n8n-nodes-langchain.lmChatAwsBedrock',
version: 1.1,
},
mistralCloud: {
name: '@n8n/n8n-nodes-langchain.lmChatMistralCloud',
version: 1,
},
};
export const NODE_NAMES = {
@ -42,6 +50,7 @@ export const NODE_NAMES = {
MEMORY: 'Memory',
RESTORE_CHAT_MEMORY: 'Restore Chat Memory',
CLEAR_CHAT_MEMORY: 'Clear Chat Memory',
MERGE: 'Merge',
} as const;
/* eslint-disable @typescript-eslint/naming-convention */

View File

@ -10,6 +10,7 @@ import {
ChatMessageId,
ChatHubCreateAgentRequest,
ChatHubUpdateAgentRequest,
ChatHubConversationsRequest,
} from '@n8n/api-types';
import { Logger } from '@n8n/backend-common';
import { AuthenticatedRequest } from '@n8n/db';
@ -22,13 +23,17 @@ import {
Delete,
Param,
Patch,
Query,
} from '@n8n/decorators';
import type { Response } from 'express';
import { strict as assert } from 'node:assert';
import { ChatHubAgentService } from './chat-hub-agent.service';
import { ChatHubAttachmentService } from './chat-hub.attachment.service';
import { ChatHubService } from './chat-hub.service';
import { ChatModelsRequestDto } from './dto/chat-models-request.dto';
import { BadRequestError } from '@/errors/response-errors/bad-request.error';
import { sanitizeFilename } from '@n8n/utils';
import { ResponseError } from '@/errors/response-errors/abstract/response.error';
@ -37,6 +42,7 @@ export class ChatHubController {
constructor(
private readonly chatService: ChatHubService,
private readonly chatAgentService: ChatHubAgentService,
private readonly chatAttachmentService: ChatHubAttachmentService,
private readonly logger: Logger,
) {}
@ -55,8 +61,9 @@ export class ChatHubController {
async getConversations(
req: AuthenticatedRequest,
_res: Response,
@Query query: ChatHubConversationsRequest,
): Promise<ChatHubConversationsResponse> {
return await this.chatService.getConversations(req.user.id);
return await this.chatService.getConversations(req.user.id, query.limit, query.cursor);
}
@Get('/conversations/:sessionId')
@ -69,6 +76,49 @@ export class ChatHubController {
return await this.chatService.getConversation(req.user.id, sessionId);
}
@Get('/conversations/:sessionId/messages/:messageId/attachments/:index')
@GlobalScope('chatHub:message')
async getMessageAttachment(
req: AuthenticatedRequest,
res: Response,
@Param('sessionId') sessionId: ChatSessionId,
@Param('messageId') messageId: ChatMessageId,
@Param('index') index: string,
) {
const attachmentIndex = Number.parseInt(index, 10);
if (isNaN(attachmentIndex)) {
throw new BadRequestError('Invalid attachment index');
}
// Verify user has access to this session
await this.chatService.getConversation(req.user.id, sessionId);
const [{ mimeType, fileName }, attachmentAsStreamOrBuffer] =
await this.chatAttachmentService.getAttachment(sessionId, messageId, attachmentIndex);
res.setHeader('Content-Type', mimeType ?? 'application/octet-stream');
if (attachmentAsStreamOrBuffer.fileSize) {
res.setHeader('Content-Length', attachmentAsStreamOrBuffer.fileSize);
}
if (fileName) {
res.setHeader('Content-Disposition', `attachment; filename="${sanitizeFilename(fileName)}"`);
}
if (attachmentAsStreamOrBuffer.type === 'buffer') {
res.send(attachmentAsStreamOrBuffer.buffer);
return;
}
return await new Promise<void>((resolve, reject) => {
attachmentAsStreamOrBuffer.stream.on('end', resolve);
attachmentAsStreamOrBuffer.stream.on('error', reject);
attachmentAsStreamOrBuffer.stream.pipe(res);
});
}
@GlobalScope('chatHub:message')
@Post('/conversations/send')
async sendMessage(

View File

@ -33,10 +33,10 @@ import {
jsonParse,
StructuredChunk,
RESPOND_TO_CHAT_NODE_TYPE,
IExecuteData,
IRunExecutionData,
INodeParameters,
INode,
type IBinaryData,
createRunExecutionData,
} from 'n8n-workflow';
@ -45,10 +45,11 @@ import { ChatHubCredentialsService, CredentialWithProjectId } from './chat-hub-c
import type { ChatHubMessage } from './chat-hub-message.entity';
import { ChatHubWorkflowService } from './chat-hub-workflow.service';
import { JSONL_STREAM_HEADERS, NODE_NAMES, PROVIDER_NODE_TYPE_MAP } from './chat-hub.constants';
import type {
import {
HumanMessagePayload,
RegenerateMessagePayload,
EditMessagePayload,
validChatTriggerParamsShape,
} from './chat-hub.types';
import { ChatHubMessageRepository } from './chat-message.repository';
import { ChatHubSessionRepository } from './chat-session.repository';
@ -64,6 +65,7 @@ import { getBase } from '@/workflow-execute-additional-data';
import { WorkflowExecutionService } from '@/workflows/workflow-execution.service';
import { WorkflowFinderService } from '@/workflows/workflow-finder.service';
import { WorkflowService } from '@/workflows/workflow.service';
import { ChatHubAttachmentService } from './chat-hub.attachment.service';
@Service()
export class ChatHubService {
@ -83,6 +85,7 @@ export class ChatHubService {
private readonly chatHubAgentService: ChatHubAgentService,
private readonly chatHubCredentialsService: ChatHubCredentialsService,
private readonly chatHubWorkflowService: ChatHubWorkflowService,
private readonly chatHubAttachmentService: ChatHubAttachmentService,
) {}
async getModels(
@ -159,6 +162,10 @@ export class ChatHubService {
return await this.fetchOllamaModels(credentials, additionalData);
case 'azureOpenAi':
return await this.fetchAzureOpenAiModels(credentials, additionalData);
case 'awsBedrock':
return await this.fetchAwsBedrockModels(credentials, additionalData);
case 'mistralCloud':
return await this.fetchMistralCloudModels(credentials, additionalData);
case 'n8n':
return await this.fetchAgentWorkflowsAsModels(user);
case 'custom-agent':
@ -189,6 +196,7 @@ export class ChatHubService {
},
createdAt: null,
updatedAt: null,
allowFileUploads: true,
})),
};
}
@ -216,6 +224,7 @@ export class ChatHubService {
},
createdAt: null,
updatedAt: null,
allowFileUploads: true,
})),
};
}
@ -273,7 +282,7 @@ export class ChatHubService {
return {
models: results.map((result) => ({
name: String(result.value),
name: result.name,
description: result.description ?? null,
model: {
provider: 'google',
@ -281,6 +290,7 @@ export class ChatHubService {
},
createdAt: null,
updatedAt: null,
allowFileUploads: true,
})),
};
}
@ -331,7 +341,7 @@ export class ChatHubService {
return {
models: results.map((result) => ({
name: String(result.value),
name: result.name,
description: result.description ?? null,
model: {
provider: 'ollama',
@ -339,6 +349,7 @@ export class ChatHubService {
},
createdAt: null,
updatedAt: null,
allowFileUploads: true,
})),
};
}
@ -355,6 +366,174 @@ export class ChatHubService {
};
}
private async fetchAwsBedrockModels(
credentials: INodeCredentials,
additionalData: IWorkflowExecuteAdditionalData,
): Promise<ChatModelsResponse['awsBedrock']> {
// From AWS Bedrock node
// https://github.com/n8n-io/n8n/blob/master/packages/%40n8n/nodes-langchain/nodes/llms/LmChatAwsBedrock/LmChatAwsBedrock.node.ts#L100
// https://github.com/n8n-io/n8n/blob/master/packages/%40n8n/nodes-langchain/nodes/llms/LmChatAwsBedrock/LmChatAwsBedrock.node.ts#L155
const foundationModelsRequest = this.nodeParametersService.getOptionsViaLoadOptions(
{
routing: {
request: {
method: 'GET',
url: '/foundation-models?&byOutputModality=TEXT&byInferenceType=ON_DEMAND',
},
output: {
postReceive: [
{
type: 'rootProperty',
properties: {
property: 'modelSummaries',
},
},
{
type: 'setKeyValue',
properties: {
name: '={{$responseItem.modelName}}',
description: '={{$responseItem.modelArn}}',
value: '={{$responseItem.modelId}}',
},
},
{
type: 'sort',
properties: {
key: 'name',
},
},
],
},
},
},
additionalData,
PROVIDER_NODE_TYPE_MAP.awsBedrock,
{},
credentials,
);
const inferenceProfileModelsRequest = this.nodeParametersService.getOptionsViaLoadOptions(
{
routing: {
request: {
method: 'GET',
url: '/inference-profiles?maxResults=1000',
},
output: {
postReceive: [
{
type: 'rootProperty',
properties: {
property: 'inferenceProfileSummaries',
},
},
{
type: 'setKeyValue',
properties: {
name: '={{$responseItem.inferenceProfileName}}',
description:
'={{$responseItem.description || $responseItem.inferenceProfileArn}}',
value: '={{$responseItem.inferenceProfileId}}',
},
},
{
type: 'sort',
properties: {
key: 'name',
},
},
],
},
},
},
additionalData,
PROVIDER_NODE_TYPE_MAP.awsBedrock,
{},
credentials,
);
const [foundationModels, inferenceProfileModels] = await Promise.all([
foundationModelsRequest,
inferenceProfileModelsRequest,
]);
return {
models: foundationModels.concat(inferenceProfileModels).map((result) => ({
name: result.name,
description: result.description ?? String(result.value),
model: {
provider: 'awsBedrock',
model: String(result.value),
},
createdAt: null,
updatedAt: null,
allowFileUploads: true,
})),
};
}
private async fetchMistralCloudModels(
credentials: INodeCredentials,
additionalData: IWorkflowExecuteAdditionalData,
): Promise<ChatModelsResponse['mistralCloud']> {
const results = await this.nodeParametersService.getOptionsViaLoadOptions(
{
routing: {
request: {
method: 'GET',
url: '/models',
},
output: {
postReceive: [
{
type: 'rootProperty',
properties: {
property: 'data',
},
},
{
type: 'filter',
properties: {
pass: "={{ !$responseItem.id.includes('embed') }}",
},
},
{
type: 'setKeyValue',
properties: {
name: '={{ $responseItem.id }}',
value: '={{ $responseItem.id }}',
},
},
{
type: 'sort',
properties: {
key: 'name',
},
},
],
},
},
},
additionalData,
PROVIDER_NODE_TYPE_MAP.mistralCloud,
{},
credentials,
);
return {
models: results.map((result) => ({
name: result.name,
description: result.description ?? String(result.value),
model: {
provider: 'mistralCloud',
model: String(result.value),
},
createdAt: null,
updatedAt: null,
})),
};
}
private async fetchAgentWorkflowsAsModels(user: User): Promise<ChatModelsResponse['n8n']> {
const nodeTypes = [CHAT_TRIGGER_NODE_TYPE];
const workflows = await this.workflowService.getWorkflowsWithNodesIncluded(
@ -374,30 +553,25 @@ export class ChatHubService {
return [];
}
if (chatTrigger.parameters.availableInChat !== true) {
const chatTriggerParams = validChatTriggerParamsShape.safeParse(
chatTrigger.parameters,
).data;
if (!chatTriggerParams) {
return [];
}
const name =
typeof chatTrigger.parameters.agentName === 'string' &&
chatTrigger.parameters.agentName.length > 0
? chatTrigger.parameters.agentName
: workflow.name;
return [
{
name: name ?? 'Unknown Agent',
description:
typeof chatTrigger.parameters.agentDescription === 'string' &&
chatTrigger.parameters.agentDescription.length > 0
? chatTrigger.parameters.agentDescription
: null,
name: chatTriggerParams.agentName ?? workflow.name ?? 'Unknown Agent',
description: chatTriggerParams.agentDescription ?? null,
model: {
provider: 'n8n',
workflowId: workflow.id,
},
createdAt: workflow.createdAt ? workflow.createdAt.toISOString() : null,
updatedAt: workflow.updatedAt ? workflow.updatedAt.toISOString() : null,
allowFileUploads: chatTriggerParams.options?.allowFileUploads ?? false,
},
];
}),
@ -449,12 +623,31 @@ export class ChatHubService {
}
async sendHumanMessage(res: Response, user: User, payload: HumanMessagePayload) {
const { sessionId, messageId, message, model, credentials, previousMessageId, tools } = payload;
const {
sessionId,
messageId,
message,
model,
credentials,
previousMessageId,
tools,
attachments,
} = payload;
const credentialId = this.getModelCredential(model, credentials);
const { executionData, workflowData } = await this.messageRepository.manager.transaction(
async (trx) => {
// Store attachments early to populate 'id' field via BinaryDataService
const processedAttachments = await this.chatHubAttachmentService.store(
sessionId,
messageId,
attachments,
);
let executionData: IRunExecutionData;
let workflowData: IWorkflowBase;
try {
const result = await this.messageRepository.manager.transaction(async (trx) => {
let session = await this.getChatSession(user, sessionId, trx);
session ??= await this.createChatSession(user, sessionId, model, credentialId, tools, trx);
@ -462,36 +655,36 @@ export class ChatHubService {
const messages = Object.fromEntries((session.messages ?? []).map((m) => [m.id, m]));
const history = this.buildMessageHistory(messages, previousMessageId);
await this.saveHumanMessage(payload, user, previousMessageId, model, undefined, trx);
await this.saveHumanMessage(
payload,
processedAttachments,
user,
previousMessageId,
model,
undefined,
trx,
);
if (model.provider === 'n8n') {
return await this.prepareCustomAgentWorkflow(user, sessionId, model.workflowId, message);
}
if (model.provider === 'custom-agent') {
return await this.prepareChatAgentWorkflow(
model.agentId,
user,
sessionId,
history,
message,
trx,
);
}
return await this.prepareBaseChatWorkflow(
return await this.prepareReplyWorkflow(
user,
sessionId,
credentials,
model,
history,
message,
undefined,
session.tools,
tools,
processedAttachments,
trx,
);
},
);
});
executionData = result.executionData;
workflowData = result.workflowData;
} catch (error) {
// Rollback stored attachments if transaction fails
await this.chatHubAttachmentService.deleteAttachments(processedAttachments);
throw error;
}
await this.executeChatWorkflowWithCleanup(
res,
@ -525,10 +718,6 @@ export class ChatHubService {
const messageToEdit = await this.getChatMessage(session.id, editId, [], trx);
if (!['ai', 'human'].includes(messageToEdit.type)) {
throw new BadRequestError('Only human and AI messages can be edited');
}
if (messageToEdit.type === 'ai') {
// AI edits just change the original message without revisioning or response generation
await this.messageRepository.updateChatMessage(editId, { content: payload.message }, trx);
@ -542,8 +731,12 @@ export class ChatHubService {
// If the message to edit isn't the original message, we want to point to the original message
const revisionOfMessageId = messageToEdit.revisionOfMessageId ?? messageToEdit.id;
// Attachments are already processed (from the original message)
const attachments = messageToEdit.attachments ?? [];
await this.saveHumanMessage(
payload,
attachments,
user,
messageToEdit.previousMessageId,
model,
@ -551,34 +744,20 @@ export class ChatHubService {
trx,
);
if (model.provider === 'n8n') {
return await this.prepareCustomAgentWorkflow(user, sessionId, model.workflowId, message);
}
if (model.provider === 'custom-agent') {
return await this.prepareChatAgentWorkflow(
model.agentId,
user,
sessionId,
history,
message,
trx,
);
}
return await this.prepareBaseChatWorkflow(
return await this.prepareReplyWorkflow(
user,
sessionId,
credentials,
model,
history,
message,
undefined,
session.tools,
attachments,
trx,
);
}
return null;
throw new BadRequestError('Only human and AI messages can be edited');
});
if (!workflow) {
@ -600,7 +779,6 @@ export class ChatHubService {
async regenerateAIMessage(res: Response, user: User, payload: RegenerateMessagePayload) {
const { sessionId, retryId, model, credentials } = payload;
const { provider } = model;
const {
workflow: { workflowData, executionData },
@ -637,37 +815,18 @@ export class ChatHubService {
// If the message being retried is itself a retry, we want to point to the original message
const retryOfMessageId = messageToRetry.retryOfMessageId ?? messageToRetry.id;
const message = lastHumanMessage ? lastHumanMessage.content : '';
let workflow;
if (provider === 'n8n') {
workflow = await this.prepareCustomAgentWorkflow(
user,
sessionId,
model.workflowId,
message,
);
} else if (provider === 'custom-agent') {
workflow = await this.prepareChatAgentWorkflow(
model.agentId,
user,
sessionId,
history,
message,
trx,
);
} else {
workflow = await this.prepareBaseChatWorkflow(
user,
sessionId,
credentials,
model,
history,
message,
undefined,
session.tools,
trx,
);
}
const attachments = lastHumanMessage.attachments ?? [];
const workflow = await this.prepareReplyWorkflow(
user,
sessionId,
credentials,
model,
history,
message,
session.tools,
attachments,
trx,
);
return {
workflow,
@ -688,6 +847,53 @@ export class ChatHubService {
);
}
private async prepareReplyWorkflow(
user: User,
sessionId: ChatSessionId,
credentials: INodeCredentials,
model: ChatHubConversationModel,
history: ChatHubMessage[],
message: string,
tools: INode[],
attachments: IBinaryData[],
trx: EntityManager,
) {
if (model.provider === 'n8n') {
return await this.prepareCustomAgentWorkflow(
user,
sessionId,
model.workflowId,
message,
attachments,
);
}
if (model.provider === 'custom-agent') {
return await this.prepareChatAgentWorkflow(
model.agentId,
user,
sessionId,
history,
message,
attachments,
trx,
);
}
return await this.prepareBaseChatWorkflow(
user,
sessionId,
credentials,
model,
history,
message,
undefined,
tools,
attachments,
trx,
);
}
private async prepareBaseChatWorkflow(
user: User,
sessionId: ChatSessionId,
@ -697,6 +903,7 @@ export class ChatHubService {
message: string,
systemMessage: string | undefined,
tools: INode[],
attachments: IBinaryData[],
trx: EntityManager,
) {
const credential = await this.chatHubCredentialsService.ensureCredentials(
@ -712,6 +919,7 @@ export class ChatHubService {
credential.projectId,
history,
message,
attachments,
credentials,
model,
systemMessage,
@ -726,6 +934,7 @@ export class ChatHubService {
sessionId: ChatSessionId,
history: ChatHubMessage[],
message: string,
attachments: IBinaryData[],
trx: EntityManager,
) {
const agent = await this.chatHubAgentService.getAgentById(agentId, user.id);
@ -772,6 +981,7 @@ export class ChatHubService {
message,
systemMessage,
tools,
attachments,
trx,
);
}
@ -781,6 +991,7 @@ export class ChatHubService {
sessionId: ChatSessionId,
workflowId: string,
message: string,
attachments: IBinaryData[],
) {
const workflowEntity = await this.workflowFinderService.findWorkflowForUser(
workflowId,
@ -813,25 +1024,12 @@ export class ChatHubService {
);
}
const nodeExecutionStack: IExecuteData[] = [
{
node: chatTriggerNode,
data: {
main: [
[
{
json: {
sessionId,
action: 'sendMessage',
chatInput: message,
},
},
],
],
},
source: null,
},
];
const nodeExecutionStack = this.chatHubWorkflowService.prepareExecutionData(
chatTriggerNode,
sessionId,
message,
attachments,
);
const executionData = createRunExecutionData({
executionData: {
@ -1341,6 +1539,7 @@ export class ChatHubService {
private async saveHumanMessage(
payload: HumanMessagePayload | EditMessagePayload,
attachments: IBinaryData[],
user: User,
previousMessageId: ChatMessageId | null,
model: ChatHubConversationModel,
@ -1357,6 +1556,7 @@ export class ChatHubService {
previousMessageId,
revisionOfMessageId,
name: user.firstName || 'User',
attachments,
...model,
},
trx,
@ -1483,24 +1683,36 @@ export class ChatHubService {
/**
* Get all conversations for a user
*/
async getConversations(userId: string): Promise<ChatHubConversationsResponse> {
const sessions = await this.sessionRepository.getManyByUserId(userId);
async getConversations(
userId: string,
limit: number,
cursor?: string,
): Promise<ChatHubConversationsResponse> {
const sessions = await this.sessionRepository.getManyByUserId(userId, limit + 1, cursor);
return sessions.map((session) => ({
id: session.id,
title: session.title,
ownerId: session.ownerId,
lastMessageAt: session.lastMessageAt?.toISOString() ?? null,
credentialId: session.credentialId,
provider: session.provider,
model: session.model,
workflowId: session.workflowId,
agentId: session.agentId,
agentName: session.agentName,
createdAt: session.createdAt.toISOString(),
updatedAt: session.updatedAt.toISOString(),
tools: session.tools,
}));
const hasMore = sessions.length > limit;
const data = hasMore ? sessions.slice(0, limit) : sessions;
const nextCursor = hasMore ? data[data.length - 1].id : null;
return {
data: data.map((session) => ({
id: session.id,
title: session.title,
ownerId: session.ownerId,
lastMessageAt: session.lastMessageAt?.toISOString() ?? null,
credentialId: session.credentialId,
provider: session.provider,
model: session.model,
workflowId: session.workflowId,
agentId: session.agentId,
agentName: session.agentName,
createdAt: session.createdAt.toISOString(),
updatedAt: session.updatedAt.toISOString(),
tools: session.tools,
})),
nextCursor,
hasMore,
};
}
/**
@ -1555,6 +1767,11 @@ export class ChatHubService {
previousMessageId: message.previousMessageId,
retryOfMessageId: message.retryOfMessageId,
revisionOfMessageId: message.revisionOfMessageId,
attachments: (message.attachments ?? []).map(({ fileName, mimeType }) => ({
fileName,
mimeType,
})),
};
}
@ -1583,6 +1800,7 @@ export class ChatHubService {
}
async deleteAllSessions() {
await this.chatHubAttachmentService.deleteAll();
const result = await this.sessionRepository.deleteAll();
return result;
}
@ -1690,6 +1908,7 @@ export class ChatHubService {
throw new NotFoundError('Session not found');
}
await this.chatHubAttachmentService.deleteAllBySessionId(sessionId);
await this.sessionRepository.deleteChatHubSession(sessionId);
}
}

View File

@ -1,5 +1,21 @@
import type { ChatHubConversationModel, ChatMessageId, ChatSessionId } from '@n8n/api-types';
import type {
ChatHubConversationModel,
ChatHubProvider,
ChatMessageId,
ChatSessionId,
ChatAttachment,
} from '@n8n/api-types';
import type { INode, INodeCredentials } from 'n8n-workflow';
import { z } from 'zod';
export interface ModelWithCredentials {
provider: ChatHubProvider;
model?: string;
workflowId?: string;
credentialId: string | null;
agentId?: string;
name?: string;
}
export interface BaseMessagePayload {
userId: string;
@ -12,6 +28,7 @@ export interface HumanMessagePayload extends BaseMessagePayload {
messageId: ChatMessageId;
message: string;
previousMessageId: ChatMessageId | null;
attachments: ChatAttachment[];
tools: INode[];
}
export interface RegenerateMessagePayload extends BaseMessagePayload {
@ -31,3 +48,14 @@ export interface MessageRecord {
message: string;
hideFromUI: boolean;
}
export const validChatTriggerParamsShape = z.object({
availableInChat: z.literal(true),
agentName: z.string().min(1).optional(),
agentDescription: z.string().min(1).optional(),
options: z
.object({
allowFileUploads: z.boolean().optional(),
})
.optional(),
});

View File

@ -5,6 +5,7 @@ import { DataSource, EntityManager, Repository } from '@n8n/typeorm';
import { ChatHubMessage } from './chat-hub-message.entity';
import { ChatHubSessionRepository } from './chat-session.repository';
import type { IBinaryData } from 'n8n-workflow';
@Service()
export class ChatHubMessageRepository extends Repository<ChatHubMessage> {
@ -33,7 +34,7 @@ export class ChatHubMessageRepository extends Repository<ChatHubMessage> {
async updateChatMessage(
id: ChatMessageId,
fields: { status?: ChatHubMessageStatus; content?: string },
fields: { status?: ChatHubMessageStatus; content?: string; attachments?: IBinaryData[] },
trx?: EntityManager,
) {
return await withTransaction(

View File

@ -2,6 +2,8 @@ import { withTransaction } from '@n8n/db';
import { Service } from '@n8n/di';
import { DataSource, EntityManager, Repository } from '@n8n/typeorm';
import { NotFoundError } from '@/errors/response-errors/not-found.error';
import { ChatHubSession } from './chat-hub-session.entity';
@Service()
@ -56,11 +58,33 @@ export class ChatHubSessionRepository extends Repository<ChatHubSession> {
});
}
async getManyByUserId(userId: string) {
return await this.find({
where: { ownerId: userId },
order: { lastMessageAt: 'DESC', id: 'ASC' },
});
async getManyByUserId(userId: string, limit: number, cursor?: string) {
const queryBuilder = this.createQueryBuilder('session')
.where('session.ownerId = :userId', { userId })
.orderBy("COALESCE(session.lastMessageAt, '1970-01-01')", 'DESC')
.addOrderBy('session.id', 'ASC');
if (cursor) {
const cursorSession = await this.findOne({
where: { id: cursor, ownerId: userId },
});
if (!cursorSession) {
throw new NotFoundError('Cursor session not found');
}
queryBuilder.andWhere(
'(session.lastMessageAt < :lastMessageAt OR (session.lastMessageAt = :lastMessageAt AND session.id > :id))',
{
lastMessageAt: cursorSession.lastMessageAt,
id: cursorSession.id,
},
);
}
queryBuilder.take(limit);
return await queryBuilder.getMany();
}
async getOneById(id: string, userId: string, trx?: EntityManager) {

View File

@ -139,6 +139,8 @@ export const maxContextWindowTokens: Record<ChatHubLLMProvider, Record<string, n
},
azureOpenAi: {},
ollama: {},
awsBedrock: {},
mistralCloud: {},
};
export const getMaxContextWindowTokens = (

View File

@ -0,0 +1,284 @@
import type { GlobalConfig } from '@n8n/config';
import { promises as fs } from 'fs';
import path from 'path';
import { DataTableFileCleanupService } from '../data-table-file-cleanup.service';
jest.mock('fs', () => ({
promises: {
unlink: jest.fn(),
readdir: jest.fn(),
stat: jest.fn(),
},
}));
describe('DataTableFileCleanupService', () => {
const uploadDir = '/mock/n8n/dataTableUploads';
const globalConfig = {
dataTable: {
cleanupIntervalMs: 60 * 1000,
fileMaxAgeMs: 2 * 60 * 1000,
uploadDir,
},
} as GlobalConfig;
const service = new DataTableFileCleanupService(globalConfig);
beforeEach(() => {
jest.clearAllMocks();
jest.spyOn(console, 'error').mockImplementation(() => {});
});
afterEach(() => {
jest.restoreAllMocks();
});
describe('deleteFile', () => {
it('should delete a file successfully', async () => {
const fileId = 'test-file-123';
const expectedPath = path.join(uploadDir, fileId);
(fs.unlink as jest.Mock).mockResolvedValue(undefined);
await service.deleteFile(fileId);
expect(fs.unlink).toHaveBeenCalledWith(expectedPath);
expect(fs.unlink).toHaveBeenCalledTimes(1);
});
it('should ignore ENOENT error when file does not exist', async () => {
const fileId = 'non-existent-file';
const error = new Error('ENOENT: no such file or directory') as NodeJS.ErrnoException;
error.code = 'ENOENT';
(fs.unlink as jest.Mock).mockRejectedValue(error);
await expect(service.deleteFile(fileId)).resolves.toBeUndefined();
});
it('should throw error for non-ENOENT errors', async () => {
const fileId = 'test-file';
const error = new Error('Permission denied') as NodeJS.ErrnoException;
error.code = 'EPERM';
(fs.unlink as jest.Mock).mockRejectedValue(error);
await expect(service.deleteFile(fileId)).rejects.toThrow('Permission denied');
});
});
describe('start and shutdown', () => {
it('should start cleanup interval', async () => {
jest.useFakeTimers();
await service.start();
expect(service['cleanupInterval']).toBeDefined();
jest.useRealTimers();
});
it('should clear interval on shutdown', async () => {
jest.useFakeTimers();
await service.start();
expect(service['cleanupInterval']).toBeDefined();
await service.shutdown();
expect(service['cleanupInterval']).toBeUndefined();
jest.useRealTimers();
});
it('should not error on shutdown if interval was never started', async () => {
await expect(service.shutdown()).resolves.toBeUndefined();
});
});
describe('cleanupOrphanedFiles', () => {
const flushPromises = async () => await new Promise(jest.requireActual('timers').setImmediate);
beforeEach(() => {
jest.useFakeTimers();
});
afterEach(() => {
jest.useRealTimers();
});
it('should delete files older than 2 minutes', async () => {
const now = Date.now();
const oldFile1 = 'old-file-1.csv';
const oldFile2 = 'old-file-2.csv';
(fs.readdir as jest.Mock).mockResolvedValue([oldFile1, oldFile2]);
(fs.stat as jest.Mock).mockResolvedValue({
mtimeMs: now - 3 * 60 * 1000, // 3 minutes ago
});
(fs.unlink as jest.Mock).mockResolvedValue(undefined);
await service.start();
// Trigger cleanup and let promises resolve
jest.advanceTimersByTime(60 * 1000);
await flushPromises();
expect(fs.readdir).toHaveBeenCalledWith(uploadDir);
expect(fs.stat).toHaveBeenCalledTimes(2);
expect(fs.unlink).toHaveBeenCalledWith(path.join(uploadDir, oldFile1));
expect(fs.unlink).toHaveBeenCalledWith(path.join(uploadDir, oldFile2));
});
it('should not delete files newer than 2 minutes', async () => {
const now = Date.now();
const newFile = 'new-file.csv';
(fs.readdir as jest.Mock).mockResolvedValue([newFile]);
(fs.stat as jest.Mock).mockResolvedValue({
mtimeMs: now - 1 * 60 * 1000, // 1 minute ago
});
await service.start();
jest.advanceTimersByTime(60 * 1000); // Trigger cleanup
await flushPromises();
expect(fs.readdir).toHaveBeenCalled();
expect(fs.stat).toHaveBeenCalled();
expect(fs.unlink).not.toHaveBeenCalled();
});
it('should handle mixed old and new files', async () => {
const now = Date.now();
const oldFile = 'old-file.csv';
const newFile = 'new-file.csv';
(fs.readdir as jest.Mock).mockResolvedValue([oldFile, newFile]);
(fs.stat as jest.Mock)
.mockResolvedValueOnce({
mtimeMs: now - 3 * 60 * 1000, // 3 minutes ago (old)
})
.mockResolvedValueOnce({
mtimeMs: now - 1 * 60 * 1000, // 1 minute ago (new)
});
(fs.unlink as jest.Mock).mockResolvedValue(undefined);
await service.start();
jest.advanceTimersByTime(60 * 1000); // Trigger cleanup
await flushPromises();
expect(fs.unlink).toHaveBeenCalledTimes(1);
expect(fs.unlink).toHaveBeenCalledWith(path.join(uploadDir, oldFile));
expect(fs.unlink).not.toHaveBeenCalledWith(path.join(uploadDir, newFile));
});
it('should handle empty upload directory', async () => {
(fs.readdir as jest.Mock).mockResolvedValue([]);
await service.start();
jest.advanceTimersByTime(60 * 1000); // Trigger cleanup
await flushPromises();
expect(fs.readdir).toHaveBeenCalled();
expect(fs.stat).not.toHaveBeenCalled();
expect(fs.unlink).not.toHaveBeenCalled();
});
it('should ignore ENOENT error if upload directory does not exist', async () => {
const error = new Error('ENOENT: no such file or directory') as NodeJS.ErrnoException;
error.code = 'ENOENT';
(fs.readdir as jest.Mock).mockRejectedValue(error);
await service.start();
jest.advanceTimersByTime(60 * 1000); // Trigger cleanup
await flushPromises();
expect(fs.readdir).toHaveBeenCalled();
expect(console.error).not.toHaveBeenCalled();
});
it('should log error for non-ENOENT readdir errors', async () => {
const error = new Error('Permission denied');
(fs.readdir as jest.Mock).mockRejectedValue(error);
await service.start();
jest.advanceTimersByTime(60 * 1000); // Trigger cleanup
await flushPromises();
expect(console.error).toHaveBeenCalledWith('Error cleaning up orphaned CSV files:', error);
});
it('should continue cleanup if individual file stat fails', async () => {
const file1 = 'file1.csv';
const file2 = 'file2.csv';
(fs.readdir as jest.Mock).mockResolvedValue([file1, file2]);
(fs.stat as jest.Mock)
.mockRejectedValueOnce(new Error('Stat failed for file1'))
.mockResolvedValueOnce({
mtimeMs: Date.now() - 3 * 60 * 1000, // file2 is old
});
(fs.unlink as jest.Mock).mockResolvedValue(undefined);
await service.start();
jest.advanceTimersByTime(60 * 1000); // Trigger cleanup
await flushPromises();
// Should still delete file2 even though file1 failed
expect(fs.unlink).toHaveBeenCalledWith(path.join(uploadDir, file2));
expect(fs.unlink).toHaveBeenCalledTimes(1);
});
it('should continue cleanup if individual file unlink fails', async () => {
const now = Date.now();
const file1 = 'file1.csv';
const file2 = 'file2.csv';
(fs.readdir as jest.Mock).mockResolvedValue([file1, file2]);
(fs.stat as jest.Mock).mockResolvedValue({
mtimeMs: now - 3 * 60 * 1000, // Both files are old
});
(fs.unlink as jest.Mock)
.mockRejectedValueOnce(new Error('Unlink failed for file1'))
.mockResolvedValueOnce(undefined);
await service.start();
jest.advanceTimersByTime(60 * 1000); // Trigger cleanup
await flushPromises();
// Should attempt to delete both files
expect(fs.unlink).toHaveBeenCalledWith(path.join(uploadDir, file1));
expect(fs.unlink).toHaveBeenCalledWith(path.join(uploadDir, file2));
expect(fs.unlink).toHaveBeenCalledTimes(2);
});
it('should run cleanup every 60 seconds', async () => {
const now = Date.now();
(fs.readdir as jest.Mock).mockResolvedValue(['old-file.csv']);
(fs.stat as jest.Mock).mockResolvedValue({
mtimeMs: now - 3 * 60 * 1000,
});
(fs.unlink as jest.Mock).mockResolvedValue(undefined);
await service.start();
// First cleanup
jest.advanceTimersByTime(60 * 1000);
await flushPromises();
expect(fs.readdir).toHaveBeenCalledTimes(1);
// Second cleanup
jest.advanceTimersByTime(60 * 1000);
await flushPromises();
expect(fs.readdir).toHaveBeenCalledTimes(2);
// Third cleanup
jest.advanceTimersByTime(60 * 1000);
await flushPromises();
expect(fs.readdir).toHaveBeenCalledTimes(3);
await service.shutdown();
});
});
});

View File

@ -0,0 +1,425 @@
import { GlobalConfig } from '@n8n/config';
import type { User } from '@n8n/db';
import { Container } from '@n8n/di';
import { createOwner, createMember } from '@test-integration/db/users';
import type { SuperAgentTest } from '@test-integration/types';
import * as utils from '@test-integration/utils';
const testServer = utils.setupTestServer({
endpointGroups: ['data-table'],
modules: ['data-table'],
});
let owner: User;
let member: User;
let authOwnerAgent: SuperAgentTest;
let authMemberAgent: SuperAgentTest;
beforeAll(async () => {
owner = await createOwner();
member = await createMember();
authOwnerAgent = testServer.authAgentFor(owner);
authMemberAgent = testServer.authAgentFor(member);
});
describe('POST /data-tables/uploads', () => {
describe('successful file uploads', () => {
test('should upload a valid CSV file and return file metadata', async () => {
const csvContent = 'name,email\nJohn Doe,john@example.com\nJane Smith,jane@example.com';
const response = await authOwnerAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(csvContent), 'test.csv')
.expect(200);
expect(response.body.data).toHaveProperty('originalName', 'test.csv');
expect(response.body.data).toHaveProperty('id');
expect(response.body.data.id).toMatch(/^[a-zA-Z0-9_-]{10}$/); // nanoid with length 10
expect(response.body.data).toHaveProperty('rowCount', 2);
expect(response.body.data).toHaveProperty('columnCount', 2);
expect(response.body.data).toHaveProperty('columns');
expect(response.body.data.columns).toEqual([
{ name: 'name', type: 'string', compatibleTypes: ['string'] },
{ name: 'email', type: 'string', compatibleTypes: ['string'] },
]);
});
test('should accept CSV file from member user', async () => {
const csvContent = 'col1,col2\nvalue1,value2';
const response = await authMemberAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(csvContent), 'data.csv')
.expect(200);
expect(response.body.data).toHaveProperty('originalName', 'data.csv');
expect(response.body.data).toHaveProperty('id');
expect(response.body.data).toHaveProperty('rowCount', 1);
expect(response.body.data).toHaveProperty('columnCount', 2);
});
test('should handle CSV files with special characters in filename', async () => {
const csvContent = 'a,b,c\n1,2,3';
const filename = 'test-file_v1.2.csv';
const response = await authOwnerAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(csvContent), filename)
.expect(200);
expect(response.body.data).toHaveProperty('originalName', filename);
expect(response.body.data).toHaveProperty('id');
expect(response.body.data).toHaveProperty('rowCount', 1);
expect(response.body.data).toHaveProperty('columnCount', 3);
expect(response.body.data.columns).toEqual([
{ name: 'a', type: 'number', compatibleTypes: ['number', 'string'] },
{ name: 'b', type: 'number', compatibleTypes: ['number', 'string'] },
{ name: 'c', type: 'number', compatibleTypes: ['number', 'string'] },
]);
});
test('should accept files within size limit', async () => {
const smallContent = 'col1,col2\nval1,val2\n';
const response = await authOwnerAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(smallContent), 'small.csv')
.expect(200);
expect(response.body.data).toHaveProperty('originalName', 'small.csv');
expect(response.body.data).toHaveProperty('id');
});
test('should accept empty CSV file', async () => {
const response = await authOwnerAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(''), 'empty.csv')
.expect(200);
expect(response.body.data).toHaveProperty('originalName', 'empty.csv');
expect(response.body.data).toHaveProperty('id');
expect(response.body.data).toHaveProperty('rowCount', 0);
expect(response.body.data).toHaveProperty('columnCount', 0);
expect(response.body.data.columns).toEqual([]);
});
test('should infer correct column types (string, number, boolean, date)', async () => {
const csvContent =
'name,age,isActive,createdDate\nJohn,30,true,2024-01-15\nJane,25,false,2024-02-20';
const response = await authOwnerAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(csvContent), 'types.csv')
.expect(200);
expect(response.body.data).toHaveProperty('originalName', 'types.csv');
expect(response.body.data).toHaveProperty('rowCount', 2);
expect(response.body.data).toHaveProperty('columnCount', 4);
expect(response.body.data.columns).toEqual([
{ name: 'name', type: 'string', compatibleTypes: ['string'] },
{ name: 'age', type: 'number', compatibleTypes: ['number', 'string'] },
{ name: 'isActive', type: 'boolean', compatibleTypes: ['boolean', 'string'] },
{ name: 'createdDate', type: 'date', compatibleTypes: ['date', 'string'] },
]);
});
});
describe('header handling', () => {
test('should use first row as headers when hasHeaders=true (default)', async () => {
const csvContent = 'FirstName,LastName,Age\nJohn,Doe,30\nJane,Smith,25';
const response = await authOwnerAgent
.post('/data-tables/uploads')
.field('hasHeaders', 'true')
.attach('file', Buffer.from(csvContent), 'with-headers.csv')
.expect(200);
expect(response.body.data).toHaveProperty('rowCount', 2);
expect(response.body.data).toHaveProperty('columnCount', 3);
expect(response.body.data.columns).toEqual([
{ name: 'FirstName', type: 'string', compatibleTypes: ['string'] },
{ name: 'LastName', type: 'string', compatibleTypes: ['string'] },
{ name: 'Age', type: 'number', compatibleTypes: ['number', 'string'] },
]);
});
test('should generate column names when hasHeaders=false', async () => {
const csvContent = 'John,Doe,30\nJane,Smith,25\nBob,Johnson,35';
const response = await authOwnerAgent
.post('/data-tables/uploads')
.field('hasHeaders', 'false')
.attach('file', Buffer.from(csvContent), 'no-headers.csv')
.expect(200);
expect(response.body.data).toHaveProperty('rowCount', 3);
expect(response.body.data).toHaveProperty('columnCount', 3);
expect(response.body.data.columns).toEqual([
{ name: 'Column_1', type: 'string', compatibleTypes: ['string'] },
{ name: 'Column_2', type: 'string', compatibleTypes: ['string'] },
{ name: 'Column_3', type: 'number', compatibleTypes: ['number', 'string'] },
]);
});
test('should include all rows in count when hasHeaders=false', async () => {
const csvContent = '100,200,300\n400,500,600';
const response = await authOwnerAgent
.post('/data-tables/uploads')
.field('hasHeaders', 'false')
.attach('file', Buffer.from(csvContent), 'no-headers-numbers.csv')
.expect(200);
expect(response.body.data).toHaveProperty('rowCount', 2); // Both rows counted as data
expect(response.body.data).toHaveProperty('columnCount', 3);
expect(response.body.data.columns).toEqual([
{ name: 'Column_1', type: 'number', compatibleTypes: ['number', 'string'] },
{ name: 'Column_2', type: 'number', compatibleTypes: ['number', 'string'] },
{ name: 'Column_3', type: 'number', compatibleTypes: ['number', 'string'] },
]);
});
test('should default to hasHeaders=true when field not provided', async () => {
const csvContent = 'col1,col2\nval1,val2';
const response = await authOwnerAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(csvContent), 'default.csv')
.expect(200);
expect(response.body.data).toHaveProperty('rowCount', 1);
expect(response.body.data.columns[0].name).toBe('col1');
expect(response.body.data.columns[1].name).toBe('col2');
});
});
describe('authentication', () => {
test('should reject unauthenticated requests', async () => {
const csvContent = 'name,value\ntest,123';
await testServer.authlessAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(csvContent), 'test.csv')
.expect(401);
});
});
describe('file validation', () => {
test('should reject request without file field', async () => {
await authOwnerAgent.post('/data-tables/uploads').send({}).expect(400);
});
test('should reject request with wrong field name', async () => {
const csvContent = 'a,b\n1,2';
await authOwnerAgent
.post('/data-tables/uploads')
.attach('wrongField', Buffer.from(csvContent), 'test.csv')
.expect(400);
});
test('should reject non-CSV files based on MIME type', async () => {
const textContent = 'This is a plain text file';
const response = await authOwnerAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(textContent), 'test.txt');
expect(response.status).toBe(400);
});
test('should reject JSON files', async () => {
const jsonContent = JSON.stringify({ name: 'test', value: 123 });
const response = await authOwnerAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(jsonContent), 'test.json');
expect(response.status).toBe(400);
});
test('should reject Excel files', async () => {
const excelBuffer = Buffer.from('PK\x03\x04'); // Excel file signature
const response = await authOwnerAgent
.post('/data-tables/uploads')
.attach('file', excelBuffer, 'test.xlsx');
expect(response.status).toBe(400);
});
});
describe('file size validation with uploadMaxFileSize', () => {
// Note: uploadMaxFileSize is set during multer initialization, so changing it
// at runtime won't affect the behavior. These tests verify the behavior
// when uploadMaxFileSize is configured via environment variable.
test('should accept small files when uploadMaxFileSize is set', async () => {
const smallContent = 'name,value\ntest,123\n';
const response = await authOwnerAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(smallContent), 'small.csv')
.expect(200);
expect(response.body.data).toHaveProperty('originalName', 'small.csv');
});
});
describe('file size validation without uploadMaxFileSize', () => {
let globalConfig: GlobalConfig;
let originalUploadMaxFileSize: number | undefined;
let originalMaxSize: number;
beforeEach(() => {
globalConfig = Container.get(GlobalConfig);
originalUploadMaxFileSize = globalConfig.dataTable.uploadMaxFileSize;
originalMaxSize = globalConfig.dataTable.maxSize;
// Unset uploadMaxFileSize to trigger remaining space check
globalConfig.dataTable.uploadMaxFileSize = undefined;
});
afterEach(() => {
// Restore original values
globalConfig.dataTable.uploadMaxFileSize = originalUploadMaxFileSize;
globalConfig.dataTable.maxSize = originalMaxSize;
});
test('should accept small files when there is remaining storage space', async () => {
// Set max size to 50MB - in test environment, database is likely empty
globalConfig.dataTable.maxSize = 50 * 1024 * 1024;
const smallContent = 'name,value\ntest,123\n';
const response = await authOwnerAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(smallContent), 'small.csv')
.expect(200);
expect(response.body.data).toHaveProperty('originalName', 'small.csv');
});
test('should reject files exceeding remaining storage space', async () => {
// Set a very small max size (1KB) to ensure the upload fails
globalConfig.dataTable.maxSize = 1024;
// Create content larger than 1KB
const largeContent = 'a,b,c\n' + '1,2,3\n'.repeat(200);
const response = await authOwnerAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(largeContent), 'large.csv');
expect(response.status).toBe(400);
expect(response.body).toHaveProperty('message');
// Error message can be either about remaining space or storage limit exceeded
const message = response.body.message as string;
expect(
message.includes('remaining storage space') || message.includes('Storage limit exceeded'),
).toBe(true);
// Verify the error message includes size units (B, KB, or MB)
expect(message).toMatch(/\d+(B|KB|MB)/);
});
});
describe('edge cases', () => {
test('should handle CSV with only headers', async () => {
const csvContent = 'column1,column2,column3';
const response = await authOwnerAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(csvContent), 'headers-only.csv')
.expect(200);
expect(response.body.data).toHaveProperty('originalName', 'headers-only.csv');
expect(response.body.data).toHaveProperty('id');
expect(response.body.data).toHaveProperty('rowCount', 0);
expect(response.body.data).toHaveProperty('columnCount', 3);
expect(response.body.data.columns).toEqual([
{ name: 'column1', type: 'string', compatibleTypes: ['string'] },
{ name: 'column2', type: 'string', compatibleTypes: ['string'] },
{ name: 'column3', type: 'string', compatibleTypes: ['string'] },
]);
});
test('should handle CSV with Unicode content', async () => {
const csvContent = 'Name,City\n日本,東京\nДмитрий,Москва';
const response = await authOwnerAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(csvContent, 'utf-8'), 'unicode.csv')
.expect(200);
expect(response.body.data).toHaveProperty('originalName', 'unicode.csv');
expect(response.body.data).toHaveProperty('id');
});
test('should handle CSV with commas in quoted values', async () => {
const csvContent = 'name,address\n"Doe, John","123 Main St, Apt 4"';
const response = await authOwnerAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(csvContent), 'quoted.csv')
.expect(200);
expect(response.body.data).toHaveProperty('originalName', 'quoted.csv');
expect(response.body.data).toHaveProperty('id');
});
test('should handle CSV with newlines in values', async () => {
const csvContent =
'name,description\n"Product A","Line 1\nLine 2"\n"Product B","Single line"';
const response = await authOwnerAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(csvContent), 'multiline.csv')
.expect(200);
expect(response.body.data).toHaveProperty('originalName', 'multiline.csv');
expect(response.body.data).toHaveProperty('id');
});
test('should handle large valid CSV files', async () => {
// Create a large but valid CSV (within size limit)
const header = 'id,name,email,city,country\n';
const rows = Array.from(
{ length: 1000 },
(_, i) => `${i},User${i},user${i}@example.com,City${i},Country${i}`,
).join('\n');
const largeContent = header + rows;
const response = await authOwnerAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(largeContent), 'large-valid.csv')
.expect(200);
expect(response.body.data).toHaveProperty('originalName', 'large-valid.csv');
expect(response.body.data).toHaveProperty('id');
});
test('should handle CSV without headers (first row treated as header)', async () => {
// CSV with just data rows, no explicit header
const csvContent = '1,2,3\n4,5,6\n7,8,9';
const response = await authOwnerAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(csvContent), 'no-headers.csv')
.expect(200);
expect(response.body.data).toHaveProperty('originalName', 'no-headers.csv');
expect(response.body.data).toHaveProperty('id');
// First row is treated as headers, so we have 2 data rows
expect(response.body.data).toHaveProperty('rowCount', 2);
expect(response.body.data).toHaveProperty('columnCount', 3);
// First row values become column names
expect(response.body.data.columns).toEqual([
{ name: '1', type: 'number', compatibleTypes: ['number', 'string'] },
{ name: '2', type: 'number', compatibleTypes: ['number', 'string'] },
{ name: '3', type: 'number', compatibleTypes: ['number', 'string'] },
]);
});
});
});

View File

@ -3761,3 +3761,429 @@ describe('PATCH /projects/:projectId/data-tables/:dataTableId/rows', () => {
},
);
});
describe('POST /projects/:projectId/data-tables - CSV Import', () => {
test('should create data table and import rows from CSV file', async () => {
// First upload a CSV file
const csvContent = 'name,age,email\nAlice,30,alice@example.com\nBob,25,bob@example.com';
const uploadResponse = await authOwnerAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(csvContent), { filename: 'test.csv', contentType: 'text/csv' })
.expect(200);
const fileId = uploadResponse.body.data.id;
// Create data table with fileId to trigger import
const payload = {
name: 'Imported Data Table',
columns: [
{ name: 'name', type: 'string' },
{ name: 'age', type: 'number' },
{ name: 'email', type: 'string' },
],
fileId,
};
const createResponse = await authOwnerAgent
.post(`/projects/${ownerProject.id}/data-tables`)
.send(payload)
.expect(200);
const dataTableId = createResponse.body.data.id;
// Verify data was imported
const rowsResponse = await authOwnerAgent
.get(`/projects/${ownerProject.id}/data-tables/${dataTableId}/rows`)
.expect(200);
expect(rowsResponse.body.data.count).toBe(2);
expect(rowsResponse.body.data.data).toHaveLength(2);
expect(rowsResponse.body.data.data).toEqual(
expect.arrayContaining([
expect.objectContaining({
name: 'Alice',
age: 30,
email: 'alice@example.com',
}),
expect.objectContaining({
name: 'Bob',
age: 25,
email: 'bob@example.com',
}),
]),
);
});
test('should map CSV columns to table columns by position', async () => {
// Upload CSV with column names that have spaces
const csvContent =
'Customer Id,Full Name,Email Address\n1001,John Doe,john@example.com\n1002,Jane Smith,jane@example.com';
const uploadResponse = await authOwnerAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(csvContent), {
filename: 'customers.csv',
contentType: 'text/csv',
})
.expect(200);
const fileId = uploadResponse.body.data.id;
// Create table with different column names (without spaces)
const payload = {
name: 'Customers',
columns: [
{ name: 'customerId', type: 'string' },
{ name: 'fullName', type: 'string' },
{ name: 'emailAddress', type: 'string' },
],
fileId,
};
const createResponse = await authOwnerAgent
.post(`/projects/${ownerProject.id}/data-tables`)
.send(payload)
.expect(200);
const dataTableId = createResponse.body.data.id;
// Verify data was mapped correctly by position
const rowsResponse = await authOwnerAgent
.get(`/projects/${ownerProject.id}/data-tables/${dataTableId}/rows`)
.expect(200);
expect(rowsResponse.body.data.count).toBe(2);
expect(rowsResponse.body.data.data).toEqual(
expect.arrayContaining([
expect.objectContaining({
customerId: '1001',
fullName: 'John Doe',
emailAddress: 'john@example.com',
}),
expect.objectContaining({
customerId: '1002',
fullName: 'Jane Smith',
emailAddress: 'jane@example.com',
}),
]),
);
});
test('should create data table with partial column mapping when schema has extra columns', async () => {
// Upload a valid CSV with 2 columns
const csvContent = 'name,age\nAlice,30';
const uploadResponse = await authOwnerAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(csvContent), { filename: 'test.csv', contentType: 'text/csv' })
.expect(200);
const fileId = uploadResponse.body.data.id;
// Create table with more columns than in CSV
// The system should map by index: CSV col 0 -> table col 0, CSV col 1 -> table col 1
// The extra table column won't have data imported
const payload = {
name: 'Partial Import Table',
columns: [
{ name: 'name', type: 'string' },
{ name: 'age', type: 'number' },
{ name: 'extra', type: 'string' }, // Extra column not in CSV
],
fileId,
};
const createResponse = await authOwnerAgent
.post(`/projects/${ownerProject.id}/data-tables`)
.send(payload)
.expect(200);
const dataTableId = createResponse.body.data.id;
// Verify rows - should have mapped only the columns that exist in CSV
const rowsResponse = await authOwnerAgent
.get(`/projects/${ownerProject.id}/data-tables/${dataTableId}/rows`)
.expect(200);
// Data should be imported with columns mapped by index position
// The 'extra' column should be null/empty since it doesn't exist in CSV
expect(rowsResponse.body.data.data[0]).toMatchObject({
name: 'Alice',
age: 30,
});
});
test('should handle empty CSV file on import', async () => {
// Upload empty CSV
const csvContent = '';
const uploadResponse = await authOwnerAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(csvContent), { filename: 'empty.csv', contentType: 'text/csv' })
.expect(200);
const fileId = uploadResponse.body.data.id;
const payload = {
name: 'Empty CSV Import',
columns: [{ name: 'name', type: 'string' }],
fileId,
};
const createResponse = await authOwnerAgent
.post(`/projects/${ownerProject.id}/data-tables`)
.send(payload);
// Should either fail or create table with no rows
if (createResponse.status === 200) {
const dataTableId = createResponse.body.data.id;
const rowsResponse = await authOwnerAgent
.get(`/projects/${ownerProject.id}/data-tables/${dataTableId}/rows`)
.expect(200);
expect(rowsResponse.body.data.count).toBe(0);
}
});
test('should handle CSV with only headers on import', async () => {
// Upload CSV with only headers
const csvContent = 'name,age,city';
const uploadResponse = await authOwnerAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(csvContent), {
filename: 'headers-only.csv',
contentType: 'text/csv',
})
.expect(200);
const fileId = uploadResponse.body.data.id;
const payload = {
name: 'Headers Only Import',
columns: [
{ name: 'name', type: 'string' },
{ name: 'age', type: 'number' },
{ name: 'city', type: 'string' },
],
fileId,
};
const createResponse = await authOwnerAgent
.post(`/projects/${ownerProject.id}/data-tables`)
.send(payload)
.expect(200);
const dataTableId = createResponse.body.data.id;
// Should create table with no rows
const rowsResponse = await authOwnerAgent
.get(`/projects/${ownerProject.id}/data-tables/${dataTableId}/rows`)
.expect(200);
expect(rowsResponse.body.data.count).toBe(0);
});
test('should import CSV file with multiple rows', async () => {
const csvContent =
'itemId,itemName,itemValue\n1,Item 1,10\n2,Item 2,20\n3,Item 3,30\n4,Item 4,40\n5,Item 5,50';
const uploadResponse = await authOwnerAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(csvContent), {
filename: 'multiple-rows.csv',
contentType: 'text/csv',
})
.expect(200);
const fileId = uploadResponse.body.data.id;
const payload = {
name: 'Multiple Rows Import',
columns: [
{ name: 'itemId', type: 'string' },
{ name: 'itemName', type: 'string' },
{ name: 'itemValue', type: 'number' },
],
fileId,
};
const createResponse = await authOwnerAgent
.post(`/projects/${ownerProject.id}/data-tables`)
.send(payload)
.expect(200);
const dataTableId = createResponse.body.data.id;
// Verify all rows were imported
const rowsResponse = await authOwnerAgent
.get(`/projects/${ownerProject.id}/data-tables/${dataTableId}/rows`)
.expect(200);
expect(rowsResponse.body.data.count).toBe(5);
});
test('should create table without import when fileId is not provided', async () => {
const payload = {
name: 'Table Without Import',
columns: [
{ name: 'col1', type: 'string' },
{ name: 'col2', type: 'number' },
],
};
const createResponse = await authOwnerAgent
.post(`/projects/${ownerProject.id}/data-tables`)
.send(payload)
.expect(200);
const dataTableId = createResponse.body.data.id;
// Should create empty table
const rowsResponse = await authOwnerAgent
.get(`/projects/${ownerProject.id}/data-tables/${dataTableId}/rows`)
.expect(200);
expect(rowsResponse.body.data.count).toBe(0);
});
test('should handle CSV with quoted values containing commas', async () => {
const csvContent =
'name,address\n"John Doe","123 Main St, Apt 4"\n"Jane Smith","456 Oak Ave, Suite 10"';
const uploadResponse = await authOwnerAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(csvContent), { filename: 'quoted.csv', contentType: 'text/csv' })
.expect(200);
const fileId = uploadResponse.body.data.id;
const payload = {
name: 'Quoted Values Import',
columns: [
{ name: 'name', type: 'string' },
{ name: 'address', type: 'string' },
],
fileId,
};
const createResponse = await authOwnerAgent
.post(`/projects/${ownerProject.id}/data-tables`)
.send(payload)
.expect(200);
const dataTableId = createResponse.body.data.id;
const rowsResponse = await authOwnerAgent
.get(`/projects/${ownerProject.id}/data-tables/${dataTableId}/rows`)
.expect(200);
expect(rowsResponse.body.data.count).toBe(2);
expect(rowsResponse.body.data.data).toEqual(
expect.arrayContaining([
expect.objectContaining({
name: 'John Doe',
address: '123 Main St, Apt 4',
}),
expect.objectContaining({
name: 'Jane Smith',
address: '456 Oak Ave, Suite 10',
}),
]),
);
});
test('should handle CSV with Unicode characters', async () => {
const csvContent = 'name,city\nJohn Müller,München\nMaría García,São Paulo';
const uploadResponse = await authOwnerAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(csvContent), { filename: 'unicode.csv', contentType: 'text/csv' })
.expect(200);
const fileId = uploadResponse.body.data.id;
const payload = {
name: 'Unicode Import',
columns: [
{ name: 'name', type: 'string' },
{ name: 'city', type: 'string' },
],
fileId,
};
const createResponse = await authOwnerAgent
.post(`/projects/${ownerProject.id}/data-tables`)
.send(payload)
.expect(200);
const dataTableId = createResponse.body.data.id;
const rowsResponse = await authOwnerAgent
.get(`/projects/${ownerProject.id}/data-tables/${dataTableId}/rows`)
.expect(200);
expect(rowsResponse.body.data.count).toBe(2);
expect(rowsResponse.body.data.data).toEqual(
expect.arrayContaining([
expect.objectContaining({
name: 'John Müller',
city: 'München',
}),
expect.objectContaining({
name: 'María García',
city: 'São Paulo',
}),
]),
);
});
test('should handle CSV with different data types', async () => {
const csvContent =
'name,age,active,joinDate\nAlice,30,true,2024-01-15\nBob,25,false,2024-02-20';
const uploadResponse = await authOwnerAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(csvContent), {
filename: 'mixed-types.csv',
contentType: 'text/csv',
})
.expect(200);
const fileId = uploadResponse.body.data.id;
const payload = {
name: 'Mixed Types Import',
columns: [
{ name: 'name', type: 'string' },
{ name: 'age', type: 'number' },
{ name: 'active', type: 'boolean' },
{ name: 'joinDate', type: 'date' },
],
fileId,
};
const createResponse = await authOwnerAgent
.post(`/projects/${ownerProject.id}/data-tables`)
.send(payload)
.expect(200);
const dataTableId = createResponse.body.data.id;
const rowsResponse = await authOwnerAgent
.get(`/projects/${ownerProject.id}/data-tables/${dataTableId}/rows`)
.expect(200);
expect(rowsResponse.body.data.count).toBe(2);
// Data types are converted based on column type definitions
expect(rowsResponse.body.data.data).toEqual(
expect.arrayContaining([
expect.objectContaining({
name: 'Alice',
age: 30,
active: true,
joinDate: expect.stringContaining('2024-01-15'),
}),
expect.objectContaining({
name: 'Bob',
age: 25,
active: false,
joinDate: expect.stringContaining('2024-02-20'),
}),
]),
);
});
});

View File

@ -6,7 +6,7 @@ export function mockDataTableSizeValidator() {
const sizeValidator = Container.get(DataTableSizeValidator);
jest.spyOn(sizeValidator, 'validateSize').mockResolvedValue();
jest.spyOn(sizeValidator, 'getCachedSizeData').mockResolvedValue({
totalBytes: 50 * 1024 * 1024, // 50MB - under the default limit
totalBytes: 0, // Start with 0 bytes to allow uploads
dataTables: {},
});
return sizeValidator;

View File

@ -0,0 +1,204 @@
import { GlobalConfig } from '@n8n/config';
import { Service } from '@n8n/di';
import { parse } from 'csv-parse';
import { createReadStream } from 'fs';
import path from 'path';
export interface CsvColumnMetadata {
name: string;
type: 'string' | 'number' | 'boolean' | 'date';
}
export interface CsvMetadata {
rowCount: number;
columnCount: number;
columns: CsvColumnMetadata[];
}
@Service()
export class CsvParserService {
private readonly uploadDir: string;
private readonly DEFAULT_COLUMN_PREFIX = 'Column_';
constructor(private readonly globalConfig: GlobalConfig) {
this.uploadDir = this.globalConfig.dataTable.uploadDir;
}
private processRowWithoutHeaders(
row: string[],
columnNames: string[],
): { rowObject: Record<string, string>; columnNames: string[] } {
let updatedColumnNames = columnNames;
if (updatedColumnNames.length === 0) {
updatedColumnNames = row.map((_, index) => `${this.DEFAULT_COLUMN_PREFIX}${index + 1}`);
}
const rowObject: Record<string, string> = {};
row.forEach((value, index) => {
rowObject[updatedColumnNames[index]] = value;
});
return { rowObject, columnNames: updatedColumnNames };
}
/**
* Parses a CSV file and returns metadata including row count, column count, and inferred column types
*/
async parseFile(fileId: string, hasHeaders: boolean = true): Promise<CsvMetadata> {
const filePath = path.join(this.uploadDir, fileId);
let rowCount = 0;
let firstDataRow: Record<string, string> | null = null;
let columnNames: string[] = [];
return await new Promise((resolve, reject) => {
const parser = parse({
columns: hasHeaders
? (header: string[]) => {
columnNames = header;
return header;
}
: false,
skip_empty_lines: true,
});
createReadStream(filePath)
.pipe(parser)
.on('data', (row: Record<string, string> | string[]) => {
rowCount++;
if (!hasHeaders && Array.isArray(row)) {
const processed = this.processRowWithoutHeaders(row, columnNames);
columnNames = processed.columnNames;
firstDataRow ??= processed.rowObject;
} else if (!Array.isArray(row)) {
firstDataRow ??= row;
}
})
.on('end', () => {
const columns = columnNames.map((columnName) => {
const detectedType = this.inferColumnType(firstDataRow?.[columnName]);
return {
name: columnName,
type: detectedType,
compatibleTypes: this.getCompatibleTypes(detectedType),
};
});
resolve({
rowCount,
columnCount: columns.length,
columns,
});
})
.on('error', reject);
});
}
/**
* Parses a CSV file and returns all rows as an array of objects
*/
async parseFileData(
fileId: string,
hasHeaders: boolean = true,
): Promise<Array<Record<string, string>>> {
const filePath = path.join(this.uploadDir, fileId);
const rows: Array<Record<string, string>> = [];
let columnNames: string[] = [];
return await new Promise((resolve, reject) => {
const parser = parse({
columns: hasHeaders ? true : false,
skip_empty_lines: true,
});
createReadStream(filePath)
.pipe(parser)
.on('data', (row: Record<string, string> | string[]) => {
if (!hasHeaders && Array.isArray(row)) {
const processed = this.processRowWithoutHeaders(row, columnNames);
columnNames = processed.columnNames;
rows.push(processed.rowObject);
} else if (!Array.isArray(row)) {
rows.push(row);
}
})
.on('end', () => {
resolve(rows);
})
.on('error', reject);
});
}
/**
* Returns the list of compatible types for a detected type
* Logic: more specific types can be converted to string, but string cannot be converted to specific types
*/
private getCompatibleTypes(
detectedType: 'string' | 'number' | 'boolean' | 'date',
): Array<'string' | 'number' | 'boolean' | 'date'> {
switch (detectedType) {
case 'date':
return ['date', 'string'];
case 'number':
return ['number', 'string'];
case 'boolean':
return ['boolean', 'string'];
case 'string':
return ['string'];
default:
return ['string'];
}
}
/**
* Infers the column type from a sample value
* Priority: boolean > number > date > string
*/
private inferColumnType(value: string | undefined): 'string' | 'number' | 'boolean' | 'date' {
if (!value?.trim()) {
return 'string';
}
const trimmedValue = value.trim();
const lowerValue = trimmedValue.toLowerCase();
if (lowerValue === 'true' || lowerValue === 'false') {
return 'boolean';
}
if (!Number.isNaN(Number(trimmedValue))) {
return 'number';
}
if (this.isDate(trimmedValue)) {
return 'date';
}
return 'string';
}
/**
* Checks if a string represents a valid date
*/
private isDate(value: string): boolean {
// Try to parse as date
const date = new Date(value);
// Check if it's a valid date and the original value looks like a date
if (!Number.isNaN(date.getTime())) {
// Additional check: make sure it looks like a date format
// This prevents strings like "123" from being interpreted as dates
const datePatterns = [
/^\d{4}-\d{2}-\d{2}/, // ISO date (YYYY-MM-DD)
/^\d{4}\/\d{2}\/\d{2}/, // YYYY/MM/DD
/^\d{2}\/\d{2}\/\d{4}/, // MM/DD/YYYY or DD/MM/YYYY
/^\d{2}-\d{2}-\d{4}/, // MM-DD-YYYY or DD-MM-YYYY
/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}/, // ISO datetime
];
return datePatterns.some((pattern) => pattern.test(value));
}
return false;
}
}

View File

@ -0,0 +1,87 @@
import { GlobalConfig } from '@n8n/config';
import { Service } from '@n8n/di';
import { promises as fs } from 'fs';
import path from 'path';
@Service()
export class DataTableFileCleanupService {
private readonly uploadDir: string;
private cleanupInterval?: NodeJS.Timeout;
constructor(private readonly globalConfig: GlobalConfig) {
this.uploadDir = this.globalConfig.dataTable.uploadDir;
}
private isErrnoException(error: unknown): error is NodeJS.ErrnoException {
return (
typeof error === 'object' &&
error !== null &&
'code' in error &&
typeof (error as { code: unknown }).code === 'string'
);
}
async start() {
// Run cleanup periodically to delete orphaned files
this.cleanupInterval = setInterval(() => {
void this.cleanupOrphanedFiles();
}, this.globalConfig.dataTable.cleanupIntervalMs);
}
async shutdown() {
if (this.cleanupInterval) {
clearInterval(this.cleanupInterval);
this.cleanupInterval = undefined;
}
}
/**
* Cleans up orphaned CSV files that exceed the configured maximum age
* These are files that were uploaded but never used to create a data table
*/
private async cleanupOrphanedFiles(): Promise<void> {
try {
const files = await fs.readdir(this.uploadDir);
const now = Date.now();
const maxAge = this.globalConfig.dataTable.fileMaxAgeMs;
for (const file of files) {
const filePath = path.join(this.uploadDir, file);
try {
const stats = await fs.stat(filePath);
const fileAge = now - stats.mtimeMs;
// Delete files older than the configured maximum age
if (fileAge > maxAge) {
await fs.unlink(filePath);
}
} catch (error) {
// Ignore errors for individual files (e.g., file already deleted)
continue;
}
}
} catch (error) {
// Ignore errors if upload directory doesn't exist yet
if (!this.isErrnoException(error) || error.code !== 'ENOENT') {
// Log other errors but don't throw - cleanup is best effort
console.error('Error cleaning up orphaned CSV files:', error);
}
}
}
/**
* Deletes a specific CSV file by its fileId
*/
async deleteFile(fileId: string): Promise<void> {
const filePath = path.join(this.uploadDir, fileId);
try {
await fs.unlink(filePath);
} catch (error) {
// Ignore errors if file doesn't exist
if (!this.isErrnoException(error) || error.code !== 'ENOENT') {
throw error;
}
}
}
}

View File

@ -5,6 +5,7 @@ import { DataTableSizeStatus, DataTablesSizeData } from 'n8n-workflow';
import { Telemetry } from '@/telemetry';
import { DataTableValidationError } from './errors/data-table-validation.error';
import { toMb } from './utils/size-utils';
@Service()
export class DataTableSizeValidator {
@ -64,7 +65,7 @@ export class DataTableSizeValidator {
});
throw new DataTableValidationError(
`Data table size limit exceeded: ${this.toMb(size.totalBytes)}MB used, limit is ${this.toMb(this.globalConfig.dataTable.maxSize)}MB`,
`Data table size limit exceeded: ${toMb(size.totalBytes)}MB used, limit is ${toMb(this.globalConfig.dataTable.maxSize)}MB`,
);
}
}
@ -87,10 +88,6 @@ export class DataTableSizeValidator {
return this.sizeToState(size.totalBytes);
}
private toMb(sizeInBytes: number): number {
return Math.round(sizeInBytes / (1024 * 1024));
}
reset() {
this.lastCheck = undefined;
this.cachedSizeData = undefined;

View File

@ -0,0 +1,50 @@
import { Post, RestController } from '@n8n/decorators';
import { Container } from '@n8n/di';
import multer from 'multer';
import { BadRequestError } from '@/errors/response-errors/bad-request.error';
import { CsvParserService } from './csv-parser.service';
import { MulterUploadMiddleware } from './multer-upload-middleware';
import { AuthenticatedRequestWithFile, hasStringProperty } from './types';
const uploadMiddleware = Container.get(MulterUploadMiddleware);
@RestController('/data-tables/uploads')
export class DataTableUploadsController {
constructor(private readonly csvParserService: CsvParserService) {}
@Post('/', {
middlewares: [uploadMiddleware.single('file')],
})
async uploadFile(req: AuthenticatedRequestWithFile, _res: Response) {
if (req.fileUploadError) {
const error = req.fileUploadError;
if (error instanceof multer.MulterError) {
throw new BadRequestError(`File upload error: ${error.message}`);
} else if (error instanceof BadRequestError) {
throw error;
} else {
throw new BadRequestError('File upload failed');
}
}
if (!req.file) {
throw new BadRequestError('No file uploaded');
}
// Extract hasHeaders parameter from request body (multer parses form fields to body), default to true
const hasHeaders =
hasStringProperty(req.body, 'hasHeaders') && req.body.hasHeaders === 'false' ? false : true;
const metadata = await this.csvParserService.parseFile(req.file.filename, hasHeaders);
return {
originalName: req.file.originalname,
id: req.file.filename,
rowCount: metadata.rowCount,
columnCount: metadata.columnCount,
columns: metadata.columns,
};
}
}

View File

@ -7,12 +7,16 @@ export class DataTableModule implements ModuleInterface {
async init() {
await import('./data-table.controller');
await import('./data-table-aggregate.controller');
await import('./data-table-uploads.controller');
const { DataTableService } = await import('./data-table.service');
await Container.get(DataTableService).start();
const { DataTableAggregateService } = await import('./data-table-aggregate.service');
await Container.get(DataTableAggregateService).start();
const { DataTableFileCleanupService } = await import('./data-table-file-cleanup.service');
await Container.get(DataTableFileCleanupService).start();
}
@OnShutdown()
@ -22,6 +26,9 @@ export class DataTableModule implements ModuleInterface {
const { DataTableAggregateService } = await import('./data-table-aggregate.service');
await Container.get(DataTableAggregateService).shutdown();
const { DataTableFileCleanupService } = await import('./data-table-file-cleanup.service');
await Container.get(DataTableFileCleanupService).shutdown();
}
async entities() {

View File

@ -28,13 +28,16 @@ import type {
} from 'n8n-workflow';
import { DATA_TABLE_SYSTEM_COLUMN_TYPE_MAP, validateFieldType } from 'n8n-workflow';
import { CsvParserService } from './csv-parser.service';
import { DataTableColumn } from './data-table-column.entity';
import { DataTableColumnRepository } from './data-table-column.repository';
import { DataTableFileCleanupService } from './data-table-file-cleanup.service';
import { DataTableRowsRepository } from './data-table-rows.repository';
import { DataTableSizeValidator } from './data-table-size-validator.service';
import { DataTableRepository } from './data-table.repository';
import { columnTypeToFieldType } from './data-table.types';
import { DataTableColumnNotFoundError } from './errors/data-table-column-not-found.error';
import { FileUploadError } from './errors/data-table-file-upload.error';
import { DataTableNameConflictError } from './errors/data-table-name-conflict.error';
import { DataTableNotFoundError } from './errors/data-table-not-found.error';
import { DataTableValidationError } from './errors/data-table-validation.error';
@ -52,6 +55,8 @@ export class DataTableService {
private readonly dataTableSizeValidator: DataTableSizeValidator,
private readonly projectRelationRepository: ProjectRelationRepository,
private readonly roleService: RoleService,
private readonly csvParserService: CsvParserService,
private readonly fileCleanupService: DataTableFileCleanupService,
) {
this.logger = this.logger.scoped('data-table');
}
@ -64,11 +69,61 @@ export class DataTableService {
const result = await this.dataTableRepository.createDataTable(projectId, dto.name, dto.columns);
if (dto.fileId) {
try {
await this.importDataFromFile(projectId, result.id, dto.fileId, dto.hasHeaders ?? true);
await this.fileCleanupService.deleteFile(dto.fileId);
} catch (error) {
await this.deleteDataTable(result.id, projectId);
throw error;
}
}
this.dataTableSizeValidator.reset();
return result;
}
private async importDataFromFile(
projectId: string,
dataTableId: string,
fileId: string,
hasHeaders: boolean,
) {
try {
const tableColumns = await this.getColumns(dataTableId, projectId);
const csvMetadata = await this.csvParserService.parseFile(fileId, hasHeaders);
const columnMapping = new Map<string, string>();
csvMetadata.columns.forEach((csvColumn, index) => {
if (tableColumns[index]) {
columnMapping.set(csvColumn.name, tableColumns[index].name);
}
});
const csvRows = await this.csvParserService.parseFileData(fileId, hasHeaders);
const transformedRows = csvRows.map((csvRow) => {
const transformedRow: DataTableRow = {};
for (const [csvColName, value] of Object.entries(csvRow)) {
const tableColName = columnMapping.get(csvColName);
if (tableColName) {
transformedRow[tableColName] = value;
}
}
return transformedRow;
});
if (transformedRows.length > 0) {
await this.insertRows(dataTableId, projectId, transformedRows);
}
} catch (error) {
this.logger.error('Failed to import data from CSV file', { error, fileId, dataTableId });
throw new FileUploadError(error instanceof Error ? error.message : 'Failed to read CSV file');
}
}
// Updates data table properties (currently limited to renaming)
async updateDataTable(dataTableId: string, projectId: string, dto: UpdateDataTableDto) {
await this.validateDataTableExists(dataTableId, projectId);

View File

@ -0,0 +1,9 @@
import { UserError } from 'n8n-workflow';
export class FileUploadError extends UserError {
constructor(msg: string) {
super(`Error uploading file: ${msg}`, {
level: 'warning',
});
}
}

View File

@ -0,0 +1,111 @@
/* eslint-disable id-denylist */
import { GlobalConfig } from '@n8n/config';
import { Service } from '@n8n/di';
import type { Request, RequestHandler } from 'express';
import { mkdir } from 'fs/promises';
import multer from 'multer';
import { nanoid } from 'nanoid';
import { BadRequestError } from '@/errors/response-errors/bad-request.error';
import { DataTableSizeValidator } from './data-table-size-validator.service';
import { DataTableRepository } from './data-table.repository';
import {
type AuthenticatedRequestWithFile,
type MulterDestinationCallback,
type MulterFilenameCallback,
type UploadMiddleware,
} from './types';
import { formatBytes } from './utils/size-utils';
const ALLOWED_MIME_TYPES = ['text/csv'];
@Service()
export class MulterUploadMiddleware implements UploadMiddleware {
private upload: multer.Multer;
private readonly uploadDir: string;
constructor(
private readonly globalConfig: GlobalConfig,
private readonly sizeValidator: DataTableSizeValidator,
private readonly dataTableRepository: DataTableRepository,
) {
this.uploadDir = this.globalConfig.dataTable.uploadDir;
void this.ensureUploadDirExists();
const storage = multer.diskStorage({
destination: (_req: Request, _file: Express.Multer.File, cb: MulterDestinationCallback) => {
cb(null, this.uploadDir);
},
filename: (_req: Request, _file: Express.Multer.File, cb: MulterFilenameCallback) => {
const filename = nanoid(10);
cb(null, filename);
},
});
this.upload = multer({
storage,
limits: this.globalConfig.dataTable.uploadMaxFileSize
? { fileSize: this.globalConfig.dataTable.uploadMaxFileSize }
: undefined,
fileFilter: async (req, file, cb: multer.FileFilterCallback) => {
if (!ALLOWED_MIME_TYPES.includes(file.mimetype)) {
cb(
new BadRequestError(
`Only the following file types are allowed: ${ALLOWED_MIME_TYPES.join(', ')}`,
),
);
return;
}
const fileSize = parseInt(req.headers['content-length'] ?? '0', 10);
// If uploadMaxFileSize is set, multer's limits will handle the rejection
if (this.globalConfig.dataTable.uploadMaxFileSize) {
cb(null, true);
return;
}
// If uploadMaxFileSize is not set, check remaining space
try {
const sizeData = await this.sizeValidator.getCachedSizeData(async () => {
return await this.dataTableRepository.findDataTablesSize();
});
const remainingSpace = Math.max(
0,
this.globalConfig.dataTable.maxSize - sizeData.totalBytes,
);
if (fileSize > remainingSpace) {
const message =
remainingSpace === 0
? `Storage limit exceeded. Current usage: ${formatBytes(sizeData.totalBytes)}, Limit: ${formatBytes(this.globalConfig.dataTable.maxSize)}`
: `File size exceeds remaining storage space. Available: ${formatBytes(remainingSpace)}, File: ${formatBytes(fileSize)}`;
cb(new BadRequestError(message));
return;
}
cb(null, true);
} catch {
cb(new BadRequestError('Failed to validate file size'));
}
},
});
}
private async ensureUploadDirExists() {
await mkdir(this.uploadDir, { recursive: true });
}
single(fieldName: string): RequestHandler {
return (req, res, next) => {
void this.upload.single(fieldName)(req, res, (error) => {
if (error) {
(req as AuthenticatedRequestWithFile).fileUploadError = error;
}
next();
});
};
}
}

View File

@ -0,0 +1,26 @@
import type { AuthenticatedRequest } from '@n8n/db';
import type { RequestHandler } from 'express';
export interface UploadMiddleware {
single(fieldName: string): RequestHandler;
}
export type MulterDestinationCallback = (error: Error | null, destination: string) => void;
export type MulterFilenameCallback = (error: Error | null, filename: string) => void;
export type AuthenticatedRequestWithFile<
RouteParams = {},
ResponseBody = {},
RequestBody = {},
RequestQuery = {},
> = AuthenticatedRequest<RouteParams, ResponseBody, RequestBody, RequestQuery> & {
file?: Express.Multer.File;
fileUploadError?: Error;
};
export function hasStringProperty<K extends string>(
obj: unknown,
key: K,
): obj is Record<K, string> & object {
return typeof obj === 'object' && obj !== null && key in obj;
}

View File

@ -0,0 +1,19 @@
/**
* Convert bytes to megabytes (rounded to nearest integer)
*/
export function toMb(sizeInBytes: number): number {
return Math.round(sizeInBytes / (1024 * 1024));
}
/**
* Format bytes to human-readable size with appropriate unit (B, KB, or MB)
*/
export function formatBytes(sizeInBytes: number): string {
if (sizeInBytes < 1024) {
return `${sizeInBytes}B`;
} else if (sizeInBytes < 1024 * 1024) {
return `${Math.round(sizeInBytes / 1024)}KB`;
} else {
return `${Math.round(sizeInBytes / (1024 * 1024))}MB`;
}
}

View File

@ -7,6 +7,7 @@ import { mock } from 'jest-mock-extended';
import { InstanceSettings } from 'n8n-core';
import { InsightsModule } from '../insights.module';
import { InsightsService } from '../insights.service';
describe('InsightsModule', () => {
let insightsModule: InsightsModule;
@ -23,11 +24,23 @@ describe('InsightsModule', () => {
beforeEach(async () => {
jest.clearAllMocks();
await testDb.truncate(['Project']);
insightsModule = Container.get(InsightsModule);
mockInstanceSettings = mock<InstanceSettings>();
Container.set(InstanceSettings, mockInstanceSettings);
Container.set(Logger, mockLogger());
Container.set(LicenseState, mock<LicenseState>());
Container.set(
InsightsService,
new InsightsService(
mock(),
mock(),
mock(),
Container.get(LicenseState),
mockInstanceSettings,
Container.get(Logger),
),
);
insightsModule = Container.get(InsightsModule);
await createTeamProject();
});

View File

@ -1,17 +1,14 @@
import type { ModuleInterface } from '@n8n/decorators';
import { BackendModule, OnShutdown } from '@n8n/decorators';
import { Container } from '@n8n/di';
import { InstanceSettings } from 'n8n-core';
@BackendModule({ name: 'insights' })
/**
* Only main- and webhook-type instances collect insights because
* only they are informed of finished workflow executions.
*/
@BackendModule({ name: 'insights', instanceTypes: ['main', 'webhook'] })
export class InsightsModule implements ModuleInterface {
async init() {
/**
* Only main- and webhook-type instances collect insights because
* only they are informed of finished workflow executions.
*/
if (Container.get(InstanceSettings).instanceType === 'worker') return;
await import('./insights.controller');
const { InsightsService } = await import('./insights.service');

View File

@ -1,6 +1,15 @@
import { ModuleRegistry, Logger } from '@n8n/backend-common';
import { type AuthenticatedRequest, WorkflowEntity } from '@n8n/db';
import { Body, Post, Get, Patch, RestController, GlobalScope, Param } from '@n8n/decorators';
import {
Body,
Post,
Get,
Patch,
RestController,
GlobalScope,
Param,
ProjectScope,
} from '@n8n/decorators';
import type { Response } from 'express';
import { UpdateMcpSettingsDto } from './dto/update-mcp-settings.dto';
@ -57,7 +66,7 @@ export class McpSettingsController {
return await this.mcpServerApiKeyService.rotateMcpServerApiKey(req.user);
}
@GlobalScope('mcp:manage')
@ProjectScope('workflow:update')
@Patch('/workflows/:workflowId/toggle-access')
async toggleWorkflowMCPAccess(
req: AuthenticatedRequest,

Some files were not shown because too many files have changed in this diff Show More