feat(core): Allow creating data tables from csv files (#21051)

This commit is contained in:
Ricardo Espinoza 2025-11-18 11:32:38 -05:00 committed by GitHub
parent 0a355ccadb
commit 2830665f7a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
27 changed files with 2428 additions and 45 deletions

View File

@ -7,4 +7,6 @@ import { dataTableNameSchema } from '../../schemas/data-table.schema';
export class CreateDataTableDto extends Z.class({
name: dataTableNameSchema,
columns: z.array(CreateDataTableColumnDto),
fileId: z.string().optional(),
hasHeaders: z.boolean().optional(),
}) {}

View File

@ -1,3 +1,6 @@
import { tmpdir } from 'node:os';
import path from 'node:path';
import { Config, Env } from '../decorators';
@Config
@ -20,4 +23,39 @@ export class DataTableConfig {
*/
@Env('N8N_DATA_TABLES_SIZE_CHECK_CACHE_DURATION_MS')
sizeCheckCacheDuration: number = 60 * 1000;
/**
* The maximum allowed file size (in bytes) for CSV uploads to data tables.
* If set, this is the hard limit for file uploads.
* If not set, the upload limit will be the remaining available storage space.
*/
@Env('N8N_DATA_TABLES_UPLOAD_MAX_FILE_SIZE_BYTES')
uploadMaxFileSize?: number;
/**
* The interval in milliseconds at which orphaned uploaded files are cleaned up.
* Defaults to 60 seconds if not explicitly set via environment variable.
*/
@Env('N8N_DATA_TABLES_CLEANUP_INTERVAL_MS')
cleanupIntervalMs: number = 60 * 1000;
/**
* The maximum age in milliseconds for uploaded files before they are considered orphaned and deleted.
* Files older than this threshold are removed during cleanup.
* Defaults to 2 minutes if not explicitly set via environment variable.
*/
@Env('N8N_DATA_TABLES_FILE_MAX_AGE_MS')
fileMaxAgeMs: number = 2 * 60 * 1000;
/**
* The directory path where uploaded CSV files are temporarily stored before being imported.
* Files in this directory are automatically cleaned up after a configurable period (fileMaxAgeMs).
* Computed as: <system-tmp-dir>/n8nDataTableUploads
* Example: /tmp/n8nDataTableUploads
*/
readonly uploadDir: string;
constructor() {
this.uploadDir = path.join(tmpdir(), 'n8nDataTableUploads');
}
}

View File

@ -1,6 +1,7 @@
import path from 'node:path';
import { Config, Env } from '../decorators';
import { getN8nFolder } from '../utils/utils';
@Config
export class InstanceSettingsConfig {
@ -29,9 +30,7 @@ export class InstanceSettingsConfig {
readonly n8nFolder: string;
constructor() {
const homeVarName = process.platform === 'win32' ? 'USERPROFILE' : 'HOME';
this.userHome = process.env.N8N_USER_FOLDER ?? process.env[homeVarName] ?? process.cwd();
this.n8nFolder = path.join(this.userHome, '.n8n');
this.n8nFolder = getN8nFolder();
this.userHome = path.dirname(this.n8nFolder);
}
}

View File

@ -0,0 +1,11 @@
import path from 'node:path';
/**
* Computes the n8n folder path based on environment variables.
* This is used by various configs that need to know the n8n installation directory.
*/
export function getN8nFolder(): string {
const homeVarName = process.platform === 'win32' ? 'USERPROFILE' : 'HOME';
const userHome = process.env.N8N_USER_FOLDER ?? process.env[homeVarName] ?? process.cwd();
return path.join(userHome, '.n8n');
}

View File

@ -1,6 +1,8 @@
import { Container } from '@n8n/di';
import fs from 'fs';
import { mock } from 'jest-mock-extended';
import { tmpdir } from 'node:os';
import path from 'node:path';
import type { UserManagementConfig } from '../src/configs/user-management.config';
import { GlobalConfig } from '../src/index';
@ -55,6 +57,9 @@ describe('GlobalConfig', () => {
dataTable: {
maxSize: 50 * 1024 * 1024,
sizeCheckCacheDuration: 60000,
cleanupIntervalMs: 60 * 1000,
fileMaxAgeMs: 2 * 60 * 1000,
uploadDir: path.join(tmpdir(), 'n8nDataTableUploads'),
},
database: {
logging: {

View File

@ -49,6 +49,12 @@ export const rawBodyReader: RequestHandler = async (req, _res, next) => {
};
export const parseBody = async (req: Request) => {
// Skip multipart requests (e.g., file uploads) - these need specialized parsing by multer.
// Reading the body stream here would consume it, making it unavailable for multer processing.
if (req.contentType?.startsWith('multipart/')) {
return;
}
await req.readRawBody();
const { rawBody, contentType, encoding } = req;
if (rawBody?.length) {

View File

@ -0,0 +1,284 @@
import type { GlobalConfig } from '@n8n/config';
import { promises as fs } from 'fs';
import path from 'path';
import { DataTableFileCleanupService } from '../data-table-file-cleanup.service';
jest.mock('fs', () => ({
promises: {
unlink: jest.fn(),
readdir: jest.fn(),
stat: jest.fn(),
},
}));
describe('DataTableFileCleanupService', () => {
const uploadDir = '/mock/n8n/dataTableUploads';
const globalConfig = {
dataTable: {
cleanupIntervalMs: 60 * 1000,
fileMaxAgeMs: 2 * 60 * 1000,
uploadDir,
},
} as GlobalConfig;
const service = new DataTableFileCleanupService(globalConfig);
beforeEach(() => {
jest.clearAllMocks();
jest.spyOn(console, 'error').mockImplementation(() => {});
});
afterEach(() => {
jest.restoreAllMocks();
});
describe('deleteFile', () => {
it('should delete a file successfully', async () => {
const fileId = 'test-file-123';
const expectedPath = path.join(uploadDir, fileId);
(fs.unlink as jest.Mock).mockResolvedValue(undefined);
await service.deleteFile(fileId);
expect(fs.unlink).toHaveBeenCalledWith(expectedPath);
expect(fs.unlink).toHaveBeenCalledTimes(1);
});
it('should ignore ENOENT error when file does not exist', async () => {
const fileId = 'non-existent-file';
const error = new Error('ENOENT: no such file or directory') as NodeJS.ErrnoException;
error.code = 'ENOENT';
(fs.unlink as jest.Mock).mockRejectedValue(error);
await expect(service.deleteFile(fileId)).resolves.toBeUndefined();
});
it('should throw error for non-ENOENT errors', async () => {
const fileId = 'test-file';
const error = new Error('Permission denied') as NodeJS.ErrnoException;
error.code = 'EPERM';
(fs.unlink as jest.Mock).mockRejectedValue(error);
await expect(service.deleteFile(fileId)).rejects.toThrow('Permission denied');
});
});
describe('start and shutdown', () => {
it('should start cleanup interval', async () => {
jest.useFakeTimers();
await service.start();
expect(service['cleanupInterval']).toBeDefined();
jest.useRealTimers();
});
it('should clear interval on shutdown', async () => {
jest.useFakeTimers();
await service.start();
expect(service['cleanupInterval']).toBeDefined();
await service.shutdown();
expect(service['cleanupInterval']).toBeUndefined();
jest.useRealTimers();
});
it('should not error on shutdown if interval was never started', async () => {
await expect(service.shutdown()).resolves.toBeUndefined();
});
});
describe('cleanupOrphanedFiles', () => {
const flushPromises = async () => await new Promise(jest.requireActual('timers').setImmediate);
beforeEach(() => {
jest.useFakeTimers();
});
afterEach(() => {
jest.useRealTimers();
});
it('should delete files older than 2 minutes', async () => {
const now = Date.now();
const oldFile1 = 'old-file-1.csv';
const oldFile2 = 'old-file-2.csv';
(fs.readdir as jest.Mock).mockResolvedValue([oldFile1, oldFile2]);
(fs.stat as jest.Mock).mockResolvedValue({
mtimeMs: now - 3 * 60 * 1000, // 3 minutes ago
});
(fs.unlink as jest.Mock).mockResolvedValue(undefined);
await service.start();
// Trigger cleanup and let promises resolve
jest.advanceTimersByTime(60 * 1000);
await flushPromises();
expect(fs.readdir).toHaveBeenCalledWith(uploadDir);
expect(fs.stat).toHaveBeenCalledTimes(2);
expect(fs.unlink).toHaveBeenCalledWith(path.join(uploadDir, oldFile1));
expect(fs.unlink).toHaveBeenCalledWith(path.join(uploadDir, oldFile2));
});
it('should not delete files newer than 2 minutes', async () => {
const now = Date.now();
const newFile = 'new-file.csv';
(fs.readdir as jest.Mock).mockResolvedValue([newFile]);
(fs.stat as jest.Mock).mockResolvedValue({
mtimeMs: now - 1 * 60 * 1000, // 1 minute ago
});
await service.start();
jest.advanceTimersByTime(60 * 1000); // Trigger cleanup
await flushPromises();
expect(fs.readdir).toHaveBeenCalled();
expect(fs.stat).toHaveBeenCalled();
expect(fs.unlink).not.toHaveBeenCalled();
});
it('should handle mixed old and new files', async () => {
const now = Date.now();
const oldFile = 'old-file.csv';
const newFile = 'new-file.csv';
(fs.readdir as jest.Mock).mockResolvedValue([oldFile, newFile]);
(fs.stat as jest.Mock)
.mockResolvedValueOnce({
mtimeMs: now - 3 * 60 * 1000, // 3 minutes ago (old)
})
.mockResolvedValueOnce({
mtimeMs: now - 1 * 60 * 1000, // 1 minute ago (new)
});
(fs.unlink as jest.Mock).mockResolvedValue(undefined);
await service.start();
jest.advanceTimersByTime(60 * 1000); // Trigger cleanup
await flushPromises();
expect(fs.unlink).toHaveBeenCalledTimes(1);
expect(fs.unlink).toHaveBeenCalledWith(path.join(uploadDir, oldFile));
expect(fs.unlink).not.toHaveBeenCalledWith(path.join(uploadDir, newFile));
});
it('should handle empty upload directory', async () => {
(fs.readdir as jest.Mock).mockResolvedValue([]);
await service.start();
jest.advanceTimersByTime(60 * 1000); // Trigger cleanup
await flushPromises();
expect(fs.readdir).toHaveBeenCalled();
expect(fs.stat).not.toHaveBeenCalled();
expect(fs.unlink).not.toHaveBeenCalled();
});
it('should ignore ENOENT error if upload directory does not exist', async () => {
const error = new Error('ENOENT: no such file or directory') as NodeJS.ErrnoException;
error.code = 'ENOENT';
(fs.readdir as jest.Mock).mockRejectedValue(error);
await service.start();
jest.advanceTimersByTime(60 * 1000); // Trigger cleanup
await flushPromises();
expect(fs.readdir).toHaveBeenCalled();
expect(console.error).not.toHaveBeenCalled();
});
it('should log error for non-ENOENT readdir errors', async () => {
const error = new Error('Permission denied');
(fs.readdir as jest.Mock).mockRejectedValue(error);
await service.start();
jest.advanceTimersByTime(60 * 1000); // Trigger cleanup
await flushPromises();
expect(console.error).toHaveBeenCalledWith('Error cleaning up orphaned CSV files:', error);
});
it('should continue cleanup if individual file stat fails', async () => {
const file1 = 'file1.csv';
const file2 = 'file2.csv';
(fs.readdir as jest.Mock).mockResolvedValue([file1, file2]);
(fs.stat as jest.Mock)
.mockRejectedValueOnce(new Error('Stat failed for file1'))
.mockResolvedValueOnce({
mtimeMs: Date.now() - 3 * 60 * 1000, // file2 is old
});
(fs.unlink as jest.Mock).mockResolvedValue(undefined);
await service.start();
jest.advanceTimersByTime(60 * 1000); // Trigger cleanup
await flushPromises();
// Should still delete file2 even though file1 failed
expect(fs.unlink).toHaveBeenCalledWith(path.join(uploadDir, file2));
expect(fs.unlink).toHaveBeenCalledTimes(1);
});
it('should continue cleanup if individual file unlink fails', async () => {
const now = Date.now();
const file1 = 'file1.csv';
const file2 = 'file2.csv';
(fs.readdir as jest.Mock).mockResolvedValue([file1, file2]);
(fs.stat as jest.Mock).mockResolvedValue({
mtimeMs: now - 3 * 60 * 1000, // Both files are old
});
(fs.unlink as jest.Mock)
.mockRejectedValueOnce(new Error('Unlink failed for file1'))
.mockResolvedValueOnce(undefined);
await service.start();
jest.advanceTimersByTime(60 * 1000); // Trigger cleanup
await flushPromises();
// Should attempt to delete both files
expect(fs.unlink).toHaveBeenCalledWith(path.join(uploadDir, file1));
expect(fs.unlink).toHaveBeenCalledWith(path.join(uploadDir, file2));
expect(fs.unlink).toHaveBeenCalledTimes(2);
});
it('should run cleanup every 60 seconds', async () => {
const now = Date.now();
(fs.readdir as jest.Mock).mockResolvedValue(['old-file.csv']);
(fs.stat as jest.Mock).mockResolvedValue({
mtimeMs: now - 3 * 60 * 1000,
});
(fs.unlink as jest.Mock).mockResolvedValue(undefined);
await service.start();
// First cleanup
jest.advanceTimersByTime(60 * 1000);
await flushPromises();
expect(fs.readdir).toHaveBeenCalledTimes(1);
// Second cleanup
jest.advanceTimersByTime(60 * 1000);
await flushPromises();
expect(fs.readdir).toHaveBeenCalledTimes(2);
// Third cleanup
jest.advanceTimersByTime(60 * 1000);
await flushPromises();
expect(fs.readdir).toHaveBeenCalledTimes(3);
await service.shutdown();
});
});
});

View File

@ -0,0 +1,425 @@
import { GlobalConfig } from '@n8n/config';
import type { User } from '@n8n/db';
import { Container } from '@n8n/di';
import { createOwner, createMember } from '@test-integration/db/users';
import type { SuperAgentTest } from '@test-integration/types';
import * as utils from '@test-integration/utils';
const testServer = utils.setupTestServer({
endpointGroups: ['data-table'],
modules: ['data-table'],
});
let owner: User;
let member: User;
let authOwnerAgent: SuperAgentTest;
let authMemberAgent: SuperAgentTest;
beforeAll(async () => {
owner = await createOwner();
member = await createMember();
authOwnerAgent = testServer.authAgentFor(owner);
authMemberAgent = testServer.authAgentFor(member);
});
describe('POST /data-tables/uploads', () => {
describe('successful file uploads', () => {
test('should upload a valid CSV file and return file metadata', async () => {
const csvContent = 'name,email\nJohn Doe,john@example.com\nJane Smith,jane@example.com';
const response = await authOwnerAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(csvContent), 'test.csv')
.expect(200);
expect(response.body.data).toHaveProperty('originalName', 'test.csv');
expect(response.body.data).toHaveProperty('id');
expect(response.body.data.id).toMatch(/^[a-zA-Z0-9_-]{10}$/); // nanoid with length 10
expect(response.body.data).toHaveProperty('rowCount', 2);
expect(response.body.data).toHaveProperty('columnCount', 2);
expect(response.body.data).toHaveProperty('columns');
expect(response.body.data.columns).toEqual([
{ name: 'name', type: 'string', compatibleTypes: ['string'] },
{ name: 'email', type: 'string', compatibleTypes: ['string'] },
]);
});
test('should accept CSV file from member user', async () => {
const csvContent = 'col1,col2\nvalue1,value2';
const response = await authMemberAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(csvContent), 'data.csv')
.expect(200);
expect(response.body.data).toHaveProperty('originalName', 'data.csv');
expect(response.body.data).toHaveProperty('id');
expect(response.body.data).toHaveProperty('rowCount', 1);
expect(response.body.data).toHaveProperty('columnCount', 2);
});
test('should handle CSV files with special characters in filename', async () => {
const csvContent = 'a,b,c\n1,2,3';
const filename = 'test-file_v1.2.csv';
const response = await authOwnerAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(csvContent), filename)
.expect(200);
expect(response.body.data).toHaveProperty('originalName', filename);
expect(response.body.data).toHaveProperty('id');
expect(response.body.data).toHaveProperty('rowCount', 1);
expect(response.body.data).toHaveProperty('columnCount', 3);
expect(response.body.data.columns).toEqual([
{ name: 'a', type: 'number', compatibleTypes: ['number', 'string'] },
{ name: 'b', type: 'number', compatibleTypes: ['number', 'string'] },
{ name: 'c', type: 'number', compatibleTypes: ['number', 'string'] },
]);
});
test('should accept files within size limit', async () => {
const smallContent = 'col1,col2\nval1,val2\n';
const response = await authOwnerAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(smallContent), 'small.csv')
.expect(200);
expect(response.body.data).toHaveProperty('originalName', 'small.csv');
expect(response.body.data).toHaveProperty('id');
});
test('should accept empty CSV file', async () => {
const response = await authOwnerAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(''), 'empty.csv')
.expect(200);
expect(response.body.data).toHaveProperty('originalName', 'empty.csv');
expect(response.body.data).toHaveProperty('id');
expect(response.body.data).toHaveProperty('rowCount', 0);
expect(response.body.data).toHaveProperty('columnCount', 0);
expect(response.body.data.columns).toEqual([]);
});
test('should infer correct column types (string, number, boolean, date)', async () => {
const csvContent =
'name,age,isActive,createdDate\nJohn,30,true,2024-01-15\nJane,25,false,2024-02-20';
const response = await authOwnerAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(csvContent), 'types.csv')
.expect(200);
expect(response.body.data).toHaveProperty('originalName', 'types.csv');
expect(response.body.data).toHaveProperty('rowCount', 2);
expect(response.body.data).toHaveProperty('columnCount', 4);
expect(response.body.data.columns).toEqual([
{ name: 'name', type: 'string', compatibleTypes: ['string'] },
{ name: 'age', type: 'number', compatibleTypes: ['number', 'string'] },
{ name: 'isActive', type: 'boolean', compatibleTypes: ['boolean', 'string'] },
{ name: 'createdDate', type: 'date', compatibleTypes: ['date', 'string'] },
]);
});
});
describe('header handling', () => {
test('should use first row as headers when hasHeaders=true (default)', async () => {
const csvContent = 'FirstName,LastName,Age\nJohn,Doe,30\nJane,Smith,25';
const response = await authOwnerAgent
.post('/data-tables/uploads')
.field('hasHeaders', 'true')
.attach('file', Buffer.from(csvContent), 'with-headers.csv')
.expect(200);
expect(response.body.data).toHaveProperty('rowCount', 2);
expect(response.body.data).toHaveProperty('columnCount', 3);
expect(response.body.data.columns).toEqual([
{ name: 'FirstName', type: 'string', compatibleTypes: ['string'] },
{ name: 'LastName', type: 'string', compatibleTypes: ['string'] },
{ name: 'Age', type: 'number', compatibleTypes: ['number', 'string'] },
]);
});
test('should generate column names when hasHeaders=false', async () => {
const csvContent = 'John,Doe,30\nJane,Smith,25\nBob,Johnson,35';
const response = await authOwnerAgent
.post('/data-tables/uploads')
.field('hasHeaders', 'false')
.attach('file', Buffer.from(csvContent), 'no-headers.csv')
.expect(200);
expect(response.body.data).toHaveProperty('rowCount', 3);
expect(response.body.data).toHaveProperty('columnCount', 3);
expect(response.body.data.columns).toEqual([
{ name: 'Column_1', type: 'string', compatibleTypes: ['string'] },
{ name: 'Column_2', type: 'string', compatibleTypes: ['string'] },
{ name: 'Column_3', type: 'number', compatibleTypes: ['number', 'string'] },
]);
});
test('should include all rows in count when hasHeaders=false', async () => {
const csvContent = '100,200,300\n400,500,600';
const response = await authOwnerAgent
.post('/data-tables/uploads')
.field('hasHeaders', 'false')
.attach('file', Buffer.from(csvContent), 'no-headers-numbers.csv')
.expect(200);
expect(response.body.data).toHaveProperty('rowCount', 2); // Both rows counted as data
expect(response.body.data).toHaveProperty('columnCount', 3);
expect(response.body.data.columns).toEqual([
{ name: 'Column_1', type: 'number', compatibleTypes: ['number', 'string'] },
{ name: 'Column_2', type: 'number', compatibleTypes: ['number', 'string'] },
{ name: 'Column_3', type: 'number', compatibleTypes: ['number', 'string'] },
]);
});
test('should default to hasHeaders=true when field not provided', async () => {
const csvContent = 'col1,col2\nval1,val2';
const response = await authOwnerAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(csvContent), 'default.csv')
.expect(200);
expect(response.body.data).toHaveProperty('rowCount', 1);
expect(response.body.data.columns[0].name).toBe('col1');
expect(response.body.data.columns[1].name).toBe('col2');
});
});
describe('authentication', () => {
test('should reject unauthenticated requests', async () => {
const csvContent = 'name,value\ntest,123';
await testServer.authlessAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(csvContent), 'test.csv')
.expect(401);
});
});
describe('file validation', () => {
test('should reject request without file field', async () => {
await authOwnerAgent.post('/data-tables/uploads').send({}).expect(400);
});
test('should reject request with wrong field name', async () => {
const csvContent = 'a,b\n1,2';
await authOwnerAgent
.post('/data-tables/uploads')
.attach('wrongField', Buffer.from(csvContent), 'test.csv')
.expect(400);
});
test('should reject non-CSV files based on MIME type', async () => {
const textContent = 'This is a plain text file';
const response = await authOwnerAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(textContent), 'test.txt');
expect(response.status).toBe(400);
});
test('should reject JSON files', async () => {
const jsonContent = JSON.stringify({ name: 'test', value: 123 });
const response = await authOwnerAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(jsonContent), 'test.json');
expect(response.status).toBe(400);
});
test('should reject Excel files', async () => {
const excelBuffer = Buffer.from('PK\x03\x04'); // Excel file signature
const response = await authOwnerAgent
.post('/data-tables/uploads')
.attach('file', excelBuffer, 'test.xlsx');
expect(response.status).toBe(400);
});
});
describe('file size validation with uploadMaxFileSize', () => {
// Note: uploadMaxFileSize is set during multer initialization, so changing it
// at runtime won't affect the behavior. These tests verify the behavior
// when uploadMaxFileSize is configured via environment variable.
test('should accept small files when uploadMaxFileSize is set', async () => {
const smallContent = 'name,value\ntest,123\n';
const response = await authOwnerAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(smallContent), 'small.csv')
.expect(200);
expect(response.body.data).toHaveProperty('originalName', 'small.csv');
});
});
describe('file size validation without uploadMaxFileSize', () => {
let globalConfig: GlobalConfig;
let originalUploadMaxFileSize: number | undefined;
let originalMaxSize: number;
beforeEach(() => {
globalConfig = Container.get(GlobalConfig);
originalUploadMaxFileSize = globalConfig.dataTable.uploadMaxFileSize;
originalMaxSize = globalConfig.dataTable.maxSize;
// Unset uploadMaxFileSize to trigger remaining space check
globalConfig.dataTable.uploadMaxFileSize = undefined;
});
afterEach(() => {
// Restore original values
globalConfig.dataTable.uploadMaxFileSize = originalUploadMaxFileSize;
globalConfig.dataTable.maxSize = originalMaxSize;
});
test('should accept small files when there is remaining storage space', async () => {
// Set max size to 50MB - in test environment, database is likely empty
globalConfig.dataTable.maxSize = 50 * 1024 * 1024;
const smallContent = 'name,value\ntest,123\n';
const response = await authOwnerAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(smallContent), 'small.csv')
.expect(200);
expect(response.body.data).toHaveProperty('originalName', 'small.csv');
});
test('should reject files exceeding remaining storage space', async () => {
// Set a very small max size (1KB) to ensure the upload fails
globalConfig.dataTable.maxSize = 1024;
// Create content larger than 1KB
const largeContent = 'a,b,c\n' + '1,2,3\n'.repeat(200);
const response = await authOwnerAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(largeContent), 'large.csv');
expect(response.status).toBe(400);
expect(response.body).toHaveProperty('message');
// Error message can be either about remaining space or storage limit exceeded
const message = response.body.message as string;
expect(
message.includes('remaining storage space') || message.includes('Storage limit exceeded'),
).toBe(true);
// Verify the error message includes size units (B, KB, or MB)
expect(message).toMatch(/\d+(B|KB|MB)/);
});
});
describe('edge cases', () => {
test('should handle CSV with only headers', async () => {
const csvContent = 'column1,column2,column3';
const response = await authOwnerAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(csvContent), 'headers-only.csv')
.expect(200);
expect(response.body.data).toHaveProperty('originalName', 'headers-only.csv');
expect(response.body.data).toHaveProperty('id');
expect(response.body.data).toHaveProperty('rowCount', 0);
expect(response.body.data).toHaveProperty('columnCount', 3);
expect(response.body.data.columns).toEqual([
{ name: 'column1', type: 'string', compatibleTypes: ['string'] },
{ name: 'column2', type: 'string', compatibleTypes: ['string'] },
{ name: 'column3', type: 'string', compatibleTypes: ['string'] },
]);
});
test('should handle CSV with Unicode content', async () => {
const csvContent = 'Name,City\n日本,東京\nДмитрий,Москва';
const response = await authOwnerAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(csvContent, 'utf-8'), 'unicode.csv')
.expect(200);
expect(response.body.data).toHaveProperty('originalName', 'unicode.csv');
expect(response.body.data).toHaveProperty('id');
});
test('should handle CSV with commas in quoted values', async () => {
const csvContent = 'name,address\n"Doe, John","123 Main St, Apt 4"';
const response = await authOwnerAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(csvContent), 'quoted.csv')
.expect(200);
expect(response.body.data).toHaveProperty('originalName', 'quoted.csv');
expect(response.body.data).toHaveProperty('id');
});
test('should handle CSV with newlines in values', async () => {
const csvContent =
'name,description\n"Product A","Line 1\nLine 2"\n"Product B","Single line"';
const response = await authOwnerAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(csvContent), 'multiline.csv')
.expect(200);
expect(response.body.data).toHaveProperty('originalName', 'multiline.csv');
expect(response.body.data).toHaveProperty('id');
});
test('should handle large valid CSV files', async () => {
// Create a large but valid CSV (within size limit)
const header = 'id,name,email,city,country\n';
const rows = Array.from(
{ length: 1000 },
(_, i) => `${i},User${i},user${i}@example.com,City${i},Country${i}`,
).join('\n');
const largeContent = header + rows;
const response = await authOwnerAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(largeContent), 'large-valid.csv')
.expect(200);
expect(response.body.data).toHaveProperty('originalName', 'large-valid.csv');
expect(response.body.data).toHaveProperty('id');
});
test('should handle CSV without headers (first row treated as header)', async () => {
// CSV with just data rows, no explicit header
const csvContent = '1,2,3\n4,5,6\n7,8,9';
const response = await authOwnerAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(csvContent), 'no-headers.csv')
.expect(200);
expect(response.body.data).toHaveProperty('originalName', 'no-headers.csv');
expect(response.body.data).toHaveProperty('id');
// First row is treated as headers, so we have 2 data rows
expect(response.body.data).toHaveProperty('rowCount', 2);
expect(response.body.data).toHaveProperty('columnCount', 3);
// First row values become column names
expect(response.body.data.columns).toEqual([
{ name: '1', type: 'number', compatibleTypes: ['number', 'string'] },
{ name: '2', type: 'number', compatibleTypes: ['number', 'string'] },
{ name: '3', type: 'number', compatibleTypes: ['number', 'string'] },
]);
});
});
});

View File

@ -3761,3 +3761,429 @@ describe('PATCH /projects/:projectId/data-tables/:dataTableId/rows', () => {
},
);
});
describe('POST /projects/:projectId/data-tables - CSV Import', () => {
test('should create data table and import rows from CSV file', async () => {
// First upload a CSV file
const csvContent = 'name,age,email\nAlice,30,alice@example.com\nBob,25,bob@example.com';
const uploadResponse = await authOwnerAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(csvContent), { filename: 'test.csv', contentType: 'text/csv' })
.expect(200);
const fileId = uploadResponse.body.data.id;
// Create data table with fileId to trigger import
const payload = {
name: 'Imported Data Table',
columns: [
{ name: 'name', type: 'string' },
{ name: 'age', type: 'number' },
{ name: 'email', type: 'string' },
],
fileId,
};
const createResponse = await authOwnerAgent
.post(`/projects/${ownerProject.id}/data-tables`)
.send(payload)
.expect(200);
const dataTableId = createResponse.body.data.id;
// Verify data was imported
const rowsResponse = await authOwnerAgent
.get(`/projects/${ownerProject.id}/data-tables/${dataTableId}/rows`)
.expect(200);
expect(rowsResponse.body.data.count).toBe(2);
expect(rowsResponse.body.data.data).toHaveLength(2);
expect(rowsResponse.body.data.data).toEqual(
expect.arrayContaining([
expect.objectContaining({
name: 'Alice',
age: 30,
email: 'alice@example.com',
}),
expect.objectContaining({
name: 'Bob',
age: 25,
email: 'bob@example.com',
}),
]),
);
});
test('should map CSV columns to table columns by position', async () => {
// Upload CSV with column names that have spaces
const csvContent =
'Customer Id,Full Name,Email Address\n1001,John Doe,john@example.com\n1002,Jane Smith,jane@example.com';
const uploadResponse = await authOwnerAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(csvContent), {
filename: 'customers.csv',
contentType: 'text/csv',
})
.expect(200);
const fileId = uploadResponse.body.data.id;
// Create table with different column names (without spaces)
const payload = {
name: 'Customers',
columns: [
{ name: 'customerId', type: 'string' },
{ name: 'fullName', type: 'string' },
{ name: 'emailAddress', type: 'string' },
],
fileId,
};
const createResponse = await authOwnerAgent
.post(`/projects/${ownerProject.id}/data-tables`)
.send(payload)
.expect(200);
const dataTableId = createResponse.body.data.id;
// Verify data was mapped correctly by position
const rowsResponse = await authOwnerAgent
.get(`/projects/${ownerProject.id}/data-tables/${dataTableId}/rows`)
.expect(200);
expect(rowsResponse.body.data.count).toBe(2);
expect(rowsResponse.body.data.data).toEqual(
expect.arrayContaining([
expect.objectContaining({
customerId: '1001',
fullName: 'John Doe',
emailAddress: 'john@example.com',
}),
expect.objectContaining({
customerId: '1002',
fullName: 'Jane Smith',
emailAddress: 'jane@example.com',
}),
]),
);
});
test('should create data table with partial column mapping when schema has extra columns', async () => {
// Upload a valid CSV with 2 columns
const csvContent = 'name,age\nAlice,30';
const uploadResponse = await authOwnerAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(csvContent), { filename: 'test.csv', contentType: 'text/csv' })
.expect(200);
const fileId = uploadResponse.body.data.id;
// Create table with more columns than in CSV
// The system should map by index: CSV col 0 -> table col 0, CSV col 1 -> table col 1
// The extra table column won't have data imported
const payload = {
name: 'Partial Import Table',
columns: [
{ name: 'name', type: 'string' },
{ name: 'age', type: 'number' },
{ name: 'extra', type: 'string' }, // Extra column not in CSV
],
fileId,
};
const createResponse = await authOwnerAgent
.post(`/projects/${ownerProject.id}/data-tables`)
.send(payload)
.expect(200);
const dataTableId = createResponse.body.data.id;
// Verify rows - should have mapped only the columns that exist in CSV
const rowsResponse = await authOwnerAgent
.get(`/projects/${ownerProject.id}/data-tables/${dataTableId}/rows`)
.expect(200);
// Data should be imported with columns mapped by index position
// The 'extra' column should be null/empty since it doesn't exist in CSV
expect(rowsResponse.body.data.data[0]).toMatchObject({
name: 'Alice',
age: 30,
});
});
test('should handle empty CSV file on import', async () => {
// Upload empty CSV
const csvContent = '';
const uploadResponse = await authOwnerAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(csvContent), { filename: 'empty.csv', contentType: 'text/csv' })
.expect(200);
const fileId = uploadResponse.body.data.id;
const payload = {
name: 'Empty CSV Import',
columns: [{ name: 'name', type: 'string' }],
fileId,
};
const createResponse = await authOwnerAgent
.post(`/projects/${ownerProject.id}/data-tables`)
.send(payload);
// Should either fail or create table with no rows
if (createResponse.status === 200) {
const dataTableId = createResponse.body.data.id;
const rowsResponse = await authOwnerAgent
.get(`/projects/${ownerProject.id}/data-tables/${dataTableId}/rows`)
.expect(200);
expect(rowsResponse.body.data.count).toBe(0);
}
});
test('should handle CSV with only headers on import', async () => {
// Upload CSV with only headers
const csvContent = 'name,age,city';
const uploadResponse = await authOwnerAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(csvContent), {
filename: 'headers-only.csv',
contentType: 'text/csv',
})
.expect(200);
const fileId = uploadResponse.body.data.id;
const payload = {
name: 'Headers Only Import',
columns: [
{ name: 'name', type: 'string' },
{ name: 'age', type: 'number' },
{ name: 'city', type: 'string' },
],
fileId,
};
const createResponse = await authOwnerAgent
.post(`/projects/${ownerProject.id}/data-tables`)
.send(payload)
.expect(200);
const dataTableId = createResponse.body.data.id;
// Should create table with no rows
const rowsResponse = await authOwnerAgent
.get(`/projects/${ownerProject.id}/data-tables/${dataTableId}/rows`)
.expect(200);
expect(rowsResponse.body.data.count).toBe(0);
});
test('should import CSV file with multiple rows', async () => {
const csvContent =
'itemId,itemName,itemValue\n1,Item 1,10\n2,Item 2,20\n3,Item 3,30\n4,Item 4,40\n5,Item 5,50';
const uploadResponse = await authOwnerAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(csvContent), {
filename: 'multiple-rows.csv',
contentType: 'text/csv',
})
.expect(200);
const fileId = uploadResponse.body.data.id;
const payload = {
name: 'Multiple Rows Import',
columns: [
{ name: 'itemId', type: 'string' },
{ name: 'itemName', type: 'string' },
{ name: 'itemValue', type: 'number' },
],
fileId,
};
const createResponse = await authOwnerAgent
.post(`/projects/${ownerProject.id}/data-tables`)
.send(payload)
.expect(200);
const dataTableId = createResponse.body.data.id;
// Verify all rows were imported
const rowsResponse = await authOwnerAgent
.get(`/projects/${ownerProject.id}/data-tables/${dataTableId}/rows`)
.expect(200);
expect(rowsResponse.body.data.count).toBe(5);
});
test('should create table without import when fileId is not provided', async () => {
const payload = {
name: 'Table Without Import',
columns: [
{ name: 'col1', type: 'string' },
{ name: 'col2', type: 'number' },
],
};
const createResponse = await authOwnerAgent
.post(`/projects/${ownerProject.id}/data-tables`)
.send(payload)
.expect(200);
const dataTableId = createResponse.body.data.id;
// Should create empty table
const rowsResponse = await authOwnerAgent
.get(`/projects/${ownerProject.id}/data-tables/${dataTableId}/rows`)
.expect(200);
expect(rowsResponse.body.data.count).toBe(0);
});
test('should handle CSV with quoted values containing commas', async () => {
const csvContent =
'name,address\n"John Doe","123 Main St, Apt 4"\n"Jane Smith","456 Oak Ave, Suite 10"';
const uploadResponse = await authOwnerAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(csvContent), { filename: 'quoted.csv', contentType: 'text/csv' })
.expect(200);
const fileId = uploadResponse.body.data.id;
const payload = {
name: 'Quoted Values Import',
columns: [
{ name: 'name', type: 'string' },
{ name: 'address', type: 'string' },
],
fileId,
};
const createResponse = await authOwnerAgent
.post(`/projects/${ownerProject.id}/data-tables`)
.send(payload)
.expect(200);
const dataTableId = createResponse.body.data.id;
const rowsResponse = await authOwnerAgent
.get(`/projects/${ownerProject.id}/data-tables/${dataTableId}/rows`)
.expect(200);
expect(rowsResponse.body.data.count).toBe(2);
expect(rowsResponse.body.data.data).toEqual(
expect.arrayContaining([
expect.objectContaining({
name: 'John Doe',
address: '123 Main St, Apt 4',
}),
expect.objectContaining({
name: 'Jane Smith',
address: '456 Oak Ave, Suite 10',
}),
]),
);
});
test('should handle CSV with Unicode characters', async () => {
const csvContent = 'name,city\nJohn Müller,München\nMaría García,São Paulo';
const uploadResponse = await authOwnerAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(csvContent), { filename: 'unicode.csv', contentType: 'text/csv' })
.expect(200);
const fileId = uploadResponse.body.data.id;
const payload = {
name: 'Unicode Import',
columns: [
{ name: 'name', type: 'string' },
{ name: 'city', type: 'string' },
],
fileId,
};
const createResponse = await authOwnerAgent
.post(`/projects/${ownerProject.id}/data-tables`)
.send(payload)
.expect(200);
const dataTableId = createResponse.body.data.id;
const rowsResponse = await authOwnerAgent
.get(`/projects/${ownerProject.id}/data-tables/${dataTableId}/rows`)
.expect(200);
expect(rowsResponse.body.data.count).toBe(2);
expect(rowsResponse.body.data.data).toEqual(
expect.arrayContaining([
expect.objectContaining({
name: 'John Müller',
city: 'München',
}),
expect.objectContaining({
name: 'María García',
city: 'São Paulo',
}),
]),
);
});
test('should handle CSV with different data types', async () => {
const csvContent =
'name,age,active,joinDate\nAlice,30,true,2024-01-15\nBob,25,false,2024-02-20';
const uploadResponse = await authOwnerAgent
.post('/data-tables/uploads')
.attach('file', Buffer.from(csvContent), {
filename: 'mixed-types.csv',
contentType: 'text/csv',
})
.expect(200);
const fileId = uploadResponse.body.data.id;
const payload = {
name: 'Mixed Types Import',
columns: [
{ name: 'name', type: 'string' },
{ name: 'age', type: 'number' },
{ name: 'active', type: 'boolean' },
{ name: 'joinDate', type: 'date' },
],
fileId,
};
const createResponse = await authOwnerAgent
.post(`/projects/${ownerProject.id}/data-tables`)
.send(payload)
.expect(200);
const dataTableId = createResponse.body.data.id;
const rowsResponse = await authOwnerAgent
.get(`/projects/${ownerProject.id}/data-tables/${dataTableId}/rows`)
.expect(200);
expect(rowsResponse.body.data.count).toBe(2);
// Data types are converted based on column type definitions
expect(rowsResponse.body.data.data).toEqual(
expect.arrayContaining([
expect.objectContaining({
name: 'Alice',
age: 30,
active: true,
joinDate: expect.stringContaining('2024-01-15'),
}),
expect.objectContaining({
name: 'Bob',
age: 25,
active: false,
joinDate: expect.stringContaining('2024-02-20'),
}),
]),
);
});
});

View File

@ -6,7 +6,7 @@ export function mockDataTableSizeValidator() {
const sizeValidator = Container.get(DataTableSizeValidator);
jest.spyOn(sizeValidator, 'validateSize').mockResolvedValue();
jest.spyOn(sizeValidator, 'getCachedSizeData').mockResolvedValue({
totalBytes: 50 * 1024 * 1024, // 50MB - under the default limit
totalBytes: 0, // Start with 0 bytes to allow uploads
dataTables: {},
});
return sizeValidator;

View File

@ -0,0 +1,204 @@
import { GlobalConfig } from '@n8n/config';
import { Service } from '@n8n/di';
import { parse } from 'csv-parse';
import { createReadStream } from 'fs';
import path from 'path';
export interface CsvColumnMetadata {
name: string;
type: 'string' | 'number' | 'boolean' | 'date';
}
export interface CsvMetadata {
rowCount: number;
columnCount: number;
columns: CsvColumnMetadata[];
}
@Service()
export class CsvParserService {
private readonly uploadDir: string;
private readonly DEFAULT_COLUMN_PREFIX = 'Column_';
constructor(private readonly globalConfig: GlobalConfig) {
this.uploadDir = this.globalConfig.dataTable.uploadDir;
}
private processRowWithoutHeaders(
row: string[],
columnNames: string[],
): { rowObject: Record<string, string>; columnNames: string[] } {
let updatedColumnNames = columnNames;
if (updatedColumnNames.length === 0) {
updatedColumnNames = row.map((_, index) => `${this.DEFAULT_COLUMN_PREFIX}${index + 1}`);
}
const rowObject: Record<string, string> = {};
row.forEach((value, index) => {
rowObject[updatedColumnNames[index]] = value;
});
return { rowObject, columnNames: updatedColumnNames };
}
/**
* Parses a CSV file and returns metadata including row count, column count, and inferred column types
*/
async parseFile(fileId: string, hasHeaders: boolean = true): Promise<CsvMetadata> {
const filePath = path.join(this.uploadDir, fileId);
let rowCount = 0;
let firstDataRow: Record<string, string> | null = null;
let columnNames: string[] = [];
return await new Promise((resolve, reject) => {
const parser = parse({
columns: hasHeaders
? (header: string[]) => {
columnNames = header;
return header;
}
: false,
skip_empty_lines: true,
});
createReadStream(filePath)
.pipe(parser)
.on('data', (row: Record<string, string> | string[]) => {
rowCount++;
if (!hasHeaders && Array.isArray(row)) {
const processed = this.processRowWithoutHeaders(row, columnNames);
columnNames = processed.columnNames;
firstDataRow ??= processed.rowObject;
} else if (!Array.isArray(row)) {
firstDataRow ??= row;
}
})
.on('end', () => {
const columns = columnNames.map((columnName) => {
const detectedType = this.inferColumnType(firstDataRow?.[columnName]);
return {
name: columnName,
type: detectedType,
compatibleTypes: this.getCompatibleTypes(detectedType),
};
});
resolve({
rowCount,
columnCount: columns.length,
columns,
});
})
.on('error', reject);
});
}
/**
* Parses a CSV file and returns all rows as an array of objects
*/
async parseFileData(
fileId: string,
hasHeaders: boolean = true,
): Promise<Array<Record<string, string>>> {
const filePath = path.join(this.uploadDir, fileId);
const rows: Array<Record<string, string>> = [];
let columnNames: string[] = [];
return await new Promise((resolve, reject) => {
const parser = parse({
columns: hasHeaders ? true : false,
skip_empty_lines: true,
});
createReadStream(filePath)
.pipe(parser)
.on('data', (row: Record<string, string> | string[]) => {
if (!hasHeaders && Array.isArray(row)) {
const processed = this.processRowWithoutHeaders(row, columnNames);
columnNames = processed.columnNames;
rows.push(processed.rowObject);
} else if (!Array.isArray(row)) {
rows.push(row);
}
})
.on('end', () => {
resolve(rows);
})
.on('error', reject);
});
}
/**
* Returns the list of compatible types for a detected type
* Logic: more specific types can be converted to string, but string cannot be converted to specific types
*/
private getCompatibleTypes(
detectedType: 'string' | 'number' | 'boolean' | 'date',
): Array<'string' | 'number' | 'boolean' | 'date'> {
switch (detectedType) {
case 'date':
return ['date', 'string'];
case 'number':
return ['number', 'string'];
case 'boolean':
return ['boolean', 'string'];
case 'string':
return ['string'];
default:
return ['string'];
}
}
/**
* Infers the column type from a sample value
* Priority: boolean > number > date > string
*/
private inferColumnType(value: string | undefined): 'string' | 'number' | 'boolean' | 'date' {
if (!value?.trim()) {
return 'string';
}
const trimmedValue = value.trim();
const lowerValue = trimmedValue.toLowerCase();
if (lowerValue === 'true' || lowerValue === 'false') {
return 'boolean';
}
if (!Number.isNaN(Number(trimmedValue))) {
return 'number';
}
if (this.isDate(trimmedValue)) {
return 'date';
}
return 'string';
}
/**
* Checks if a string represents a valid date
*/
private isDate(value: string): boolean {
// Try to parse as date
const date = new Date(value);
// Check if it's a valid date and the original value looks like a date
if (!Number.isNaN(date.getTime())) {
// Additional check: make sure it looks like a date format
// This prevents strings like "123" from being interpreted as dates
const datePatterns = [
/^\d{4}-\d{2}-\d{2}/, // ISO date (YYYY-MM-DD)
/^\d{4}\/\d{2}\/\d{2}/, // YYYY/MM/DD
/^\d{2}\/\d{2}\/\d{4}/, // MM/DD/YYYY or DD/MM/YYYY
/^\d{2}-\d{2}-\d{4}/, // MM-DD-YYYY or DD-MM-YYYY
/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}/, // ISO datetime
];
return datePatterns.some((pattern) => pattern.test(value));
}
return false;
}
}

View File

@ -0,0 +1,87 @@
import { GlobalConfig } from '@n8n/config';
import { Service } from '@n8n/di';
import { promises as fs } from 'fs';
import path from 'path';
@Service()
export class DataTableFileCleanupService {
private readonly uploadDir: string;
private cleanupInterval?: NodeJS.Timeout;
constructor(private readonly globalConfig: GlobalConfig) {
this.uploadDir = this.globalConfig.dataTable.uploadDir;
}
private isErrnoException(error: unknown): error is NodeJS.ErrnoException {
return (
typeof error === 'object' &&
error !== null &&
'code' in error &&
typeof (error as { code: unknown }).code === 'string'
);
}
async start() {
// Run cleanup periodically to delete orphaned files
this.cleanupInterval = setInterval(() => {
void this.cleanupOrphanedFiles();
}, this.globalConfig.dataTable.cleanupIntervalMs);
}
async shutdown() {
if (this.cleanupInterval) {
clearInterval(this.cleanupInterval);
this.cleanupInterval = undefined;
}
}
/**
* Cleans up orphaned CSV files that exceed the configured maximum age
* These are files that were uploaded but never used to create a data table
*/
private async cleanupOrphanedFiles(): Promise<void> {
try {
const files = await fs.readdir(this.uploadDir);
const now = Date.now();
const maxAge = this.globalConfig.dataTable.fileMaxAgeMs;
for (const file of files) {
const filePath = path.join(this.uploadDir, file);
try {
const stats = await fs.stat(filePath);
const fileAge = now - stats.mtimeMs;
// Delete files older than the configured maximum age
if (fileAge > maxAge) {
await fs.unlink(filePath);
}
} catch (error) {
// Ignore errors for individual files (e.g., file already deleted)
continue;
}
}
} catch (error) {
// Ignore errors if upload directory doesn't exist yet
if (!this.isErrnoException(error) || error.code !== 'ENOENT') {
// Log other errors but don't throw - cleanup is best effort
console.error('Error cleaning up orphaned CSV files:', error);
}
}
}
/**
* Deletes a specific CSV file by its fileId
*/
async deleteFile(fileId: string): Promise<void> {
const filePath = path.join(this.uploadDir, fileId);
try {
await fs.unlink(filePath);
} catch (error) {
// Ignore errors if file doesn't exist
if (!this.isErrnoException(error) || error.code !== 'ENOENT') {
throw error;
}
}
}
}

View File

@ -5,6 +5,7 @@ import { DataTableSizeStatus, DataTablesSizeData } from 'n8n-workflow';
import { Telemetry } from '@/telemetry';
import { DataTableValidationError } from './errors/data-table-validation.error';
import { toMb } from './utils/size-utils';
@Service()
export class DataTableSizeValidator {
@ -64,7 +65,7 @@ export class DataTableSizeValidator {
});
throw new DataTableValidationError(
`Data table size limit exceeded: ${this.toMb(size.totalBytes)}MB used, limit is ${this.toMb(this.globalConfig.dataTable.maxSize)}MB`,
`Data table size limit exceeded: ${toMb(size.totalBytes)}MB used, limit is ${toMb(this.globalConfig.dataTable.maxSize)}MB`,
);
}
}
@ -87,10 +88,6 @@ export class DataTableSizeValidator {
return this.sizeToState(size.totalBytes);
}
private toMb(sizeInBytes: number): number {
return Math.round(sizeInBytes / (1024 * 1024));
}
reset() {
this.lastCheck = undefined;
this.cachedSizeData = undefined;

View File

@ -0,0 +1,50 @@
import { Post, RestController } from '@n8n/decorators';
import { Container } from '@n8n/di';
import multer from 'multer';
import { BadRequestError } from '@/errors/response-errors/bad-request.error';
import { CsvParserService } from './csv-parser.service';
import { MulterUploadMiddleware } from './multer-upload-middleware';
import { AuthenticatedRequestWithFile, hasStringProperty } from './types';
const uploadMiddleware = Container.get(MulterUploadMiddleware);
@RestController('/data-tables/uploads')
export class DataTableUploadsController {
constructor(private readonly csvParserService: CsvParserService) {}
@Post('/', {
middlewares: [uploadMiddleware.single('file')],
})
async uploadFile(req: AuthenticatedRequestWithFile, _res: Response) {
if (req.fileUploadError) {
const error = req.fileUploadError;
if (error instanceof multer.MulterError) {
throw new BadRequestError(`File upload error: ${error.message}`);
} else if (error instanceof BadRequestError) {
throw error;
} else {
throw new BadRequestError('File upload failed');
}
}
if (!req.file) {
throw new BadRequestError('No file uploaded');
}
// Extract hasHeaders parameter from request body (multer parses form fields to body), default to true
const hasHeaders =
hasStringProperty(req.body, 'hasHeaders') && req.body.hasHeaders === 'false' ? false : true;
const metadata = await this.csvParserService.parseFile(req.file.filename, hasHeaders);
return {
originalName: req.file.originalname,
id: req.file.filename,
rowCount: metadata.rowCount,
columnCount: metadata.columnCount,
columns: metadata.columns,
};
}
}

View File

@ -7,12 +7,16 @@ export class DataTableModule implements ModuleInterface {
async init() {
await import('./data-table.controller');
await import('./data-table-aggregate.controller');
await import('./data-table-uploads.controller');
const { DataTableService } = await import('./data-table.service');
await Container.get(DataTableService).start();
const { DataTableAggregateService } = await import('./data-table-aggregate.service');
await Container.get(DataTableAggregateService).start();
const { DataTableFileCleanupService } = await import('./data-table-file-cleanup.service');
await Container.get(DataTableFileCleanupService).start();
}
@OnShutdown()
@ -22,6 +26,9 @@ export class DataTableModule implements ModuleInterface {
const { DataTableAggregateService } = await import('./data-table-aggregate.service');
await Container.get(DataTableAggregateService).shutdown();
const { DataTableFileCleanupService } = await import('./data-table-file-cleanup.service');
await Container.get(DataTableFileCleanupService).shutdown();
}
async entities() {

View File

@ -28,13 +28,16 @@ import type {
} from 'n8n-workflow';
import { DATA_TABLE_SYSTEM_COLUMN_TYPE_MAP, validateFieldType } from 'n8n-workflow';
import { CsvParserService } from './csv-parser.service';
import { DataTableColumn } from './data-table-column.entity';
import { DataTableColumnRepository } from './data-table-column.repository';
import { DataTableFileCleanupService } from './data-table-file-cleanup.service';
import { DataTableRowsRepository } from './data-table-rows.repository';
import { DataTableSizeValidator } from './data-table-size-validator.service';
import { DataTableRepository } from './data-table.repository';
import { columnTypeToFieldType } from './data-table.types';
import { DataTableColumnNotFoundError } from './errors/data-table-column-not-found.error';
import { FileUploadError } from './errors/data-table-file-upload.error';
import { DataTableNameConflictError } from './errors/data-table-name-conflict.error';
import { DataTableNotFoundError } from './errors/data-table-not-found.error';
import { DataTableValidationError } from './errors/data-table-validation.error';
@ -52,6 +55,8 @@ export class DataTableService {
private readonly dataTableSizeValidator: DataTableSizeValidator,
private readonly projectRelationRepository: ProjectRelationRepository,
private readonly roleService: RoleService,
private readonly csvParserService: CsvParserService,
private readonly fileCleanupService: DataTableFileCleanupService,
) {
this.logger = this.logger.scoped('data-table');
}
@ -64,11 +69,61 @@ export class DataTableService {
const result = await this.dataTableRepository.createDataTable(projectId, dto.name, dto.columns);
if (dto.fileId) {
try {
await this.importDataFromFile(projectId, result.id, dto.fileId, dto.hasHeaders ?? true);
await this.fileCleanupService.deleteFile(dto.fileId);
} catch (error) {
await this.deleteDataTable(result.id, projectId);
throw error;
}
}
this.dataTableSizeValidator.reset();
return result;
}
private async importDataFromFile(
projectId: string,
dataTableId: string,
fileId: string,
hasHeaders: boolean,
) {
try {
const tableColumns = await this.getColumns(dataTableId, projectId);
const csvMetadata = await this.csvParserService.parseFile(fileId, hasHeaders);
const columnMapping = new Map<string, string>();
csvMetadata.columns.forEach((csvColumn, index) => {
if (tableColumns[index]) {
columnMapping.set(csvColumn.name, tableColumns[index].name);
}
});
const csvRows = await this.csvParserService.parseFileData(fileId, hasHeaders);
const transformedRows = csvRows.map((csvRow) => {
const transformedRow: DataTableRow = {};
for (const [csvColName, value] of Object.entries(csvRow)) {
const tableColName = columnMapping.get(csvColName);
if (tableColName) {
transformedRow[tableColName] = value;
}
}
return transformedRow;
});
if (transformedRows.length > 0) {
await this.insertRows(dataTableId, projectId, transformedRows);
}
} catch (error) {
this.logger.error('Failed to import data from CSV file', { error, fileId, dataTableId });
throw new FileUploadError(error instanceof Error ? error.message : 'Failed to read CSV file');
}
}
// Updates data table properties (currently limited to renaming)
async updateDataTable(dataTableId: string, projectId: string, dto: UpdateDataTableDto) {
await this.validateDataTableExists(dataTableId, projectId);

View File

@ -0,0 +1,9 @@
import { UserError } from 'n8n-workflow';
export class FileUploadError extends UserError {
constructor(msg: string) {
super(`Error uploading file: ${msg}`, {
level: 'warning',
});
}
}

View File

@ -0,0 +1,111 @@
/* eslint-disable id-denylist */
import { GlobalConfig } from '@n8n/config';
import { Service } from '@n8n/di';
import type { Request, RequestHandler } from 'express';
import { mkdir } from 'fs/promises';
import multer from 'multer';
import { nanoid } from 'nanoid';
import { BadRequestError } from '@/errors/response-errors/bad-request.error';
import { DataTableSizeValidator } from './data-table-size-validator.service';
import { DataTableRepository } from './data-table.repository';
import {
type AuthenticatedRequestWithFile,
type MulterDestinationCallback,
type MulterFilenameCallback,
type UploadMiddleware,
} from './types';
import { formatBytes } from './utils/size-utils';
const ALLOWED_MIME_TYPES = ['text/csv'];
@Service()
export class MulterUploadMiddleware implements UploadMiddleware {
private upload: multer.Multer;
private readonly uploadDir: string;
constructor(
private readonly globalConfig: GlobalConfig,
private readonly sizeValidator: DataTableSizeValidator,
private readonly dataTableRepository: DataTableRepository,
) {
this.uploadDir = this.globalConfig.dataTable.uploadDir;
void this.ensureUploadDirExists();
const storage = multer.diskStorage({
destination: (_req: Request, _file: Express.Multer.File, cb: MulterDestinationCallback) => {
cb(null, this.uploadDir);
},
filename: (_req: Request, _file: Express.Multer.File, cb: MulterFilenameCallback) => {
const filename = nanoid(10);
cb(null, filename);
},
});
this.upload = multer({
storage,
limits: this.globalConfig.dataTable.uploadMaxFileSize
? { fileSize: this.globalConfig.dataTable.uploadMaxFileSize }
: undefined,
fileFilter: async (req, file, cb: multer.FileFilterCallback) => {
if (!ALLOWED_MIME_TYPES.includes(file.mimetype)) {
cb(
new BadRequestError(
`Only the following file types are allowed: ${ALLOWED_MIME_TYPES.join(', ')}`,
),
);
return;
}
const fileSize = parseInt(req.headers['content-length'] ?? '0', 10);
// If uploadMaxFileSize is set, multer's limits will handle the rejection
if (this.globalConfig.dataTable.uploadMaxFileSize) {
cb(null, true);
return;
}
// If uploadMaxFileSize is not set, check remaining space
try {
const sizeData = await this.sizeValidator.getCachedSizeData(async () => {
return await this.dataTableRepository.findDataTablesSize();
});
const remainingSpace = Math.max(
0,
this.globalConfig.dataTable.maxSize - sizeData.totalBytes,
);
if (fileSize > remainingSpace) {
const message =
remainingSpace === 0
? `Storage limit exceeded. Current usage: ${formatBytes(sizeData.totalBytes)}, Limit: ${formatBytes(this.globalConfig.dataTable.maxSize)}`
: `File size exceeds remaining storage space. Available: ${formatBytes(remainingSpace)}, File: ${formatBytes(fileSize)}`;
cb(new BadRequestError(message));
return;
}
cb(null, true);
} catch {
cb(new BadRequestError('Failed to validate file size'));
}
},
});
}
private async ensureUploadDirExists() {
await mkdir(this.uploadDir, { recursive: true });
}
single(fieldName: string): RequestHandler {
return (req, res, next) => {
void this.upload.single(fieldName)(req, res, (error) => {
if (error) {
(req as AuthenticatedRequestWithFile).fileUploadError = error;
}
next();
});
};
}
}

View File

@ -0,0 +1,26 @@
import type { AuthenticatedRequest } from '@n8n/db';
import type { RequestHandler } from 'express';
export interface UploadMiddleware {
single(fieldName: string): RequestHandler;
}
export type MulterDestinationCallback = (error: Error | null, destination: string) => void;
export type MulterFilenameCallback = (error: Error | null, filename: string) => void;
export type AuthenticatedRequestWithFile<
RouteParams = {},
ResponseBody = {},
RequestBody = {},
RequestQuery = {},
> = AuthenticatedRequest<RouteParams, ResponseBody, RequestBody, RequestQuery> & {
file?: Express.Multer.File;
fileUploadError?: Error;
};
export function hasStringProperty<K extends string>(
obj: unknown,
key: K,
): obj is Record<K, string> & object {
return typeof obj === 'object' && obj !== null && key in obj;
}

View File

@ -0,0 +1,19 @@
/**
* Convert bytes to megabytes (rounded to nearest integer)
*/
export function toMb(sizeInBytes: number): number {
return Math.round(sizeInBytes / (1024 * 1024));
}
/**
* Format bytes to human-readable size with appropriate unit (B, KB, or MB)
*/
export function formatBytes(sizeInBytes: number): string {
if (sizeInBytes < 1024) {
return `${sizeInBytes}B`;
} else if (sizeInBytes < 1024 * 1024) {
return `${Math.round(sizeInBytes / 1024)}KB`;
} else {
return `${Math.round(sizeInBytes / (1024 * 1024))}MB`;
}
}

View File

@ -42,6 +42,7 @@
"generic.any": "Any",
"generic.allow": "Allow",
"generic.deny": "Deny",
"generic.back": "Back",
"generic.cancel": "Cancel",
"generic.open": "Open",
"generic.openResource": "Open {resource}",
@ -3263,10 +3264,28 @@
"dataTable.card.column.count": "{count} column | {count} columns",
"dataTable.add.title": "Create new data table",
"dataTable.add.button.label": "Create data table",
"dataTable.add.fromScratch": "From scratch",
"dataTable.add.importCsv": "Import CSV",
"dataTable.add.input.name.label": "Data table name",
"dataTable.add.input.name.placeholder": "Enter data table name",
"dataTable.add.error": "Error creating data table",
"dataTable.delete.confirm.title": "Delete data table",
"dataTable.upload.uploading": "Uploading and processing CSV file...",
"dataTable.upload.selectFile": "Waiting for file selection...",
"dataTable.upload.dropOrClick": "Drop file here or click to upload",
"dataTable.upload.csvOnly": "CSV files only",
"dataTable.upload.hasHeaders": "My CSV file contains a header row",
"dataTable.upload.uploadButton": "Upload CSV",
"dataTable.upload.success": "'{fileName}' has been uploaded successfully. We found {columnCount} column and {rowCount} row | '{fileName}' has been uploaded successfully. We found {columnCount} columns and {rowCount} rows",
"dataTable.upload.error": "Error uploading CSV file",
"dataTable.import.columnsFound": "Columns found",
"dataTable.import.columnName": "Column Name",
"dataTable.import.columnsDescription": "Review and adjust the column names and types detected from your CSV file",
"dataTable.import.columnType": "Data Type",
"dataTable.import.columnNamePlaceholder": "Enter column name",
"dataTable.import.duplicateColumnName": "Column name must be unique",
"dataTable.import.systemColumnName": "{columnName} is a reserved column",
"dataTable.import.invalidColumnName": "Only alphabetical and non-leading numbers and underscores allowed",
"dataTable.delete.confirm.message": "Are you sure you want to delete the data table '{name}'? This action cannot be undone.",
"dataTable.delete.error": "Error deleting data table",
"dataTable.rename.error": "Error renaming data table",

View File

@ -1,19 +1,44 @@
<script lang="ts" setup>
import { useI18n } from '@n8n/i18n';
import { onMounted, ref } from 'vue';
import { onMounted, ref, computed } from 'vue';
import { useDataTableStore } from '@/features/core/dataTable/dataTable.store';
import { useUIStore } from '@/app/stores/ui.store';
import { useToast } from '@/app/composables/useToast';
import { useRoute, useRouter } from 'vue-router';
import { DATA_TABLE_DETAILS, PROJECT_DATA_TABLES } from '@/features/core/dataTable/constants';
import { useTelemetry } from '@/app/composables/useTelemetry';
import { dataTableColumnNameSchema } from '@n8n/api-types';
import { DATA_TABLE_SYSTEM_COLUMNS } from 'n8n-workflow';
import { N8nButton, N8nInput, N8nInputLabel } from '@n8n/design-system';
import {
N8nButton,
N8nCheckbox,
N8nIcon,
N8nInput,
N8nInputLabel,
N8nSelect,
N8nOption,
N8nText,
} from '@n8n/design-system';
import Modal from '@/app/components/Modal.vue';
import { ElUpload, ElRadio, ElRadioGroup } from 'element-plus';
import type { UploadFile } from 'element-plus';
type Props = {
modalName: string;
};
type CreationMode = 'select' | 'scratch' | 'import' | 'file-selected';
type ColumnType = 'string' | 'number' | 'boolean' | 'date';
interface CsvColumn {
name: string;
type: ColumnType;
compatibleTypes: ColumnType[];
typeOptions: Array<{ label: string; value: string }>;
error?: string;
}
const props = defineProps<Props>();
const dataTableStore = useDataTableStore();
@ -25,8 +50,82 @@ const i18n = useI18n();
const toast = useToast();
const telemetry = useTelemetry();
const creationMode = ref<CreationMode>('select');
const dataTableName = ref('');
const inputRef = ref<HTMLInputElement | null>(null);
const selectedFile = ref<File | null>(null);
const uploadedFileId = ref<string | null>(null);
const uploadedFileName = ref<string>('');
const csvColumns = ref<CsvColumn[]>([]);
const csvRowCount = ref<number>(0);
const csvColumnCount = ref<number>(0);
const isUploading = ref(false);
const hasHeaders = ref(true);
const isUploadHovered = ref(false);
const allColumnTypeOptions = [
{ label: 'String', value: 'string' },
{ label: 'Number', value: 'number' },
{ label: 'Boolean', value: 'boolean' },
{ label: 'Datetime', value: 'date' },
];
const isColumnType = (value: unknown): value is ColumnType => {
return allColumnTypeOptions.some((option) => option.value === value);
};
const getColumnTypeOptions = (compatibleTypes: ColumnType[]) => {
if (!compatibleTypes || compatibleTypes.length === 0) {
return allColumnTypeOptions;
}
return allColumnTypeOptions.filter((option) =>
compatibleTypes.includes(option.value as ColumnType),
);
};
const validateColumnName = (columnName: string): string | undefined => {
if (DATA_TABLE_SYSTEM_COLUMNS.includes(columnName)) {
return i18n.baseText('dataTable.import.systemColumnName', {
interpolate: { columnName },
});
}
const result = dataTableColumnNameSchema.safeParse(columnName);
if (!result.success) {
return i18n.baseText('dataTable.import.invalidColumnName');
}
return undefined;
};
const hasValidationErrors = computed(() => {
if (creationMode.value !== 'import') return false;
return csvColumns.value.some((column) => column.error !== undefined);
});
const hasDuplicateNames = computed(() => {
if (creationMode.value !== 'import') return false;
const names = csvColumns.value.map((col) => col.name.toLowerCase());
return names.length !== new Set(names).size;
});
const modalTitle = computed(() => {
if (creationMode.value === 'import') {
return 'Set data table columns';
}
return i18n.baseText('dataTable.add.title');
});
const isCreateDisabled = computed(() => {
if (creationMode.value === 'import') {
return (
!dataTableName.value ||
!uploadedFileId.value ||
hasValidationErrors.value ||
hasDuplicateNames.value
);
}
return true;
});
onMounted(() => {
setTimeout(() => {
@ -35,32 +134,146 @@ onMounted(() => {
}, 0);
});
const selectedOption = ref<'scratch' | 'import'>('scratch');
const proceedFromSelect = async () => {
if (!selectedOption.value || !dataTableName.value) return;
if (selectedOption.value === 'scratch') {
await onSubmit();
} else if (selectedOption.value === 'import') {
if (!selectedFile.value) return;
await uploadFile();
}
};
const onColumnNameChange = (index: number) => {
const column = csvColumns.value[index];
if (!column) return;
column.error = validateColumnName(column.name);
const isDuplicate = csvColumns.value.some(
(col, idx) => idx !== index && col.name.toLowerCase() === column.name.toLowerCase(),
);
if (isDuplicate && !column.error) {
column.error = i18n.baseText('dataTable.import.duplicateColumnName');
}
csvColumns.value.forEach((col, idx) => {
if (idx !== index) {
const otherIsDuplicate = csvColumns.value.some(
(c, i) => i !== idx && c.name.toLowerCase() === col.name.toLowerCase(),
);
const validationError = validateColumnName(col.name);
if (otherIsDuplicate && !validationError) {
col.error = i18n.baseText('dataTable.import.duplicateColumnName');
} else {
col.error = validationError;
}
}
});
};
const reset = (clearTableName = false) => {
if (clearTableName) {
dataTableName.value = '';
}
selectedFile.value = null;
uploadedFileId.value = null;
uploadedFileName.value = '';
csvColumns.value = [];
csvRowCount.value = 0;
csvColumnCount.value = 0;
selectedOption.value = 'scratch';
creationMode.value = 'select';
};
const handleFileChange = (uploadFile: UploadFile) => {
if (uploadFile.raw) {
selectedFile.value = uploadFile.raw;
}
};
const uploadFile = async () => {
if (!selectedFile.value) return;
isUploading.value = true;
creationMode.value = 'import';
try {
const uploadResponse = await dataTableStore.uploadCsvFile(selectedFile.value, hasHeaders.value);
uploadedFileId.value = uploadResponse.id;
uploadedFileName.value = uploadResponse.originalName;
csvRowCount.value = uploadResponse.rowCount;
csvColumnCount.value = uploadResponse.columnCount;
csvColumns.value = uploadResponse.columns.map((col) => {
const compatibleTypes = (col.compatibleTypes || [col.type]).filter(isColumnType);
const sanitizedName = col.name.replace(/\s+/g, '_');
const colType = isColumnType(col.type) ? col.type : 'string';
return {
name: sanitizedName,
type: colType,
compatibleTypes,
typeOptions: getColumnTypeOptions(compatibleTypes),
error: validateColumnName(sanitizedName),
};
});
if (!dataTableName.value) {
const fileName = selectedFile.value.name.replace(/\.csv$/i, '');
dataTableName.value = fileName;
}
} catch (error) {
toast.showError(error, i18n.baseText('dataTable.upload.error'));
reset();
} finally {
isUploading.value = false;
}
};
const onSubmit = async () => {
try {
const newDataTable = await dataTableStore.createDataTable(
dataTableName.value,
route.params.projectId as string,
);
telemetry.track('User created data table', {
data_table_id: newDataTable.id,
data_table_project_id: newDataTable.project?.id,
});
dataTableName.value = '';
uiStore.closeModal(props.modalName);
void router.push({
name: DATA_TABLE_DETAILS,
params: {
id: newDataTable.id,
},
});
let newDataTable;
if (selectedOption.value === 'scratch') {
newDataTable = await dataTableStore.createDataTable(
dataTableName.value,
route.params.projectId as string,
);
} else if (creationMode.value === 'import' && uploadedFileId.value) {
newDataTable = await dataTableStore.createDataTable(
dataTableName.value,
route.params.projectId as string,
csvColumns.value.map((col) => ({ name: col.name, type: col.type })),
uploadedFileId.value,
hasHeaders.value,
);
}
if (newDataTable) {
telemetry.track('User created data table', {
data_table_id: newDataTable.id,
data_table_project_id: newDataTable.project?.id,
creation_mode: selectedOption.value,
});
reset(true);
uiStore.closeModal(props.modalName);
void router.push({
name: DATA_TABLE_DETAILS,
params: {
id: newDataTable.id,
},
});
}
} catch (error) {
toast.showError(error, i18n.baseText('dataTable.add.error'));
}
};
const onCancel = () => {
uiStore.closeModal(props.modalName);
redirectToDataTables();
const goBack = () => {
creationMode.value = 'select';
};
const redirectToDataTables = () => {
@ -69,41 +282,174 @@ const redirectToDataTables = () => {
</script>
<template>
<Modal :name="props.modalName" :center="true" width="540px" :before-close="redirectToDataTables">
<Modal
:name="props.modalName"
:center="true"
:width="creationMode === 'import' ? '700px' : '540px'"
:min-height="creationMode === 'import' ? '600px' : undefined"
:before-close="redirectToDataTables"
>
<template #header>
<div :class="$style.header">
<h2>{{ i18n.baseText('dataTable.add.title') }}</h2>
<h2>{{ modalTitle }}</h2>
</div>
</template>
<template #content>
<div :class="$style.content">
<div v-if="creationMode === 'select'" :class="$style.selectionContent">
<N8nInputLabel
:label="i18n.baseText('dataTable.add.input.name.label')"
:required="true"
input-name="dataTableName"
input-name="dataTableNameSelect"
>
<N8nInput
ref="inputRef"
v-model="dataTableName"
type="text"
:placeholder="i18n.baseText('dataTable.add.input.name.placeholder')"
data-test-id="data-table-name-input"
name="dataTableName"
@keydown.enter="onSubmit"
data-test-id="data-table-name-input-select"
name="dataTableNameSelect"
/>
</N8nInputLabel>
<ElRadioGroup v-model="selectedOption" :class="$style.radioGroup">
<ElRadio label="scratch" data-test-id="create-from-scratch-option">
{{ i18n.baseText('dataTable.add.fromScratch') }}
</ElRadio>
<ElRadio label="import" data-test-id="import-csv-option">
{{ i18n.baseText('dataTable.add.importCsv') }}
</ElRadio>
</ElRadioGroup>
<div v-if="selectedOption === 'import'" :class="$style.uploadSection">
<ElUpload
:class="$style.uploadDemo"
drag
:auto-upload="false"
:show-file-list="false"
accept=".csv"
:on-change="handleFileChange"
@mouseenter="isUploadHovered = true"
@mouseleave="isUploadHovered = false"
>
<N8nIcon
icon="file"
:size="24"
:color="isUploadHovered ? 'text-dark' : 'text-light'"
:class="$style.uploadIcon"
/>
<N8nText v-if="selectedFile" :color="isUploadHovered ? 'text-dark' : 'text-light'">
{{ selectedFile?.name }}
</N8nText>
<N8nText v-else size="medium" :color="isUploadHovered ? 'text-dark' : 'text-light'">
{{ i18n.baseText('dataTable.upload.dropOrClick') }}
</N8nText>
</ElUpload>
<N8nCheckbox
v-model="hasHeaders"
:label="i18n.baseText('dataTable.upload.hasHeaders')"
data-test-id="has-headers-checkbox"
/>
</div>
</div>
<div v-else-if="creationMode === 'import'" :class="$style.content">
<div v-if="isUploading" :class="$style.uploadingMessage">
{{ i18n.baseText('dataTable.upload.uploading') }}
</div>
<div v-else-if="!uploadedFileId" :class="$style.uploadingMessage">
{{ i18n.baseText('dataTable.upload.selectFile') }}
</div>
<div v-else-if="uploadedFileId && csvColumns.length > 0" :class="$style.importContent">
<div :class="$style.successNotice">
{{
i18n.baseText('dataTable.upload.success', {
adjustToNumber: csvRowCount,
interpolate: {
fileName: uploadedFileName,
columnCount: csvColumnCount,
rowCount: csvRowCount,
},
})
}}
</div>
<div :class="$style.columnHeaders">
<div :class="$style.columnHeaderLabel">
{{ i18n.baseText('dataTable.import.columnName') }}
</div>
<div :class="$style.columnHeaderLabel">
{{ i18n.baseText('dataTable.import.columnType') }}
</div>
</div>
<div :class="$style.columnsContainer">
<div v-for="(column, index) in csvColumns" :key="index" :class="$style.columnItem">
<div :class="$style.columnInputWrapper">
<N8nInput
v-model="column.name"
:placeholder="i18n.baseText('dataTable.import.columnNamePlaceholder')"
:data-test-id="`column-name-${index}`"
:class="{ [$style.inputError]: column.error }"
@update:model-value="onColumnNameChange(index)"
/>
<div v-if="column.error" :class="$style.columnErrorMessage">
{{ column.error }}
</div>
</div>
<div :class="$style.columnTypeWrapper">
<N8nSelect
v-model="column.type"
:disabled="column.typeOptions.length === 1"
:data-test-id="`column-type-${index}`"
>
<N8nOption
v-for="option in column.typeOptions"
:key="option.value"
:value="option.value"
:label="option.label"
/>
</N8nSelect>
</div>
</div>
</div>
</div>
</div>
</template>
<template #footer>
<div :class="$style.footer">
<N8nButton
v-if="creationMode === 'select'"
type="secondary"
size="large"
:label="i18n.baseText('generic.cancel')"
data-test-id="cancel-add-data-table-button"
@click="onCancel"
data-test-id="cancel-select-button"
@click="redirectToDataTables"
/>
<N8nButton
:disabled="!dataTableName"
v-if="creationMode === 'select'"
size="large"
:disabled="
!dataTableName || !selectedOption || (selectedOption === 'import' && !selectedFile)
"
:label="i18n.baseText('generic.create')"
data-test-id="proceed-from-select-button"
@click="proceedFromSelect"
/>
<N8nButton
v-if="creationMode === 'import'"
type="secondary"
size="large"
:label="i18n.baseText('generic.back')"
data-test-id="back-button"
@click="goBack"
/>
<N8nButton
v-if="creationMode === 'import'"
size="large"
:disabled="isCreateDisabled"
:label="i18n.baseText('generic.create')"
data-test-id="confirm-add-data-table-button"
@click="onSubmit"
@ -123,10 +469,181 @@ const redirectToDataTables = () => {
flex-direction: column;
}
.selectionContent {
display: flex;
flex-direction: column;
gap: var(--spacing--md);
}
.radioGroup {
display: flex;
flex-direction: column;
gap: var(--spacing--xs);
:global(.el-radio) {
height: auto;
margin-right: 0;
}
:global(.el-radio__input.is-checked .el-radio__inner) {
background-color: var(--color--primary);
border-color: var(--color--primary);
}
:global(.el-radio__inner) {
width: 16px;
height: 16px;
}
:global(.el-radio__input:hover .el-radio__inner) {
border-color: var(--color--foreground);
}
:global(.el-radio__input.is-checked:hover .el-radio__inner) {
border-color: var(--color--primary);
}
:global(.el-radio__label) {
font-size: var(--font-size--sm);
color: var(--color--text) !important;
padding-left: var(--spacing--xs);
}
:global(.el-radio.is-checked .el-radio__label) {
color: var(--color--text) !important;
}
}
.uploadSection {
display: flex;
flex-direction: column;
gap: var(--spacing--sm);
}
.uploadingMessage {
padding: var(--spacing--lg);
text-align: center;
color: var(--color--text--tint-1);
}
.importContent {
display: flex;
flex-direction: column;
gap: var(--spacing--md);
}
.successNotice {
padding: var(--spacing--sm) var(--spacing--md);
background-color: var(--color--success--tint-4);
border-radius: var(--radius);
color: var(--color--success--shade-1);
font-size: var(--font-size--sm);
line-height: var(--line-height--lg);
}
.columnHeaders {
display: grid;
grid-template-columns: 1fr 1fr;
gap: var(--spacing--md);
padding: 0 var(--spacing--2xs);
}
.columnHeaderLabel {
font-size: var(--font-size--sm);
font-weight: var(--font-weight--regular);
color: var(--color--text--tint-1);
}
.columnsContainer {
display: flex;
flex-direction: column;
gap: var(--spacing--md);
max-height: 400px;
overflow-y: auto;
padding: var(--spacing--2xs);
}
.columnItem {
display: grid;
grid-template-columns: 1fr 1fr;
gap: var(--spacing--md);
align-items: start;
}
.columnInputWrapper {
display: flex;
flex-direction: column;
gap: var(--spacing--4xs);
}
.columnTypeWrapper {
display: flex;
align-items: center;
}
.inputError {
border-color: var(--color--danger) !important;
&:focus {
box-shadow: 0 0 0 2px var(--color--danger--tint-3) !important;
}
}
.columnErrorMessage {
font-size: var(--font-size--3xs);
color: var(--color--danger);
line-height: var(--line-height--sm);
}
.footer {
display: flex;
gap: var(--spacing--2xs);
justify-content: flex-end;
margin-top: var(--spacing--lg);
}
.fileSelectedContainer {
display: flex;
flex-direction: column;
gap: var(--spacing--lg);
padding: var(--spacing--lg) 0;
}
.uploadDemo {
width: 100%;
:global(.el-upload) {
width: 100%;
border-radius: var(--radius--lg);
}
:global(.el-upload-dragger) {
width: 100%;
padding: var(--spacing--2xl) var(--spacing--lg);
border: 1px solid var(--color--foreground);
background-color: var(--color--background-base);
border-radius: var(--radius--lg);
transition: all 0.2s ease;
display: flex;
flex-direction: column;
align-items: center;
justify-content: center;
&:hover {
background-color: var(--color--background);
}
}
:global(input[type='file']) {
display: none !important;
}
}
.uploadIcon {
margin-bottom: var(--spacing--sm);
}
.fileName {
font-weight: var(--font-weight--regular);
}
</style>

View File

@ -39,6 +39,8 @@ export const createDataTableApi = async (
name: string,
projectId: string,
columns?: DataTableColumnCreatePayload[],
fileId?: string,
hasHeaders: boolean = true,
) => {
return await makeRestApiRequest<DataTable>(
context,
@ -47,6 +49,8 @@ export const createDataTableApi = async (
{
name,
columns: columns ?? [],
hasHeaders,
...(fileId ? { fileId } : {}),
},
);
};
@ -218,3 +222,21 @@ export const fetchDataTableGlobalLimitInBytes = async (context: IRestApiContext)
'/data-tables-global/limits',
);
};
export const uploadCsvFileApi = async (
context: IRestApiContext,
file: File,
hasHeaders: boolean = true,
) => {
const formData = new FormData();
formData.append('file', file);
formData.append('hasHeaders', String(hasHeaders));
return await makeRestApiRequest<{
originalName: string;
id: string;
rowCount: number;
columnCount: number;
columns: Array<{ name: string; type: string; compatibleTypes: string[] }>;
}>(context, 'POST', '/data-tables/uploads', formData);
};

View File

@ -85,12 +85,44 @@ describe('dataTable.store', () => {
rootStore.restApiContext,
'New Table',
'p1',
undefined,
undefined,
true,
);
expect(dataTableStore.dataTables[0]).toEqual(mockTable);
expect(dataTableStore.totalCount).toBe(1);
expect(result).toBe(mockTable);
});
it('should create data table with CSV import parameters', async () => {
const mockTable = createTable({ id: 'dt-1', name: 'Imported Table' });
const columns = [
{ name: 'col1', type: 'string' as const },
{ name: 'col2', type: 'number' as const },
];
const fileId = 'file123';
vi.spyOn(dataTableApi, 'createDataTableApi').mockResolvedValue(mockTable);
const result = await dataTableStore.createDataTable(
'Imported Table',
'p1',
columns,
fileId,
false,
);
expect(dataTableApi.createDataTableApi).toHaveBeenCalledWith(
rootStore.restApiContext,
'Imported Table',
'p1',
columns,
fileId,
false,
);
expect(dataTableStore.dataTables[0]).toEqual(mockTable);
expect(result).toBe(mockTable);
});
it('should fetch and attach project if missing', async () => {
const mockTable = createTable({ id: 'dt-1', name: 'Table' });
const mockProject = {

View File

@ -15,6 +15,7 @@ import {
updateDataTableRowsApi,
deleteDataTableRowsApi,
fetchDataTableGlobalLimitInBytes,
uploadCsvFileApi,
} from '@/features/core/dataTable/dataTable.api';
import type {
DataTable,
@ -69,8 +70,21 @@ export const useDataTableStore = defineStore(DATA_TABLE_STORE, () => {
totalCount.value = response.count;
};
const createDataTable = async (name: string, projectId: string) => {
const newTable = await createDataTableApi(rootStore.restApiContext, name, projectId);
const createDataTable = async (
name: string,
projectId: string,
columns?: DataTableColumnCreatePayload[],
fileId?: string,
hasHeaders: boolean = true,
) => {
const newTable = await createDataTableApi(
rootStore.restApiContext,
name,
projectId,
columns,
fileId,
hasHeaders,
);
if (!newTable.project && projectId) {
const project = await projectStore.fetchProject(projectId);
if (project) {
@ -82,6 +96,10 @@ export const useDataTableStore = defineStore(DATA_TABLE_STORE, () => {
return newTable;
};
const uploadCsvFile = async (file: File, hasHeaders: boolean = true) => {
return await uploadCsvFileApi(rootStore.restApiContext, file, hasHeaders);
};
const deleteDataTable = async (dataTableId: string, projectId: string) => {
const deleted = await deleteDataTableApi(rootStore.restApiContext, dataTableId, projectId);
if (deleted) {
@ -265,6 +283,7 @@ export const useDataTableStore = defineStore(DATA_TABLE_STORE, () => {
dataTableSizes,
maxSizeMB,
createDataTable,
uploadCsvFile,
deleteDataTable,
updateDataTable,
fetchDataTableDetails,

View File

@ -6,7 +6,8 @@ export class DataTableComposer {
async createNewDataTable(name: string) {
const nameInput = this.n8n.dataTable.getNewDataTableNameInput();
await nameInput.fill(name);
await this.n8n.dataTable.getNewDataTableConfirmButton().click();
await this.n8n.dataTable.getFromScratchOption().click();
await this.n8n.dataTable.getProceedFromSelectButton().click();
}
/**

View File

@ -25,7 +25,19 @@ export class DataTableView extends BasePage {
}
getNewDataTableNameInput() {
return this.page.getByTestId('data-table-name-input');
return this.page.getByTestId('data-table-name-input-select');
}
getFromScratchOption() {
return this.page.getByTestId('create-from-scratch-option');
}
getImportCsvOption() {
return this.page.getByTestId('import-csv-option');
}
getProceedFromSelectButton() {
return this.page.getByTestId('proceed-from-select-button');
}
getNewDataTableConfirmButton() {