mirror of
https://github.com/n8n-io/n8n.git
synced 2025-11-20 17:46:34 +00:00
feat(core): Allow creating data tables from csv files (#21051)
This commit is contained in:
parent
0a355ccadb
commit
2830665f7a
@ -7,4 +7,6 @@ import { dataTableNameSchema } from '../../schemas/data-table.schema';
|
||||
export class CreateDataTableDto extends Z.class({
|
||||
name: dataTableNameSchema,
|
||||
columns: z.array(CreateDataTableColumnDto),
|
||||
fileId: z.string().optional(),
|
||||
hasHeaders: z.boolean().optional(),
|
||||
}) {}
|
||||
|
||||
@ -1,3 +1,6 @@
|
||||
import { tmpdir } from 'node:os';
|
||||
import path from 'node:path';
|
||||
|
||||
import { Config, Env } from '../decorators';
|
||||
|
||||
@Config
|
||||
@ -20,4 +23,39 @@ export class DataTableConfig {
|
||||
*/
|
||||
@Env('N8N_DATA_TABLES_SIZE_CHECK_CACHE_DURATION_MS')
|
||||
sizeCheckCacheDuration: number = 60 * 1000;
|
||||
|
||||
/**
|
||||
* The maximum allowed file size (in bytes) for CSV uploads to data tables.
|
||||
* If set, this is the hard limit for file uploads.
|
||||
* If not set, the upload limit will be the remaining available storage space.
|
||||
*/
|
||||
@Env('N8N_DATA_TABLES_UPLOAD_MAX_FILE_SIZE_BYTES')
|
||||
uploadMaxFileSize?: number;
|
||||
|
||||
/**
|
||||
* The interval in milliseconds at which orphaned uploaded files are cleaned up.
|
||||
* Defaults to 60 seconds if not explicitly set via environment variable.
|
||||
*/
|
||||
@Env('N8N_DATA_TABLES_CLEANUP_INTERVAL_MS')
|
||||
cleanupIntervalMs: number = 60 * 1000;
|
||||
|
||||
/**
|
||||
* The maximum age in milliseconds for uploaded files before they are considered orphaned and deleted.
|
||||
* Files older than this threshold are removed during cleanup.
|
||||
* Defaults to 2 minutes if not explicitly set via environment variable.
|
||||
*/
|
||||
@Env('N8N_DATA_TABLES_FILE_MAX_AGE_MS')
|
||||
fileMaxAgeMs: number = 2 * 60 * 1000;
|
||||
|
||||
/**
|
||||
* The directory path where uploaded CSV files are temporarily stored before being imported.
|
||||
* Files in this directory are automatically cleaned up after a configurable period (fileMaxAgeMs).
|
||||
* Computed as: <system-tmp-dir>/n8nDataTableUploads
|
||||
* Example: /tmp/n8nDataTableUploads
|
||||
*/
|
||||
readonly uploadDir: string;
|
||||
|
||||
constructor() {
|
||||
this.uploadDir = path.join(tmpdir(), 'n8nDataTableUploads');
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
import path from 'node:path';
|
||||
|
||||
import { Config, Env } from '../decorators';
|
||||
import { getN8nFolder } from '../utils/utils';
|
||||
|
||||
@Config
|
||||
export class InstanceSettingsConfig {
|
||||
@ -29,9 +30,7 @@ export class InstanceSettingsConfig {
|
||||
readonly n8nFolder: string;
|
||||
|
||||
constructor() {
|
||||
const homeVarName = process.platform === 'win32' ? 'USERPROFILE' : 'HOME';
|
||||
this.userHome = process.env.N8N_USER_FOLDER ?? process.env[homeVarName] ?? process.cwd();
|
||||
|
||||
this.n8nFolder = path.join(this.userHome, '.n8n');
|
||||
this.n8nFolder = getN8nFolder();
|
||||
this.userHome = path.dirname(this.n8nFolder);
|
||||
}
|
||||
}
|
||||
|
||||
11
packages/@n8n/config/src/utils/utils.ts
Normal file
11
packages/@n8n/config/src/utils/utils.ts
Normal file
@ -0,0 +1,11 @@
|
||||
import path from 'node:path';
|
||||
|
||||
/**
|
||||
* Computes the n8n folder path based on environment variables.
|
||||
* This is used by various configs that need to know the n8n installation directory.
|
||||
*/
|
||||
export function getN8nFolder(): string {
|
||||
const homeVarName = process.platform === 'win32' ? 'USERPROFILE' : 'HOME';
|
||||
const userHome = process.env.N8N_USER_FOLDER ?? process.env[homeVarName] ?? process.cwd();
|
||||
return path.join(userHome, '.n8n');
|
||||
}
|
||||
@ -1,6 +1,8 @@
|
||||
import { Container } from '@n8n/di';
|
||||
import fs from 'fs';
|
||||
import { mock } from 'jest-mock-extended';
|
||||
import { tmpdir } from 'node:os';
|
||||
import path from 'node:path';
|
||||
|
||||
import type { UserManagementConfig } from '../src/configs/user-management.config';
|
||||
import { GlobalConfig } from '../src/index';
|
||||
@ -55,6 +57,9 @@ describe('GlobalConfig', () => {
|
||||
dataTable: {
|
||||
maxSize: 50 * 1024 * 1024,
|
||||
sizeCheckCacheDuration: 60000,
|
||||
cleanupIntervalMs: 60 * 1000,
|
||||
fileMaxAgeMs: 2 * 60 * 1000,
|
||||
uploadDir: path.join(tmpdir(), 'n8nDataTableUploads'),
|
||||
},
|
||||
database: {
|
||||
logging: {
|
||||
|
||||
@ -49,6 +49,12 @@ export const rawBodyReader: RequestHandler = async (req, _res, next) => {
|
||||
};
|
||||
|
||||
export const parseBody = async (req: Request) => {
|
||||
// Skip multipart requests (e.g., file uploads) - these need specialized parsing by multer.
|
||||
// Reading the body stream here would consume it, making it unavailable for multer processing.
|
||||
if (req.contentType?.startsWith('multipart/')) {
|
||||
return;
|
||||
}
|
||||
|
||||
await req.readRawBody();
|
||||
const { rawBody, contentType, encoding } = req;
|
||||
if (rawBody?.length) {
|
||||
|
||||
@ -0,0 +1,284 @@
|
||||
import type { GlobalConfig } from '@n8n/config';
|
||||
import { promises as fs } from 'fs';
|
||||
import path from 'path';
|
||||
|
||||
import { DataTableFileCleanupService } from '../data-table-file-cleanup.service';
|
||||
|
||||
jest.mock('fs', () => ({
|
||||
promises: {
|
||||
unlink: jest.fn(),
|
||||
readdir: jest.fn(),
|
||||
stat: jest.fn(),
|
||||
},
|
||||
}));
|
||||
|
||||
describe('DataTableFileCleanupService', () => {
|
||||
const uploadDir = '/mock/n8n/dataTableUploads';
|
||||
|
||||
const globalConfig = {
|
||||
dataTable: {
|
||||
cleanupIntervalMs: 60 * 1000,
|
||||
fileMaxAgeMs: 2 * 60 * 1000,
|
||||
uploadDir,
|
||||
},
|
||||
} as GlobalConfig;
|
||||
|
||||
const service = new DataTableFileCleanupService(globalConfig);
|
||||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
jest.spyOn(console, 'error').mockImplementation(() => {});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
jest.restoreAllMocks();
|
||||
});
|
||||
|
||||
describe('deleteFile', () => {
|
||||
it('should delete a file successfully', async () => {
|
||||
const fileId = 'test-file-123';
|
||||
const expectedPath = path.join(uploadDir, fileId);
|
||||
|
||||
(fs.unlink as jest.Mock).mockResolvedValue(undefined);
|
||||
|
||||
await service.deleteFile(fileId);
|
||||
|
||||
expect(fs.unlink).toHaveBeenCalledWith(expectedPath);
|
||||
expect(fs.unlink).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('should ignore ENOENT error when file does not exist', async () => {
|
||||
const fileId = 'non-existent-file';
|
||||
const error = new Error('ENOENT: no such file or directory') as NodeJS.ErrnoException;
|
||||
error.code = 'ENOENT';
|
||||
|
||||
(fs.unlink as jest.Mock).mockRejectedValue(error);
|
||||
|
||||
await expect(service.deleteFile(fileId)).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it('should throw error for non-ENOENT errors', async () => {
|
||||
const fileId = 'test-file';
|
||||
const error = new Error('Permission denied') as NodeJS.ErrnoException;
|
||||
error.code = 'EPERM';
|
||||
|
||||
(fs.unlink as jest.Mock).mockRejectedValue(error);
|
||||
|
||||
await expect(service.deleteFile(fileId)).rejects.toThrow('Permission denied');
|
||||
});
|
||||
});
|
||||
|
||||
describe('start and shutdown', () => {
|
||||
it('should start cleanup interval', async () => {
|
||||
jest.useFakeTimers();
|
||||
|
||||
await service.start();
|
||||
|
||||
expect(service['cleanupInterval']).toBeDefined();
|
||||
|
||||
jest.useRealTimers();
|
||||
});
|
||||
|
||||
it('should clear interval on shutdown', async () => {
|
||||
jest.useFakeTimers();
|
||||
|
||||
await service.start();
|
||||
expect(service['cleanupInterval']).toBeDefined();
|
||||
|
||||
await service.shutdown();
|
||||
|
||||
expect(service['cleanupInterval']).toBeUndefined();
|
||||
|
||||
jest.useRealTimers();
|
||||
});
|
||||
|
||||
it('should not error on shutdown if interval was never started', async () => {
|
||||
await expect(service.shutdown()).resolves.toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('cleanupOrphanedFiles', () => {
|
||||
const flushPromises = async () => await new Promise(jest.requireActual('timers').setImmediate);
|
||||
|
||||
beforeEach(() => {
|
||||
jest.useFakeTimers();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
jest.useRealTimers();
|
||||
});
|
||||
|
||||
it('should delete files older than 2 minutes', async () => {
|
||||
const now = Date.now();
|
||||
const oldFile1 = 'old-file-1.csv';
|
||||
const oldFile2 = 'old-file-2.csv';
|
||||
|
||||
(fs.readdir as jest.Mock).mockResolvedValue([oldFile1, oldFile2]);
|
||||
(fs.stat as jest.Mock).mockResolvedValue({
|
||||
mtimeMs: now - 3 * 60 * 1000, // 3 minutes ago
|
||||
});
|
||||
(fs.unlink as jest.Mock).mockResolvedValue(undefined);
|
||||
|
||||
await service.start();
|
||||
|
||||
// Trigger cleanup and let promises resolve
|
||||
jest.advanceTimersByTime(60 * 1000);
|
||||
await flushPromises();
|
||||
|
||||
expect(fs.readdir).toHaveBeenCalledWith(uploadDir);
|
||||
expect(fs.stat).toHaveBeenCalledTimes(2);
|
||||
expect(fs.unlink).toHaveBeenCalledWith(path.join(uploadDir, oldFile1));
|
||||
expect(fs.unlink).toHaveBeenCalledWith(path.join(uploadDir, oldFile2));
|
||||
});
|
||||
|
||||
it('should not delete files newer than 2 minutes', async () => {
|
||||
const now = Date.now();
|
||||
const newFile = 'new-file.csv';
|
||||
|
||||
(fs.readdir as jest.Mock).mockResolvedValue([newFile]);
|
||||
(fs.stat as jest.Mock).mockResolvedValue({
|
||||
mtimeMs: now - 1 * 60 * 1000, // 1 minute ago
|
||||
});
|
||||
|
||||
await service.start();
|
||||
jest.advanceTimersByTime(60 * 1000); // Trigger cleanup
|
||||
await flushPromises();
|
||||
|
||||
expect(fs.readdir).toHaveBeenCalled();
|
||||
expect(fs.stat).toHaveBeenCalled();
|
||||
expect(fs.unlink).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should handle mixed old and new files', async () => {
|
||||
const now = Date.now();
|
||||
const oldFile = 'old-file.csv';
|
||||
const newFile = 'new-file.csv';
|
||||
|
||||
(fs.readdir as jest.Mock).mockResolvedValue([oldFile, newFile]);
|
||||
(fs.stat as jest.Mock)
|
||||
.mockResolvedValueOnce({
|
||||
mtimeMs: now - 3 * 60 * 1000, // 3 minutes ago (old)
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
mtimeMs: now - 1 * 60 * 1000, // 1 minute ago (new)
|
||||
});
|
||||
(fs.unlink as jest.Mock).mockResolvedValue(undefined);
|
||||
|
||||
await service.start();
|
||||
jest.advanceTimersByTime(60 * 1000); // Trigger cleanup
|
||||
await flushPromises();
|
||||
|
||||
expect(fs.unlink).toHaveBeenCalledTimes(1);
|
||||
expect(fs.unlink).toHaveBeenCalledWith(path.join(uploadDir, oldFile));
|
||||
expect(fs.unlink).not.toHaveBeenCalledWith(path.join(uploadDir, newFile));
|
||||
});
|
||||
|
||||
it('should handle empty upload directory', async () => {
|
||||
(fs.readdir as jest.Mock).mockResolvedValue([]);
|
||||
|
||||
await service.start();
|
||||
jest.advanceTimersByTime(60 * 1000); // Trigger cleanup
|
||||
await flushPromises();
|
||||
|
||||
expect(fs.readdir).toHaveBeenCalled();
|
||||
expect(fs.stat).not.toHaveBeenCalled();
|
||||
expect(fs.unlink).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should ignore ENOENT error if upload directory does not exist', async () => {
|
||||
const error = new Error('ENOENT: no such file or directory') as NodeJS.ErrnoException;
|
||||
error.code = 'ENOENT';
|
||||
(fs.readdir as jest.Mock).mockRejectedValue(error);
|
||||
|
||||
await service.start();
|
||||
jest.advanceTimersByTime(60 * 1000); // Trigger cleanup
|
||||
await flushPromises();
|
||||
|
||||
expect(fs.readdir).toHaveBeenCalled();
|
||||
expect(console.error).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should log error for non-ENOENT readdir errors', async () => {
|
||||
const error = new Error('Permission denied');
|
||||
(fs.readdir as jest.Mock).mockRejectedValue(error);
|
||||
|
||||
await service.start();
|
||||
jest.advanceTimersByTime(60 * 1000); // Trigger cleanup
|
||||
await flushPromises();
|
||||
|
||||
expect(console.error).toHaveBeenCalledWith('Error cleaning up orphaned CSV files:', error);
|
||||
});
|
||||
|
||||
it('should continue cleanup if individual file stat fails', async () => {
|
||||
const file1 = 'file1.csv';
|
||||
const file2 = 'file2.csv';
|
||||
|
||||
(fs.readdir as jest.Mock).mockResolvedValue([file1, file2]);
|
||||
(fs.stat as jest.Mock)
|
||||
.mockRejectedValueOnce(new Error('Stat failed for file1'))
|
||||
.mockResolvedValueOnce({
|
||||
mtimeMs: Date.now() - 3 * 60 * 1000, // file2 is old
|
||||
});
|
||||
(fs.unlink as jest.Mock).mockResolvedValue(undefined);
|
||||
|
||||
await service.start();
|
||||
jest.advanceTimersByTime(60 * 1000); // Trigger cleanup
|
||||
await flushPromises();
|
||||
|
||||
// Should still delete file2 even though file1 failed
|
||||
expect(fs.unlink).toHaveBeenCalledWith(path.join(uploadDir, file2));
|
||||
expect(fs.unlink).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('should continue cleanup if individual file unlink fails', async () => {
|
||||
const now = Date.now();
|
||||
const file1 = 'file1.csv';
|
||||
const file2 = 'file2.csv';
|
||||
|
||||
(fs.readdir as jest.Mock).mockResolvedValue([file1, file2]);
|
||||
(fs.stat as jest.Mock).mockResolvedValue({
|
||||
mtimeMs: now - 3 * 60 * 1000, // Both files are old
|
||||
});
|
||||
(fs.unlink as jest.Mock)
|
||||
.mockRejectedValueOnce(new Error('Unlink failed for file1'))
|
||||
.mockResolvedValueOnce(undefined);
|
||||
|
||||
await service.start();
|
||||
jest.advanceTimersByTime(60 * 1000); // Trigger cleanup
|
||||
await flushPromises();
|
||||
|
||||
// Should attempt to delete both files
|
||||
expect(fs.unlink).toHaveBeenCalledWith(path.join(uploadDir, file1));
|
||||
expect(fs.unlink).toHaveBeenCalledWith(path.join(uploadDir, file2));
|
||||
expect(fs.unlink).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('should run cleanup every 60 seconds', async () => {
|
||||
const now = Date.now();
|
||||
(fs.readdir as jest.Mock).mockResolvedValue(['old-file.csv']);
|
||||
(fs.stat as jest.Mock).mockResolvedValue({
|
||||
mtimeMs: now - 3 * 60 * 1000,
|
||||
});
|
||||
(fs.unlink as jest.Mock).mockResolvedValue(undefined);
|
||||
|
||||
await service.start();
|
||||
|
||||
// First cleanup
|
||||
jest.advanceTimersByTime(60 * 1000);
|
||||
await flushPromises();
|
||||
expect(fs.readdir).toHaveBeenCalledTimes(1);
|
||||
|
||||
// Second cleanup
|
||||
jest.advanceTimersByTime(60 * 1000);
|
||||
await flushPromises();
|
||||
expect(fs.readdir).toHaveBeenCalledTimes(2);
|
||||
|
||||
// Third cleanup
|
||||
jest.advanceTimersByTime(60 * 1000);
|
||||
await flushPromises();
|
||||
expect(fs.readdir).toHaveBeenCalledTimes(3);
|
||||
|
||||
await service.shutdown();
|
||||
});
|
||||
});
|
||||
});
|
||||
@ -0,0 +1,425 @@
|
||||
import { GlobalConfig } from '@n8n/config';
|
||||
import type { User } from '@n8n/db';
|
||||
import { Container } from '@n8n/di';
|
||||
|
||||
import { createOwner, createMember } from '@test-integration/db/users';
|
||||
import type { SuperAgentTest } from '@test-integration/types';
|
||||
import * as utils from '@test-integration/utils';
|
||||
|
||||
const testServer = utils.setupTestServer({
|
||||
endpointGroups: ['data-table'],
|
||||
modules: ['data-table'],
|
||||
});
|
||||
|
||||
let owner: User;
|
||||
let member: User;
|
||||
let authOwnerAgent: SuperAgentTest;
|
||||
let authMemberAgent: SuperAgentTest;
|
||||
|
||||
beforeAll(async () => {
|
||||
owner = await createOwner();
|
||||
member = await createMember();
|
||||
|
||||
authOwnerAgent = testServer.authAgentFor(owner);
|
||||
authMemberAgent = testServer.authAgentFor(member);
|
||||
});
|
||||
|
||||
describe('POST /data-tables/uploads', () => {
|
||||
describe('successful file uploads', () => {
|
||||
test('should upload a valid CSV file and return file metadata', async () => {
|
||||
const csvContent = 'name,email\nJohn Doe,john@example.com\nJane Smith,jane@example.com';
|
||||
|
||||
const response = await authOwnerAgent
|
||||
.post('/data-tables/uploads')
|
||||
.attach('file', Buffer.from(csvContent), 'test.csv')
|
||||
.expect(200);
|
||||
|
||||
expect(response.body.data).toHaveProperty('originalName', 'test.csv');
|
||||
expect(response.body.data).toHaveProperty('id');
|
||||
expect(response.body.data.id).toMatch(/^[a-zA-Z0-9_-]{10}$/); // nanoid with length 10
|
||||
expect(response.body.data).toHaveProperty('rowCount', 2);
|
||||
expect(response.body.data).toHaveProperty('columnCount', 2);
|
||||
expect(response.body.data).toHaveProperty('columns');
|
||||
expect(response.body.data.columns).toEqual([
|
||||
{ name: 'name', type: 'string', compatibleTypes: ['string'] },
|
||||
{ name: 'email', type: 'string', compatibleTypes: ['string'] },
|
||||
]);
|
||||
});
|
||||
|
||||
test('should accept CSV file from member user', async () => {
|
||||
const csvContent = 'col1,col2\nvalue1,value2';
|
||||
|
||||
const response = await authMemberAgent
|
||||
.post('/data-tables/uploads')
|
||||
.attach('file', Buffer.from(csvContent), 'data.csv')
|
||||
.expect(200);
|
||||
|
||||
expect(response.body.data).toHaveProperty('originalName', 'data.csv');
|
||||
expect(response.body.data).toHaveProperty('id');
|
||||
expect(response.body.data).toHaveProperty('rowCount', 1);
|
||||
expect(response.body.data).toHaveProperty('columnCount', 2);
|
||||
});
|
||||
|
||||
test('should handle CSV files with special characters in filename', async () => {
|
||||
const csvContent = 'a,b,c\n1,2,3';
|
||||
const filename = 'test-file_v1.2.csv';
|
||||
|
||||
const response = await authOwnerAgent
|
||||
.post('/data-tables/uploads')
|
||||
.attach('file', Buffer.from(csvContent), filename)
|
||||
.expect(200);
|
||||
|
||||
expect(response.body.data).toHaveProperty('originalName', filename);
|
||||
expect(response.body.data).toHaveProperty('id');
|
||||
expect(response.body.data).toHaveProperty('rowCount', 1);
|
||||
expect(response.body.data).toHaveProperty('columnCount', 3);
|
||||
expect(response.body.data.columns).toEqual([
|
||||
{ name: 'a', type: 'number', compatibleTypes: ['number', 'string'] },
|
||||
{ name: 'b', type: 'number', compatibleTypes: ['number', 'string'] },
|
||||
{ name: 'c', type: 'number', compatibleTypes: ['number', 'string'] },
|
||||
]);
|
||||
});
|
||||
|
||||
test('should accept files within size limit', async () => {
|
||||
const smallContent = 'col1,col2\nval1,val2\n';
|
||||
|
||||
const response = await authOwnerAgent
|
||||
.post('/data-tables/uploads')
|
||||
.attach('file', Buffer.from(smallContent), 'small.csv')
|
||||
.expect(200);
|
||||
|
||||
expect(response.body.data).toHaveProperty('originalName', 'small.csv');
|
||||
expect(response.body.data).toHaveProperty('id');
|
||||
});
|
||||
|
||||
test('should accept empty CSV file', async () => {
|
||||
const response = await authOwnerAgent
|
||||
.post('/data-tables/uploads')
|
||||
.attach('file', Buffer.from(''), 'empty.csv')
|
||||
.expect(200);
|
||||
|
||||
expect(response.body.data).toHaveProperty('originalName', 'empty.csv');
|
||||
expect(response.body.data).toHaveProperty('id');
|
||||
expect(response.body.data).toHaveProperty('rowCount', 0);
|
||||
expect(response.body.data).toHaveProperty('columnCount', 0);
|
||||
expect(response.body.data.columns).toEqual([]);
|
||||
});
|
||||
|
||||
test('should infer correct column types (string, number, boolean, date)', async () => {
|
||||
const csvContent =
|
||||
'name,age,isActive,createdDate\nJohn,30,true,2024-01-15\nJane,25,false,2024-02-20';
|
||||
|
||||
const response = await authOwnerAgent
|
||||
.post('/data-tables/uploads')
|
||||
.attach('file', Buffer.from(csvContent), 'types.csv')
|
||||
.expect(200);
|
||||
|
||||
expect(response.body.data).toHaveProperty('originalName', 'types.csv');
|
||||
expect(response.body.data).toHaveProperty('rowCount', 2);
|
||||
expect(response.body.data).toHaveProperty('columnCount', 4);
|
||||
expect(response.body.data.columns).toEqual([
|
||||
{ name: 'name', type: 'string', compatibleTypes: ['string'] },
|
||||
{ name: 'age', type: 'number', compatibleTypes: ['number', 'string'] },
|
||||
{ name: 'isActive', type: 'boolean', compatibleTypes: ['boolean', 'string'] },
|
||||
{ name: 'createdDate', type: 'date', compatibleTypes: ['date', 'string'] },
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('header handling', () => {
|
||||
test('should use first row as headers when hasHeaders=true (default)', async () => {
|
||||
const csvContent = 'FirstName,LastName,Age\nJohn,Doe,30\nJane,Smith,25';
|
||||
|
||||
const response = await authOwnerAgent
|
||||
.post('/data-tables/uploads')
|
||||
.field('hasHeaders', 'true')
|
||||
.attach('file', Buffer.from(csvContent), 'with-headers.csv')
|
||||
.expect(200);
|
||||
|
||||
expect(response.body.data).toHaveProperty('rowCount', 2);
|
||||
expect(response.body.data).toHaveProperty('columnCount', 3);
|
||||
expect(response.body.data.columns).toEqual([
|
||||
{ name: 'FirstName', type: 'string', compatibleTypes: ['string'] },
|
||||
{ name: 'LastName', type: 'string', compatibleTypes: ['string'] },
|
||||
{ name: 'Age', type: 'number', compatibleTypes: ['number', 'string'] },
|
||||
]);
|
||||
});
|
||||
|
||||
test('should generate column names when hasHeaders=false', async () => {
|
||||
const csvContent = 'John,Doe,30\nJane,Smith,25\nBob,Johnson,35';
|
||||
|
||||
const response = await authOwnerAgent
|
||||
.post('/data-tables/uploads')
|
||||
.field('hasHeaders', 'false')
|
||||
.attach('file', Buffer.from(csvContent), 'no-headers.csv')
|
||||
.expect(200);
|
||||
|
||||
expect(response.body.data).toHaveProperty('rowCount', 3);
|
||||
expect(response.body.data).toHaveProperty('columnCount', 3);
|
||||
expect(response.body.data.columns).toEqual([
|
||||
{ name: 'Column_1', type: 'string', compatibleTypes: ['string'] },
|
||||
{ name: 'Column_2', type: 'string', compatibleTypes: ['string'] },
|
||||
{ name: 'Column_3', type: 'number', compatibleTypes: ['number', 'string'] },
|
||||
]);
|
||||
});
|
||||
|
||||
test('should include all rows in count when hasHeaders=false', async () => {
|
||||
const csvContent = '100,200,300\n400,500,600';
|
||||
|
||||
const response = await authOwnerAgent
|
||||
.post('/data-tables/uploads')
|
||||
.field('hasHeaders', 'false')
|
||||
.attach('file', Buffer.from(csvContent), 'no-headers-numbers.csv')
|
||||
.expect(200);
|
||||
|
||||
expect(response.body.data).toHaveProperty('rowCount', 2); // Both rows counted as data
|
||||
expect(response.body.data).toHaveProperty('columnCount', 3);
|
||||
expect(response.body.data.columns).toEqual([
|
||||
{ name: 'Column_1', type: 'number', compatibleTypes: ['number', 'string'] },
|
||||
{ name: 'Column_2', type: 'number', compatibleTypes: ['number', 'string'] },
|
||||
{ name: 'Column_3', type: 'number', compatibleTypes: ['number', 'string'] },
|
||||
]);
|
||||
});
|
||||
|
||||
test('should default to hasHeaders=true when field not provided', async () => {
|
||||
const csvContent = 'col1,col2\nval1,val2';
|
||||
|
||||
const response = await authOwnerAgent
|
||||
.post('/data-tables/uploads')
|
||||
.attach('file', Buffer.from(csvContent), 'default.csv')
|
||||
.expect(200);
|
||||
|
||||
expect(response.body.data).toHaveProperty('rowCount', 1);
|
||||
expect(response.body.data.columns[0].name).toBe('col1');
|
||||
expect(response.body.data.columns[1].name).toBe('col2');
|
||||
});
|
||||
});
|
||||
|
||||
describe('authentication', () => {
|
||||
test('should reject unauthenticated requests', async () => {
|
||||
const csvContent = 'name,value\ntest,123';
|
||||
|
||||
await testServer.authlessAgent
|
||||
.post('/data-tables/uploads')
|
||||
.attach('file', Buffer.from(csvContent), 'test.csv')
|
||||
.expect(401);
|
||||
});
|
||||
});
|
||||
|
||||
describe('file validation', () => {
|
||||
test('should reject request without file field', async () => {
|
||||
await authOwnerAgent.post('/data-tables/uploads').send({}).expect(400);
|
||||
});
|
||||
|
||||
test('should reject request with wrong field name', async () => {
|
||||
const csvContent = 'a,b\n1,2';
|
||||
|
||||
await authOwnerAgent
|
||||
.post('/data-tables/uploads')
|
||||
.attach('wrongField', Buffer.from(csvContent), 'test.csv')
|
||||
.expect(400);
|
||||
});
|
||||
|
||||
test('should reject non-CSV files based on MIME type', async () => {
|
||||
const textContent = 'This is a plain text file';
|
||||
|
||||
const response = await authOwnerAgent
|
||||
.post('/data-tables/uploads')
|
||||
.attach('file', Buffer.from(textContent), 'test.txt');
|
||||
|
||||
expect(response.status).toBe(400);
|
||||
});
|
||||
|
||||
test('should reject JSON files', async () => {
|
||||
const jsonContent = JSON.stringify({ name: 'test', value: 123 });
|
||||
|
||||
const response = await authOwnerAgent
|
||||
.post('/data-tables/uploads')
|
||||
.attach('file', Buffer.from(jsonContent), 'test.json');
|
||||
|
||||
expect(response.status).toBe(400);
|
||||
});
|
||||
|
||||
test('should reject Excel files', async () => {
|
||||
const excelBuffer = Buffer.from('PK\x03\x04'); // Excel file signature
|
||||
|
||||
const response = await authOwnerAgent
|
||||
.post('/data-tables/uploads')
|
||||
.attach('file', excelBuffer, 'test.xlsx');
|
||||
|
||||
expect(response.status).toBe(400);
|
||||
});
|
||||
});
|
||||
|
||||
describe('file size validation with uploadMaxFileSize', () => {
|
||||
// Note: uploadMaxFileSize is set during multer initialization, so changing it
|
||||
// at runtime won't affect the behavior. These tests verify the behavior
|
||||
// when uploadMaxFileSize is configured via environment variable.
|
||||
|
||||
test('should accept small files when uploadMaxFileSize is set', async () => {
|
||||
const smallContent = 'name,value\ntest,123\n';
|
||||
|
||||
const response = await authOwnerAgent
|
||||
.post('/data-tables/uploads')
|
||||
.attach('file', Buffer.from(smallContent), 'small.csv')
|
||||
.expect(200);
|
||||
|
||||
expect(response.body.data).toHaveProperty('originalName', 'small.csv');
|
||||
});
|
||||
});
|
||||
|
||||
describe('file size validation without uploadMaxFileSize', () => {
|
||||
let globalConfig: GlobalConfig;
|
||||
let originalUploadMaxFileSize: number | undefined;
|
||||
let originalMaxSize: number;
|
||||
|
||||
beforeEach(() => {
|
||||
globalConfig = Container.get(GlobalConfig);
|
||||
originalUploadMaxFileSize = globalConfig.dataTable.uploadMaxFileSize;
|
||||
originalMaxSize = globalConfig.dataTable.maxSize;
|
||||
|
||||
// Unset uploadMaxFileSize to trigger remaining space check
|
||||
globalConfig.dataTable.uploadMaxFileSize = undefined;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
// Restore original values
|
||||
globalConfig.dataTable.uploadMaxFileSize = originalUploadMaxFileSize;
|
||||
globalConfig.dataTable.maxSize = originalMaxSize;
|
||||
});
|
||||
|
||||
test('should accept small files when there is remaining storage space', async () => {
|
||||
// Set max size to 50MB - in test environment, database is likely empty
|
||||
globalConfig.dataTable.maxSize = 50 * 1024 * 1024;
|
||||
|
||||
const smallContent = 'name,value\ntest,123\n';
|
||||
|
||||
const response = await authOwnerAgent
|
||||
.post('/data-tables/uploads')
|
||||
.attach('file', Buffer.from(smallContent), 'small.csv')
|
||||
.expect(200);
|
||||
|
||||
expect(response.body.data).toHaveProperty('originalName', 'small.csv');
|
||||
});
|
||||
|
||||
test('should reject files exceeding remaining storage space', async () => {
|
||||
// Set a very small max size (1KB) to ensure the upload fails
|
||||
globalConfig.dataTable.maxSize = 1024;
|
||||
|
||||
// Create content larger than 1KB
|
||||
const largeContent = 'a,b,c\n' + '1,2,3\n'.repeat(200);
|
||||
|
||||
const response = await authOwnerAgent
|
||||
.post('/data-tables/uploads')
|
||||
.attach('file', Buffer.from(largeContent), 'large.csv');
|
||||
|
||||
expect(response.status).toBe(400);
|
||||
expect(response.body).toHaveProperty('message');
|
||||
// Error message can be either about remaining space or storage limit exceeded
|
||||
const message = response.body.message as string;
|
||||
expect(
|
||||
message.includes('remaining storage space') || message.includes('Storage limit exceeded'),
|
||||
).toBe(true);
|
||||
// Verify the error message includes size units (B, KB, or MB)
|
||||
expect(message).toMatch(/\d+(B|KB|MB)/);
|
||||
});
|
||||
});
|
||||
|
||||
describe('edge cases', () => {
|
||||
test('should handle CSV with only headers', async () => {
|
||||
const csvContent = 'column1,column2,column3';
|
||||
|
||||
const response = await authOwnerAgent
|
||||
.post('/data-tables/uploads')
|
||||
.attach('file', Buffer.from(csvContent), 'headers-only.csv')
|
||||
.expect(200);
|
||||
|
||||
expect(response.body.data).toHaveProperty('originalName', 'headers-only.csv');
|
||||
expect(response.body.data).toHaveProperty('id');
|
||||
expect(response.body.data).toHaveProperty('rowCount', 0);
|
||||
expect(response.body.data).toHaveProperty('columnCount', 3);
|
||||
expect(response.body.data.columns).toEqual([
|
||||
{ name: 'column1', type: 'string', compatibleTypes: ['string'] },
|
||||
{ name: 'column2', type: 'string', compatibleTypes: ['string'] },
|
||||
{ name: 'column3', type: 'string', compatibleTypes: ['string'] },
|
||||
]);
|
||||
});
|
||||
|
||||
test('should handle CSV with Unicode content', async () => {
|
||||
const csvContent = 'Name,City\n日本,東京\nДмитрий,Москва';
|
||||
|
||||
const response = await authOwnerAgent
|
||||
.post('/data-tables/uploads')
|
||||
.attach('file', Buffer.from(csvContent, 'utf-8'), 'unicode.csv')
|
||||
.expect(200);
|
||||
|
||||
expect(response.body.data).toHaveProperty('originalName', 'unicode.csv');
|
||||
expect(response.body.data).toHaveProperty('id');
|
||||
});
|
||||
|
||||
test('should handle CSV with commas in quoted values', async () => {
|
||||
const csvContent = 'name,address\n"Doe, John","123 Main St, Apt 4"';
|
||||
|
||||
const response = await authOwnerAgent
|
||||
.post('/data-tables/uploads')
|
||||
.attach('file', Buffer.from(csvContent), 'quoted.csv')
|
||||
.expect(200);
|
||||
|
||||
expect(response.body.data).toHaveProperty('originalName', 'quoted.csv');
|
||||
expect(response.body.data).toHaveProperty('id');
|
||||
});
|
||||
|
||||
test('should handle CSV with newlines in values', async () => {
|
||||
const csvContent =
|
||||
'name,description\n"Product A","Line 1\nLine 2"\n"Product B","Single line"';
|
||||
|
||||
const response = await authOwnerAgent
|
||||
.post('/data-tables/uploads')
|
||||
.attach('file', Buffer.from(csvContent), 'multiline.csv')
|
||||
.expect(200);
|
||||
|
||||
expect(response.body.data).toHaveProperty('originalName', 'multiline.csv');
|
||||
expect(response.body.data).toHaveProperty('id');
|
||||
});
|
||||
|
||||
test('should handle large valid CSV files', async () => {
|
||||
// Create a large but valid CSV (within size limit)
|
||||
const header = 'id,name,email,city,country\n';
|
||||
const rows = Array.from(
|
||||
{ length: 1000 },
|
||||
(_, i) => `${i},User${i},user${i}@example.com,City${i},Country${i}`,
|
||||
).join('\n');
|
||||
const largeContent = header + rows;
|
||||
|
||||
const response = await authOwnerAgent
|
||||
.post('/data-tables/uploads')
|
||||
.attach('file', Buffer.from(largeContent), 'large-valid.csv')
|
||||
.expect(200);
|
||||
|
||||
expect(response.body.data).toHaveProperty('originalName', 'large-valid.csv');
|
||||
expect(response.body.data).toHaveProperty('id');
|
||||
});
|
||||
|
||||
test('should handle CSV without headers (first row treated as header)', async () => {
|
||||
// CSV with just data rows, no explicit header
|
||||
const csvContent = '1,2,3\n4,5,6\n7,8,9';
|
||||
|
||||
const response = await authOwnerAgent
|
||||
.post('/data-tables/uploads')
|
||||
.attach('file', Buffer.from(csvContent), 'no-headers.csv')
|
||||
.expect(200);
|
||||
|
||||
expect(response.body.data).toHaveProperty('originalName', 'no-headers.csv');
|
||||
expect(response.body.data).toHaveProperty('id');
|
||||
// First row is treated as headers, so we have 2 data rows
|
||||
expect(response.body.data).toHaveProperty('rowCount', 2);
|
||||
expect(response.body.data).toHaveProperty('columnCount', 3);
|
||||
// First row values become column names
|
||||
expect(response.body.data.columns).toEqual([
|
||||
{ name: '1', type: 'number', compatibleTypes: ['number', 'string'] },
|
||||
{ name: '2', type: 'number', compatibleTypes: ['number', 'string'] },
|
||||
{ name: '3', type: 'number', compatibleTypes: ['number', 'string'] },
|
||||
]);
|
||||
});
|
||||
});
|
||||
});
|
||||
@ -3761,3 +3761,429 @@ describe('PATCH /projects/:projectId/data-tables/:dataTableId/rows', () => {
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe('POST /projects/:projectId/data-tables - CSV Import', () => {
|
||||
test('should create data table and import rows from CSV file', async () => {
|
||||
// First upload a CSV file
|
||||
const csvContent = 'name,age,email\nAlice,30,alice@example.com\nBob,25,bob@example.com';
|
||||
const uploadResponse = await authOwnerAgent
|
||||
.post('/data-tables/uploads')
|
||||
.attach('file', Buffer.from(csvContent), { filename: 'test.csv', contentType: 'text/csv' })
|
||||
.expect(200);
|
||||
|
||||
const fileId = uploadResponse.body.data.id;
|
||||
|
||||
// Create data table with fileId to trigger import
|
||||
const payload = {
|
||||
name: 'Imported Data Table',
|
||||
columns: [
|
||||
{ name: 'name', type: 'string' },
|
||||
{ name: 'age', type: 'number' },
|
||||
{ name: 'email', type: 'string' },
|
||||
],
|
||||
fileId,
|
||||
};
|
||||
|
||||
const createResponse = await authOwnerAgent
|
||||
.post(`/projects/${ownerProject.id}/data-tables`)
|
||||
.send(payload)
|
||||
.expect(200);
|
||||
|
||||
const dataTableId = createResponse.body.data.id;
|
||||
|
||||
// Verify data was imported
|
||||
const rowsResponse = await authOwnerAgent
|
||||
.get(`/projects/${ownerProject.id}/data-tables/${dataTableId}/rows`)
|
||||
.expect(200);
|
||||
|
||||
expect(rowsResponse.body.data.count).toBe(2);
|
||||
expect(rowsResponse.body.data.data).toHaveLength(2);
|
||||
expect(rowsResponse.body.data.data).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
name: 'Alice',
|
||||
age: 30,
|
||||
email: 'alice@example.com',
|
||||
}),
|
||||
expect.objectContaining({
|
||||
name: 'Bob',
|
||||
age: 25,
|
||||
email: 'bob@example.com',
|
||||
}),
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
test('should map CSV columns to table columns by position', async () => {
|
||||
// Upload CSV with column names that have spaces
|
||||
const csvContent =
|
||||
'Customer Id,Full Name,Email Address\n1001,John Doe,john@example.com\n1002,Jane Smith,jane@example.com';
|
||||
const uploadResponse = await authOwnerAgent
|
||||
.post('/data-tables/uploads')
|
||||
.attach('file', Buffer.from(csvContent), {
|
||||
filename: 'customers.csv',
|
||||
contentType: 'text/csv',
|
||||
})
|
||||
.expect(200);
|
||||
|
||||
const fileId = uploadResponse.body.data.id;
|
||||
|
||||
// Create table with different column names (without spaces)
|
||||
const payload = {
|
||||
name: 'Customers',
|
||||
columns: [
|
||||
{ name: 'customerId', type: 'string' },
|
||||
{ name: 'fullName', type: 'string' },
|
||||
{ name: 'emailAddress', type: 'string' },
|
||||
],
|
||||
fileId,
|
||||
};
|
||||
|
||||
const createResponse = await authOwnerAgent
|
||||
.post(`/projects/${ownerProject.id}/data-tables`)
|
||||
.send(payload)
|
||||
.expect(200);
|
||||
|
||||
const dataTableId = createResponse.body.data.id;
|
||||
|
||||
// Verify data was mapped correctly by position
|
||||
const rowsResponse = await authOwnerAgent
|
||||
.get(`/projects/${ownerProject.id}/data-tables/${dataTableId}/rows`)
|
||||
.expect(200);
|
||||
|
||||
expect(rowsResponse.body.data.count).toBe(2);
|
||||
expect(rowsResponse.body.data.data).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
customerId: '1001',
|
||||
fullName: 'John Doe',
|
||||
emailAddress: 'john@example.com',
|
||||
}),
|
||||
expect.objectContaining({
|
||||
customerId: '1002',
|
||||
fullName: 'Jane Smith',
|
||||
emailAddress: 'jane@example.com',
|
||||
}),
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
test('should create data table with partial column mapping when schema has extra columns', async () => {
|
||||
// Upload a valid CSV with 2 columns
|
||||
const csvContent = 'name,age\nAlice,30';
|
||||
const uploadResponse = await authOwnerAgent
|
||||
.post('/data-tables/uploads')
|
||||
.attach('file', Buffer.from(csvContent), { filename: 'test.csv', contentType: 'text/csv' })
|
||||
.expect(200);
|
||||
|
||||
const fileId = uploadResponse.body.data.id;
|
||||
|
||||
// Create table with more columns than in CSV
|
||||
// The system should map by index: CSV col 0 -> table col 0, CSV col 1 -> table col 1
|
||||
// The extra table column won't have data imported
|
||||
const payload = {
|
||||
name: 'Partial Import Table',
|
||||
columns: [
|
||||
{ name: 'name', type: 'string' },
|
||||
{ name: 'age', type: 'number' },
|
||||
{ name: 'extra', type: 'string' }, // Extra column not in CSV
|
||||
],
|
||||
fileId,
|
||||
};
|
||||
|
||||
const createResponse = await authOwnerAgent
|
||||
.post(`/projects/${ownerProject.id}/data-tables`)
|
||||
.send(payload)
|
||||
.expect(200);
|
||||
|
||||
const dataTableId = createResponse.body.data.id;
|
||||
|
||||
// Verify rows - should have mapped only the columns that exist in CSV
|
||||
const rowsResponse = await authOwnerAgent
|
||||
.get(`/projects/${ownerProject.id}/data-tables/${dataTableId}/rows`)
|
||||
.expect(200);
|
||||
|
||||
// Data should be imported with columns mapped by index position
|
||||
// The 'extra' column should be null/empty since it doesn't exist in CSV
|
||||
expect(rowsResponse.body.data.data[0]).toMatchObject({
|
||||
name: 'Alice',
|
||||
age: 30,
|
||||
});
|
||||
});
|
||||
|
||||
test('should handle empty CSV file on import', async () => {
|
||||
// Upload empty CSV
|
||||
const csvContent = '';
|
||||
const uploadResponse = await authOwnerAgent
|
||||
.post('/data-tables/uploads')
|
||||
.attach('file', Buffer.from(csvContent), { filename: 'empty.csv', contentType: 'text/csv' })
|
||||
.expect(200);
|
||||
|
||||
const fileId = uploadResponse.body.data.id;
|
||||
|
||||
const payload = {
|
||||
name: 'Empty CSV Import',
|
||||
columns: [{ name: 'name', type: 'string' }],
|
||||
fileId,
|
||||
};
|
||||
|
||||
const createResponse = await authOwnerAgent
|
||||
.post(`/projects/${ownerProject.id}/data-tables`)
|
||||
.send(payload);
|
||||
|
||||
// Should either fail or create table with no rows
|
||||
if (createResponse.status === 200) {
|
||||
const dataTableId = createResponse.body.data.id;
|
||||
const rowsResponse = await authOwnerAgent
|
||||
.get(`/projects/${ownerProject.id}/data-tables/${dataTableId}/rows`)
|
||||
.expect(200);
|
||||
|
||||
expect(rowsResponse.body.data.count).toBe(0);
|
||||
}
|
||||
});
|
||||
|
||||
test('should handle CSV with only headers on import', async () => {
|
||||
// Upload CSV with only headers
|
||||
const csvContent = 'name,age,city';
|
||||
const uploadResponse = await authOwnerAgent
|
||||
.post('/data-tables/uploads')
|
||||
.attach('file', Buffer.from(csvContent), {
|
||||
filename: 'headers-only.csv',
|
||||
contentType: 'text/csv',
|
||||
})
|
||||
.expect(200);
|
||||
|
||||
const fileId = uploadResponse.body.data.id;
|
||||
|
||||
const payload = {
|
||||
name: 'Headers Only Import',
|
||||
columns: [
|
||||
{ name: 'name', type: 'string' },
|
||||
{ name: 'age', type: 'number' },
|
||||
{ name: 'city', type: 'string' },
|
||||
],
|
||||
fileId,
|
||||
};
|
||||
|
||||
const createResponse = await authOwnerAgent
|
||||
.post(`/projects/${ownerProject.id}/data-tables`)
|
||||
.send(payload)
|
||||
.expect(200);
|
||||
|
||||
const dataTableId = createResponse.body.data.id;
|
||||
|
||||
// Should create table with no rows
|
||||
const rowsResponse = await authOwnerAgent
|
||||
.get(`/projects/${ownerProject.id}/data-tables/${dataTableId}/rows`)
|
||||
.expect(200);
|
||||
|
||||
expect(rowsResponse.body.data.count).toBe(0);
|
||||
});
|
||||
|
||||
test('should import CSV file with multiple rows', async () => {
|
||||
const csvContent =
|
||||
'itemId,itemName,itemValue\n1,Item 1,10\n2,Item 2,20\n3,Item 3,30\n4,Item 4,40\n5,Item 5,50';
|
||||
|
||||
const uploadResponse = await authOwnerAgent
|
||||
.post('/data-tables/uploads')
|
||||
.attach('file', Buffer.from(csvContent), {
|
||||
filename: 'multiple-rows.csv',
|
||||
contentType: 'text/csv',
|
||||
})
|
||||
.expect(200);
|
||||
|
||||
const fileId = uploadResponse.body.data.id;
|
||||
|
||||
const payload = {
|
||||
name: 'Multiple Rows Import',
|
||||
columns: [
|
||||
{ name: 'itemId', type: 'string' },
|
||||
{ name: 'itemName', type: 'string' },
|
||||
{ name: 'itemValue', type: 'number' },
|
||||
],
|
||||
fileId,
|
||||
};
|
||||
|
||||
const createResponse = await authOwnerAgent
|
||||
.post(`/projects/${ownerProject.id}/data-tables`)
|
||||
.send(payload)
|
||||
.expect(200);
|
||||
|
||||
const dataTableId = createResponse.body.data.id;
|
||||
|
||||
// Verify all rows were imported
|
||||
const rowsResponse = await authOwnerAgent
|
||||
.get(`/projects/${ownerProject.id}/data-tables/${dataTableId}/rows`)
|
||||
.expect(200);
|
||||
|
||||
expect(rowsResponse.body.data.count).toBe(5);
|
||||
});
|
||||
|
||||
test('should create table without import when fileId is not provided', async () => {
|
||||
const payload = {
|
||||
name: 'Table Without Import',
|
||||
columns: [
|
||||
{ name: 'col1', type: 'string' },
|
||||
{ name: 'col2', type: 'number' },
|
||||
],
|
||||
};
|
||||
|
||||
const createResponse = await authOwnerAgent
|
||||
.post(`/projects/${ownerProject.id}/data-tables`)
|
||||
.send(payload)
|
||||
.expect(200);
|
||||
|
||||
const dataTableId = createResponse.body.data.id;
|
||||
|
||||
// Should create empty table
|
||||
const rowsResponse = await authOwnerAgent
|
||||
.get(`/projects/${ownerProject.id}/data-tables/${dataTableId}/rows`)
|
||||
.expect(200);
|
||||
|
||||
expect(rowsResponse.body.data.count).toBe(0);
|
||||
});
|
||||
|
||||
test('should handle CSV with quoted values containing commas', async () => {
|
||||
const csvContent =
|
||||
'name,address\n"John Doe","123 Main St, Apt 4"\n"Jane Smith","456 Oak Ave, Suite 10"';
|
||||
const uploadResponse = await authOwnerAgent
|
||||
.post('/data-tables/uploads')
|
||||
.attach('file', Buffer.from(csvContent), { filename: 'quoted.csv', contentType: 'text/csv' })
|
||||
.expect(200);
|
||||
|
||||
const fileId = uploadResponse.body.data.id;
|
||||
|
||||
const payload = {
|
||||
name: 'Quoted Values Import',
|
||||
columns: [
|
||||
{ name: 'name', type: 'string' },
|
||||
{ name: 'address', type: 'string' },
|
||||
],
|
||||
fileId,
|
||||
};
|
||||
|
||||
const createResponse = await authOwnerAgent
|
||||
.post(`/projects/${ownerProject.id}/data-tables`)
|
||||
.send(payload)
|
||||
.expect(200);
|
||||
|
||||
const dataTableId = createResponse.body.data.id;
|
||||
|
||||
const rowsResponse = await authOwnerAgent
|
||||
.get(`/projects/${ownerProject.id}/data-tables/${dataTableId}/rows`)
|
||||
.expect(200);
|
||||
|
||||
expect(rowsResponse.body.data.count).toBe(2);
|
||||
expect(rowsResponse.body.data.data).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
name: 'John Doe',
|
||||
address: '123 Main St, Apt 4',
|
||||
}),
|
||||
expect.objectContaining({
|
||||
name: 'Jane Smith',
|
||||
address: '456 Oak Ave, Suite 10',
|
||||
}),
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
test('should handle CSV with Unicode characters', async () => {
|
||||
const csvContent = 'name,city\nJohn Müller,München\nMaría García,São Paulo';
|
||||
const uploadResponse = await authOwnerAgent
|
||||
.post('/data-tables/uploads')
|
||||
.attach('file', Buffer.from(csvContent), { filename: 'unicode.csv', contentType: 'text/csv' })
|
||||
.expect(200);
|
||||
|
||||
const fileId = uploadResponse.body.data.id;
|
||||
|
||||
const payload = {
|
||||
name: 'Unicode Import',
|
||||
columns: [
|
||||
{ name: 'name', type: 'string' },
|
||||
{ name: 'city', type: 'string' },
|
||||
],
|
||||
fileId,
|
||||
};
|
||||
|
||||
const createResponse = await authOwnerAgent
|
||||
.post(`/projects/${ownerProject.id}/data-tables`)
|
||||
.send(payload)
|
||||
.expect(200);
|
||||
|
||||
const dataTableId = createResponse.body.data.id;
|
||||
|
||||
const rowsResponse = await authOwnerAgent
|
||||
.get(`/projects/${ownerProject.id}/data-tables/${dataTableId}/rows`)
|
||||
.expect(200);
|
||||
|
||||
expect(rowsResponse.body.data.count).toBe(2);
|
||||
expect(rowsResponse.body.data.data).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
name: 'John Müller',
|
||||
city: 'München',
|
||||
}),
|
||||
expect.objectContaining({
|
||||
name: 'María García',
|
||||
city: 'São Paulo',
|
||||
}),
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
test('should handle CSV with different data types', async () => {
|
||||
const csvContent =
|
||||
'name,age,active,joinDate\nAlice,30,true,2024-01-15\nBob,25,false,2024-02-20';
|
||||
const uploadResponse = await authOwnerAgent
|
||||
.post('/data-tables/uploads')
|
||||
.attach('file', Buffer.from(csvContent), {
|
||||
filename: 'mixed-types.csv',
|
||||
contentType: 'text/csv',
|
||||
})
|
||||
.expect(200);
|
||||
|
||||
const fileId = uploadResponse.body.data.id;
|
||||
|
||||
const payload = {
|
||||
name: 'Mixed Types Import',
|
||||
columns: [
|
||||
{ name: 'name', type: 'string' },
|
||||
{ name: 'age', type: 'number' },
|
||||
{ name: 'active', type: 'boolean' },
|
||||
{ name: 'joinDate', type: 'date' },
|
||||
],
|
||||
fileId,
|
||||
};
|
||||
|
||||
const createResponse = await authOwnerAgent
|
||||
.post(`/projects/${ownerProject.id}/data-tables`)
|
||||
.send(payload)
|
||||
.expect(200);
|
||||
|
||||
const dataTableId = createResponse.body.data.id;
|
||||
|
||||
const rowsResponse = await authOwnerAgent
|
||||
.get(`/projects/${ownerProject.id}/data-tables/${dataTableId}/rows`)
|
||||
.expect(200);
|
||||
|
||||
expect(rowsResponse.body.data.count).toBe(2);
|
||||
// Data types are converted based on column type definitions
|
||||
expect(rowsResponse.body.data.data).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
name: 'Alice',
|
||||
age: 30,
|
||||
active: true,
|
||||
joinDate: expect.stringContaining('2024-01-15'),
|
||||
}),
|
||||
expect.objectContaining({
|
||||
name: 'Bob',
|
||||
age: 25,
|
||||
active: false,
|
||||
joinDate: expect.stringContaining('2024-02-20'),
|
||||
}),
|
||||
]),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@ -6,7 +6,7 @@ export function mockDataTableSizeValidator() {
|
||||
const sizeValidator = Container.get(DataTableSizeValidator);
|
||||
jest.spyOn(sizeValidator, 'validateSize').mockResolvedValue();
|
||||
jest.spyOn(sizeValidator, 'getCachedSizeData').mockResolvedValue({
|
||||
totalBytes: 50 * 1024 * 1024, // 50MB - under the default limit
|
||||
totalBytes: 0, // Start with 0 bytes to allow uploads
|
||||
dataTables: {},
|
||||
});
|
||||
return sizeValidator;
|
||||
|
||||
204
packages/cli/src/modules/data-table/csv-parser.service.ts
Normal file
204
packages/cli/src/modules/data-table/csv-parser.service.ts
Normal file
@ -0,0 +1,204 @@
|
||||
import { GlobalConfig } from '@n8n/config';
|
||||
import { Service } from '@n8n/di';
|
||||
import { parse } from 'csv-parse';
|
||||
import { createReadStream } from 'fs';
|
||||
import path from 'path';
|
||||
|
||||
export interface CsvColumnMetadata {
|
||||
name: string;
|
||||
type: 'string' | 'number' | 'boolean' | 'date';
|
||||
}
|
||||
|
||||
export interface CsvMetadata {
|
||||
rowCount: number;
|
||||
columnCount: number;
|
||||
columns: CsvColumnMetadata[];
|
||||
}
|
||||
|
||||
@Service()
|
||||
export class CsvParserService {
|
||||
private readonly uploadDir: string;
|
||||
|
||||
private readonly DEFAULT_COLUMN_PREFIX = 'Column_';
|
||||
|
||||
constructor(private readonly globalConfig: GlobalConfig) {
|
||||
this.uploadDir = this.globalConfig.dataTable.uploadDir;
|
||||
}
|
||||
|
||||
private processRowWithoutHeaders(
|
||||
row: string[],
|
||||
columnNames: string[],
|
||||
): { rowObject: Record<string, string>; columnNames: string[] } {
|
||||
let updatedColumnNames = columnNames;
|
||||
if (updatedColumnNames.length === 0) {
|
||||
updatedColumnNames = row.map((_, index) => `${this.DEFAULT_COLUMN_PREFIX}${index + 1}`);
|
||||
}
|
||||
const rowObject: Record<string, string> = {};
|
||||
row.forEach((value, index) => {
|
||||
rowObject[updatedColumnNames[index]] = value;
|
||||
});
|
||||
return { rowObject, columnNames: updatedColumnNames };
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses a CSV file and returns metadata including row count, column count, and inferred column types
|
||||
*/
|
||||
async parseFile(fileId: string, hasHeaders: boolean = true): Promise<CsvMetadata> {
|
||||
const filePath = path.join(this.uploadDir, fileId);
|
||||
let rowCount = 0;
|
||||
let firstDataRow: Record<string, string> | null = null;
|
||||
let columnNames: string[] = [];
|
||||
|
||||
return await new Promise((resolve, reject) => {
|
||||
const parser = parse({
|
||||
columns: hasHeaders
|
||||
? (header: string[]) => {
|
||||
columnNames = header;
|
||||
return header;
|
||||
}
|
||||
: false,
|
||||
skip_empty_lines: true,
|
||||
});
|
||||
|
||||
createReadStream(filePath)
|
||||
.pipe(parser)
|
||||
.on('data', (row: Record<string, string> | string[]) => {
|
||||
rowCount++;
|
||||
|
||||
if (!hasHeaders && Array.isArray(row)) {
|
||||
const processed = this.processRowWithoutHeaders(row, columnNames);
|
||||
columnNames = processed.columnNames;
|
||||
firstDataRow ??= processed.rowObject;
|
||||
} else if (!Array.isArray(row)) {
|
||||
firstDataRow ??= row;
|
||||
}
|
||||
})
|
||||
.on('end', () => {
|
||||
const columns = columnNames.map((columnName) => {
|
||||
const detectedType = this.inferColumnType(firstDataRow?.[columnName]);
|
||||
return {
|
||||
name: columnName,
|
||||
type: detectedType,
|
||||
compatibleTypes: this.getCompatibleTypes(detectedType),
|
||||
};
|
||||
});
|
||||
|
||||
resolve({
|
||||
rowCount,
|
||||
columnCount: columns.length,
|
||||
columns,
|
||||
});
|
||||
})
|
||||
.on('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses a CSV file and returns all rows as an array of objects
|
||||
*/
|
||||
async parseFileData(
|
||||
fileId: string,
|
||||
hasHeaders: boolean = true,
|
||||
): Promise<Array<Record<string, string>>> {
|
||||
const filePath = path.join(this.uploadDir, fileId);
|
||||
|
||||
const rows: Array<Record<string, string>> = [];
|
||||
let columnNames: string[] = [];
|
||||
|
||||
return await new Promise((resolve, reject) => {
|
||||
const parser = parse({
|
||||
columns: hasHeaders ? true : false,
|
||||
skip_empty_lines: true,
|
||||
});
|
||||
|
||||
createReadStream(filePath)
|
||||
.pipe(parser)
|
||||
.on('data', (row: Record<string, string> | string[]) => {
|
||||
if (!hasHeaders && Array.isArray(row)) {
|
||||
const processed = this.processRowWithoutHeaders(row, columnNames);
|
||||
columnNames = processed.columnNames;
|
||||
rows.push(processed.rowObject);
|
||||
} else if (!Array.isArray(row)) {
|
||||
rows.push(row);
|
||||
}
|
||||
})
|
||||
.on('end', () => {
|
||||
resolve(rows);
|
||||
})
|
||||
.on('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the list of compatible types for a detected type
|
||||
* Logic: more specific types can be converted to string, but string cannot be converted to specific types
|
||||
*/
|
||||
private getCompatibleTypes(
|
||||
detectedType: 'string' | 'number' | 'boolean' | 'date',
|
||||
): Array<'string' | 'number' | 'boolean' | 'date'> {
|
||||
switch (detectedType) {
|
||||
case 'date':
|
||||
return ['date', 'string'];
|
||||
case 'number':
|
||||
return ['number', 'string'];
|
||||
case 'boolean':
|
||||
return ['boolean', 'string'];
|
||||
case 'string':
|
||||
return ['string'];
|
||||
default:
|
||||
return ['string'];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Infers the column type from a sample value
|
||||
* Priority: boolean > number > date > string
|
||||
*/
|
||||
private inferColumnType(value: string | undefined): 'string' | 'number' | 'boolean' | 'date' {
|
||||
if (!value?.trim()) {
|
||||
return 'string';
|
||||
}
|
||||
|
||||
const trimmedValue = value.trim();
|
||||
const lowerValue = trimmedValue.toLowerCase();
|
||||
|
||||
if (lowerValue === 'true' || lowerValue === 'false') {
|
||||
return 'boolean';
|
||||
}
|
||||
|
||||
if (!Number.isNaN(Number(trimmedValue))) {
|
||||
return 'number';
|
||||
}
|
||||
|
||||
if (this.isDate(trimmedValue)) {
|
||||
return 'date';
|
||||
}
|
||||
|
||||
return 'string';
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if a string represents a valid date
|
||||
*/
|
||||
private isDate(value: string): boolean {
|
||||
// Try to parse as date
|
||||
const date = new Date(value);
|
||||
|
||||
// Check if it's a valid date and the original value looks like a date
|
||||
if (!Number.isNaN(date.getTime())) {
|
||||
// Additional check: make sure it looks like a date format
|
||||
// This prevents strings like "123" from being interpreted as dates
|
||||
const datePatterns = [
|
||||
/^\d{4}-\d{2}-\d{2}/, // ISO date (YYYY-MM-DD)
|
||||
/^\d{4}\/\d{2}\/\d{2}/, // YYYY/MM/DD
|
||||
/^\d{2}\/\d{2}\/\d{4}/, // MM/DD/YYYY or DD/MM/YYYY
|
||||
/^\d{2}-\d{2}-\d{4}/, // MM-DD-YYYY or DD-MM-YYYY
|
||||
/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}/, // ISO datetime
|
||||
];
|
||||
|
||||
return datePatterns.some((pattern) => pattern.test(value));
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,87 @@
|
||||
import { GlobalConfig } from '@n8n/config';
|
||||
import { Service } from '@n8n/di';
|
||||
import { promises as fs } from 'fs';
|
||||
import path from 'path';
|
||||
|
||||
@Service()
|
||||
export class DataTableFileCleanupService {
|
||||
private readonly uploadDir: string;
|
||||
|
||||
private cleanupInterval?: NodeJS.Timeout;
|
||||
|
||||
constructor(private readonly globalConfig: GlobalConfig) {
|
||||
this.uploadDir = this.globalConfig.dataTable.uploadDir;
|
||||
}
|
||||
|
||||
private isErrnoException(error: unknown): error is NodeJS.ErrnoException {
|
||||
return (
|
||||
typeof error === 'object' &&
|
||||
error !== null &&
|
||||
'code' in error &&
|
||||
typeof (error as { code: unknown }).code === 'string'
|
||||
);
|
||||
}
|
||||
|
||||
async start() {
|
||||
// Run cleanup periodically to delete orphaned files
|
||||
this.cleanupInterval = setInterval(() => {
|
||||
void this.cleanupOrphanedFiles();
|
||||
}, this.globalConfig.dataTable.cleanupIntervalMs);
|
||||
}
|
||||
|
||||
async shutdown() {
|
||||
if (this.cleanupInterval) {
|
||||
clearInterval(this.cleanupInterval);
|
||||
this.cleanupInterval = undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Cleans up orphaned CSV files that exceed the configured maximum age
|
||||
* These are files that were uploaded but never used to create a data table
|
||||
*/
|
||||
private async cleanupOrphanedFiles(): Promise<void> {
|
||||
try {
|
||||
const files = await fs.readdir(this.uploadDir);
|
||||
const now = Date.now();
|
||||
const maxAge = this.globalConfig.dataTable.fileMaxAgeMs;
|
||||
|
||||
for (const file of files) {
|
||||
const filePath = path.join(this.uploadDir, file);
|
||||
try {
|
||||
const stats = await fs.stat(filePath);
|
||||
const fileAge = now - stats.mtimeMs;
|
||||
|
||||
// Delete files older than the configured maximum age
|
||||
if (fileAge > maxAge) {
|
||||
await fs.unlink(filePath);
|
||||
}
|
||||
} catch (error) {
|
||||
// Ignore errors for individual files (e.g., file already deleted)
|
||||
continue;
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
// Ignore errors if upload directory doesn't exist yet
|
||||
if (!this.isErrnoException(error) || error.code !== 'ENOENT') {
|
||||
// Log other errors but don't throw - cleanup is best effort
|
||||
console.error('Error cleaning up orphaned CSV files:', error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes a specific CSV file by its fileId
|
||||
*/
|
||||
async deleteFile(fileId: string): Promise<void> {
|
||||
const filePath = path.join(this.uploadDir, fileId);
|
||||
try {
|
||||
await fs.unlink(filePath);
|
||||
} catch (error) {
|
||||
// Ignore errors if file doesn't exist
|
||||
if (!this.isErrnoException(error) || error.code !== 'ENOENT') {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -5,6 +5,7 @@ import { DataTableSizeStatus, DataTablesSizeData } from 'n8n-workflow';
|
||||
import { Telemetry } from '@/telemetry';
|
||||
|
||||
import { DataTableValidationError } from './errors/data-table-validation.error';
|
||||
import { toMb } from './utils/size-utils';
|
||||
|
||||
@Service()
|
||||
export class DataTableSizeValidator {
|
||||
@ -64,7 +65,7 @@ export class DataTableSizeValidator {
|
||||
});
|
||||
|
||||
throw new DataTableValidationError(
|
||||
`Data table size limit exceeded: ${this.toMb(size.totalBytes)}MB used, limit is ${this.toMb(this.globalConfig.dataTable.maxSize)}MB`,
|
||||
`Data table size limit exceeded: ${toMb(size.totalBytes)}MB used, limit is ${toMb(this.globalConfig.dataTable.maxSize)}MB`,
|
||||
);
|
||||
}
|
||||
}
|
||||
@ -87,10 +88,6 @@ export class DataTableSizeValidator {
|
||||
return this.sizeToState(size.totalBytes);
|
||||
}
|
||||
|
||||
private toMb(sizeInBytes: number): number {
|
||||
return Math.round(sizeInBytes / (1024 * 1024));
|
||||
}
|
||||
|
||||
reset() {
|
||||
this.lastCheck = undefined;
|
||||
this.cachedSizeData = undefined;
|
||||
|
||||
@ -0,0 +1,50 @@
|
||||
import { Post, RestController } from '@n8n/decorators';
|
||||
import { Container } from '@n8n/di';
|
||||
import multer from 'multer';
|
||||
|
||||
import { BadRequestError } from '@/errors/response-errors/bad-request.error';
|
||||
|
||||
import { CsvParserService } from './csv-parser.service';
|
||||
import { MulterUploadMiddleware } from './multer-upload-middleware';
|
||||
import { AuthenticatedRequestWithFile, hasStringProperty } from './types';
|
||||
|
||||
const uploadMiddleware = Container.get(MulterUploadMiddleware);
|
||||
|
||||
@RestController('/data-tables/uploads')
|
||||
export class DataTableUploadsController {
|
||||
constructor(private readonly csvParserService: CsvParserService) {}
|
||||
|
||||
@Post('/', {
|
||||
middlewares: [uploadMiddleware.single('file')],
|
||||
})
|
||||
async uploadFile(req: AuthenticatedRequestWithFile, _res: Response) {
|
||||
if (req.fileUploadError) {
|
||||
const error = req.fileUploadError;
|
||||
if (error instanceof multer.MulterError) {
|
||||
throw new BadRequestError(`File upload error: ${error.message}`);
|
||||
} else if (error instanceof BadRequestError) {
|
||||
throw error;
|
||||
} else {
|
||||
throw new BadRequestError('File upload failed');
|
||||
}
|
||||
}
|
||||
|
||||
if (!req.file) {
|
||||
throw new BadRequestError('No file uploaded');
|
||||
}
|
||||
|
||||
// Extract hasHeaders parameter from request body (multer parses form fields to body), default to true
|
||||
const hasHeaders =
|
||||
hasStringProperty(req.body, 'hasHeaders') && req.body.hasHeaders === 'false' ? false : true;
|
||||
|
||||
const metadata = await this.csvParserService.parseFile(req.file.filename, hasHeaders);
|
||||
|
||||
return {
|
||||
originalName: req.file.originalname,
|
||||
id: req.file.filename,
|
||||
rowCount: metadata.rowCount,
|
||||
columnCount: metadata.columnCount,
|
||||
columns: metadata.columns,
|
||||
};
|
||||
}
|
||||
}
|
||||
@ -7,12 +7,16 @@ export class DataTableModule implements ModuleInterface {
|
||||
async init() {
|
||||
await import('./data-table.controller');
|
||||
await import('./data-table-aggregate.controller');
|
||||
await import('./data-table-uploads.controller');
|
||||
|
||||
const { DataTableService } = await import('./data-table.service');
|
||||
await Container.get(DataTableService).start();
|
||||
|
||||
const { DataTableAggregateService } = await import('./data-table-aggregate.service');
|
||||
await Container.get(DataTableAggregateService).start();
|
||||
|
||||
const { DataTableFileCleanupService } = await import('./data-table-file-cleanup.service');
|
||||
await Container.get(DataTableFileCleanupService).start();
|
||||
}
|
||||
|
||||
@OnShutdown()
|
||||
@ -22,6 +26,9 @@ export class DataTableModule implements ModuleInterface {
|
||||
|
||||
const { DataTableAggregateService } = await import('./data-table-aggregate.service');
|
||||
await Container.get(DataTableAggregateService).shutdown();
|
||||
|
||||
const { DataTableFileCleanupService } = await import('./data-table-file-cleanup.service');
|
||||
await Container.get(DataTableFileCleanupService).shutdown();
|
||||
}
|
||||
|
||||
async entities() {
|
||||
|
||||
@ -28,13 +28,16 @@ import type {
|
||||
} from 'n8n-workflow';
|
||||
import { DATA_TABLE_SYSTEM_COLUMN_TYPE_MAP, validateFieldType } from 'n8n-workflow';
|
||||
|
||||
import { CsvParserService } from './csv-parser.service';
|
||||
import { DataTableColumn } from './data-table-column.entity';
|
||||
import { DataTableColumnRepository } from './data-table-column.repository';
|
||||
import { DataTableFileCleanupService } from './data-table-file-cleanup.service';
|
||||
import { DataTableRowsRepository } from './data-table-rows.repository';
|
||||
import { DataTableSizeValidator } from './data-table-size-validator.service';
|
||||
import { DataTableRepository } from './data-table.repository';
|
||||
import { columnTypeToFieldType } from './data-table.types';
|
||||
import { DataTableColumnNotFoundError } from './errors/data-table-column-not-found.error';
|
||||
import { FileUploadError } from './errors/data-table-file-upload.error';
|
||||
import { DataTableNameConflictError } from './errors/data-table-name-conflict.error';
|
||||
import { DataTableNotFoundError } from './errors/data-table-not-found.error';
|
||||
import { DataTableValidationError } from './errors/data-table-validation.error';
|
||||
@ -52,6 +55,8 @@ export class DataTableService {
|
||||
private readonly dataTableSizeValidator: DataTableSizeValidator,
|
||||
private readonly projectRelationRepository: ProjectRelationRepository,
|
||||
private readonly roleService: RoleService,
|
||||
private readonly csvParserService: CsvParserService,
|
||||
private readonly fileCleanupService: DataTableFileCleanupService,
|
||||
) {
|
||||
this.logger = this.logger.scoped('data-table');
|
||||
}
|
||||
@ -64,11 +69,61 @@ export class DataTableService {
|
||||
|
||||
const result = await this.dataTableRepository.createDataTable(projectId, dto.name, dto.columns);
|
||||
|
||||
if (dto.fileId) {
|
||||
try {
|
||||
await this.importDataFromFile(projectId, result.id, dto.fileId, dto.hasHeaders ?? true);
|
||||
await this.fileCleanupService.deleteFile(dto.fileId);
|
||||
} catch (error) {
|
||||
await this.deleteDataTable(result.id, projectId);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
this.dataTableSizeValidator.reset();
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private async importDataFromFile(
|
||||
projectId: string,
|
||||
dataTableId: string,
|
||||
fileId: string,
|
||||
hasHeaders: boolean,
|
||||
) {
|
||||
try {
|
||||
const tableColumns = await this.getColumns(dataTableId, projectId);
|
||||
|
||||
const csvMetadata = await this.csvParserService.parseFile(fileId, hasHeaders);
|
||||
|
||||
const columnMapping = new Map<string, string>();
|
||||
csvMetadata.columns.forEach((csvColumn, index) => {
|
||||
if (tableColumns[index]) {
|
||||
columnMapping.set(csvColumn.name, tableColumns[index].name);
|
||||
}
|
||||
});
|
||||
|
||||
const csvRows = await this.csvParserService.parseFileData(fileId, hasHeaders);
|
||||
|
||||
const transformedRows = csvRows.map((csvRow) => {
|
||||
const transformedRow: DataTableRow = {};
|
||||
for (const [csvColName, value] of Object.entries(csvRow)) {
|
||||
const tableColName = columnMapping.get(csvColName);
|
||||
if (tableColName) {
|
||||
transformedRow[tableColName] = value;
|
||||
}
|
||||
}
|
||||
return transformedRow;
|
||||
});
|
||||
|
||||
if (transformedRows.length > 0) {
|
||||
await this.insertRows(dataTableId, projectId, transformedRows);
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.error('Failed to import data from CSV file', { error, fileId, dataTableId });
|
||||
throw new FileUploadError(error instanceof Error ? error.message : 'Failed to read CSV file');
|
||||
}
|
||||
}
|
||||
|
||||
// Updates data table properties (currently limited to renaming)
|
||||
async updateDataTable(dataTableId: string, projectId: string, dto: UpdateDataTableDto) {
|
||||
await this.validateDataTableExists(dataTableId, projectId);
|
||||
|
||||
@ -0,0 +1,9 @@
|
||||
import { UserError } from 'n8n-workflow';
|
||||
|
||||
export class FileUploadError extends UserError {
|
||||
constructor(msg: string) {
|
||||
super(`Error uploading file: ${msg}`, {
|
||||
level: 'warning',
|
||||
});
|
||||
}
|
||||
}
|
||||
111
packages/cli/src/modules/data-table/multer-upload-middleware.ts
Normal file
111
packages/cli/src/modules/data-table/multer-upload-middleware.ts
Normal file
@ -0,0 +1,111 @@
|
||||
/* eslint-disable id-denylist */
|
||||
import { GlobalConfig } from '@n8n/config';
|
||||
import { Service } from '@n8n/di';
|
||||
import type { Request, RequestHandler } from 'express';
|
||||
import { mkdir } from 'fs/promises';
|
||||
import multer from 'multer';
|
||||
import { nanoid } from 'nanoid';
|
||||
|
||||
import { BadRequestError } from '@/errors/response-errors/bad-request.error';
|
||||
|
||||
import { DataTableSizeValidator } from './data-table-size-validator.service';
|
||||
import { DataTableRepository } from './data-table.repository';
|
||||
import {
|
||||
type AuthenticatedRequestWithFile,
|
||||
type MulterDestinationCallback,
|
||||
type MulterFilenameCallback,
|
||||
type UploadMiddleware,
|
||||
} from './types';
|
||||
import { formatBytes } from './utils/size-utils';
|
||||
|
||||
const ALLOWED_MIME_TYPES = ['text/csv'];
|
||||
|
||||
@Service()
|
||||
export class MulterUploadMiddleware implements UploadMiddleware {
|
||||
private upload: multer.Multer;
|
||||
|
||||
private readonly uploadDir: string;
|
||||
|
||||
constructor(
|
||||
private readonly globalConfig: GlobalConfig,
|
||||
private readonly sizeValidator: DataTableSizeValidator,
|
||||
private readonly dataTableRepository: DataTableRepository,
|
||||
) {
|
||||
this.uploadDir = this.globalConfig.dataTable.uploadDir;
|
||||
|
||||
void this.ensureUploadDirExists();
|
||||
|
||||
const storage = multer.diskStorage({
|
||||
destination: (_req: Request, _file: Express.Multer.File, cb: MulterDestinationCallback) => {
|
||||
cb(null, this.uploadDir);
|
||||
},
|
||||
filename: (_req: Request, _file: Express.Multer.File, cb: MulterFilenameCallback) => {
|
||||
const filename = nanoid(10);
|
||||
cb(null, filename);
|
||||
},
|
||||
});
|
||||
|
||||
this.upload = multer({
|
||||
storage,
|
||||
limits: this.globalConfig.dataTable.uploadMaxFileSize
|
||||
? { fileSize: this.globalConfig.dataTable.uploadMaxFileSize }
|
||||
: undefined,
|
||||
fileFilter: async (req, file, cb: multer.FileFilterCallback) => {
|
||||
if (!ALLOWED_MIME_TYPES.includes(file.mimetype)) {
|
||||
cb(
|
||||
new BadRequestError(
|
||||
`Only the following file types are allowed: ${ALLOWED_MIME_TYPES.join(', ')}`,
|
||||
),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const fileSize = parseInt(req.headers['content-length'] ?? '0', 10);
|
||||
|
||||
// If uploadMaxFileSize is set, multer's limits will handle the rejection
|
||||
if (this.globalConfig.dataTable.uploadMaxFileSize) {
|
||||
cb(null, true);
|
||||
return;
|
||||
}
|
||||
|
||||
// If uploadMaxFileSize is not set, check remaining space
|
||||
try {
|
||||
const sizeData = await this.sizeValidator.getCachedSizeData(async () => {
|
||||
return await this.dataTableRepository.findDataTablesSize();
|
||||
});
|
||||
const remainingSpace = Math.max(
|
||||
0,
|
||||
this.globalConfig.dataTable.maxSize - sizeData.totalBytes,
|
||||
);
|
||||
|
||||
if (fileSize > remainingSpace) {
|
||||
const message =
|
||||
remainingSpace === 0
|
||||
? `Storage limit exceeded. Current usage: ${formatBytes(sizeData.totalBytes)}, Limit: ${formatBytes(this.globalConfig.dataTable.maxSize)}`
|
||||
: `File size exceeds remaining storage space. Available: ${formatBytes(remainingSpace)}, File: ${formatBytes(fileSize)}`;
|
||||
cb(new BadRequestError(message));
|
||||
return;
|
||||
}
|
||||
cb(null, true);
|
||||
} catch {
|
||||
cb(new BadRequestError('Failed to validate file size'));
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
private async ensureUploadDirExists() {
|
||||
await mkdir(this.uploadDir, { recursive: true });
|
||||
}
|
||||
|
||||
single(fieldName: string): RequestHandler {
|
||||
return (req, res, next) => {
|
||||
void this.upload.single(fieldName)(req, res, (error) => {
|
||||
if (error) {
|
||||
(req as AuthenticatedRequestWithFile).fileUploadError = error;
|
||||
}
|
||||
next();
|
||||
});
|
||||
};
|
||||
}
|
||||
}
|
||||
26
packages/cli/src/modules/data-table/types.ts
Normal file
26
packages/cli/src/modules/data-table/types.ts
Normal file
@ -0,0 +1,26 @@
|
||||
import type { AuthenticatedRequest } from '@n8n/db';
|
||||
import type { RequestHandler } from 'express';
|
||||
|
||||
export interface UploadMiddleware {
|
||||
single(fieldName: string): RequestHandler;
|
||||
}
|
||||
|
||||
export type MulterDestinationCallback = (error: Error | null, destination: string) => void;
|
||||
export type MulterFilenameCallback = (error: Error | null, filename: string) => void;
|
||||
|
||||
export type AuthenticatedRequestWithFile<
|
||||
RouteParams = {},
|
||||
ResponseBody = {},
|
||||
RequestBody = {},
|
||||
RequestQuery = {},
|
||||
> = AuthenticatedRequest<RouteParams, ResponseBody, RequestBody, RequestQuery> & {
|
||||
file?: Express.Multer.File;
|
||||
fileUploadError?: Error;
|
||||
};
|
||||
|
||||
export function hasStringProperty<K extends string>(
|
||||
obj: unknown,
|
||||
key: K,
|
||||
): obj is Record<K, string> & object {
|
||||
return typeof obj === 'object' && obj !== null && key in obj;
|
||||
}
|
||||
19
packages/cli/src/modules/data-table/utils/size-utils.ts
Normal file
19
packages/cli/src/modules/data-table/utils/size-utils.ts
Normal file
@ -0,0 +1,19 @@
|
||||
/**
|
||||
* Convert bytes to megabytes (rounded to nearest integer)
|
||||
*/
|
||||
export function toMb(sizeInBytes: number): number {
|
||||
return Math.round(sizeInBytes / (1024 * 1024));
|
||||
}
|
||||
|
||||
/**
|
||||
* Format bytes to human-readable size with appropriate unit (B, KB, or MB)
|
||||
*/
|
||||
export function formatBytes(sizeInBytes: number): string {
|
||||
if (sizeInBytes < 1024) {
|
||||
return `${sizeInBytes}B`;
|
||||
} else if (sizeInBytes < 1024 * 1024) {
|
||||
return `${Math.round(sizeInBytes / 1024)}KB`;
|
||||
} else {
|
||||
return `${Math.round(sizeInBytes / (1024 * 1024))}MB`;
|
||||
}
|
||||
}
|
||||
@ -42,6 +42,7 @@
|
||||
"generic.any": "Any",
|
||||
"generic.allow": "Allow",
|
||||
"generic.deny": "Deny",
|
||||
"generic.back": "Back",
|
||||
"generic.cancel": "Cancel",
|
||||
"generic.open": "Open",
|
||||
"generic.openResource": "Open {resource}",
|
||||
@ -3263,10 +3264,28 @@
|
||||
"dataTable.card.column.count": "{count} column | {count} columns",
|
||||
"dataTable.add.title": "Create new data table",
|
||||
"dataTable.add.button.label": "Create data table",
|
||||
"dataTable.add.fromScratch": "From scratch",
|
||||
"dataTable.add.importCsv": "Import CSV",
|
||||
"dataTable.add.input.name.label": "Data table name",
|
||||
"dataTable.add.input.name.placeholder": "Enter data table name",
|
||||
"dataTable.add.error": "Error creating data table",
|
||||
"dataTable.delete.confirm.title": "Delete data table",
|
||||
"dataTable.upload.uploading": "Uploading and processing CSV file...",
|
||||
"dataTable.upload.selectFile": "Waiting for file selection...",
|
||||
"dataTable.upload.dropOrClick": "Drop file here or click to upload",
|
||||
"dataTable.upload.csvOnly": "CSV files only",
|
||||
"dataTable.upload.hasHeaders": "My CSV file contains a header row",
|
||||
"dataTable.upload.uploadButton": "Upload CSV",
|
||||
"dataTable.upload.success": "'{fileName}' has been uploaded successfully. We found {columnCount} column and {rowCount} row | '{fileName}' has been uploaded successfully. We found {columnCount} columns and {rowCount} rows",
|
||||
"dataTable.upload.error": "Error uploading CSV file",
|
||||
"dataTable.import.columnsFound": "Columns found",
|
||||
"dataTable.import.columnName": "Column Name",
|
||||
"dataTable.import.columnsDescription": "Review and adjust the column names and types detected from your CSV file",
|
||||
"dataTable.import.columnType": "Data Type",
|
||||
"dataTable.import.columnNamePlaceholder": "Enter column name",
|
||||
"dataTable.import.duplicateColumnName": "Column name must be unique",
|
||||
"dataTable.import.systemColumnName": "{columnName} is a reserved column",
|
||||
"dataTable.import.invalidColumnName": "Only alphabetical and non-leading numbers and underscores allowed",
|
||||
"dataTable.delete.confirm.message": "Are you sure you want to delete the data table '{name}'? This action cannot be undone.",
|
||||
"dataTable.delete.error": "Error deleting data table",
|
||||
"dataTable.rename.error": "Error renaming data table",
|
||||
|
||||
@ -1,19 +1,44 @@
|
||||
<script lang="ts" setup>
|
||||
import { useI18n } from '@n8n/i18n';
|
||||
import { onMounted, ref } from 'vue';
|
||||
import { onMounted, ref, computed } from 'vue';
|
||||
import { useDataTableStore } from '@/features/core/dataTable/dataTable.store';
|
||||
import { useUIStore } from '@/app/stores/ui.store';
|
||||
import { useToast } from '@/app/composables/useToast';
|
||||
import { useRoute, useRouter } from 'vue-router';
|
||||
import { DATA_TABLE_DETAILS, PROJECT_DATA_TABLES } from '@/features/core/dataTable/constants';
|
||||
import { useTelemetry } from '@/app/composables/useTelemetry';
|
||||
import { dataTableColumnNameSchema } from '@n8n/api-types';
|
||||
import { DATA_TABLE_SYSTEM_COLUMNS } from 'n8n-workflow';
|
||||
|
||||
import { N8nButton, N8nInput, N8nInputLabel } from '@n8n/design-system';
|
||||
import {
|
||||
N8nButton,
|
||||
N8nCheckbox,
|
||||
N8nIcon,
|
||||
N8nInput,
|
||||
N8nInputLabel,
|
||||
N8nSelect,
|
||||
N8nOption,
|
||||
N8nText,
|
||||
} from '@n8n/design-system';
|
||||
import Modal from '@/app/components/Modal.vue';
|
||||
import { ElUpload, ElRadio, ElRadioGroup } from 'element-plus';
|
||||
import type { UploadFile } from 'element-plus';
|
||||
|
||||
type Props = {
|
||||
modalName: string;
|
||||
};
|
||||
|
||||
type CreationMode = 'select' | 'scratch' | 'import' | 'file-selected';
|
||||
type ColumnType = 'string' | 'number' | 'boolean' | 'date';
|
||||
|
||||
interface CsvColumn {
|
||||
name: string;
|
||||
type: ColumnType;
|
||||
compatibleTypes: ColumnType[];
|
||||
typeOptions: Array<{ label: string; value: string }>;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
const props = defineProps<Props>();
|
||||
|
||||
const dataTableStore = useDataTableStore();
|
||||
@ -25,8 +50,82 @@ const i18n = useI18n();
|
||||
const toast = useToast();
|
||||
const telemetry = useTelemetry();
|
||||
|
||||
const creationMode = ref<CreationMode>('select');
|
||||
const dataTableName = ref('');
|
||||
const inputRef = ref<HTMLInputElement | null>(null);
|
||||
const selectedFile = ref<File | null>(null);
|
||||
const uploadedFileId = ref<string | null>(null);
|
||||
const uploadedFileName = ref<string>('');
|
||||
const csvColumns = ref<CsvColumn[]>([]);
|
||||
const csvRowCount = ref<number>(0);
|
||||
const csvColumnCount = ref<number>(0);
|
||||
const isUploading = ref(false);
|
||||
const hasHeaders = ref(true);
|
||||
const isUploadHovered = ref(false);
|
||||
|
||||
const allColumnTypeOptions = [
|
||||
{ label: 'String', value: 'string' },
|
||||
{ label: 'Number', value: 'number' },
|
||||
{ label: 'Boolean', value: 'boolean' },
|
||||
{ label: 'Datetime', value: 'date' },
|
||||
];
|
||||
|
||||
const isColumnType = (value: unknown): value is ColumnType => {
|
||||
return allColumnTypeOptions.some((option) => option.value === value);
|
||||
};
|
||||
|
||||
const getColumnTypeOptions = (compatibleTypes: ColumnType[]) => {
|
||||
if (!compatibleTypes || compatibleTypes.length === 0) {
|
||||
return allColumnTypeOptions;
|
||||
}
|
||||
return allColumnTypeOptions.filter((option) =>
|
||||
compatibleTypes.includes(option.value as ColumnType),
|
||||
);
|
||||
};
|
||||
|
||||
const validateColumnName = (columnName: string): string | undefined => {
|
||||
if (DATA_TABLE_SYSTEM_COLUMNS.includes(columnName)) {
|
||||
return i18n.baseText('dataTable.import.systemColumnName', {
|
||||
interpolate: { columnName },
|
||||
});
|
||||
}
|
||||
|
||||
const result = dataTableColumnNameSchema.safeParse(columnName);
|
||||
if (!result.success) {
|
||||
return i18n.baseText('dataTable.import.invalidColumnName');
|
||||
}
|
||||
return undefined;
|
||||
};
|
||||
|
||||
const hasValidationErrors = computed(() => {
|
||||
if (creationMode.value !== 'import') return false;
|
||||
return csvColumns.value.some((column) => column.error !== undefined);
|
||||
});
|
||||
|
||||
const hasDuplicateNames = computed(() => {
|
||||
if (creationMode.value !== 'import') return false;
|
||||
const names = csvColumns.value.map((col) => col.name.toLowerCase());
|
||||
return names.length !== new Set(names).size;
|
||||
});
|
||||
|
||||
const modalTitle = computed(() => {
|
||||
if (creationMode.value === 'import') {
|
||||
return 'Set data table columns';
|
||||
}
|
||||
return i18n.baseText('dataTable.add.title');
|
||||
});
|
||||
|
||||
const isCreateDisabled = computed(() => {
|
||||
if (creationMode.value === 'import') {
|
||||
return (
|
||||
!dataTableName.value ||
|
||||
!uploadedFileId.value ||
|
||||
hasValidationErrors.value ||
|
||||
hasDuplicateNames.value
|
||||
);
|
||||
}
|
||||
return true;
|
||||
});
|
||||
|
||||
onMounted(() => {
|
||||
setTimeout(() => {
|
||||
@ -35,32 +134,146 @@ onMounted(() => {
|
||||
}, 0);
|
||||
});
|
||||
|
||||
const selectedOption = ref<'scratch' | 'import'>('scratch');
|
||||
|
||||
const proceedFromSelect = async () => {
|
||||
if (!selectedOption.value || !dataTableName.value) return;
|
||||
|
||||
if (selectedOption.value === 'scratch') {
|
||||
await onSubmit();
|
||||
} else if (selectedOption.value === 'import') {
|
||||
if (!selectedFile.value) return;
|
||||
await uploadFile();
|
||||
}
|
||||
};
|
||||
|
||||
const onColumnNameChange = (index: number) => {
|
||||
const column = csvColumns.value[index];
|
||||
if (!column) return;
|
||||
|
||||
column.error = validateColumnName(column.name);
|
||||
|
||||
const isDuplicate = csvColumns.value.some(
|
||||
(col, idx) => idx !== index && col.name.toLowerCase() === column.name.toLowerCase(),
|
||||
);
|
||||
|
||||
if (isDuplicate && !column.error) {
|
||||
column.error = i18n.baseText('dataTable.import.duplicateColumnName');
|
||||
}
|
||||
|
||||
csvColumns.value.forEach((col, idx) => {
|
||||
if (idx !== index) {
|
||||
const otherIsDuplicate = csvColumns.value.some(
|
||||
(c, i) => i !== idx && c.name.toLowerCase() === col.name.toLowerCase(),
|
||||
);
|
||||
const validationError = validateColumnName(col.name);
|
||||
if (otherIsDuplicate && !validationError) {
|
||||
col.error = i18n.baseText('dataTable.import.duplicateColumnName');
|
||||
} else {
|
||||
col.error = validationError;
|
||||
}
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
const reset = (clearTableName = false) => {
|
||||
if (clearTableName) {
|
||||
dataTableName.value = '';
|
||||
}
|
||||
selectedFile.value = null;
|
||||
uploadedFileId.value = null;
|
||||
uploadedFileName.value = '';
|
||||
csvColumns.value = [];
|
||||
csvRowCount.value = 0;
|
||||
csvColumnCount.value = 0;
|
||||
selectedOption.value = 'scratch';
|
||||
creationMode.value = 'select';
|
||||
};
|
||||
|
||||
const handleFileChange = (uploadFile: UploadFile) => {
|
||||
if (uploadFile.raw) {
|
||||
selectedFile.value = uploadFile.raw;
|
||||
}
|
||||
};
|
||||
|
||||
const uploadFile = async () => {
|
||||
if (!selectedFile.value) return;
|
||||
|
||||
isUploading.value = true;
|
||||
creationMode.value = 'import';
|
||||
|
||||
try {
|
||||
const uploadResponse = await dataTableStore.uploadCsvFile(selectedFile.value, hasHeaders.value);
|
||||
uploadedFileId.value = uploadResponse.id;
|
||||
uploadedFileName.value = uploadResponse.originalName;
|
||||
csvRowCount.value = uploadResponse.rowCount;
|
||||
csvColumnCount.value = uploadResponse.columnCount;
|
||||
csvColumns.value = uploadResponse.columns.map((col) => {
|
||||
const compatibleTypes = (col.compatibleTypes || [col.type]).filter(isColumnType);
|
||||
const sanitizedName = col.name.replace(/\s+/g, '_');
|
||||
const colType = isColumnType(col.type) ? col.type : 'string';
|
||||
return {
|
||||
name: sanitizedName,
|
||||
type: colType,
|
||||
compatibleTypes,
|
||||
typeOptions: getColumnTypeOptions(compatibleTypes),
|
||||
error: validateColumnName(sanitizedName),
|
||||
};
|
||||
});
|
||||
|
||||
if (!dataTableName.value) {
|
||||
const fileName = selectedFile.value.name.replace(/\.csv$/i, '');
|
||||
dataTableName.value = fileName;
|
||||
}
|
||||
} catch (error) {
|
||||
toast.showError(error, i18n.baseText('dataTable.upload.error'));
|
||||
reset();
|
||||
} finally {
|
||||
isUploading.value = false;
|
||||
}
|
||||
};
|
||||
|
||||
const onSubmit = async () => {
|
||||
try {
|
||||
const newDataTable = await dataTableStore.createDataTable(
|
||||
dataTableName.value,
|
||||
route.params.projectId as string,
|
||||
);
|
||||
telemetry.track('User created data table', {
|
||||
data_table_id: newDataTable.id,
|
||||
data_table_project_id: newDataTable.project?.id,
|
||||
});
|
||||
dataTableName.value = '';
|
||||
uiStore.closeModal(props.modalName);
|
||||
void router.push({
|
||||
name: DATA_TABLE_DETAILS,
|
||||
params: {
|
||||
id: newDataTable.id,
|
||||
},
|
||||
});
|
||||
let newDataTable;
|
||||
|
||||
if (selectedOption.value === 'scratch') {
|
||||
newDataTable = await dataTableStore.createDataTable(
|
||||
dataTableName.value,
|
||||
route.params.projectId as string,
|
||||
);
|
||||
} else if (creationMode.value === 'import' && uploadedFileId.value) {
|
||||
newDataTable = await dataTableStore.createDataTable(
|
||||
dataTableName.value,
|
||||
route.params.projectId as string,
|
||||
csvColumns.value.map((col) => ({ name: col.name, type: col.type })),
|
||||
uploadedFileId.value,
|
||||
hasHeaders.value,
|
||||
);
|
||||
}
|
||||
|
||||
if (newDataTable) {
|
||||
telemetry.track('User created data table', {
|
||||
data_table_id: newDataTable.id,
|
||||
data_table_project_id: newDataTable.project?.id,
|
||||
creation_mode: selectedOption.value,
|
||||
});
|
||||
reset(true);
|
||||
uiStore.closeModal(props.modalName);
|
||||
void router.push({
|
||||
name: DATA_TABLE_DETAILS,
|
||||
params: {
|
||||
id: newDataTable.id,
|
||||
},
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
toast.showError(error, i18n.baseText('dataTable.add.error'));
|
||||
}
|
||||
};
|
||||
|
||||
const onCancel = () => {
|
||||
uiStore.closeModal(props.modalName);
|
||||
redirectToDataTables();
|
||||
const goBack = () => {
|
||||
creationMode.value = 'select';
|
||||
};
|
||||
|
||||
const redirectToDataTables = () => {
|
||||
@ -69,41 +282,174 @@ const redirectToDataTables = () => {
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<Modal :name="props.modalName" :center="true" width="540px" :before-close="redirectToDataTables">
|
||||
<Modal
|
||||
:name="props.modalName"
|
||||
:center="true"
|
||||
:width="creationMode === 'import' ? '700px' : '540px'"
|
||||
:min-height="creationMode === 'import' ? '600px' : undefined"
|
||||
:before-close="redirectToDataTables"
|
||||
>
|
||||
<template #header>
|
||||
<div :class="$style.header">
|
||||
<h2>{{ i18n.baseText('dataTable.add.title') }}</h2>
|
||||
<h2>{{ modalTitle }}</h2>
|
||||
</div>
|
||||
</template>
|
||||
<template #content>
|
||||
<div :class="$style.content">
|
||||
<div v-if="creationMode === 'select'" :class="$style.selectionContent">
|
||||
<N8nInputLabel
|
||||
:label="i18n.baseText('dataTable.add.input.name.label')"
|
||||
:required="true"
|
||||
input-name="dataTableName"
|
||||
input-name="dataTableNameSelect"
|
||||
>
|
||||
<N8nInput
|
||||
ref="inputRef"
|
||||
v-model="dataTableName"
|
||||
type="text"
|
||||
:placeholder="i18n.baseText('dataTable.add.input.name.placeholder')"
|
||||
data-test-id="data-table-name-input"
|
||||
name="dataTableName"
|
||||
@keydown.enter="onSubmit"
|
||||
data-test-id="data-table-name-input-select"
|
||||
name="dataTableNameSelect"
|
||||
/>
|
||||
</N8nInputLabel>
|
||||
<ElRadioGroup v-model="selectedOption" :class="$style.radioGroup">
|
||||
<ElRadio label="scratch" data-test-id="create-from-scratch-option">
|
||||
{{ i18n.baseText('dataTable.add.fromScratch') }}
|
||||
</ElRadio>
|
||||
<ElRadio label="import" data-test-id="import-csv-option">
|
||||
{{ i18n.baseText('dataTable.add.importCsv') }}
|
||||
</ElRadio>
|
||||
</ElRadioGroup>
|
||||
|
||||
<div v-if="selectedOption === 'import'" :class="$style.uploadSection">
|
||||
<ElUpload
|
||||
:class="$style.uploadDemo"
|
||||
drag
|
||||
:auto-upload="false"
|
||||
:show-file-list="false"
|
||||
accept=".csv"
|
||||
:on-change="handleFileChange"
|
||||
@mouseenter="isUploadHovered = true"
|
||||
@mouseleave="isUploadHovered = false"
|
||||
>
|
||||
<N8nIcon
|
||||
icon="file"
|
||||
:size="24"
|
||||
:color="isUploadHovered ? 'text-dark' : 'text-light'"
|
||||
:class="$style.uploadIcon"
|
||||
/>
|
||||
<N8nText v-if="selectedFile" :color="isUploadHovered ? 'text-dark' : 'text-light'">
|
||||
{{ selectedFile?.name }}
|
||||
</N8nText>
|
||||
<N8nText v-else size="medium" :color="isUploadHovered ? 'text-dark' : 'text-light'">
|
||||
{{ i18n.baseText('dataTable.upload.dropOrClick') }}
|
||||
</N8nText>
|
||||
</ElUpload>
|
||||
|
||||
<N8nCheckbox
|
||||
v-model="hasHeaders"
|
||||
:label="i18n.baseText('dataTable.upload.hasHeaders')"
|
||||
data-test-id="has-headers-checkbox"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div v-else-if="creationMode === 'import'" :class="$style.content">
|
||||
<div v-if="isUploading" :class="$style.uploadingMessage">
|
||||
{{ i18n.baseText('dataTable.upload.uploading') }}
|
||||
</div>
|
||||
|
||||
<div v-else-if="!uploadedFileId" :class="$style.uploadingMessage">
|
||||
{{ i18n.baseText('dataTable.upload.selectFile') }}
|
||||
</div>
|
||||
|
||||
<div v-else-if="uploadedFileId && csvColumns.length > 0" :class="$style.importContent">
|
||||
<div :class="$style.successNotice">
|
||||
{{
|
||||
i18n.baseText('dataTable.upload.success', {
|
||||
adjustToNumber: csvRowCount,
|
||||
interpolate: {
|
||||
fileName: uploadedFileName,
|
||||
columnCount: csvColumnCount,
|
||||
rowCount: csvRowCount,
|
||||
},
|
||||
})
|
||||
}}
|
||||
</div>
|
||||
|
||||
<div :class="$style.columnHeaders">
|
||||
<div :class="$style.columnHeaderLabel">
|
||||
{{ i18n.baseText('dataTable.import.columnName') }}
|
||||
</div>
|
||||
<div :class="$style.columnHeaderLabel">
|
||||
{{ i18n.baseText('dataTable.import.columnType') }}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div :class="$style.columnsContainer">
|
||||
<div v-for="(column, index) in csvColumns" :key="index" :class="$style.columnItem">
|
||||
<div :class="$style.columnInputWrapper">
|
||||
<N8nInput
|
||||
v-model="column.name"
|
||||
:placeholder="i18n.baseText('dataTable.import.columnNamePlaceholder')"
|
||||
:data-test-id="`column-name-${index}`"
|
||||
:class="{ [$style.inputError]: column.error }"
|
||||
@update:model-value="onColumnNameChange(index)"
|
||||
/>
|
||||
<div v-if="column.error" :class="$style.columnErrorMessage">
|
||||
{{ column.error }}
|
||||
</div>
|
||||
</div>
|
||||
<div :class="$style.columnTypeWrapper">
|
||||
<N8nSelect
|
||||
v-model="column.type"
|
||||
:disabled="column.typeOptions.length === 1"
|
||||
:data-test-id="`column-type-${index}`"
|
||||
>
|
||||
<N8nOption
|
||||
v-for="option in column.typeOptions"
|
||||
:key="option.value"
|
||||
:value="option.value"
|
||||
:label="option.label"
|
||||
/>
|
||||
</N8nSelect>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
<template #footer>
|
||||
<div :class="$style.footer">
|
||||
<N8nButton
|
||||
v-if="creationMode === 'select'"
|
||||
type="secondary"
|
||||
size="large"
|
||||
:label="i18n.baseText('generic.cancel')"
|
||||
data-test-id="cancel-add-data-table-button"
|
||||
@click="onCancel"
|
||||
data-test-id="cancel-select-button"
|
||||
@click="redirectToDataTables"
|
||||
/>
|
||||
<N8nButton
|
||||
:disabled="!dataTableName"
|
||||
v-if="creationMode === 'select'"
|
||||
size="large"
|
||||
:disabled="
|
||||
!dataTableName || !selectedOption || (selectedOption === 'import' && !selectedFile)
|
||||
"
|
||||
:label="i18n.baseText('generic.create')"
|
||||
data-test-id="proceed-from-select-button"
|
||||
@click="proceedFromSelect"
|
||||
/>
|
||||
|
||||
<N8nButton
|
||||
v-if="creationMode === 'import'"
|
||||
type="secondary"
|
||||
size="large"
|
||||
:label="i18n.baseText('generic.back')"
|
||||
data-test-id="back-button"
|
||||
@click="goBack"
|
||||
/>
|
||||
<N8nButton
|
||||
v-if="creationMode === 'import'"
|
||||
size="large"
|
||||
:disabled="isCreateDisabled"
|
||||
:label="i18n.baseText('generic.create')"
|
||||
data-test-id="confirm-add-data-table-button"
|
||||
@click="onSubmit"
|
||||
@ -123,10 +469,181 @@ const redirectToDataTables = () => {
|
||||
flex-direction: column;
|
||||
}
|
||||
|
||||
.selectionContent {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: var(--spacing--md);
|
||||
}
|
||||
|
||||
.radioGroup {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: var(--spacing--xs);
|
||||
|
||||
:global(.el-radio) {
|
||||
height: auto;
|
||||
margin-right: 0;
|
||||
}
|
||||
|
||||
:global(.el-radio__input.is-checked .el-radio__inner) {
|
||||
background-color: var(--color--primary);
|
||||
border-color: var(--color--primary);
|
||||
}
|
||||
|
||||
:global(.el-radio__inner) {
|
||||
width: 16px;
|
||||
height: 16px;
|
||||
}
|
||||
|
||||
:global(.el-radio__input:hover .el-radio__inner) {
|
||||
border-color: var(--color--foreground);
|
||||
}
|
||||
|
||||
:global(.el-radio__input.is-checked:hover .el-radio__inner) {
|
||||
border-color: var(--color--primary);
|
||||
}
|
||||
|
||||
:global(.el-radio__label) {
|
||||
font-size: var(--font-size--sm);
|
||||
color: var(--color--text) !important;
|
||||
padding-left: var(--spacing--xs);
|
||||
}
|
||||
|
||||
:global(.el-radio.is-checked .el-radio__label) {
|
||||
color: var(--color--text) !important;
|
||||
}
|
||||
}
|
||||
|
||||
.uploadSection {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: var(--spacing--sm);
|
||||
}
|
||||
|
||||
.uploadingMessage {
|
||||
padding: var(--spacing--lg);
|
||||
text-align: center;
|
||||
color: var(--color--text--tint-1);
|
||||
}
|
||||
|
||||
.importContent {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: var(--spacing--md);
|
||||
}
|
||||
|
||||
.successNotice {
|
||||
padding: var(--spacing--sm) var(--spacing--md);
|
||||
background-color: var(--color--success--tint-4);
|
||||
border-radius: var(--radius);
|
||||
color: var(--color--success--shade-1);
|
||||
font-size: var(--font-size--sm);
|
||||
line-height: var(--line-height--lg);
|
||||
}
|
||||
|
||||
.columnHeaders {
|
||||
display: grid;
|
||||
grid-template-columns: 1fr 1fr;
|
||||
gap: var(--spacing--md);
|
||||
padding: 0 var(--spacing--2xs);
|
||||
}
|
||||
|
||||
.columnHeaderLabel {
|
||||
font-size: var(--font-size--sm);
|
||||
font-weight: var(--font-weight--regular);
|
||||
color: var(--color--text--tint-1);
|
||||
}
|
||||
|
||||
.columnsContainer {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: var(--spacing--md);
|
||||
max-height: 400px;
|
||||
overflow-y: auto;
|
||||
padding: var(--spacing--2xs);
|
||||
}
|
||||
|
||||
.columnItem {
|
||||
display: grid;
|
||||
grid-template-columns: 1fr 1fr;
|
||||
gap: var(--spacing--md);
|
||||
align-items: start;
|
||||
}
|
||||
|
||||
.columnInputWrapper {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: var(--spacing--4xs);
|
||||
}
|
||||
|
||||
.columnTypeWrapper {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
.inputError {
|
||||
border-color: var(--color--danger) !important;
|
||||
|
||||
&:focus {
|
||||
box-shadow: 0 0 0 2px var(--color--danger--tint-3) !important;
|
||||
}
|
||||
}
|
||||
|
||||
.columnErrorMessage {
|
||||
font-size: var(--font-size--3xs);
|
||||
color: var(--color--danger);
|
||||
line-height: var(--line-height--sm);
|
||||
}
|
||||
|
||||
.footer {
|
||||
display: flex;
|
||||
gap: var(--spacing--2xs);
|
||||
justify-content: flex-end;
|
||||
margin-top: var(--spacing--lg);
|
||||
}
|
||||
|
||||
.fileSelectedContainer {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: var(--spacing--lg);
|
||||
padding: var(--spacing--lg) 0;
|
||||
}
|
||||
|
||||
.uploadDemo {
|
||||
width: 100%;
|
||||
|
||||
:global(.el-upload) {
|
||||
width: 100%;
|
||||
border-radius: var(--radius--lg);
|
||||
}
|
||||
|
||||
:global(.el-upload-dragger) {
|
||||
width: 100%;
|
||||
padding: var(--spacing--2xl) var(--spacing--lg);
|
||||
border: 1px solid var(--color--foreground);
|
||||
background-color: var(--color--background-base);
|
||||
border-radius: var(--radius--lg);
|
||||
transition: all 0.2s ease;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
|
||||
&:hover {
|
||||
background-color: var(--color--background);
|
||||
}
|
||||
}
|
||||
|
||||
:global(input[type='file']) {
|
||||
display: none !important;
|
||||
}
|
||||
}
|
||||
|
||||
.uploadIcon {
|
||||
margin-bottom: var(--spacing--sm);
|
||||
}
|
||||
|
||||
.fileName {
|
||||
font-weight: var(--font-weight--regular);
|
||||
}
|
||||
</style>
|
||||
|
||||
@ -39,6 +39,8 @@ export const createDataTableApi = async (
|
||||
name: string,
|
||||
projectId: string,
|
||||
columns?: DataTableColumnCreatePayload[],
|
||||
fileId?: string,
|
||||
hasHeaders: boolean = true,
|
||||
) => {
|
||||
return await makeRestApiRequest<DataTable>(
|
||||
context,
|
||||
@ -47,6 +49,8 @@ export const createDataTableApi = async (
|
||||
{
|
||||
name,
|
||||
columns: columns ?? [],
|
||||
hasHeaders,
|
||||
...(fileId ? { fileId } : {}),
|
||||
},
|
||||
);
|
||||
};
|
||||
@ -218,3 +222,21 @@ export const fetchDataTableGlobalLimitInBytes = async (context: IRestApiContext)
|
||||
'/data-tables-global/limits',
|
||||
);
|
||||
};
|
||||
|
||||
export const uploadCsvFileApi = async (
|
||||
context: IRestApiContext,
|
||||
file: File,
|
||||
hasHeaders: boolean = true,
|
||||
) => {
|
||||
const formData = new FormData();
|
||||
formData.append('file', file);
|
||||
formData.append('hasHeaders', String(hasHeaders));
|
||||
|
||||
return await makeRestApiRequest<{
|
||||
originalName: string;
|
||||
id: string;
|
||||
rowCount: number;
|
||||
columnCount: number;
|
||||
columns: Array<{ name: string; type: string; compatibleTypes: string[] }>;
|
||||
}>(context, 'POST', '/data-tables/uploads', formData);
|
||||
};
|
||||
|
||||
@ -85,12 +85,44 @@ describe('dataTable.store', () => {
|
||||
rootStore.restApiContext,
|
||||
'New Table',
|
||||
'p1',
|
||||
undefined,
|
||||
undefined,
|
||||
true,
|
||||
);
|
||||
expect(dataTableStore.dataTables[0]).toEqual(mockTable);
|
||||
expect(dataTableStore.totalCount).toBe(1);
|
||||
expect(result).toBe(mockTable);
|
||||
});
|
||||
|
||||
it('should create data table with CSV import parameters', async () => {
|
||||
const mockTable = createTable({ id: 'dt-1', name: 'Imported Table' });
|
||||
const columns = [
|
||||
{ name: 'col1', type: 'string' as const },
|
||||
{ name: 'col2', type: 'number' as const },
|
||||
];
|
||||
const fileId = 'file123';
|
||||
vi.spyOn(dataTableApi, 'createDataTableApi').mockResolvedValue(mockTable);
|
||||
|
||||
const result = await dataTableStore.createDataTable(
|
||||
'Imported Table',
|
||||
'p1',
|
||||
columns,
|
||||
fileId,
|
||||
false,
|
||||
);
|
||||
|
||||
expect(dataTableApi.createDataTableApi).toHaveBeenCalledWith(
|
||||
rootStore.restApiContext,
|
||||
'Imported Table',
|
||||
'p1',
|
||||
columns,
|
||||
fileId,
|
||||
false,
|
||||
);
|
||||
expect(dataTableStore.dataTables[0]).toEqual(mockTable);
|
||||
expect(result).toBe(mockTable);
|
||||
});
|
||||
|
||||
it('should fetch and attach project if missing', async () => {
|
||||
const mockTable = createTable({ id: 'dt-1', name: 'Table' });
|
||||
const mockProject = {
|
||||
|
||||
@ -15,6 +15,7 @@ import {
|
||||
updateDataTableRowsApi,
|
||||
deleteDataTableRowsApi,
|
||||
fetchDataTableGlobalLimitInBytes,
|
||||
uploadCsvFileApi,
|
||||
} from '@/features/core/dataTable/dataTable.api';
|
||||
import type {
|
||||
DataTable,
|
||||
@ -69,8 +70,21 @@ export const useDataTableStore = defineStore(DATA_TABLE_STORE, () => {
|
||||
totalCount.value = response.count;
|
||||
};
|
||||
|
||||
const createDataTable = async (name: string, projectId: string) => {
|
||||
const newTable = await createDataTableApi(rootStore.restApiContext, name, projectId);
|
||||
const createDataTable = async (
|
||||
name: string,
|
||||
projectId: string,
|
||||
columns?: DataTableColumnCreatePayload[],
|
||||
fileId?: string,
|
||||
hasHeaders: boolean = true,
|
||||
) => {
|
||||
const newTable = await createDataTableApi(
|
||||
rootStore.restApiContext,
|
||||
name,
|
||||
projectId,
|
||||
columns,
|
||||
fileId,
|
||||
hasHeaders,
|
||||
);
|
||||
if (!newTable.project && projectId) {
|
||||
const project = await projectStore.fetchProject(projectId);
|
||||
if (project) {
|
||||
@ -82,6 +96,10 @@ export const useDataTableStore = defineStore(DATA_TABLE_STORE, () => {
|
||||
return newTable;
|
||||
};
|
||||
|
||||
const uploadCsvFile = async (file: File, hasHeaders: boolean = true) => {
|
||||
return await uploadCsvFileApi(rootStore.restApiContext, file, hasHeaders);
|
||||
};
|
||||
|
||||
const deleteDataTable = async (dataTableId: string, projectId: string) => {
|
||||
const deleted = await deleteDataTableApi(rootStore.restApiContext, dataTableId, projectId);
|
||||
if (deleted) {
|
||||
@ -265,6 +283,7 @@ export const useDataTableStore = defineStore(DATA_TABLE_STORE, () => {
|
||||
dataTableSizes,
|
||||
maxSizeMB,
|
||||
createDataTable,
|
||||
uploadCsvFile,
|
||||
deleteDataTable,
|
||||
updateDataTable,
|
||||
fetchDataTableDetails,
|
||||
|
||||
@ -6,7 +6,8 @@ export class DataTableComposer {
|
||||
async createNewDataTable(name: string) {
|
||||
const nameInput = this.n8n.dataTable.getNewDataTableNameInput();
|
||||
await nameInput.fill(name);
|
||||
await this.n8n.dataTable.getNewDataTableConfirmButton().click();
|
||||
await this.n8n.dataTable.getFromScratchOption().click();
|
||||
await this.n8n.dataTable.getProceedFromSelectButton().click();
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@ -25,7 +25,19 @@ export class DataTableView extends BasePage {
|
||||
}
|
||||
|
||||
getNewDataTableNameInput() {
|
||||
return this.page.getByTestId('data-table-name-input');
|
||||
return this.page.getByTestId('data-table-name-input-select');
|
||||
}
|
||||
|
||||
getFromScratchOption() {
|
||||
return this.page.getByTestId('create-from-scratch-option');
|
||||
}
|
||||
|
||||
getImportCsvOption() {
|
||||
return this.page.getByTestId('import-csv-option');
|
||||
}
|
||||
|
||||
getProceedFromSelectButton() {
|
||||
return this.page.getByTestId('proceed-from-select-button');
|
||||
}
|
||||
|
||||
getNewDataTableConfirmButton() {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user