diff --git a/.gitignore b/.gitignore index 25d9e41dd..b05fe8575 100644 --- a/.gitignore +++ b/.gitignore @@ -73,3 +73,5 @@ docs/.cache-loader .cursorrules .cursor + +next-env.d.ts diff --git a/backend/apps/cloud/src/app.module.ts b/backend/apps/cloud/src/app.module.ts index 6068b9f0d..fa8ac2128 100644 --- a/backend/apps/cloud/src/app.module.ts +++ b/backend/apps/cloud/src/app.module.ts @@ -32,8 +32,10 @@ import { AppController } from './app.controller' import { isPrimaryNode, isPrimaryClusterNode } from './common/utils' import { OrganisationModule } from './organisation/organisation.module' import { RevenueModule } from './revenue/revenue.module' +import { BullModule } from '@nestjs/bullmq' import { ToolsModule } from './tools/tools.module' import { PendingInvitationModule } from './pending-invitation/pending-invitation.module' +import { DataImportModule } from './data-import/data-import.module' const modules = [ SentryModule.forRoot(), @@ -71,6 +73,14 @@ const modules = [ I18nModule.forRootAsync(getI18nConfig()), ScheduleModule.forRoot(), NestjsFormDataModule.config({ isGlobal: true }), + BullModule.forRoot({ + connection: { + host: process.env.REDIS_HOST, + port: Number(process.env.REDIS_PORT), + password: process.env.REDIS_PASSWORD, + username: process.env.REDIS_USER, + }, + }), BlogModule, UserModule, MailerModule, @@ -92,6 +102,7 @@ const modules = [ RevenueModule, ToolsModule, PendingInvitationModule, + DataImportModule, ] @Module({ diff --git a/backend/apps/cloud/src/data-import/data-import.controller.ts b/backend/apps/cloud/src/data-import/data-import.controller.ts new file mode 100644 index 000000000..5edb0ff75 --- /dev/null +++ b/backend/apps/cloud/src/data-import/data-import.controller.ts @@ -0,0 +1,240 @@ +import * as fs from 'fs' +import * as os from 'os' +import * as path from 'path' +import { randomUUID } from 'crypto' +import { + Controller, + Post, + Get, + Delete, + Param, + ParseIntPipe, + Body, + UseGuards, + UseInterceptors, + UploadedFile, + BadRequestException, + ServiceUnavailableException, + Query, + Logger, + Headers, +} from '@nestjs/common' +import { FileInterceptor } from '@nestjs/platform-express' +import { diskStorage } from 'multer' +import { ApiBearerAuth, ApiTags, ApiConsumes, ApiBody } from '@nestjs/swagger' +import { InjectQueue } from '@nestjs/bullmq' +import { Queue } from 'bullmq' + +import { Auth } from '../auth/decorators' +import { CurrentUserId } from '../auth/decorators/current-user-id.decorator' +import { AuthenticationGuard } from '../auth/guards/authentication.guard' +import { ProjectService } from '../project/project.service' +import { DataImportService } from './data-import.service' +import { UploadImportDto } from './dto' +import { getMapper, SUPPORTED_PROVIDERS } from './mappers' +import { DataImportJobData, DATA_IMPORT_QUEUE } from './data-import.processor' + +const MAX_FILE_SIZE = 100 * 1024 * 1024 // 100 MB +const IMPORT_TMP_DIR = path.resolve(os.tmpdir(), 'swetrix-imports') + +@ApiTags('Data Import') +@UseGuards(AuthenticationGuard) +@Controller('data-import') +export class DataImportController { + private readonly logger = new Logger(DataImportController.name) + + constructor( + private readonly dataImportService: DataImportService, + private readonly projectService: ProjectService, + @InjectQueue(DATA_IMPORT_QUEUE) + private readonly importQueue: Queue, + ) {} + + @Post(':projectId/upload') + @Auth() + @ApiConsumes('multipart/form-data') + @ApiBearerAuth() + @ApiBody({ + schema: { + type: 'object', + properties: { + provider: { type: 'string', enum: [...SUPPORTED_PROVIDERS] }, + file: { type: 'string', format: 'binary' }, + }, + }, + }) + @UseInterceptors( + FileInterceptor('file', { + limits: { fileSize: MAX_FILE_SIZE }, + storage: diskStorage({ + destination: (_req, _file, cb) => { + fs.mkdirSync(IMPORT_TMP_DIR, { recursive: true }) + cb(null, IMPORT_TMP_DIR) + }, + filename: (_req, file, cb) => { + const ext = path.extname(file.originalname).toLowerCase() + cb(null, `${Date.now()}-${randomUUID()}${ext}`) + }, + }), + }), + ) + async upload( + @Param('projectId') projectId: string, + @CurrentUserId() uid: string, + @UploadedFile() file: Express.Multer.File, + @Body() dto: UploadImportDto, + ) { + const project = await this.projectService.getFullProject(projectId) + this.projectService.allowedToManage(project, uid) + + if (!file?.filename) { + throw new BadRequestException('No file uploaded') + } + + const mapper = getMapper(dto.provider) + if (!mapper) { + this.cleanupFile(file.filename) + throw new BadRequestException( + `Unsupported provider: ${dto.provider}. Supported: ${SUPPORTED_PROVIDERS.join(', ')}`, + ) + } + + const ext = path.extname(file.originalname).toLowerCase() + if (ext !== mapper.expectedFileExtension) { + this.cleanupFile(file.filename) + throw new BadRequestException( + `Invalid file type. Expected a ${mapper.expectedFileExtension} file for ${dto.provider} import.`, + ) + } + + let dataImport + try { + dataImport = await this.dataImportService.create(projectId, dto.provider) + } catch (error) { + this.cleanupFile(file.filename) + throw error + } + + try { + await this.importQueue.add( + 'process-import', + { + id: dataImport.id, + importId: dataImport.importId, + projectId, + provider: dto.provider, + fileName: file.filename, + }, + { + attempts: 1, + removeOnComplete: true, + removeOnFail: false, + }, + ) + } catch (error) { + this.logger.error(`Failed to enqueue import job: ${error.message}`) + await this.dataImportService.deleteImportRecord(dataImport.id) + this.cleanupFile(file.filename) + throw new ServiceUnavailableException( + 'Failed to start import. Please try again later.', + ) + } + + return dataImport + } + + @Get(':projectId') + @Auth() + @ApiBearerAuth() + async list( + @Param('projectId') projectId: string, + @CurrentUserId() uid: string, + ) { + const project = await this.projectService.getFullProject(projectId) + this.projectService.allowedToManage(project, uid) + return this.dataImportService.findAllByProject(projectId) + } + + @Get(':projectId/has-imported-data') + @Auth(true, true) + @ApiBearerAuth() + async hasImportedData( + @Param('projectId') projectId: string, + @CurrentUserId() uid: string | null, + @Headers() headers: { 'x-password'?: string }, + @Query('from') from: string, + @Query('to') to: string, + ) { + const project = await this.projectService.getFullProject(projectId) + this.projectService.allowedToView(project, uid, headers['x-password']) + + if (!from || !to) { + throw new BadRequestException('"from" and "to" query params are required') + } + + const hasImported = await this.dataImportService.hasImportedDataInRange( + projectId, + from, + to, + ) + + return { hasImportedData: hasImported } + } + + @Get(':projectId/:importId') + @Auth() + @ApiBearerAuth() + async getOne( + @Param('projectId') projectId: string, + @Param('importId', ParseIntPipe) importId: number, + @CurrentUserId() uid: string, + ) { + const project = await this.projectService.getFullProject(projectId) + this.projectService.allowedToManage(project, uid) + return this.dataImportService.findOne(projectId, importId) + } + + @Delete(':projectId/:importId') + @Auth() + @ApiBearerAuth() + async deleteImport( + @Param('projectId') projectId: string, + @Param('importId', ParseIntPipe) importId: number, + @CurrentUserId() uid: string, + ) { + const project = await this.projectService.getFullProject(projectId) + this.projectService.allowedToManage(project, uid) + await this.dataImportService.deleteImport(projectId, importId) + return { success: true } + } + + private cleanupFile(fileName: string): void { + try { + const resolvedPath = this.getImportFilePath(fileName) + if (!resolvedPath) { + this.logger.warn( + `Skipped cleanup for invalid import file name: ${fileName}`, + ) + return + } + + if (fs.existsSync(resolvedPath) && fs.lstatSync(resolvedPath).isFile()) { + fs.unlinkSync(resolvedPath) + } + } catch { + // + } + } + + private getImportFilePath(fileName: string): string | null { + if ( + !fileName || + fileName !== path.basename(fileName) || + /[\\/]/.test(fileName) + ) { + return null + } + + return path.join(IMPORT_TMP_DIR, fileName) + } +} diff --git a/backend/apps/cloud/src/data-import/data-import.module.ts b/backend/apps/cloud/src/data-import/data-import.module.ts new file mode 100644 index 000000000..593e1c0ee --- /dev/null +++ b/backend/apps/cloud/src/data-import/data-import.module.ts @@ -0,0 +1,21 @@ +import { Module, forwardRef } from '@nestjs/common' +import { TypeOrmModule } from '@nestjs/typeorm' +import { BullModule } from '@nestjs/bullmq' + +import { ProjectModule } from '../project/project.module' +import { DataImport } from './entity/data-import.entity' +import { DataImportService } from './data-import.service' +import { DataImportController } from './data-import.controller' +import { DataImportProcessor, DATA_IMPORT_QUEUE } from './data-import.processor' + +@Module({ + imports: [ + TypeOrmModule.forFeature([DataImport]), + BullModule.registerQueue({ name: DATA_IMPORT_QUEUE }), + forwardRef(() => ProjectModule), + ], + providers: [DataImportService, DataImportProcessor], + controllers: [DataImportController], + exports: [DataImportService], +}) +export class DataImportModule {} diff --git a/backend/apps/cloud/src/data-import/data-import.processor.ts b/backend/apps/cloud/src/data-import/data-import.processor.ts new file mode 100644 index 000000000..bb68b50fb --- /dev/null +++ b/backend/apps/cloud/src/data-import/data-import.processor.ts @@ -0,0 +1,226 @@ +import * as fs from 'fs' +import * as os from 'os' +import * as path from 'path' +import { Processor, WorkerHost } from '@nestjs/bullmq' +import { Logger } from '@nestjs/common' +import { Job } from 'bullmq' + +import { DataImportService } from './data-import.service' +import { DataImportStatus } from './entity/data-import.entity' +import { getMapper } from './mappers' +import { ImportError } from './mappers/mapper.interface' +import { clickhouse } from '../common/integrations/clickhouse' + +const CLICKHOUSE_DB = process.env.CLICKHOUSE_DATABASE || 'analytics' +const BATCH_SIZE = 5000 +const IMPORT_TMP_DIR = path.resolve(os.tmpdir(), 'swetrix-imports') + +export const DATA_IMPORT_QUEUE = 'data-import' + +export interface DataImportJobData { + id: number + importId: number + projectId: string + provider: string + fileName: string +} + +@Processor(DATA_IMPORT_QUEUE, { concurrency: 1 }) +export class DataImportProcessor extends WorkerHost { + private readonly logger = new Logger(DataImportProcessor.name) + + constructor(private readonly dataImportService: DataImportService) { + super() + } + + async process(job: Job): Promise { + const { id, importId, projectId, provider, fileName } = job.data + const filePath = this.getImportFilePath(fileName) + + try { + this.logger.log( + `Processing import ${id} (importId=${importId}) for project ${projectId} (provider: ${provider})`, + ) + + const dataImport = await this.dataImportService.findById(id) + + if (!dataImport) { + this.logger.warn(`Import record ${id} not found, skipping`) + return + } + + if (dataImport.status === DataImportStatus.COMPLETED) { + this.logger.log(`Import ${id} already completed, skipping reprocessing`) + return + } + + if (dataImport.status === DataImportStatus.PROCESSING) { + this.logger.warn( + `Import ${id} has status PROCESSING (stalled job replay), cleaning up partial data before reprocessing`, + ) + await this.dataImportService.cleanupImportedRows(projectId, importId) + } + + await this.dataImportService.markProcessing(id) + + if (!filePath) { + await this.dataImportService.markFailed( + id, + 'Invalid import file reference. Please upload the file again.', + ) + return + } + + const mapper = getMapper(provider) + if (!mapper) { + await this.dataImportService.markFailed( + id, + `Unsupported provider: ${provider}`, + ) + return + } + + let importedRows = 0 + let totalRows = 0 + let minDate: string | null = null + let maxDate: string | null = null + + const analyticsBatch: Record[] = [] + const customEVBatch: Record[] = [] + + try { + for await (const row of mapper.createRowStream( + filePath, + projectId, + importId, + )) { + totalRows++ + + const created = row.data.created as string + if (created) { + if (!minDate || created < minDate) minDate = created + if (!maxDate || created > maxDate) maxDate = created + } + + if (row.table === 'analytics') { + analyticsBatch.push(row.data) + } else { + customEVBatch.push(row.data) + } + + if (analyticsBatch.length >= BATCH_SIZE) { + await this.flushBatch('analytics', analyticsBatch) + importedRows += analyticsBatch.length + analyticsBatch.length = 0 + } + + if (customEVBatch.length >= BATCH_SIZE) { + await this.flushBatch('customEV', customEVBatch) + importedRows += customEVBatch.length + customEVBatch.length = 0 + } + + if (totalRows % 10000 === 0) { + const progress = { + importedRows: + importedRows + analyticsBatch.length + customEVBatch.length, + totalRows, + } + + try { + await job.updateProgress(progress) + } catch (progressError) { + const errorMessage = + progressError instanceof Error + ? progressError.message + : String(progressError) + this.logger.error( + `Failed to update progress for import ${id} (importId=${importId}, importedRows=${progress.importedRows}, totalRows=${progress.totalRows}): ${errorMessage}`, + ) + } + } + } + + if (analyticsBatch.length > 0) { + await this.flushBatch('analytics', analyticsBatch) + importedRows += analyticsBatch.length + } + + if (customEVBatch.length > 0) { + await this.flushBatch('customEV', customEVBatch) + importedRows += customEVBatch.length + } + + await this.dataImportService.markCompleted(id, { + importedRows, + invalidRows: 0, + totalRows: importedRows, + dateFrom: minDate ? new Date(minDate) : null, + dateTo: maxDate ? new Date(maxDate) : null, + }) + + this.logger.log(`Import ${id} completed: ${importedRows} rows imported`) + } catch (error) { + this.logger.error(`Import ${id} failed: ${error.message}`, error.stack) + + try { + await this.dataImportService.cleanupImportedRows(projectId, importId) + } catch (cleanupError) { + this.logger.error( + `Failed to rollback partial import ${id}: ${cleanupError.message}`, + ) + } + + const userMessage = + error instanceof ImportError + ? error.message + : 'An unexpected error occurred while processing the import. Please try again or contact support.' + + await this.dataImportService.markFailed(id, userMessage) + } + } finally { + this.cleanupFile(fileName) + } + } + + private async flushBatch( + table: 'analytics' | 'customEV', + batch: Record[], + ): Promise { + await clickhouse.insert({ + table: `${CLICKHOUSE_DB}.${table}`, + values: batch, + format: 'JSONEachRow', + }) + } + + private cleanupFile(fileName: string): void { + try { + const resolvedPath = this.getImportFilePath(fileName) + if (!resolvedPath) { + this.logger.warn( + `Skipped cleanup for invalid import file name: ${fileName}`, + ) + return + } + + if (fs.existsSync(resolvedPath) && fs.lstatSync(resolvedPath).isFile()) { + fs.unlinkSync(resolvedPath) + } + } catch { + this.logger.warn(`Failed to clean up temp file: ${fileName}`) + } + } + + private getImportFilePath(fileName: string): string | null { + if ( + !fileName || + fileName !== path.basename(fileName) || + /[\\/]/.test(fileName) + ) { + return null + } + + return path.join(IMPORT_TMP_DIR, fileName) + } +} diff --git a/backend/apps/cloud/src/data-import/data-import.service.ts b/backend/apps/cloud/src/data-import/data-import.service.ts new file mode 100644 index 000000000..63f60b59e --- /dev/null +++ b/backend/apps/cloud/src/data-import/data-import.service.ts @@ -0,0 +1,217 @@ +import { + Injectable, + BadRequestException, + NotFoundException, + Logger, +} from '@nestjs/common' +import { InjectRepository } from '@nestjs/typeorm' +import { DataSource, Repository } from 'typeorm' + +import { + DataImport, + DataImportStatus, + MAX_IMPORT_ID, +} from './entity/data-import.entity' +import { clickhouse } from '../common/integrations/clickhouse' +import { Project } from '../project/entity/project.entity' + +const CLICKHOUSE_DB = process.env.CLICKHOUSE_DATABASE || 'analytics' + +@Injectable() +export class DataImportService { + private readonly logger = new Logger(DataImportService.name) + + constructor( + @InjectRepository(DataImport) + private readonly dataImportRepository: Repository, + private readonly dataSource: DataSource, + ) {} + + private async getNextImportId( + repository: Repository, + projectId: string, + ): Promise { + const existingImportIds = await repository + .createQueryBuilder('di') + .select('di.importId', 'importId') + .where('di.projectId = :projectId', { projectId }) + .orderBy('di.importId', 'ASC') + .getRawMany<{ importId: number | string }>() + + let next = 1 + + for (const { importId } of existingImportIds) { + const currentImportId = Number(importId) + + if (currentImportId < next) { + continue + } + + if (currentImportId === next) { + next += 1 + continue + } + + break + } + + if (next > MAX_IMPORT_ID) { + throw new BadRequestException( + `This project has reached the maximum number of imports (${MAX_IMPORT_ID}). Please delete an old import before creating a new one.`, + ) + } + + return next + } + + async create(projectId: string, provider: string): Promise { + return this.dataSource.transaction(async (manager) => { + await manager.getRepository(Project).findOneOrFail({ + where: { id: projectId }, + lock: { mode: 'pessimistic_write' }, + }) + + const repository = manager.getRepository(DataImport) + const active = await repository.findOne({ + where: [ + { projectId, status: DataImportStatus.PENDING }, + { projectId, status: DataImportStatus.PROCESSING }, + ], + }) + + if (active) { + throw new BadRequestException( + 'An import is already in progress for this project. Please wait for it to complete before starting a new one.', + ) + } + + const importId = await this.getNextImportId(repository, projectId) + + const dataImport = repository.create({ + projectId, + importId, + provider, + status: DataImportStatus.PENDING, + }) + + return repository.save(dataImport) + }) + } + + async findAllByProject(projectId: string): Promise { + return this.dataImportRepository.find({ + where: { projectId }, + order: { createdAt: 'DESC' }, + }) + } + + async findById(id: number): Promise { + return this.dataImportRepository.findOneBy({ id }) + } + + async findOne(projectId: string, importId: number): Promise { + const dataImport = await this.dataImportRepository.findOne({ + where: { importId, projectId }, + }) + + if (!dataImport) { + throw new NotFoundException('Import not found') + } + + return dataImport + } + + async updateStatus( + id: number, + status: DataImportStatus, + extra?: Partial, + ): Promise { + await this.dataImportRepository.update(id, { status, ...extra }) + } + + async markProcessing(id: number): Promise { + await this.updateStatus(id, DataImportStatus.PROCESSING) + } + + async markCompleted( + id: number, + stats: { + importedRows: number + invalidRows: number + totalRows: number + dateFrom: Date | null + dateTo: Date | null + }, + ): Promise { + await this.updateStatus(id, DataImportStatus.COMPLETED, { + ...stats, + finishedAt: new Date(), + }) + } + + async markFailed(id: number, errorMessage: string): Promise { + await this.updateStatus(id, DataImportStatus.FAILED, { + errorMessage, + finishedAt: new Date(), + }) + } + + async cleanupImportedRows( + projectId: string, + importId: number, + ): Promise { + await clickhouse.command({ + query: `ALTER TABLE ${CLICKHOUSE_DB}.analytics DELETE WHERE pid = {pid:FixedString(12)} AND importID = {importID:UInt8}`, + query_params: { pid: projectId, importID: importId }, + }) + await clickhouse.command({ + query: `ALTER TABLE ${CLICKHOUSE_DB}.customEV DELETE WHERE pid = {pid:FixedString(12)} AND importID = {importID:UInt8}`, + query_params: { pid: projectId, importID: importId }, + }) + } + + async deleteImport(projectId: string, importId: number): Promise { + const dataImport = await this.findOne(projectId, importId) + + if ( + dataImport.status === DataImportStatus.PENDING || + dataImport.status === DataImportStatus.PROCESSING + ) { + throw new BadRequestException( + 'Cannot delete an import that is still in progress.', + ) + } + + try { + await this.cleanupImportedRows(projectId, dataImport.importId) + } catch (error) { + this.logger.error( + `Failed to delete imported data from ClickHouse for import ${dataImport.importId}: ${error.message}`, + ) + throw new BadRequestException( + 'Failed to remove imported data. Please try again later.', + ) + } + + await this.dataImportRepository.delete(dataImport.id) + } + + async deleteImportRecord(id: number): Promise { + await this.dataImportRepository.delete(id) + } + + async hasImportedDataInRange( + projectId: string, + from: string, + to: string, + ): Promise { + const result = await clickhouse + .query({ + query: `SELECT count() as cnt FROM ${CLICKHOUSE_DB}.analytics WHERE pid = {pid:FixedString(12)} AND importID IS NOT NULL AND created >= {from:DateTime} AND created <= {to:DateTime} LIMIT 1`, + query_params: { pid: projectId, from, to }, + }) + .then((resultSet) => resultSet.json<{ cnt: string }>()) + + return Number(result.data?.[0]?.cnt) > 0 + } +} diff --git a/backend/apps/cloud/src/data-import/dto/index.ts b/backend/apps/cloud/src/data-import/dto/index.ts new file mode 100644 index 000000000..0221d0e71 --- /dev/null +++ b/backend/apps/cloud/src/data-import/dto/index.ts @@ -0,0 +1 @@ +export { UploadImportDto } from './upload-import.dto' diff --git a/backend/apps/cloud/src/data-import/dto/upload-import.dto.ts b/backend/apps/cloud/src/data-import/dto/upload-import.dto.ts new file mode 100644 index 000000000..25413d40e --- /dev/null +++ b/backend/apps/cloud/src/data-import/dto/upload-import.dto.ts @@ -0,0 +1,15 @@ +import { IsString, IsIn } from 'class-validator' +import { ApiProperty } from '@nestjs/swagger' + +import { SUPPORTED_PROVIDERS } from '../mappers' + +export class UploadImportDto { + @ApiProperty({ + description: 'Analytics provider to import from', + enum: SUPPORTED_PROVIDERS, + example: 'umami', + }) + @IsString() + @IsIn(SUPPORTED_PROVIDERS) + provider: (typeof SUPPORTED_PROVIDERS)[number] +} diff --git a/backend/apps/cloud/src/data-import/entity/data-import.entity.ts b/backend/apps/cloud/src/data-import/entity/data-import.entity.ts new file mode 100644 index 000000000..a6c362fb7 --- /dev/null +++ b/backend/apps/cloud/src/data-import/entity/data-import.entity.ts @@ -0,0 +1,69 @@ +import { + Entity, + Column, + PrimaryGeneratedColumn, + ManyToOne, + JoinColumn, + CreateDateColumn, +} from 'typeorm' + +import { Project } from '../../project/entity/project.entity' + +export enum DataImportStatus { + PENDING = 'pending', + PROCESSING = 'processing', + COMPLETED = 'completed', + FAILED = 'failed', +} + +export const MAX_IMPORT_ID = 255 + +@Entity('data_import') +export class DataImport { + @PrimaryGeneratedColumn() + id: number + + @Column('tinyint', { unsigned: true }) + importId: number + + @Column('varchar', { length: 12 }) + projectId: string + + @ManyToOne(() => Project, { onDelete: 'CASCADE' }) + @JoinColumn({ name: 'projectId' }) + project: Project + + @Column('varchar', { length: 30 }) + provider: string + + @Column({ + type: 'enum', + enum: DataImportStatus, + default: DataImportStatus.PENDING, + }) + status: DataImportStatus + + @Column({ type: 'date', nullable: true }) + dateFrom: Date | null + + @Column({ type: 'date', nullable: true }) + dateTo: Date | null + + @Column({ default: 0 }) + totalRows: number + + @Column({ default: 0 }) + importedRows: number + + @Column({ default: 0 }) + invalidRows: number + + @Column('text', { nullable: true }) + errorMessage: string | null + + @CreateDateColumn() + createdAt: Date + + @Column({ type: 'datetime', nullable: true }) + finishedAt: Date | null +} diff --git a/backend/apps/cloud/src/data-import/mappers/index.ts b/backend/apps/cloud/src/data-import/mappers/index.ts new file mode 100644 index 000000000..5302f3e7c --- /dev/null +++ b/backend/apps/cloud/src/data-import/mappers/index.ts @@ -0,0 +1,12 @@ +import { ImportMapper } from './mapper.interface' +import { UmamiMapper } from './umami.mapper' + +export const SUPPORTED_PROVIDERS = ['umami'] as const + +const mappers: Record = { + umami: new UmamiMapper(), +} + +export function getMapper(provider: string): ImportMapper | undefined { + return mappers[provider] +} diff --git a/backend/apps/cloud/src/data-import/mappers/mapper.interface.ts b/backend/apps/cloud/src/data-import/mappers/mapper.interface.ts new file mode 100644 index 000000000..b7bdc9bf0 --- /dev/null +++ b/backend/apps/cloud/src/data-import/mappers/mapper.interface.ts @@ -0,0 +1,22 @@ +export class ImportError extends Error { + constructor(message: string) { + super(message) + this.name = 'ImportError' + } +} + +export interface AnalyticsImportRow { + table: 'analytics' | 'customEV' + data: Record +} + +export interface ImportMapper { + readonly provider: string + readonly expectedFileExtension: string + + createRowStream( + filePath: string, + pid: string, + importID: number, + ): AsyncIterable +} diff --git a/backend/apps/cloud/src/data-import/mappers/umami.mapper.ts b/backend/apps/cloud/src/data-import/mappers/umami.mapper.ts new file mode 100644 index 000000000..038ce257f --- /dev/null +++ b/backend/apps/cloud/src/data-import/mappers/umami.mapper.ts @@ -0,0 +1,408 @@ +import * as fs from 'fs' +import * as path from 'path' +import * as os from 'os' +import * as crypto from 'crypto' +import { Unzip, UnzipInflate } from 'fflate' +import { parse } from 'csv-parse' + +import { + ImportMapper, + ImportError, + AnalyticsImportRow, +} from './mapper.interface' + +const WEBSITE_EVENT_CSV = 'website_event.csv' +const MAX_UMAMI_CSV_BYTES = 512 * 1024 * 1024 + +const BROWSER_MAP: Record = { + chrome: 'Chrome', + opera: 'Opera', + crios: 'Mobile Chrome', + firefox: 'Firefox', + facebook: 'Facebook', + safari: 'Safari', + ios: 'Mobile Safari', + 'ios-webview': 'Mobile Safari', + 'edge-chromium': 'Edge', + samsung: 'Samsung Internet', + yandexbrowser: 'Yandex', + 'edge-ios': 'Edge', + 'chromium-webview': 'Chrome WebView', + fxios: 'Mobile Firefox', + edge: 'Edge', +} + +const MOBILE_BROWSER_VARIANTS: Record = { + Chrome: 'Mobile Chrome', + Firefox: 'Mobile Firefox', + Safari: 'Mobile Safari', +} + +const OS_MAP: Record = { + 'mac os': 'macOS', + 'windows 10': 'Windows', + 'windows 7': 'Windows', + 'windows 8': 'Windows', + 'windows 8.1': 'Windows', + 'windows vista': 'Windows', + 'windows xp': 'Windows', + 'windows server 2003': 'Windows', + 'android os': 'Android', + 'chrome os': 'Chrome OS', + 'chromium os': 'Chrome OS', +} + +const DEVICE_MAP: Record = { + laptop: 'desktop', + desktop: 'desktop', + mobile: 'mobile', + tablet: 'tablet', +} + +function normalizeNull(value: string | undefined): string | null { + if (!value || value === '' || value === '\\N') return null + return value +} + +function truncate(value: string | null, maxLen: number): string | null { + if (!value) return null + return value.length > maxLen ? value.slice(0, maxLen) : value +} + +function mapBrowser(raw: string | null, device: string | null): string | null { + if (!raw) return null + + const key = raw.toLowerCase() + let mapped = BROWSER_MAP[key] + + if (!mapped) { + mapped = raw.charAt(0).toUpperCase() + raw.slice(1) + } + + const isMobile = device === 'mobile' || device === 'tablet' + if (isMobile && MOBILE_BROWSER_VARIANTS[mapped]) { + return MOBILE_BROWSER_VARIANTS[mapped] + } + + return mapped +} + +function mapOS(raw: string | null): string | null { + if (!raw) return null + const key = raw.toLowerCase() + return OS_MAP[key] ?? raw +} + +function mapDevice(raw: string | null): string { + if (!raw) return 'desktop' + const key = raw.toLowerCase() + return DEVICE_MAP[key] ?? 'desktop' +} + +function buildReferrer( + domain: string | null, + path: string | null, + query: string | null, + hostname: string | null, +): string | null { + if (!domain) return null + + let ref = `https://${domain}` + if (path) ref += path + if (query) ref += query.startsWith('?') ? query : `?${query}` + + if (hostname) { + const cleanHost = hostname.replace(/^www\./, '') + const cleanDomain = domain.replace(/^www\./, '') + if (cleanDomain === cleanHost) return null + } + + return ref +} + +const DATE_REGEX = + /^\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01]) ([01]\d|2[0-3]):([0-5]\d):([0-5]\d)$/ + +function isValidTimestamp(ts: string): boolean { + return DATE_REGEX.test(ts) +} + +function sessionIdToPsid(sessionId: string): string { + return crypto + .createHash('sha256') + .update(sessionId) + .digest() + .readBigUInt64BE(0) + .toString() +} + +export class UmamiMapper implements ImportMapper { + readonly provider = 'umami' + readonly expectedFileExtension = '.zip' + + private async extractCsvFromZip(zipPath: string): Promise { + const tempCsv = path.join(os.tmpdir(), `swetrix-umami-${Date.now()}.csv`) + + return new Promise((resolve, reject) => { + const zipStream = fs.createReadStream(zipPath) + const unzipper = new Unzip() + + unzipper.register(UnzipInflate) + + let settled = false + let foundEntry = false + let fileDescriptor: number | null = null + let tempFileCreated = false + let extractedBytes = 0 + let terminateEntry: (() => void) | null = null + + const cleanupTempCsv = () => { + if (fileDescriptor !== null) { + try { + fs.closeSync(fileDescriptor) + } catch { + // + } + fileDescriptor = null + } + + if (tempFileCreated) { + try { + fs.unlinkSync(tempCsv) + } catch { + // + } + } + } + + const fail = (error: Error) => { + if (settled) return + settled = true + terminateEntry?.() + zipStream.destroy() + cleanupTempCsv() + reject(error) + } + + const succeed = () => { + if (settled) return + settled = true + if (fileDescriptor !== null) { + try { + fs.closeSync(fileDescriptor) + } catch (error) { + cleanupTempCsv() + reject(error) + return + } + fileDescriptor = null + } + zipStream.destroy() + resolve(tempCsv) + } + + unzipper.onfile = (file) => { + if (path.basename(file.name) !== WEBSITE_EVENT_CSV || foundEntry) return + + foundEntry = true + + if ( + typeof file.originalSize === 'number' && + Number.isFinite(file.originalSize) && + file.originalSize > MAX_UMAMI_CSV_BYTES + ) { + fail( + new ImportError( + `${WEBSITE_EVENT_CSV} exceeds the ${MAX_UMAMI_CSV_BYTES} byte limit.`, + ), + ) + return + } + + try { + fileDescriptor = fs.openSync(tempCsv, 'wx') + tempFileCreated = true + } catch (error) { + fail(error as Error) + return + } + + terminateEntry = file.terminate + file.ondata = (error, chunk, final) => { + if (error) { + fail(error) + return + } + + extractedBytes += chunk.length + if (extractedBytes > MAX_UMAMI_CSV_BYTES) { + fail( + new ImportError( + `${WEBSITE_EVENT_CSV} exceeds the ${MAX_UMAMI_CSV_BYTES} byte limit.`, + ), + ) + return + } + + try { + fs.writeSync(fileDescriptor!, chunk) + } catch (writeError) { + fail(writeError as Error) + return + } + + if (final) { + succeed() + } + } + + try { + file.start() + } catch (error) { + fail(error as Error) + } + } + + zipStream.on('data', (chunk: Buffer) => { + if (settled) return + + try { + unzipper.push( + new Uint8Array(chunk.buffer, chunk.byteOffset, chunk.byteLength), + ) + } catch (error) { + fail(error as Error) + } + }) + + zipStream.on('end', () => { + if (settled) return + + try { + unzipper.push(new Uint8Array(0), true) + } catch (error) { + fail(error as Error) + return + } + + if (!foundEntry) { + fail( + new ImportError( + `ZIP does not contain ${WEBSITE_EVENT_CSV}. Please upload the export ZIP from Umami.`, + ), + ) + } + }) + + zipStream.on('error', (error) => { + fail(error) + }) + }) + } + + async *createRowStream( + filePath: string, + pid: string, + importID: number, + ): AsyncIterable { + const csvPath = await this.extractCsvFromZip(filePath) + + let cleanedUp = false + const cleanup = () => { + if (!cleanedUp) { + cleanedUp = true + try { + fs.unlinkSync(csvPath) + } catch { + // + } + } + } + + try { + const parser = fs.createReadStream(csvPath).pipe( + parse({ + columns: true, + skip_empty_lines: true, + trim: true, + relax_quotes: true, + relax_column_count: true, + }), + ) + + for await (const row of parser) { + const sessionId = normalizeNull(row.session_id) + if (!sessionId) continue + + const createdAt = normalizeNull(row.created_at) + if (!createdAt || !isValidTimestamp(createdAt)) continue + + const eventType = normalizeNull(row.event_type) + if (!eventType || (eventType !== '1' && eventType !== '2')) continue + + const psid = sessionIdToPsid(sessionId) + + const rawDevice = normalizeNull(row.device) + const device = mapDevice(rawDevice) + const browser = mapBrowser(normalizeNull(row.browser), rawDevice) + const os = mapOS(normalizeNull(row.os)) + + const hostname = truncate(normalizeNull(row.hostname), 253) + const referrerDomain = normalizeNull(row.referrer_domain) + const referrerPath = normalizeNull(row.referrer_path) + const referrerQuery = normalizeNull(row.referrer_query) + const ref = buildReferrer( + referrerDomain, + referrerPath, + referrerQuery, + hostname, + ) + + const country = normalizeNull(row.country) + const cc = country && /^[A-Z]{2}$/.test(country) ? country : null + + const region = normalizeNull(row.region) + + const baseData: Record = { + psid, + profileId: null, + pid, + host: hostname, + pg: truncate(normalizeNull(row.url_path), 2048), + dv: device, + br: truncate(browser, 30), + brv: null, + os: truncate(os, 25), + osv: null, + lc: truncate(normalizeNull(row.language), 35), + ref: truncate(ref, 2048), + so: truncate(normalizeNull(row.utm_source), 256), + me: truncate(normalizeNull(row.utm_medium), 256), + ca: truncate(normalizeNull(row.utm_campaign), 256), + te: truncate(normalizeNull(row.utm_term), 256), + co: truncate(normalizeNull(row.utm_content), 256), + cc, + rg: null, + rgc: region && region.length <= 10 ? region : null, + ct: truncate(normalizeNull(row.city), 60), + 'meta.key': [], + 'meta.value': [], + importID, + created: createdAt, + } + + if (eventType === '1') { + yield { table: 'analytics', data: baseData } + } else { + const eventName = truncate(normalizeNull(row.event_name), 256) || '' + yield { + table: 'customEV', + data: { ...baseData, ev: eventName }, + } + } + } + } finally { + cleanup() + } + } +} diff --git a/backend/apps/community/src/app.module.ts b/backend/apps/community/src/app.module.ts index 73481f249..15b07ab64 100644 --- a/backend/apps/community/src/app.module.ts +++ b/backend/apps/community/src/app.module.ts @@ -16,9 +16,11 @@ import { CaptchaModule } from './captcha/captcha.module' import { getI18nConfig } from './configs' import { AuthModule } from './auth/auth.module' import { TwoFactorAuthModule } from './twoFactorAuth/twoFactorAuth.module' +import { BullModule } from '@nestjs/bullmq' import { AppController } from './app.controller' import { isPrimaryNode } from './common/utils' import { MailerModule } from './mailer/mailer.module' +import { DataImportModule } from './data-import/data-import.module' const modules = [ ConfigModule.forRoot({ @@ -63,6 +65,14 @@ const modules = [ I18nModule.forRootAsync(getI18nConfig()), ScheduleModule.forRoot(), NestjsFormDataModule.config({ isGlobal: true }), + BullModule.forRoot({ + connection: { + host: process.env.REDIS_HOST, + port: Number(process.env.REDIS_PORT), + password: process.env.REDIS_PASSWORD, + username: process.env.REDIS_USER, + }, + }), UserModule, MailerModule, ProjectModule, @@ -73,6 +83,7 @@ const modules = [ CaptchaModule, AuthModule, TwoFactorAuthModule, + DataImportModule, ] @Module({ diff --git a/backend/apps/community/src/data-import/data-import.controller.ts b/backend/apps/community/src/data-import/data-import.controller.ts new file mode 100644 index 000000000..ed7b259d1 --- /dev/null +++ b/backend/apps/community/src/data-import/data-import.controller.ts @@ -0,0 +1,229 @@ +import * as fs from 'fs' +import * as os from 'os' +import * as path from 'path' +import { randomUUID } from 'crypto' +import { + Controller, + Post, + Get, + Delete, + Param, + ParseIntPipe, + Body, + UseInterceptors, + UploadedFile, + BadRequestException, + ServiceUnavailableException, + Query, + Logger, +} from '@nestjs/common' +import { FileInterceptor } from '@nestjs/platform-express' +import { diskStorage } from 'multer' +import { ApiTags, ApiConsumes, ApiBody } from '@nestjs/swagger' +import { InjectQueue } from '@nestjs/bullmq' +import { Queue } from 'bullmq' + +import { CurrentUserId } from '../auth/decorators/current-user-id.decorator' +import { Auth } from '../auth/decorators' +import { ProjectService } from '../project/project.service' +import { DataImportService } from './data-import.service' +import { UploadImportDto } from './dto' +import { getMapper, SUPPORTED_PROVIDERS } from './mappers' +import { DataImportJobData, DATA_IMPORT_QUEUE } from './data-import.processor' + +const MAX_FILE_SIZE = 100 * 1024 * 1024 // 100 MB +const IMPORT_TMP_DIR = path.resolve(os.tmpdir(), 'swetrix-imports') + +@ApiTags('Data Import') +@Controller('data-import') +export class DataImportController { + private readonly logger = new Logger(DataImportController.name) + + constructor( + private readonly dataImportService: DataImportService, + private readonly projectService: ProjectService, + @InjectQueue(DATA_IMPORT_QUEUE) + private readonly importQueue: Queue, + ) {} + + @Post(':projectId/upload') + @Auth() + @ApiConsumes('multipart/form-data') + @ApiBody({ + schema: { + type: 'object', + properties: { + provider: { type: 'string', enum: [...SUPPORTED_PROVIDERS] }, + file: { type: 'string', format: 'binary' }, + }, + }, + }) + @UseInterceptors( + FileInterceptor('file', { + limits: { fileSize: MAX_FILE_SIZE }, + storage: diskStorage({ + destination: (_req, _file, cb) => { + fs.mkdirSync(IMPORT_TMP_DIR, { recursive: true }) + cb(null, IMPORT_TMP_DIR) + }, + filename: (_req, file, cb) => { + const ext = path.extname(file.originalname).toLowerCase() + cb(null, `${Date.now()}-${randomUUID()}${ext}`) + }, + }), + }), + ) + async upload( + @Param('projectId') projectId: string, + @CurrentUserId() uid: string, + @UploadedFile() file: Express.Multer.File, + @Body() dto: UploadImportDto, + ) { + const project = await this.projectService.getFullProject(projectId) + this.projectService.allowedToManage(project, uid) + + if (!file?.filename) { + throw new BadRequestException('No file uploaded') + } + + const mapper = getMapper(dto.provider) + if (!mapper) { + this.cleanupFile(file.filename) + throw new BadRequestException( + `Unsupported provider: ${dto.provider}. Supported: ${SUPPORTED_PROVIDERS.join(', ')}`, + ) + } + + const ext = path.extname(file.originalname).toLowerCase() + if (ext !== mapper.expectedFileExtension) { + this.cleanupFile(file.filename) + throw new BadRequestException( + `Invalid file type. Expected a ${mapper.expectedFileExtension} file for ${dto.provider} import.`, + ) + } + + let dataImport + try { + dataImport = await this.dataImportService.create(projectId, dto.provider) + } catch (error) { + this.cleanupFile(file.filename) + throw error + } + + try { + await this.importQueue.add( + 'process-import', + { + importId: dataImport.id, + projectId, + provider: dto.provider, + fileName: file.filename, + }, + { + attempts: 1, + removeOnComplete: true, + removeOnFail: false, + }, + ) + } catch (error) { + this.logger.error(`Failed to enqueue import job: ${error.message}`) + await this.dataImportService.deleteImportRecord(projectId, dataImport.id) + this.cleanupFile(file.filename) + throw new ServiceUnavailableException( + 'Failed to start import. Please try again later.', + ) + } + + return dataImport + } + + @Get(':projectId') + @Auth() + async list( + @Param('projectId') projectId: string, + @CurrentUserId() uid: string, + ) { + const project = await this.projectService.getFullProject(projectId) + this.projectService.allowedToManage(project, uid) + return this.dataImportService.findAllByProject(projectId) + } + + @Get(':projectId/has-imported-data') + @Auth() + async hasImportedData( + @Param('projectId') projectId: string, + @CurrentUserId() uid: string, + @Query('from') from: string, + @Query('to') to: string, + ) { + const project = await this.projectService.getFullProject(projectId) + this.projectService.allowedToView(project, uid) + + if (!from || !to) { + throw new BadRequestException('"from" and "to" query params are required') + } + + const hasImported = await this.dataImportService.hasImportedDataInRange( + projectId, + from, + to, + ) + + return { hasImportedData: hasImported } + } + + @Get(':projectId/:importId') + @Auth() + async getOne( + @Param('projectId') projectId: string, + @Param('importId', ParseIntPipe) importId: number, + @CurrentUserId() uid: string, + ) { + const project = await this.projectService.getFullProject(projectId) + this.projectService.allowedToManage(project, uid) + return this.dataImportService.findOne(projectId, importId) + } + + @Delete(':projectId/:importId') + @Auth() + async deleteImport( + @Param('projectId') projectId: string, + @Param('importId', ParseIntPipe) importId: number, + @CurrentUserId() uid: string, + ) { + const project = await this.projectService.getFullProject(projectId) + this.projectService.allowedToManage(project, uid) + await this.dataImportService.deleteImport(projectId, importId) + return { success: true } + } + + private cleanupFile(fileName: string): void { + try { + const resolvedPath = this.getImportFilePath(fileName) + if (!resolvedPath) { + this.logger.warn( + `Skipped cleanup for invalid import file name: ${fileName}`, + ) + return + } + + if (fs.existsSync(resolvedPath) && fs.lstatSync(resolvedPath).isFile()) { + fs.unlinkSync(resolvedPath) + } + } catch { + // + } + } + + private getImportFilePath(fileName: string): string | null { + if ( + !fileName || + fileName !== path.basename(fileName) || + /[\\/]/.test(fileName) + ) { + return null + } + + return path.join(IMPORT_TMP_DIR, fileName) + } +} diff --git a/backend/apps/community/src/data-import/data-import.module.ts b/backend/apps/community/src/data-import/data-import.module.ts new file mode 100644 index 000000000..90b6b48ee --- /dev/null +++ b/backend/apps/community/src/data-import/data-import.module.ts @@ -0,0 +1,18 @@ +import { Module, forwardRef } from '@nestjs/common' +import { BullModule } from '@nestjs/bullmq' + +import { ProjectModule } from '../project/project.module' +import { DataImportService } from './data-import.service' +import { DataImportController } from './data-import.controller' +import { DataImportProcessor, DATA_IMPORT_QUEUE } from './data-import.processor' + +@Module({ + imports: [ + BullModule.registerQueue({ name: DATA_IMPORT_QUEUE }), + forwardRef(() => ProjectModule), + ], + providers: [DataImportService, DataImportProcessor], + controllers: [DataImportController], + exports: [DataImportService], +}) +export class DataImportModule {} diff --git a/backend/apps/community/src/data-import/data-import.processor.ts b/backend/apps/community/src/data-import/data-import.processor.ts new file mode 100644 index 000000000..0d0646d1b --- /dev/null +++ b/backend/apps/community/src/data-import/data-import.processor.ts @@ -0,0 +1,230 @@ +import * as fs from 'fs' +import * as os from 'os' +import * as path from 'path' +import { Processor, WorkerHost } from '@nestjs/bullmq' +import { Logger } from '@nestjs/common' +import { Job } from 'bullmq' + +import { DataImportService } from './data-import.service' +import { DataImportStatus } from './entity/data-import.entity' +import { getMapper } from './mappers' +import { clickhouse } from '../common/integrations/clickhouse' + +const CLICKHOUSE_DB = process.env.CLICKHOUSE_DATABASE || 'analytics' +const BATCH_SIZE = 5000 +const IMPORT_TMP_DIR = path.resolve(os.tmpdir(), 'swetrix-imports') + +export const DATA_IMPORT_QUEUE = 'data-import' + +export interface DataImportJobData { + importId: number + projectId: string + provider: string + fileName: string +} + +@Processor(DATA_IMPORT_QUEUE, { concurrency: 1 }) +export class DataImportProcessor extends WorkerHost { + private readonly logger = new Logger(DataImportProcessor.name) + + constructor(private readonly dataImportService: DataImportService) { + super() + } + + async process(job: Job): Promise { + const { importId, projectId, provider, fileName } = job.data + const filePath = this.getImportFilePath(fileName) + + try { + this.logger.log( + `Processing import ${importId} for project ${projectId} (provider: ${provider})`, + ) + + const dataImport = await this.dataImportService.findOne( + projectId, + importId, + ) + + if (dataImport.status === DataImportStatus.COMPLETED) { + this.logger.log( + `Import ${importId} already completed, skipping reprocessing`, + ) + return + } + + if (dataImport.status === DataImportStatus.PROCESSING) { + this.logger.warn( + `Import ${importId} has status PROCESSING (stalled job replay), cleaning up partial data before reprocessing`, + ) + await this.dataImportService.cleanupImportedRows(projectId, importId) + } + + await this.dataImportService.markProcessing(importId, projectId) + + if (!filePath) { + await this.dataImportService.markFailed( + importId, + projectId, + 'Invalid import file reference. Please upload the file again.', + ) + return + } + + const mapper = getMapper(provider) + if (!mapper) { + await this.dataImportService.markFailed( + importId, + projectId, + `Unsupported provider: ${provider}`, + ) + return + } + + let importedRows = 0 + let totalRows = 0 + let minDate: string | null = null + let maxDate: string | null = null + + const analyticsBatch: Record[] = [] + const customEVBatch: Record[] = [] + + try { + for await (const row of mapper.createRowStream( + filePath, + projectId, + importId, + )) { + totalRows++ + + const created = row.data.created as string + if (created) { + if (!minDate || created < minDate) minDate = created + if (!maxDate || created > maxDate) maxDate = created + } + + if (row.table === 'analytics') { + analyticsBatch.push(row.data) + } else { + customEVBatch.push(row.data) + } + + if (analyticsBatch.length >= BATCH_SIZE) { + await this.flushBatch('analytics', analyticsBatch) + importedRows += analyticsBatch.length + analyticsBatch.length = 0 + } + + if (customEVBatch.length >= BATCH_SIZE) { + await this.flushBatch('customEV', customEVBatch) + importedRows += customEVBatch.length + customEVBatch.length = 0 + } + + if (totalRows % 10000 === 0) { + const progress = { + importedRows: + importedRows + analyticsBatch.length + customEVBatch.length, + totalRows, + } + + try { + await job.updateProgress(progress) + } catch (progressError) { + const errorMessage = + progressError instanceof Error + ? progressError.message + : String(progressError) + this.logger.error( + `Failed to update progress for import ${importId} (projectId=${projectId}, importedRows=${progress.importedRows}, totalRows=${progress.totalRows}): ${errorMessage}`, + ) + } + } + } + + if (analyticsBatch.length > 0) { + await this.flushBatch('analytics', analyticsBatch) + importedRows += analyticsBatch.length + } + + if (customEVBatch.length > 0) { + await this.flushBatch('customEV', customEVBatch) + importedRows += customEVBatch.length + } + + await this.dataImportService.markCompleted(importId, projectId, { + importedRows, + invalidRows: 0, + totalRows: importedRows, + dateFrom: minDate ? minDate.split(' ')[0] : null, + dateTo: maxDate ? maxDate.split(' ')[0] : null, + }) + + this.logger.log( + `Import ${importId} completed: ${importedRows} rows imported`, + ) + } catch (error) { + this.logger.error( + `Import ${importId} failed: ${error.message}`, + error.stack, + ) + + try { + await this.dataImportService.cleanupImportedRows(projectId, importId) + } catch (cleanupError) { + this.logger.error( + `Failed to rollback partial import ${importId}: ${cleanupError.message}`, + ) + } + + await this.dataImportService.markFailed( + importId, + projectId, + error.message, + ) + } + } finally { + this.cleanupFile(fileName) + } + } + + private async flushBatch( + table: 'analytics' | 'customEV', + batch: Record[], + ): Promise { + await clickhouse.insert({ + table: `${CLICKHOUSE_DB}.${table}`, + values: batch, + format: 'JSONEachRow', + }) + } + + private cleanupFile(fileName: string): void { + try { + const resolvedPath = this.getImportFilePath(fileName) + if (!resolvedPath) { + this.logger.warn( + `Skipped cleanup for invalid import file name: ${fileName}`, + ) + return + } + + if (fs.existsSync(resolvedPath) && fs.lstatSync(resolvedPath).isFile()) { + fs.unlinkSync(resolvedPath) + } + } catch { + this.logger.warn(`Failed to clean up temp file: ${fileName}`) + } + } + + private getImportFilePath(fileName: string): string | null { + if ( + !fileName || + fileName !== path.basename(fileName) || + /[\\/]/.test(fileName) + ) { + return null + } + + return path.join(IMPORT_TMP_DIR, fileName) + } +} diff --git a/backend/apps/community/src/data-import/data-import.service.ts b/backend/apps/community/src/data-import/data-import.service.ts new file mode 100644 index 000000000..ac9be0c50 --- /dev/null +++ b/backend/apps/community/src/data-import/data-import.service.ts @@ -0,0 +1,340 @@ +import { + Injectable, + BadRequestException, + NotFoundException, + Logger, +} from '@nestjs/common' +import { randomUUID } from 'crypto' + +import { + DataImport, + DataImportStatus, + MAX_IMPORT_ID, +} from './entity/data-import.entity' +import { clickhouse } from '../common/integrations/clickhouse' +import { redis } from '../common/constants' + +const CLICKHOUSE_DB = process.env.CLICKHOUSE_DATABASE || 'analytics' +const IMPORT_IN_PROGRESS_ERROR = + 'An import is already in progress for this project. Please wait for it to complete before starting a new one.' +const IMPORT_CREATE_LOCK_TTL_MS = 10_000 +const IMPORT_CREATE_LOCK_WAIT_MS = 5_000 +const IMPORT_CREATE_LOCK_RETRY_MS = 100 + +@Injectable() +export class DataImportService { + private readonly logger = new Logger(DataImportService.name) + + private getCreateLockKey(projectId: string): string { + return `data-import:create:${projectId}` + } + + private async acquireCreateLock(projectId: string): Promise { + const token = randomUUID() + const lockKey = this.getCreateLockKey(projectId) + const deadline = Date.now() + IMPORT_CREATE_LOCK_WAIT_MS + + while (Date.now() < deadline) { + const acquired = await redis.set( + lockKey, + token, + 'PX', + IMPORT_CREATE_LOCK_TTL_MS, + 'NX', + ) + + if (acquired === 'OK') { + return token + } + + await new Promise((resolve) => + setTimeout(resolve, IMPORT_CREATE_LOCK_RETRY_MS), + ) + } + + throw new BadRequestException(IMPORT_IN_PROGRESS_ERROR) + } + + private async releaseCreateLock( + projectId: string, + token: string, + ): Promise { + await redis.eval( + `if redis.call("GET", KEYS[1]) == ARGV[1] then + return redis.call("DEL", KEYS[1]) + end + return 0`, + 1, + this.getCreateLockKey(projectId), + token, + ) + } + + private async getNextId(projectId: string): Promise { + const result = await clickhouse + .query({ + query: `SELECT id FROM ${CLICKHOUSE_DB}.data_import FINAL WHERE projectId = {projectId:FixedString(12)} ORDER BY id ASC`, + query_params: { projectId }, + }) + .then((rs) => rs.json<{ id: number | string }>()) + + let next = 1 + + for (const row of result.data || []) { + const currentId = Number(row.id) + + if (currentId < next) { + continue + } + + if (currentId === next) { + next += 1 + continue + } + + break + } + + if (next > MAX_IMPORT_ID) { + throw new BadRequestException( + `This project has reached the maximum number of imports (${MAX_IMPORT_ID}). Please delete an old import before creating a new one.`, + ) + } + + return next + } + + async create(projectId: string, provider: string): Promise { + const lockToken = await this.acquireCreateLock(projectId) + + try { + const active = await this.findActiveByProject(projectId) + if (active) { + throw new BadRequestException(IMPORT_IN_PROGRESS_ERROR) + } + + const id = await this.getNextId(projectId) + + const now = new Date() + .toISOString() + .replace('T', ' ') + .replace('Z', '') + .slice(0, 19) + + await clickhouse.insert({ + table: `${CLICKHOUSE_DB}.data_import`, + values: [ + { + id, + projectId, + provider, + status: 'pending', + dateFrom: null, + dateTo: null, + totalRows: 0, + importedRows: 0, + invalidRows: 0, + errorMessage: null, + createdAt: now, + finishedAt: null, + version: 1, + }, + ], + format: 'JSONEachRow', + }) + + return { + id, + projectId, + provider, + status: DataImportStatus.PENDING, + dateFrom: null, + dateTo: null, + totalRows: 0, + importedRows: 0, + invalidRows: 0, + errorMessage: null, + createdAt: now, + finishedAt: null, + version: 1, + } + } finally { + await this.releaseCreateLock(projectId, lockToken) + } + } + + private async findActiveByProject( + projectId: string, + ): Promise { + const result = await clickhouse + .query({ + query: `SELECT * FROM ${CLICKHOUSE_DB}.data_import FINAL WHERE projectId = {projectId:FixedString(12)} AND status IN ('pending', 'processing') LIMIT 1`, + query_params: { projectId }, + }) + .then((rs) => rs.json()) + + return result.data?.[0] || null + } + + async findAllByProject(projectId: string): Promise { + const result = await clickhouse + .query({ + query: `SELECT * FROM ${CLICKHOUSE_DB}.data_import FINAL WHERE projectId = {projectId:FixedString(12)} ORDER BY createdAt DESC`, + query_params: { projectId }, + }) + .then((rs) => rs.json()) + + return result.data || [] + } + + async findOne(projectId: string, importId: number): Promise { + const result = await clickhouse + .query({ + query: `SELECT * FROM ${CLICKHOUSE_DB}.data_import FINAL WHERE projectId = {projectId:FixedString(12)} AND id = {id:UInt8} LIMIT 1`, + query_params: { projectId, id: importId }, + }) + .then((rs) => rs.json()) + + const dataImport = result.data?.[0] + if (!dataImport) { + throw new NotFoundException('Import not found') + } + + return dataImport + } + + async updateStatus( + id: number, + projectId: string, + updates: Partial, + ): Promise { + const existing = await this.findOne(projectId, id) + const nextVersion = (existing.version || 0) + 1 + + await clickhouse.insert({ + table: `${CLICKHOUSE_DB}.data_import`, + values: [ + { + ...existing, + ...updates, + id, + projectId, + version: nextVersion, + }, + ], + format: 'JSONEachRow', + }) + } + + async markProcessing(id: number, projectId: string): Promise { + await this.updateStatus(id, projectId, { + status: DataImportStatus.PROCESSING, + }) + } + + async markCompleted( + id: number, + projectId: string, + stats: { + importedRows: number + invalidRows: number + totalRows: number + dateFrom: string | null + dateTo: string | null + }, + ): Promise { + const now = new Date() + .toISOString() + .replace('T', ' ') + .replace('Z', '') + .slice(0, 19) + await this.updateStatus(id, projectId, { + status: DataImportStatus.COMPLETED, + importedRows: stats.importedRows, + invalidRows: stats.invalidRows, + totalRows: stats.totalRows, + dateFrom: stats.dateFrom, + dateTo: stats.dateTo, + finishedAt: now, + }) + } + + async markFailed( + id: number, + projectId: string, + errorMessage: string, + ): Promise { + const now = new Date() + .toISOString() + .replace('T', ' ') + .replace('Z', '') + .slice(0, 19) + await this.updateStatus(id, projectId, { + status: DataImportStatus.FAILED, + errorMessage, + finishedAt: now, + }) + } + + async cleanupImportedRows( + projectId: string, + importId: number, + ): Promise { + await clickhouse.command({ + query: `ALTER TABLE ${CLICKHOUSE_DB}.analytics DELETE WHERE pid = {pid:FixedString(12)} AND importID = {importID:UInt8}`, + query_params: { pid: projectId, importID: importId }, + }) + await clickhouse.command({ + query: `ALTER TABLE ${CLICKHOUSE_DB}.customEV DELETE WHERE pid = {pid:FixedString(12)} AND importID = {importID:UInt8}`, + query_params: { pid: projectId, importID: importId }, + }) + } + + async deleteImport(projectId: string, importId: number): Promise { + const dataImport = await this.findOne(projectId, importId) + + if ( + dataImport.status === DataImportStatus.PENDING || + dataImport.status === DataImportStatus.PROCESSING + ) { + throw new BadRequestException( + 'Cannot delete an import that is still in progress.', + ) + } + + try { + await this.cleanupImportedRows(projectId, importId) + } catch (error) { + this.logger.error( + `Failed to delete imported data from ClickHouse for import ${importId}: ${error.message}`, + ) + throw new BadRequestException( + 'Failed to remove imported data. Please try again later.', + ) + } + + await this.deleteImportRecord(projectId, importId) + } + + async deleteImportRecord(projectId: string, importId: number): Promise { + await clickhouse.command({ + query: `ALTER TABLE ${CLICKHOUSE_DB}.data_import DELETE WHERE projectId = {projectId:FixedString(12)} AND id = {id:UInt8}`, + query_params: { projectId, id: importId }, + }) + } + + async hasImportedDataInRange( + projectId: string, + from: string, + to: string, + ): Promise { + const result = await clickhouse + .query({ + query: `SELECT count() as cnt FROM ${CLICKHOUSE_DB}.analytics WHERE pid = {pid:FixedString(12)} AND importID IS NOT NULL AND created >= {from:DateTime} AND created <= {to:DateTime} LIMIT 1`, + query_params: { pid: projectId, from, to }, + }) + .then((rs) => rs.json<{ cnt: string }>()) + + return Number(result.data?.[0]?.cnt) > 0 + } +} diff --git a/backend/apps/community/src/data-import/dto/index.ts b/backend/apps/community/src/data-import/dto/index.ts new file mode 100644 index 000000000..0221d0e71 --- /dev/null +++ b/backend/apps/community/src/data-import/dto/index.ts @@ -0,0 +1 @@ +export { UploadImportDto } from './upload-import.dto' diff --git a/backend/apps/community/src/data-import/dto/upload-import.dto.ts b/backend/apps/community/src/data-import/dto/upload-import.dto.ts new file mode 100644 index 000000000..25413d40e --- /dev/null +++ b/backend/apps/community/src/data-import/dto/upload-import.dto.ts @@ -0,0 +1,15 @@ +import { IsString, IsIn } from 'class-validator' +import { ApiProperty } from '@nestjs/swagger' + +import { SUPPORTED_PROVIDERS } from '../mappers' + +export class UploadImportDto { + @ApiProperty({ + description: 'Analytics provider to import from', + enum: SUPPORTED_PROVIDERS, + example: 'umami', + }) + @IsString() + @IsIn(SUPPORTED_PROVIDERS) + provider: (typeof SUPPORTED_PROVIDERS)[number] +} diff --git a/backend/apps/community/src/data-import/entity/data-import.entity.ts b/backend/apps/community/src/data-import/entity/data-import.entity.ts new file mode 100644 index 000000000..2c1ba7817 --- /dev/null +++ b/backend/apps/community/src/data-import/entity/data-import.entity.ts @@ -0,0 +1,24 @@ +export enum DataImportStatus { + PENDING = 'pending', + PROCESSING = 'processing', + COMPLETED = 'completed', + FAILED = 'failed', +} + +export const MAX_IMPORT_ID = 255 + +export class DataImport { + id: number + projectId: string + provider: string + status: DataImportStatus + dateFrom: string | null + dateTo: string | null + totalRows: number + importedRows: number + invalidRows: number + errorMessage: string | null + createdAt: string + finishedAt: string | null + version: number +} diff --git a/backend/apps/community/src/data-import/mappers/index.ts b/backend/apps/community/src/data-import/mappers/index.ts new file mode 100644 index 000000000..5302f3e7c --- /dev/null +++ b/backend/apps/community/src/data-import/mappers/index.ts @@ -0,0 +1,12 @@ +import { ImportMapper } from './mapper.interface' +import { UmamiMapper } from './umami.mapper' + +export const SUPPORTED_PROVIDERS = ['umami'] as const + +const mappers: Record = { + umami: new UmamiMapper(), +} + +export function getMapper(provider: string): ImportMapper | undefined { + return mappers[provider] +} diff --git a/backend/apps/community/src/data-import/mappers/mapper.interface.ts b/backend/apps/community/src/data-import/mappers/mapper.interface.ts new file mode 100644 index 000000000..68cc5c3e8 --- /dev/null +++ b/backend/apps/community/src/data-import/mappers/mapper.interface.ts @@ -0,0 +1,15 @@ +export interface AnalyticsImportRow { + table: 'analytics' | 'customEV' + data: Record +} + +export interface ImportMapper { + readonly provider: string + readonly expectedFileExtension: string + + createRowStream( + filePath: string, + pid: string, + importID: number, + ): AsyncIterable +} diff --git a/backend/apps/community/src/data-import/mappers/umami.mapper.ts b/backend/apps/community/src/data-import/mappers/umami.mapper.ts new file mode 100644 index 000000000..aa152e8aa --- /dev/null +++ b/backend/apps/community/src/data-import/mappers/umami.mapper.ts @@ -0,0 +1,404 @@ +import * as fs from 'fs' +import * as path from 'path' +import * as os from 'os' +import * as crypto from 'crypto' +import { Unzip, UnzipInflate } from 'fflate' +import { parse } from 'csv-parse' + +import { ImportMapper, AnalyticsImportRow } from './mapper.interface' + +const WEBSITE_EVENT_CSV = 'website_event.csv' +const MAX_UMAMI_CSV_BYTES = 512 * 1024 * 1024 + +const BROWSER_MAP: Record = { + chrome: 'Chrome', + opera: 'Opera', + crios: 'Mobile Chrome', + firefox: 'Firefox', + facebook: 'Facebook', + safari: 'Safari', + ios: 'Mobile Safari', + 'ios-webview': 'Mobile Safari', + 'edge-chromium': 'Edge', + samsung: 'Samsung Internet', + yandexbrowser: 'Yandex', + 'edge-ios': 'Edge', + 'chromium-webview': 'Chrome WebView', + fxios: 'Mobile Firefox', + edge: 'Edge', +} + +const MOBILE_BROWSER_VARIANTS: Record = { + Chrome: 'Mobile Chrome', + Firefox: 'Mobile Firefox', + Safari: 'Mobile Safari', +} + +const OS_MAP: Record = { + 'mac os': 'macOS', + 'windows 10': 'Windows', + 'windows 7': 'Windows', + 'windows 8': 'Windows', + 'windows 8.1': 'Windows', + 'windows vista': 'Windows', + 'windows xp': 'Windows', + 'windows server 2003': 'Windows', + 'android os': 'Android', + 'chrome os': 'Chrome OS', + 'chromium os': 'Chrome OS', +} + +const DEVICE_MAP: Record = { + laptop: 'desktop', + desktop: 'desktop', + mobile: 'mobile', + tablet: 'tablet', +} + +function normalizeNull(value: string | undefined): string | null { + if (!value || value === '' || value === '\\N') return null + return value +} + +function truncate(value: string | null, maxLen: number): string | null { + if (!value) return null + return value.length > maxLen ? value.slice(0, maxLen) : value +} + +function mapBrowser(raw: string | null, device: string | null): string | null { + if (!raw) return null + + const key = raw.toLowerCase() + let mapped = BROWSER_MAP[key] + + if (!mapped) { + mapped = raw.charAt(0).toUpperCase() + raw.slice(1) + } + + const isMobile = device === 'mobile' || device === 'tablet' + if (isMobile && MOBILE_BROWSER_VARIANTS[mapped]) { + return MOBILE_BROWSER_VARIANTS[mapped] + } + + return mapped +} + +function mapOS(raw: string | null): string | null { + if (!raw) return null + const key = raw.toLowerCase() + return OS_MAP[key] ?? raw +} + +function mapDevice(raw: string | null): string { + if (!raw) return 'desktop' + const key = raw.toLowerCase() + return DEVICE_MAP[key] ?? 'desktop' +} + +function buildReferrer( + domain: string | null, + path: string | null, + query: string | null, + hostname: string | null, +): string | null { + if (!domain) return null + + let ref = `https://${domain}` + if (path) ref += path + if (query) ref += query.startsWith('?') ? query : `?${query}` + + if (hostname) { + const cleanHost = hostname.replace(/^www\./, '') + const cleanDomain = domain.replace(/^www\./, '') + if (cleanDomain === cleanHost) return null + } + + return ref +} + +const DATE_REGEX = + /^\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01]) ([01]\d|2[0-3]):([0-5]\d):([0-5]\d)$/ + +function isValidTimestamp(ts: string): boolean { + return DATE_REGEX.test(ts) +} + +function sessionIdToPsid(sessionId: string): string { + return crypto + .createHash('sha256') + .update(sessionId) + .digest() + .readBigUInt64BE(0) + .toString() +} + +export class UmamiMapper implements ImportMapper { + readonly provider = 'umami' + readonly expectedFileExtension = '.zip' + + private async extractCsvFromZip(zipPath: string): Promise { + const tempCsv = path.join(os.tmpdir(), `swetrix-umami-${Date.now()}.csv`) + + return new Promise((resolve, reject) => { + const zipStream = fs.createReadStream(zipPath) + const unzipper = new Unzip() + + unzipper.register(UnzipInflate) + + let settled = false + let foundEntry = false + let fileDescriptor: number | null = null + let tempFileCreated = false + let extractedBytes = 0 + let terminateEntry: (() => void) | null = null + + const cleanupTempCsv = () => { + if (fileDescriptor !== null) { + try { + fs.closeSync(fileDescriptor) + } catch { + // + } + fileDescriptor = null + } + + if (tempFileCreated) { + try { + fs.unlinkSync(tempCsv) + } catch { + // + } + } + } + + const fail = (error: Error) => { + if (settled) return + settled = true + terminateEntry?.() + zipStream.destroy() + cleanupTempCsv() + reject(error) + } + + const succeed = () => { + if (settled) return + settled = true + if (fileDescriptor !== null) { + try { + fs.closeSync(fileDescriptor) + } catch (error) { + cleanupTempCsv() + reject(error) + return + } + fileDescriptor = null + } + zipStream.destroy() + resolve(tempCsv) + } + + unzipper.onfile = (file) => { + if (path.basename(file.name) !== WEBSITE_EVENT_CSV || foundEntry) return + + foundEntry = true + + if ( + typeof file.originalSize === 'number' && + Number.isFinite(file.originalSize) && + file.originalSize > MAX_UMAMI_CSV_BYTES + ) { + fail( + new Error( + `${WEBSITE_EVENT_CSV} exceeds the ${MAX_UMAMI_CSV_BYTES} byte limit.`, + ), + ) + return + } + + try { + fileDescriptor = fs.openSync(tempCsv, 'wx') + tempFileCreated = true + } catch (error) { + fail(error as Error) + return + } + + terminateEntry = file.terminate + file.ondata = (error, chunk, final) => { + if (error) { + fail(error) + return + } + + extractedBytes += chunk.length + if (extractedBytes > MAX_UMAMI_CSV_BYTES) { + fail( + new Error( + `${WEBSITE_EVENT_CSV} exceeds the ${MAX_UMAMI_CSV_BYTES} byte limit.`, + ), + ) + return + } + + try { + fs.writeSync(fileDescriptor!, chunk) + } catch (writeError) { + fail(writeError as Error) + return + } + + if (final) { + succeed() + } + } + + try { + file.start() + } catch (error) { + fail(error as Error) + } + } + + zipStream.on('data', (chunk: Buffer) => { + if (settled) return + + try { + unzipper.push( + new Uint8Array(chunk.buffer, chunk.byteOffset, chunk.byteLength), + ) + } catch (error) { + fail(error as Error) + } + }) + + zipStream.on('end', () => { + if (settled) return + + try { + unzipper.push(new Uint8Array(0), true) + } catch (error) { + fail(error as Error) + return + } + + if (!foundEntry) { + fail( + new Error( + `ZIP does not contain ${WEBSITE_EVENT_CSV}. Please upload the export ZIP from Umami.`, + ), + ) + } + }) + + zipStream.on('error', (error) => { + fail(error) + }) + }) + } + + async *createRowStream( + filePath: string, + pid: string, + importID: number, + ): AsyncIterable { + const csvPath = await this.extractCsvFromZip(filePath) + + let cleanedUp = false + const cleanup = () => { + if (!cleanedUp) { + cleanedUp = true + try { + fs.unlinkSync(csvPath) + } catch { + // + } + } + } + + try { + const parser = fs.createReadStream(csvPath).pipe( + parse({ + columns: true, + skip_empty_lines: true, + trim: true, + relax_quotes: true, + relax_column_count: true, + }), + ) + + for await (const row of parser) { + const sessionId = normalizeNull(row.session_id) + if (!sessionId) continue + + const createdAt = normalizeNull(row.created_at) + if (!createdAt || !isValidTimestamp(createdAt)) continue + + const eventType = normalizeNull(row.event_type) + if (!eventType || (eventType !== '1' && eventType !== '2')) continue + + const psid = sessionIdToPsid(sessionId) + + const rawDevice = normalizeNull(row.device) + const device = mapDevice(rawDevice) + const browser = mapBrowser(normalizeNull(row.browser), rawDevice) + const os = mapOS(normalizeNull(row.os)) + + const hostname = truncate(normalizeNull(row.hostname), 253) + const referrerDomain = normalizeNull(row.referrer_domain) + const referrerPath = normalizeNull(row.referrer_path) + const referrerQuery = normalizeNull(row.referrer_query) + const ref = buildReferrer( + referrerDomain, + referrerPath, + referrerQuery, + hostname, + ) + + const country = normalizeNull(row.country) + const cc = country && /^[A-Z]{2}$/.test(country) ? country : null + + const region = normalizeNull(row.region) + + const baseData: Record = { + psid, + profileId: null, + pid, + host: hostname, + pg: truncate(normalizeNull(row.url_path), 2048), + dv: device, + br: truncate(browser, 30), + brv: null, + os: truncate(os, 25), + osv: null, + lc: truncate(normalizeNull(row.language), 35), + ref: truncate(ref, 2048), + so: truncate(normalizeNull(row.utm_source), 256), + me: truncate(normalizeNull(row.utm_medium), 256), + ca: truncate(normalizeNull(row.utm_campaign), 256), + te: truncate(normalizeNull(row.utm_term), 256), + co: truncate(normalizeNull(row.utm_content), 256), + cc, + rg: null, + rgc: region && region.length <= 10 ? region : null, + ct: truncate(normalizeNull(row.city), 60), + 'meta.key': [], + 'meta.value': [], + importID, + created: createdAt, + } + + if (eventType === '1') { + yield { table: 'analytics', data: baseData } + } else { + const eventName = truncate(normalizeNull(row.event_name), 256) || '' + yield { + table: 'customEV', + data: { ...baseData, ev: eventName }, + } + } + } + } finally { + cleanup() + } + } +} diff --git a/backend/knip.jsonc b/backend/knip.jsonc index 34cc19be7..096b8195c 100644 --- a/backend/knip.jsonc +++ b/backend/knip.jsonc @@ -1,10 +1,8 @@ { "$schema": "https://unpkg.com/knip@5/schema.json", - "ignoreDependencies": [ - "@faker-js/faker", "@types/ioredis", "@types/multer" - ], + "ignoreDependencies": ["@faker-js/faker", "@types/ioredis"], "ignore": [], "exclude": ["enumMembers"], "entry": ["./apps/cloud/src/main.ts", "./apps/community/src/main.ts"], - "project": ["./apps/cloud/src/**/*.ts", "./apps/community/src/**/*.ts"] + "project": ["./apps/cloud/src/**/*.ts", "./apps/community/src/**/*.ts"], } diff --git a/backend/migrations/clickhouse/2026_03_31_import_id.js b/backend/migrations/clickhouse/2026_03_31_import_id.js new file mode 100644 index 000000000..b22295395 --- /dev/null +++ b/backend/migrations/clickhouse/2026_03_31_import_id.js @@ -0,0 +1,8 @@ +const { queriesRunner, dbName } = require('./setup') + +const queries = [ + `ALTER TABLE ${dbName}.analytics ADD COLUMN IF NOT EXISTS importID Nullable(UInt8) AFTER \`meta.value\``, + `ALTER TABLE ${dbName}.customEV ADD COLUMN IF NOT EXISTS importID Nullable(UInt8) AFTER \`meta.value\``, +] + +queriesRunner(queries) diff --git a/backend/migrations/clickhouse/initialise_database.js b/backend/migrations/clickhouse/initialise_database.js index 655f4a21a..c93de76d9 100644 --- a/backend/migrations/clickhouse/initialise_database.js +++ b/backend/migrations/clickhouse/initialise_database.js @@ -30,6 +30,7 @@ const CLICKHOUSE_INIT_QUERIES = [ ct Nullable(String) CODEC(ZSTD(3)), \`meta.key\` Array(String) CODEC(ZSTD(3)), \`meta.value\` Array(String) CODEC(ZSTD(3)), + importID Nullable(UInt8), created DateTime('UTC') CODEC(Delta(4), LZ4) ) ENGINE = MergeTree() @@ -63,6 +64,7 @@ const CLICKHOUSE_INIT_QUERIES = [ ct Nullable(String) CODEC(ZSTD(3)), \`meta.key\` Array(String) CODEC(ZSTD(3)), \`meta.value\` Array(String) CODEC(ZSTD(3)), + importID Nullable(UInt8), created DateTime('UTC') CODEC(Delta(4), LZ4) ) ENGINE = MergeTree() diff --git a/backend/migrations/clickhouse/initialise_selfhosted.js b/backend/migrations/clickhouse/initialise_selfhosted.js index f0b6eff39..c2d840a46 100644 --- a/backend/migrations/clickhouse/initialise_selfhosted.js +++ b/backend/migrations/clickhouse/initialise_selfhosted.js @@ -178,6 +178,25 @@ const CLICKHOUSE_INIT_QUERIES = [ ) ENGINE = MergeTree() PRIMARY KEY rotation;`, + + `CREATE TABLE IF NOT EXISTS ${dbName}.data_import + ( + id UInt8, + projectId FixedString(12), + provider LowCardinality(String), + status Enum8('pending' = 1, 'processing' = 2, 'completed' = 3, 'failed' = 4) DEFAULT 'pending', + dateFrom Nullable(Date), + dateTo Nullable(Date), + totalRows UInt32 DEFAULT 0, + importedRows UInt32 DEFAULT 0, + invalidRows UInt32 DEFAULT 0, + errorMessage Nullable(String) CODEC(ZSTD(3)), + createdAt DateTime('UTC') DEFAULT now(), + finishedAt Nullable(DateTime('UTC')), + version UInt32 DEFAULT 1 + ) + ENGINE = ReplacingMergeTree(version) + ORDER BY (projectId, id);`, ] const initialiseSelfhosted = async () => { diff --git a/backend/migrations/clickhouse/selfhosted_2026_03_31_data_import.js b/backend/migrations/clickhouse/selfhosted_2026_03_31_data_import.js new file mode 100644 index 000000000..4334445fc --- /dev/null +++ b/backend/migrations/clickhouse/selfhosted_2026_03_31_data_import.js @@ -0,0 +1,24 @@ +const { queriesRunner, dbName } = require('./setup') + +const queries = [ + `CREATE TABLE IF NOT EXISTS ${dbName}.data_import + ( + id UInt8, + projectId FixedString(12), + provider LowCardinality(String), + status Enum8('pending' = 1, 'processing' = 2, 'completed' = 3, 'failed' = 4) DEFAULT 'pending', + dateFrom Nullable(Date), + dateTo Nullable(Date), + totalRows UInt32 DEFAULT 0, + importedRows UInt32 DEFAULT 0, + invalidRows UInt32 DEFAULT 0, + errorMessage Nullable(String) CODEC(ZSTD(3)), + createdAt DateTime('UTC') DEFAULT now(), + finishedAt Nullable(DateTime('UTC')), + version UInt32 DEFAULT 1 + ) + ENGINE = ReplacingMergeTree(version) + ORDER BY (projectId, id);`, +] + +queriesRunner(queries) diff --git a/backend/migrations/mysql/2026_03_31_data_import.sql b/backend/migrations/mysql/2026_03_31_data_import.sql new file mode 100644 index 000000000..09b4b3028 --- /dev/null +++ b/backend/migrations/mysql/2026_03_31_data_import.sql @@ -0,0 +1,19 @@ +-- Data import table for tracking imports from third-party analytics providers +CREATE TABLE IF NOT EXISTS `data_import` ( + `id` int NOT NULL AUTO_INCREMENT, + `importId` tinyint UNSIGNED NOT NULL, + `projectId` varchar(12) NOT NULL, + `provider` varchar(30) NOT NULL, + `status` enum('pending', 'processing', 'completed', 'failed') NOT NULL DEFAULT 'pending', + `dateFrom` date DEFAULT NULL, + `dateTo` date DEFAULT NULL, + `totalRows` int NOT NULL DEFAULT 0, + `importedRows` int NOT NULL DEFAULT 0, + `invalidRows` int NOT NULL DEFAULT 0, + `errorMessage` text DEFAULT NULL, + `createdAt` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + `finishedAt` datetime(6) DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `FK_data_import_project` (`projectId`), + CONSTRAINT `FK_data_import_project` FOREIGN KEY (`projectId`) REFERENCES `project` (`id`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; diff --git a/backend/package-lock.json b/backend/package-lock.json index fedbaef1f..678548a9a 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -12,6 +12,7 @@ "@googleapis/searchconsole": "^6.0.1", "@nestjs-modules/mailer": "^2.0.2", "@nestjs/axios": "^4.0.1", + "@nestjs/bullmq": "^11.0.4", "@nestjs/common": "^11.1.15", "@nestjs/config": "^4.0.3", "@nestjs/core": "^11.1.15", @@ -35,15 +36,18 @@ "axios": "^1.13.6", "bcrypt": "^6.0.0", "body-parser": "^2.2.2", + "bullmq": "^5.71.1", "class-transformer": "^0.5.1", "class-validator": "^0.14.3", "cookie-parser": "^1.4.7", "countries-and-timezones": "^3.8.0", "cross-env": "^10.1.0", "crypto-js": "^4.2.0", + "csv-parse": "^6.2.1", "dayjs": "^1.11.19", "dotenv": "^17.3.1", "express": "^5.2.1", + "fflate": "^0.8.2", "front-matter": "^4.0.2", "google-auth-library": "^10.6.1", "handlebars": "^4.7.8", @@ -1224,6 +1228,84 @@ "integrity": "sha512-xgAyonlVVS+q7Vc7qLW0UrJU7rSFcETRWsqdXZtjzRU8dF+6CkozTK4V4y1LwOX7j8r/vHphjDeMeGI4tNGeGA==", "license": "MIT" }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-arm64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.3.tgz", + "integrity": "sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.3.tgz", + "integrity": "sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.3.tgz", + "integrity": "sha512-fg0uy/dG/nZEXfYilKoRe7yALaNmHoYeIoJuJ7KJ+YyU2bvY8vPv27f7UKhGRpY6euFYqEVhxCFZgAUNQBM3nw==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.3.tgz", + "integrity": "sha512-YxQL+ax0XqBJDZiKimS2XQaf+2wDGVa1enVRGzEvLLVFeqa5kx2bWbtcSXgsxjQB7nRqqIGFIcLteF/sHeVtQg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.3.tgz", + "integrity": "sha512-cvwNfbP07pKUfq1uH+S6KJ7dT9K8WOE4ZiAcsrSes+UY55E/0jLYc+vq+DO7jlmqRb5zAggExKm0H7O/CBaesg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.3.tgz", + "integrity": "sha512-x0fWaQtYp4E6sktbsdAqnehxDgEc/VwM7uLsRCYWaiGu0ykYdZPiS8zCWdnjHwyiumousxfBm4SO31eXqwEZhQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/@napi-rs/wasm-runtime": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.0.tgz", @@ -1283,6 +1365,34 @@ "rxjs": "^7.0.0" } }, + "node_modules/@nestjs/bull-shared": { + "version": "11.0.4", + "resolved": "https://registry.npmjs.org/@nestjs/bull-shared/-/bull-shared-11.0.4.tgz", + "integrity": "sha512-VBJcDHSAzxQnpcDfA0kt9MTGUD1XZzfByV70su0W0eDCQ9aqIEBlzWRW21tv9FG9dIut22ysgDidshdjlnczLw==", + "license": "MIT", + "dependencies": { + "tslib": "2.8.1" + }, + "peerDependencies": { + "@nestjs/common": "^10.0.0 || ^11.0.0", + "@nestjs/core": "^10.0.0 || ^11.0.0" + } + }, + "node_modules/@nestjs/bullmq": { + "version": "11.0.4", + "resolved": "https://registry.npmjs.org/@nestjs/bullmq/-/bullmq-11.0.4.tgz", + "integrity": "sha512-wBzK9raAVG0/6NTMdvLGM4/FQ1lsB35/pYS8L6a0SDgkTiLpd7mAjQ8R692oMx5s7IjvgntaZOuTUrKYLNfIkA==", + "license": "MIT", + "dependencies": { + "@nestjs/bull-shared": "^11.0.4", + "tslib": "2.8.1" + }, + "peerDependencies": { + "@nestjs/common": "^10.0.0 || ^11.0.0", + "@nestjs/core": "^10.0.0 || ^11.0.0", + "bullmq": "^3.0.0 || ^4.0.0 || ^5.0.0" + } + }, "node_modules/@nestjs/cli": { "version": "11.0.16", "resolved": "https://registry.npmjs.org/@nestjs/cli/-/cli-11.0.16.tgz", @@ -5215,6 +5325,34 @@ "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", "license": "MIT" }, + "node_modules/bullmq": { + "version": "5.71.1", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-5.71.1.tgz", + "integrity": "sha512-kOBfdcsHmO6wwmIjpersoVdYQ7jkjTgky4Yop0loc7QwSdgxliSzD69U9ijZuRrkyCJwz5p5eqxeGeQkJ0YGZQ==", + "license": "MIT", + "dependencies": { + "cron-parser": "4.9.0", + "ioredis": "5.10.1", + "msgpackr": "1.11.5", + "node-abort-controller": "3.1.1", + "semver": "7.7.4", + "tslib": "2.8.1", + "uuid": "11.1.0" + } + }, + "node_modules/bullmq/node_modules/uuid": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-11.1.0.tgz", + "integrity": "sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist/esm/bin/uuid" + } + }, "node_modules/busboy": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", @@ -5884,6 +6022,18 @@ "url": "https://ko-fi.com/intcreator" } }, + "node_modules/cron-parser": { + "version": "4.9.0", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.9.0.tgz", + "integrity": "sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==", + "license": "MIT", + "dependencies": { + "luxon": "^3.2.1" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/cross-env": { "version": "10.1.0", "resolved": "https://registry.npmjs.org/cross-env/-/cross-env-10.1.0.tgz", @@ -5951,6 +6101,12 @@ "url": "https://github.com/sponsors/fb55" } }, + "node_modules/csv-parse": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-6.2.1.tgz", + "integrity": "sha512-LRLMV+UCyfMokp8Wb411duBf1gaBKJfOfBWU9eHMJ+b+cJYZsNu3AFmjJf3+yPGd59Exz1TsMjaSFyxnYB9+IQ==", + "license": "MIT" + }, "node_modules/dayjs": { "version": "1.11.19", "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.19.tgz", @@ -6907,6 +7063,12 @@ "node": "^12.20 || >= 14.13" } }, + "node_modules/fflate": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.8.2.tgz", + "integrity": "sha512-cPJU47OaAoCbg0pBvzsgpTPhmhqI5eJjh/JIu8tPj5q+T7iLvW/JAYUqmE7KOB4R1ZyEhzBaIQpQpardBF5z8A==", + "license": "MIT" + }, "node_modules/file-type": { "version": "21.3.0", "resolved": "https://registry.npmjs.org/file-type/-/file-type-21.3.0.tgz", @@ -7797,9 +7959,9 @@ "optional": true }, "node_modules/ioredis": { - "version": "5.10.0", - "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.10.0.tgz", - "integrity": "sha512-HVBe9OFuqs+Z6n64q09PQvP1/R4Bm+30PAyyD4wIEqssh3v9L21QjCVk4kRLucMBcDokJTcLjsGeVRlq/nH6DA==", + "version": "5.10.1", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.10.1.tgz", + "integrity": "sha512-HuEDBTI70aYdx1v6U97SbNx9F1+svQKBDo30o0b9fw055LMepzpOOd0Ccg9Q6tbqmBSJaMuY0fB7yw9/vjBYCA==", "license": "MIT", "dependencies": { "@ioredis/commands": "1.5.1", @@ -9567,6 +9729,37 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "license": "MIT" }, + "node_modules/msgpackr": { + "version": "1.11.5", + "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.11.5.tgz", + "integrity": "sha512-UjkUHN0yqp9RWKy0Lplhh+wlpdt9oQBYgULZOiFhV3VclSF1JnSQWZ5r9gORQlNYaUKQoR8itv7g7z1xDDuACA==", + "license": "MIT", + "optionalDependencies": { + "msgpackr-extract": "^3.0.2" + } + }, + "node_modules/msgpackr-extract": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.3.tgz", + "integrity": "sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "dependencies": { + "node-gyp-build-optional-packages": "5.2.2" + }, + "bin": { + "download-msgpackr-prebuilds": "bin/download-prebuilds.js" + }, + "optionalDependencies": { + "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.3", + "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.3" + } + }, "node_modules/multer": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/multer/-/multer-2.1.0.tgz", @@ -9951,7 +10144,6 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/node-abort-controller/-/node-abort-controller-3.1.1.tgz", "integrity": "sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==", - "dev": true, "license": "MIT" }, "node_modules/node-addon-api": { @@ -10024,6 +10216,21 @@ "node-gyp-build-test": "build-test.js" } }, + "node_modules/node-gyp-build-optional-packages": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.2.2.tgz", + "integrity": "sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==", + "license": "MIT", + "optional": true, + "dependencies": { + "detect-libc": "^2.0.1" + }, + "bin": { + "node-gyp-build-optional-packages": "bin.js", + "node-gyp-build-optional-packages-optional": "optional.js", + "node-gyp-build-optional-packages-test": "build-test.js" + } + }, "node_modules/node-releases": { "version": "2.0.27", "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.27.tgz", @@ -11622,9 +11829,9 @@ } }, "node_modules/semver": { - "version": "7.7.3", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", - "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", + "version": "7.7.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", + "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", "license": "ISC", "bin": { "semver": "bin/semver.js" diff --git a/backend/package.json b/backend/package.json index a6494b71f..b99406990 100644 --- a/backend/package.json +++ b/backend/package.json @@ -33,6 +33,7 @@ "@googleapis/searchconsole": "^6.0.1", "@nestjs-modules/mailer": "^2.0.2", "@nestjs/axios": "^4.0.1", + "@nestjs/bullmq": "^11.0.4", "@nestjs/common": "^11.1.15", "@nestjs/config": "^4.0.3", "@nestjs/core": "^11.1.15", @@ -56,15 +57,18 @@ "axios": "^1.13.6", "bcrypt": "^6.0.0", "body-parser": "^2.2.2", + "bullmq": "^5.71.1", "class-transformer": "^0.5.1", "class-validator": "^0.14.3", "cookie-parser": "^1.4.7", "countries-and-timezones": "^3.8.0", "cross-env": "^10.1.0", "crypto-js": "^4.2.0", + "csv-parse": "^6.2.1", "dayjs": "^1.11.19", "dotenv": "^17.3.1", "express": "^5.2.1", + "fflate": "^0.8.2", "front-matter": "^4.0.2", "google-auth-library": "^10.6.1", "handlebars": "^4.7.8", diff --git a/docs/content/docs/data-import.mdx b/docs/content/docs/data-import.mdx new file mode 100644 index 000000000..cce446950 --- /dev/null +++ b/docs/content/docs/data-import.mdx @@ -0,0 +1,110 @@ +--- +title: Data Import +description: Import historical analytics data from third-party platforms into Swetrix +--- + +# Data Import + +Swetrix allows you to import historical analytics data from other platforms so you can switch without losing your history. Imported data appears alongside your live-tracked data in dashboards, charts, and reports. It's tagged internally so it can be identified and, if needed, removed later without affecting your live-tracked data. + +## Supported Providers + +- Umami + +## Importing from Umami + + + +### Export Your Data from Umami + +1. Log in to your Umami instance +2. Go to **Settings** → **Data** +3. Click on the export button +4. Select a website you want to export and a date range, then click export - Umami will email you a ZIP file with the exported data + +Umami export data page + +You can learn more on how to export your data from Umami on their [data export documentation](https://docs.umami.is/docs/cloud/export-data). + + + +### Import into Swetrix + +1. Go to your project's **Settings** page in Swetrix +2. Click the **Import data** tab + +Swetrix import data tab + +3. Click the **Umami** provider card + +Swetrix import data tab with provider cards + +4. In the upload modal, either drag and drop the `.zip` file or click to browse +5. Wait for the upload and processing to complete + +Upload the `.zip` file from Umami directly - do not extract it first. + + + +### Monitor Progress + +After uploading, the import will appear in the **Import history** table below the provider cards. You can see: + +- **Status** — Pending, Processing, Completed, or Failed +- **Date range** — The time span of the imported data +- **Rows** — How many analytics events were imported + +The page auto-refreshes while an import is processing. You can navigate away and come back later. + + + + +## Viewing Imported Data + +Once an import completes, the data is immediately available in your analytics dashboard. When you select a date range that includes imported data, a small **"Imported"** badge appears near the period selector to let you know. + +Dashboard with imported data indicator badge + +## Deleting an Import + +If you want to remove imported data: + +1. Go to **Settings** → **Import data** +2. Find the import in the history table +3. Click the trash icon +4. Confirm the deletion + +This permanently removes all analytics rows associated with that import. Your live-tracked data is not affected. + +## Limitations + +- **File size**: Maximum 100 MB per upload +- **One at a time**: Only one import can be active (pending or processing) per project at a time +- **Timestamps**: Events with invalid timestamps are skipped during import + +## Troubleshooting + +### Import stuck in "Processing" + +Large files may take several minutes to process. If an import seems stuck, wait up to 30 minutes. If it still hasn't completed, the import may have failed - check for error messages in the import history. + +### "An import is already in progress" + +Only one import can run at a time per project. Wait for the current import to complete or fail before starting a new one. + +### Invalid file type error + +Make sure you're uploading the correct file format for your provider. For Umami, upload the `.zip` file from the export directly - do not extract it first. + +### Data not appearing in charts + +After a successful import, select a date range that covers the imported data's time period. The imported data indicator badge should appear to confirm it's included. diff --git a/docs/content/docs/meta.json b/docs/content/docs/meta.json index 29e1e77e7..039242a12 100644 --- a/docs/content/docs/meta.json +++ b/docs/content/docs/meta.json @@ -13,6 +13,7 @@ "---Settings & Billing---", "analytics-dashboard", "sitesettings", + "data-import", "accountsettings", "billing", "---Developers---", diff --git a/docs/next-env.d.ts b/docs/next-env.d.ts deleted file mode 100644 index 9edff1c7c..000000000 --- a/docs/next-env.d.ts +++ /dev/null @@ -1,6 +0,0 @@ -/// -/// -import "./.next/types/routes.d.ts"; - -// NOTE: This file should not be edited -// see https://nextjs.org/docs/app/api-reference/config/typescript for more information. diff --git a/docs/public/img/sitesettings/data-import/imported-data-tooltip.png b/docs/public/img/sitesettings/data-import/imported-data-tooltip.png new file mode 100644 index 000000000..45b1235f5 Binary files /dev/null and b/docs/public/img/sitesettings/data-import/imported-data-tooltip.png differ diff --git a/docs/public/img/sitesettings/data-import/swetrix-import-data-tab.png b/docs/public/img/sitesettings/data-import/swetrix-import-data-tab.png new file mode 100644 index 000000000..a90fbcc5a Binary files /dev/null and b/docs/public/img/sitesettings/data-import/swetrix-import-data-tab.png differ diff --git a/docs/public/img/sitesettings/data-import/swetrix-umami-upload-modal.png b/docs/public/img/sitesettings/data-import/swetrix-umami-upload-modal.png new file mode 100644 index 000000000..17824bfb0 Binary files /dev/null and b/docs/public/img/sitesettings/data-import/swetrix-umami-upload-modal.png differ diff --git a/docs/public/img/sitesettings/data-import/umami-export-data.png b/docs/public/img/sitesettings/data-import/umami-export-data.png new file mode 100644 index 000000000..a3430cedb Binary files /dev/null and b/docs/public/img/sitesettings/data-import/umami-export-data.png differ diff --git a/web/app/api/api.server.ts b/web/app/api/api.server.ts index e621f725f..0b3452878 100644 --- a/web/app/api/api.server.ts +++ b/web/app/api/api.server.ts @@ -46,7 +46,7 @@ async function fetchWithTimeout( interface ServerFetchOptions { method?: 'GET' | 'POST' | 'PUT' | 'DELETE' | 'PATCH' - body?: Record + body?: Record | FormData headers?: Record /** Skip authentication - for public endpoints */ skipAuth?: boolean @@ -75,8 +75,10 @@ export async function serverFetch( const cleanEndpoint = endpoint.startsWith('/') ? endpoint.slice(1) : endpoint const url = `${API_URL}${cleanEndpoint}` + const isFormData = body instanceof FormData + const fetchHeaders: Record = { - 'Content-Type': 'application/json', + ...(isFormData ? {} : { 'Content-Type': 'application/json' }), ...customHeaders, } @@ -97,11 +99,17 @@ export async function serverFetch( } } + const serializedBody = body + ? isFormData + ? body + : JSON.stringify(body) + : undefined + try { const response = await fetchWithTimeout(url, { method, headers: fetchHeaders, - body: body ? JSON.stringify(body) : undefined, + body: serializedBody, }) if ((response.status === 401 || response.status === 403) && !skipAuth) { @@ -114,7 +122,7 @@ export async function serverFetch( const retryResponse = await fetchWithTimeout(url, { method, headers: fetchHeaders, - body: body ? JSON.stringify(body) : undefined, + body: serializedBody, }) if (retryResponse.ok) { diff --git a/web/app/lib/models/Project.ts b/web/app/lib/models/Project.ts index 2eb44f2d4..f761bb03c 100644 --- a/web/app/lib/models/Project.ts +++ b/web/app/lib/models/Project.ts @@ -172,6 +172,22 @@ export interface AnalyticsFunnel { dropoffPercStep: number } +export interface DataImport { + id: number + importId: number + projectId: string + provider: string + status: 'pending' | 'processing' | 'completed' | 'failed' + dateFrom: string | null + dateTo: string | null + totalRows: number + importedRows: number + invalidRows: number + errorMessage: string | null + createdAt: string + finishedAt: string | null +} + export interface Project { id: string name: string diff --git a/web/app/pages/Project/Settings/ProjectSettings.tsx b/web/app/pages/Project/Settings/ProjectSettings.tsx index 4af2725eb..6977b7585 100644 --- a/web/app/pages/Project/Settings/ProjectSettings.tsx +++ b/web/app/pages/Project/Settings/ProjectSettings.tsx @@ -21,6 +21,7 @@ import { NoteIcon, ShieldCheckIcon, CurrencyDollarIcon, + DownloadIcon, } from '@phosphor-icons/react' import React, { useState, useEffect, useMemo, useRef } from 'react' import { Trans, useTranslation } from 'react-i18next' @@ -64,6 +65,7 @@ import DangerZone from './tabs/DangerZone' import General from './tabs/General' import Revenue from './tabs/Revenue' import Shields from './tabs/Shields' +import DataImportTab from './components/DataImportTab' const MAX_NAME_LENGTH = 50 const MAX_ORIGINS_LENGTH = 300 @@ -355,6 +357,7 @@ const ProjectSettings = () => { | 'emails' | 'people' | 'annotations' + | 'import' | 'danger' const tabs = useMemo( @@ -424,6 +427,14 @@ const ProjectSettings = () => { iconColor: 'text-indigo-500', visible: true, }, + { + id: 'import', + label: t('project.settings.tabs.import'), + description: t('project.settings.tabs.importDesc'), + icon: DownloadIcon, + iconColor: 'text-cyan-500', + visible: true, + }, { id: 'annotations', label: t('project.settings.tabs.annotations'), @@ -1045,6 +1056,17 @@ const ProjectSettings = () => { /> ) : null} + {activeTab === 'import' && activeTabConfig ? ( + <> + + + + ) : null} {activeTab === 'integrations' && activeTabConfig ? (
diff --git a/web/app/pages/Project/Settings/components/DataImportTab.tsx b/web/app/pages/Project/Settings/components/DataImportTab.tsx new file mode 100644 index 000000000..8367ebb02 --- /dev/null +++ b/web/app/pages/Project/Settings/components/DataImportTab.tsx @@ -0,0 +1,509 @@ +import { useState, useEffect, useCallback, useRef } from 'react' +import { + UploadIcon, + TrashIcon, + CheckCircleIcon, + XCircleIcon, + ClockIcon, + SpinnerIcon, + FileTextIcon, +} from '@phosphor-icons/react' +import cx from 'clsx' +import { toast } from 'sonner' +import dayjs from 'dayjs' +import { Trans, useTranslation } from 'react-i18next' +import { useFetcher } from 'react-router' + +import { DataImport } from '~/lib/models/Project' +import type { ProjectSettingsActionData } from '~/routes/projects.settings.$id' +import Modal from '~/ui/Modal' +import Loader from '~/ui/Loader' +import FileUpload from '~/ui/FileUpload' +import { Text } from '~/ui/Text' +import UmamiSVG from '~/ui/icons/Umami' + +const PROVIDERS = ['umami'] as const + +const STATUS_ICONS = { + pending: ClockIcon, + processing: SpinnerIcon, + completed: CheckCircleIcon, + failed: XCircleIcon, +} as const + +const STATUS_CLASSES = { + pending: + 'bg-amber-50 text-amber-700 ring-amber-600/20 dark:bg-amber-500/10 dark:text-amber-400 dark:ring-amber-500/20', + processing: + 'bg-blue-50 text-blue-700 ring-blue-600/20 dark:bg-blue-500/10 dark:text-blue-400 dark:ring-blue-500/20', + completed: + 'bg-emerald-50 text-emerald-700 ring-emerald-600/20 dark:bg-emerald-500/10 dark:text-emerald-400 dark:ring-emerald-500/20', + failed: + 'bg-red-50 text-red-700 ring-red-600/20 dark:bg-red-500/10 dark:text-red-400 dark:ring-red-500/20', +} as const + +function StatusBadge({ + status, + label, +}: { + status: DataImport['status'] + label: string +}) { + const Icon = STATUS_ICONS[status] + + return ( + + + {label} + + ) +} + +interface DataImportTabProps { + projectId: string +} + +export default function DataImportTab({ projectId }: DataImportTabProps) { + const { t } = useTranslation('common') + const [imports, setImports] = useState([]) + const [loading, setLoading] = useState(true) + const [selectedProvider, setSelectedProvider] = useState(null) + const [deleteTarget, setDeleteTarget] = useState(null) + const pollRef = useRef | null>(null) + + const listFetcher = useFetcher() + const uploadFetcher = useFetcher() + const deleteFetcher = useFetcher() + + const settingsAction = `/projects/settings/${projectId}` + + const statusLabels: Record = { + pending: t('project.settings.dataImport.statusPending'), + processing: t('project.settings.dataImport.statusProcessing'), + completed: t('project.settings.dataImport.statusCompleted'), + failed: t('project.settings.dataImport.statusFailed'), + } + + const fetchImports = useCallback(() => { + listFetcher.submit( + { intent: 'list-data-imports' }, + { method: 'POST', action: settingsAction }, + ) + }, [listFetcher, settingsAction]) + + useEffect(() => { + fetchImports() + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []) + + useEffect(() => { + if (listFetcher.state !== 'idle' || !listFetcher.data) return + if (listFetcher.data.intent !== 'list-data-imports') return + + if (listFetcher.data.error) { + toast.error( + listFetcher.data.error || t('project.settings.dataImport.loadFailed'), + ) + } else if (listFetcher.data.dataImports) { + setImports(listFetcher.data.dataImports) + } + setLoading(false) + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [listFetcher.state, listFetcher.data]) + + useEffect(() => { + const hasActive = imports.some( + (i) => i.status === 'pending' || i.status === 'processing', + ) + + if (hasActive) { + pollRef.current = setInterval(fetchImports, 3000) + } else if (pollRef.current) { + clearInterval(pollRef.current) + pollRef.current = null + } + + return () => { + if (pollRef.current) { + clearInterval(pollRef.current) + } + } + }, [imports, fetchImports]) + + useEffect(() => { + if (uploadFetcher.state !== 'idle' || !uploadFetcher.data) return + if (uploadFetcher.data.intent !== 'upload-data-import') return + + if (uploadFetcher.data.success) { + toast.success(t('project.settings.dataImport.importStarted')) + setSelectedProvider(null) + fetchImports() + } else if (uploadFetcher.data.error) { + toast.error( + uploadFetcher.data.error || + t('project.settings.dataImport.uploadFailed'), + ) + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [uploadFetcher.state, uploadFetcher.data]) + + useEffect(() => { + if (deleteFetcher.state !== 'idle' || !deleteFetcher.data) return + if (deleteFetcher.data.intent !== 'delete-data-import') return + + if (deleteFetcher.data.success) { + toast.success(t('project.settings.dataImport.importDeleted')) + setDeleteTarget(null) + fetchImports() + } else if (deleteFetcher.data.error) { + toast.error( + deleteFetcher.data.error || + t('project.settings.dataImport.deleteFailed'), + ) + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [deleteFetcher.state, deleteFetcher.data]) + + const handleUpload = (file: File) => { + if (!selectedProvider) return + + const formData = new FormData() + formData.append('intent', 'upload-data-import') + formData.append('file', file) + formData.append('provider', selectedProvider) + + uploadFetcher.submit(formData, { + method: 'POST', + action: settingsAction, + encType: 'multipart/form-data', + }) + } + + const handleDelete = () => { + if (!deleteTarget) return + + deleteFetcher.submit( + { intent: 'delete-data-import', importId: String(deleteTarget.importId) }, + { method: 'POST', action: settingsAction }, + ) + } + + const uploading = uploadFetcher.state !== 'idle' + const deleting = deleteFetcher.state !== 'idle' + + const providerFileType = selectedProvider + ? t(`project.settings.dataImport.${selectedProvider}.fileType`) + : '' + + return ( +
+
+ + {t('project.settings.dataImport.importFrom')} + + + {t('project.settings.dataImport.importFromDesc')} + + +
+ {PROVIDERS.map((providerId) => { + const Icon = providerId === 'umami' ? UmamiSVG : FileTextIcon + const hasActive = imports.some( + (i) => i.status === 'pending' || i.status === 'processing', + ) + const name = + providerId.charAt(0).toUpperCase() + providerId.slice(1) + const fileType = t( + `project.settings.dataImport.${providerId}.fileType`, + ) + + return ( + + ) + })} +
+
+ +
+ + {t('project.settings.dataImport.importHistory')} + + + {loading ? ( +
+ +
+ ) : imports.length === 0 ? ( +
+ + + {t('project.settings.dataImport.noImports')} + +
+ ) : ( +
+ + + + + + + + + + + + {imports.map((imp) => ( + + + + + + + + + ))} + +
+ {t('project.settings.dataImport.provider')} + + {t('project.settings.dataImport.status')} + + {t('project.settings.dataImport.dateRange')} + + {t('project.settings.dataImport.rows')} + + {t('project.settings.dataImport.created')} + +
+ {imp.provider} + + + {imp.status === 'failed' && imp.errorMessage && ( + + {imp.errorMessage} + + )} + + {imp.dateFrom && imp.dateTo ? ( + <> + {dayjs(imp.dateFrom).format('MMM D, YYYY')} + {' — '} + {dayjs(imp.dateTo).format('MMM D, YYYY')} + + ) : ( + '—' + )} + + {imp.importedRows > 0 + ? imp.importedRows.toLocaleString() + : '—'} + + {dayjs(imp.createdAt).format('MMM D, YYYY HH:mm')} + + {(imp.status === 'completed' || + imp.status === 'failed') && ( +
+ +
+ )} +
+
+ )} +
+ + { + if (!uploading) setSelectedProvider(null) + }} + title={t('project.settings.dataImport.importFromProvider', { + provider: selectedProvider + ? selectedProvider.charAt(0).toUpperCase() + + selectedProvider.slice(1) + : '', + })} + size='medium' + message={ +
+ + + ), + }} + /> + + + + + {t('project.settings.dataImport.clickToUpload')} + {' '} + {t('project.settings.dataImport.orDragDrop')} + + ) + } + hint={ + uploading + ? undefined + : t('project.settings.dataImport.maxFileSize', { + fileType: providerFileType.toUpperCase(), + maxSize: 100, + }) + } + /> +
+ } + closeText={t('common.cancel')} + /> + + { + if (!deleting) setDeleteTarget(null) + }} + type='warning' + title={t('project.settings.dataImport.deleteImport')} + message={ + + {t('project.settings.dataImport.deleteImportConfirm', { + count: deleteTarget?.importedRows || 0, + })} + + } + submitText={t('common.delete')} + closeText={t('common.cancel')} + submitType='danger' + onSubmit={handleDelete} + isLoading={deleting} + /> +
+ ) +} diff --git a/web/app/pages/Project/View/ViewProject.tsx b/web/app/pages/Project/View/ViewProject.tsx index cb81011d9..e2e0a66dc 100644 --- a/web/app/pages/Project/View/ViewProject.tsx +++ b/web/app/pages/Project/View/ViewProject.tsx @@ -225,6 +225,7 @@ const defaultViewProjectContext: ViewProjectContextType = { setShowFiltersSearch: () => {}, resetDateRange: () => {}, refreshStats: async () => {}, + refCalendar: { current: null } as any, refCalendarCompare: { current: null } as any, diff --git a/web/app/pages/Project/tabs/Traffic/TrafficView.tsx b/web/app/pages/Project/tabs/Traffic/TrafficView.tsx index c7cb9a1ea..cfab31886 100644 --- a/web/app/pages/Project/tabs/Traffic/TrafficView.tsx +++ b/web/app/pages/Project/tabs/Traffic/TrafficView.tsx @@ -6,7 +6,7 @@ import _isEmpty from 'lodash/isEmpty' import _keys from 'lodash/keys' import _map from 'lodash/map' import _some from 'lodash/some' -import { ProhibitIcon, EyeIcon } from '@phosphor-icons/react' +import { ProhibitIcon, EyeIcon, DownloadIcon } from '@phosphor-icons/react' import React, { useState, useEffect, @@ -96,6 +96,7 @@ import Checkbox from '~/ui/Checkbox' import Dropdown from '~/ui/Dropdown' import Loader from '~/ui/Loader' import LoadingBar from '~/ui/LoadingBar' +import Tooltip from '~/ui/Tooltip' import { getLocaleDisplayName, nLocaleFormatter } from '~/utils/generic' import { groupRefEntries } from '~/utils/referrers' import routes from '~/utils/routes' @@ -222,6 +223,29 @@ function TrafficDataResolver({ ) } +function HasImportedIndicator() { + const { t } = useTranslation('common') + const { hasImportedData: hasImportedDataPromise } = + useLoaderData() + + const hasImportedData = hasImportedDataPromise + ? use(hasImportedDataPromise) + : false + + if (!hasImportedData) return null + + return ( + + + + } + /> + ) +} + function TrafficViewWrapper(props: TrafficViewProps) { return ( @@ -1055,6 +1079,9 @@ const TrafficViewInner = ({ onSwitch={setChartTypeOnClick} type={chartType} /> + + +
{!_isEmpty(overall) ? (
diff --git a/web/app/routes/projects.$id.tsx b/web/app/routes/projects.$id.tsx index f2abddb91..e724719ea 100644 --- a/web/app/routes/projects.$id.tsx +++ b/web/app/routes/projects.$id.tsx @@ -63,6 +63,51 @@ import { hasAuthTokens, } from '~/utils/session.server' +function computeDateRangeForPeriod( + period: string, + from?: string, + to?: string, +): { from: string; to: string } | null { + if (from && to) { + return { from: `${from} 00:00:00`, to: `${to} 23:59:59` } + } + + const now = new Date() + const pad = (n: number) => String(n).padStart(2, '0') + const fmt = (d: Date) => + `${d.getUTCFullYear()}-${pad(d.getUTCMonth() + 1)}-${pad(d.getUTCDate())} ${pad(d.getUTCHours())}:${pad(d.getUTCMinutes())}:${pad(d.getUTCSeconds())}` + + const subtract = (ms: number) => new Date(now.getTime() - ms) + const MS_HOUR = 3_600_000 + const MS_DAY = 86_400_000 + + const periodMap: Record { from: string; to: string }> = { + '1h': () => ({ from: fmt(subtract(MS_HOUR)), to: fmt(now) }), + today: () => { + const start = new Date(now) + start.setUTCHours(0, 0, 0, 0) + return { from: fmt(start), to: fmt(now) } + }, + yesterday: () => { + const dayStart = new Date(now.getTime() - MS_DAY) + dayStart.setUTCHours(0, 0, 0, 0) + const dayEnd = new Date(now.getTime() - MS_DAY) + dayEnd.setUTCHours(23, 59, 59, 999) + return { from: fmt(dayStart), to: fmt(dayEnd) } + }, + '1d': () => ({ from: fmt(subtract(MS_DAY)), to: fmt(now) }), + '7d': () => ({ from: fmt(subtract(7 * MS_DAY)), to: fmt(now) }), + '4w': () => ({ from: fmt(subtract(28 * MS_DAY)), to: fmt(now) }), + '3M': () => ({ from: fmt(subtract(90 * MS_DAY)), to: fmt(now) }), + '12M': () => ({ from: fmt(subtract(365 * MS_DAY)), to: fmt(now) }), + '24M': () => ({ from: fmt(subtract(730 * MS_DAY)), to: fmt(now) }), + all: () => ({ from: '2020-01-01 00:00:00', to: fmt(now) }), + } + + const compute = periodMap[period] + return compute ? compute() : null +} + function formatDateForBackend(dateStr: string): string { if (!dateStr) return '' @@ -269,6 +314,8 @@ export interface ProjectLoaderData { alertsData?: Promise // Funnels data funnelData?: Promise + // Data import + hasImportedData?: Promise } export async function loader({ request, params }: LoaderFunctionArgs) { @@ -386,6 +433,7 @@ export async function loader({ request, params }: LoaderFunctionArgs) { let goalsData: Promise | undefined let alertsData: Promise | undefined let funnelData: Promise | undefined + let hasImportedData: Promise | undefined // Only fetch data if project is valid and not locked if (project && !project.isLocked && projectId) { @@ -510,6 +558,19 @@ export async function loader({ request, params }: LoaderFunctionArgs) { ).then((res) => res.data) } } + + const importRange = computeDateRangeForPeriod(period, from, to) + if (importRange) { + hasImportedData = serverFetch<{ hasImportedData: boolean }>( + request, + `data-import/${projectId}/has-imported-data?from=${encodeURIComponent(importRange.from)}&to=${encodeURIComponent(importRange.to)}`, + { + headers: password ? { 'x-password': password } : undefined, + }, + ) + .then((res) => res.data?.hasImportedData === true) + .catch(() => false) + } } return data( @@ -532,6 +593,7 @@ export async function loader({ request, params }: LoaderFunctionArgs) { goalsData, alertsData, funnelData, + hasImportedData, }, { headers: createHeadersWithCookies(cookies) }, ) diff --git a/web/app/routes/projects.settings.$id.tsx b/web/app/routes/projects.settings.$id.tsx index 3e8af894c..3edf4578a 100644 --- a/web/app/routes/projects.settings.$id.tsx +++ b/web/app/routes/projects.settings.$id.tsx @@ -8,7 +8,7 @@ import { data, redirect } from 'react-router' import { serverFetch } from '~/api/api.server' import { getOgImageUrl } from '~/lib/constants' -import { Project } from '~/lib/models/Project' +import { Project, DataImport } from '~/lib/models/Project' import { Subscriber } from '~/lib/models/Subscriber' import ProjectSettings from '~/pages/Project/Settings' import { getDescription, getPreviewImage, getTitle } from '~/utils/seo' @@ -91,6 +91,8 @@ export interface ProjectSettingsActionData { gscAuthUrl?: string gscStatus?: GscStatus gscProperties?: GscProperty[] + dataImports?: DataImport[] + dataImport?: unknown } export async function action({ request, params }: ActionFunctionArgs) { @@ -769,6 +771,91 @@ export async function action({ request, params }: ActionFunctionArgs) { ) } + case 'list-data-imports': { + const result = await serverFetch( + request, + `data-import/${id}`, + ) + + if (result.error) { + return data( + { intent, error: result.error as string }, + { status: 400 }, + ) + } + + return data( + { + intent, + success: true, + dataImports: (result.data as DataImport[]) || [], + }, + { headers: createHeadersWithCookies(result.cookies) }, + ) + } + + case 'upload-data-import': { + const file = formData.get('file') as File + const provider = formData.get('provider')?.toString() + + if (!file || !provider) { + return data( + { intent, error: 'File and provider are required' }, + { status: 400 }, + ) + } + + const proxyForm = new FormData() + proxyForm.append('file', file, file.name) + proxyForm.append('provider', provider) + + const result = await serverFetch(request, `data-import/${id}/upload`, { + method: 'POST', + body: proxyForm, + }) + + if (result.error) { + return data( + { intent, error: result.error as string }, + { status: result.status }, + ) + } + + return data( + { intent, success: true, dataImport: result.data }, + { headers: createHeadersWithCookies(result.cookies) }, + ) + } + + case 'delete-data-import': { + const importId = formData.get('importId')?.toString() + + if (!importId) { + return data( + { intent, error: 'Import ID is required' }, + { status: 400 }, + ) + } + + const result = await serverFetch( + request, + `data-import/${id}/${importId}`, + { method: 'DELETE' }, + ) + + if (result.error) { + return data( + { intent, error: result.error as string }, + { status: 400 }, + ) + } + + return data( + { intent, success: true }, + { headers: createHeadersWithCookies(result.cookies) }, + ) + } + default: return data( { error: 'Unknown action' }, diff --git a/web/app/ui/FileUpload.tsx b/web/app/ui/FileUpload.tsx new file mode 100644 index 000000000..ac039337b --- /dev/null +++ b/web/app/ui/FileUpload.tsx @@ -0,0 +1,138 @@ +import { useCallback, useId, useRef, useState } from 'react' +import { UploadIcon, SpinnerIcon } from '@phosphor-icons/react' + +import { cn } from '~/utils/generic' + +import { Text } from './Text' + +interface FileUploadProps { + accept?: string + disabled?: boolean + loading?: boolean + onFile: (file: File) => void + label?: React.ReactNode + hint?: React.ReactNode + className?: string +} + +export default function FileUpload({ + accept, + disabled, + loading, + onFile, + label, + hint, + className, +}: FileUploadProps) { + const inputId = useId() + const inputRef = useRef(null) + const [dragOver, setDragOver] = useState(false) + + const isDisabled = disabled || loading + + const handleFile = useCallback( + (file: File | undefined) => { + if (!file || isDisabled) return + onFile(file) + }, + [isDisabled, onFile], + ) + + const handleDragOver = useCallback( + (e: React.DragEvent) => { + e.preventDefault() + e.stopPropagation() + if (!isDisabled) setDragOver(true) + }, + [isDisabled], + ) + + const handleDragLeave = useCallback((e: React.DragEvent) => { + e.preventDefault() + e.stopPropagation() + setDragOver(false) + }, []) + + const handleDrop = useCallback( + (e: React.DragEvent) => { + e.preventDefault() + e.stopPropagation() + setDragOver(false) + if (isDisabled) return + handleFile(e.dataTransfer.files?.[0]) + }, + [isDisabled, handleFile], + ) + + const handleChange = useCallback( + (e: React.ChangeEvent) => { + handleFile(e.target.files?.[0]) + e.target.value = '' + }, + [handleFile], + ) + + return ( + + ) +} diff --git a/web/app/ui/icons/Umami.tsx b/web/app/ui/icons/Umami.tsx new file mode 100644 index 000000000..18d017436 --- /dev/null +++ b/web/app/ui/icons/Umami.tsx @@ -0,0 +1,20 @@ +const UmamiSVG = ({ className }: { className?: string }) => ( + + + + + + + + +) + +export default UmamiSVG diff --git a/web/public/locales/en.json b/web/public/locales/en.json index 96bd490c2..af8e90e00 100644 --- a/web/public/locales/en.json +++ b/web/public/locales/en.json @@ -1702,7 +1702,9 @@ "integrations": "Integrations", "integrationsDesc": "Connect external services to enrich your analytics.", "revenue": "Revenue", - "revenueDesc": "Track revenue and attribute sales to campaigns." + "revenueDesc": "Track revenue and attribute sales to campaigns.", + "import": "Import data", + "importDesc": "Import analytics data from other platforms" }, "revenue": { "title": "Revenue Tracking", @@ -1734,6 +1736,40 @@ "paddleExample": "Paddle Checkout Integration:" } }, + "dataImport": { + "importFrom": "Import from", + "importFromDesc": "Select an analytics provider to import historical data from.", + "importHistory": "Import history", + "noImports": "No imports yet. Select a provider above to get started.", + "provider": "Provider", + "status": "Status", + "dateRange": "Date range", + "rows": "Rows", + "created": "Created", + "deleteImport": "Delete import", + "deleteImportConfirm": "This will permanently remove all imported data ({{count}} rows) from your analytics. This action cannot be undone.", + "importStarted": "Import started. Your data is being processed.", + "importDeleted": "Import deleted successfully", + "loadFailed": "Failed to load imports", + "uploadFailed": "Failed to upload file. Please try again.", + "deleteFailed": "Failed to delete import", + "clickToUpload": "Click to upload", + "orDragDrop": "or drag and drop", + "maxFileSize": "{{fileType}} file, max {{maxSize}} MB", + "uploadingProcessing": "Uploading and processing...", + "importFromProvider": "Import from {{provider}}", + "statsIncludeImported": "Stats include imported data", + "imported": "Imported", + "statusPending": "Pending", + "statusProcessing": "Processing", + "statusCompleted": "Completed", + "statusFailed": "Failed", + "uploadInstructions": "Please select your export file. Once started, the data will be imported in the background. Please allow for some time to complete. To learn more how to export 3rd party providers data, <1>see our docs.", + "umami": { + "description": "Import pageviews and custom events from Umami export", + "fileType": ".zip" + } + }, "captcha": { "title": "CAPTCHA Integration", "description": "Enable CAPTCHA protection for your forms and actions. Generate a secret key to integrate Swetrix CAPTCHA with your application.",