Merge Philipinho/storage

storage module

server/.env.example

@@ -0,0 +1,23 @@
+PORT=3001
+DEBUG_MODE=true
+NODE_ENV=production
+
+JWT_SECRET_KEY=ba8642edbed7f6c450e46875e8c835c7e417031abe1f7b03f3e56bb7481706d8
+JWT_TOKEN_EXPIRES_IN=30d
+
+DATABASE_URL="postgresql://postgres:password@localhost:5432/dc?schema=public"
+
+# local | s3
+STORAGE_DRIVER=local
+
+# local config
+LOCAL_STORAGE_PATH=/storage
+
+# S3 Config
+AWS_S3_ACCESS_KEY_ID=
+AWS_S3_SECRET_ACCESS_KEY=
+AWS_S3_REGION=
+AWS_S3_BUCKET=
+AWS_S3_ENDPOINT=
+AWS_S3_URL=
+AWS_S3_USE_PATH_STYLE_ENDPOINT=false

server/package.json

@@ -27,6 +27,8 @@
     "migration:show": "npm run typeorm migration:show"
   },
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.417.0",
+    "@aws-sdk/s3-request-presigner": "^3.418.0",
     "@hocuspocus/server": "^2.5.0",
     "@hocuspocus/transformer": "^2.5.0",
     "@nestjs/common": "^10.0.0",
@@ -42,6 +44,8 @@
     "class-transformer": "^0.5.1",
     "class-validator": "^0.14.0",
     "fastify": "^4.22.2",
+    "fs-extra": "^11.1.1",
+    "mime-types": "^2.1.35",
     "pg": "^8.11.3",
     "reflect-metadata": "^0.1.13",
     "rxjs": "^7.8.1",
@@ -55,7 +59,9 @@
     "@nestjs/testing": "^10.0.0",
     "@types/bcrypt": "^5.0.0",
     "@types/express": "^4.17.17",
+    "@types/fs-extra": "^11.0.2",
     "@types/jest": "^29.5.2",
+    "@types/mime-types": "^2.1.1",
     "@types/node": "^20.3.1",
     "@types/supertest": "^2.0.12",
     "@types/uuid": "^9.0.3",

server/src/core/attachment/attachment.module.ts

@@ -0,0 +1,7 @@
+import { Module } from '@nestjs/common';
+import { AttachmentService } from './attachment.service';
+
+@Module({
+  providers: [AttachmentService],
+})
+export class AttachmentModule {}

server/src/core/attachment/attachment.service.spec.ts

@@ -0,0 +1,18 @@
+import { Test, TestingModule } from '@nestjs/testing';
+import { AttachmentService } from './attachment.service';
+
+describe('AttachmentService', () => {
+  let service: AttachmentService;
+
+  beforeEach(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      providers: [AttachmentService],
+    }).compile();
+
+    service = module.get<AttachmentService>(AttachmentService);
+  });
+
+  it('should be defined', () => {
+    expect(service).toBeDefined();
+  });
+});

server/src/core/attachment/attachment.service.ts

@@ -0,0 +1,4 @@
+import { Injectable } from '@nestjs/common';
+
+@Injectable()
+export class AttachmentService {}

server/src/core/core.module.ts

@@ -3,8 +3,20 @@ import { UserModule } from './user/user.module';
 import { AuthModule } from './auth/auth.module';
 import { WorkspaceModule } from './workspace/workspace.module';
 import { PageModule } from './page/page.module';
+import { StorageModule } from './storage/storage.module';
+import { AttachmentModule } from './attachment/attachment.module';
+import { EnvironmentModule } from '../environment/environment.module';
 
 @Module({
-  imports: [UserModule, AuthModule, WorkspaceModule, PageModule],
+  imports: [
+    UserModule,
+    AuthModule,
+    WorkspaceModule,
+    PageModule,
+    StorageModule.forRootAsync({
+      imports: [EnvironmentModule],
+    }),
+    AttachmentModule,
+  ],
 })
 export class CoreModule {}

server/src/core/storage/constants/storage.constants.ts

@@ -0,0 +1,2 @@
+export const STORAGE_DRIVER_TOKEN = 'STORAGE_DRIVER_TOKEN';
+export const STORAGE_CONFIG_TOKEN = 'STORAGE_CONFIG_TOKEN';

server/src/core/storage/drivers/index.ts

@@ -0,0 +1,2 @@
+export { LocalDriver } from './local.driver';
+export { S3Driver } from './s3.driver';

server/src/core/storage/drivers/local.driver.ts

@@ -0,0 +1,71 @@
+import {
+  StorageDriver,
+  LocalStorageConfig,
+  StorageOption,
+} from '../interfaces';
+import { join } from 'path';
+import * as fs from 'fs-extra';
+
+export class LocalDriver implements StorageDriver {
+  private readonly config: LocalStorageConfig;
+
+  constructor(config: LocalStorageConfig) {
+    this.config = config;
+  }
+
+  private _fullPath(filePath: string): string {
+    return join(this.config.storagePath, filePath);
+  }
+
+  async upload(filePath: string, file: Buffer): Promise<void> {
+    try {
+      await fs.outputFile(this._fullPath(filePath), file);
+    } catch (error) {
+      throw new Error(`Failed to upload file: ${error.message}`);
+    }
+  }
+
+  async read(filePath: string): Promise<Buffer> {
+    try {
+      return await fs.readFile(this._fullPath(filePath));
+    } catch (error) {
+      throw new Error(`Failed to read file: ${error.message}`);
+    }
+  }
+
+  async exists(filePath: string): Promise<boolean> {
+    try {
+      return await fs.pathExists(this._fullPath(filePath));
+    } catch (error) {
+      throw new Error(`Failed to check file existence: ${error.message}`);
+    }
+  }
+
+  async getSignedUrl(filePath: string, expireIn: number): Promise<string> {
+    throw new Error('Signed URLs are not supported for local storage.');
+  }
+
+  getUrl(filePath: string): string {
+    return this._fullPath(filePath);
+  }
+
+  async delete(filePath: string): Promise<void> {
+    try {
+      await fs.remove(this._fullPath(filePath));
+    } catch (error) {
+      throw new Error(`Failed to delete file: ${error.message}`);
+    }
+  }
+
+  getDriver(): typeof fs {
+    return fs;
+  }
+
+  getDriverName(): string {
+    return StorageOption.LOCAL;
+  }
+
+  getConfig(): Record<string, any> {
+    return this.config;
+  }
+}

server/src/core/storage/drivers/s3.driver.ts

@@ -0,0 +1,115 @@
+import { S3StorageConfig, StorageDriver, StorageOption } from '../interfaces';
+import {
+  DeleteObjectCommand,
+  GetObjectCommand,
+  HeadObjectCommand,
+  NoSuchKey,
+  PutObjectCommand,
+  S3Client,
+} from '@aws-sdk/client-s3';
+import { streamToBuffer } from '../storage.utils';
+import { Readable } from 'stream';
+import * as mime from 'mime-types';
+import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
+
+export class S3Driver implements StorageDriver {
+  private readonly s3Client: S3Client;
+  private readonly config: S3StorageConfig;
+
+  constructor(config: S3StorageConfig) {
+    this.config = config;
+    this.s3Client = new S3Client(config);
+  }
+
+  async upload(filePath: string, file: Buffer): Promise<void> {
+    try {
+      const contentType =
+        mime.contentType(filePath) || 'application/octet-stream';
+
+      const command = new PutObjectCommand({
+        Bucket: this.config.bucket,
+        Key: filePath,
+        Body: file,
+        ContentType: contentType,
+        // ACL: "public-read",
+      });
+
+      await this.s3Client.send(command);
+      // we can get the path from location
+
+      console.log(`File uploaded successfully: ${filePath}`);
+    } catch (error) {
+      throw new Error(`Failed to upload file: ${error.message}`);
+    }
+  }
+
+  async read(filePath: string): Promise<Buffer> {
+    try {
+      const command = new GetObjectCommand({
+        Bucket: this.config.bucket,
+        Key: filePath,
+      });
+
+      const response = await this.s3Client.send(command);
+
+      return streamToBuffer(response.Body as Readable);
+    } catch (error) {
+      throw new Error(`Failed to read file from S3: ${error.message}`);
+    }
+  }
+
+  async exists(filePath: string): Promise<boolean> {
+    try {
+      const command = new HeadObjectCommand({
+        Bucket: this.config.bucket,
+        Key: filePath,
+      });
+
+      await this.s3Client.send(command);
+      return true;
+    } catch (err) {
+      if (err instanceof NoSuchKey) {
+        return false;
+      }
+      throw err;
+    }
+  }
+  getUrl(filePath: string): string {
+    return `${this.config.endpoint}/${this.config.bucket}/${filePath}`;
+  }
+
+  async getSignedUrl(filePath: string, expiresIn: number): Promise<string> {
+    const command = new GetObjectCommand({
+      Bucket: this.config.bucket,
+      Key: filePath,
+    });
+    return await getSignedUrl(this.s3Client, command, { expiresIn });
+  }
+
+  async delete(filePath: string): Promise<void> {
+    try {
+      const command = new DeleteObjectCommand({
+        Bucket: this.config.bucket,
+        Key: filePath,
+      });
+
+      await this.s3Client.send(command);
+    } catch (err) {
+      throw new Error(
+        `Error deleting file ${filePath} from S3. ${err.message}`,
+      );
+    }
+  }
+
+  getDriver(): S3Client {
+    return this.s3Client;
+  }
+
+  getDriverName(): string {
+    return StorageOption.S3;
+  }
+
+  getConfig(): Record<string, any> {
+    return this.config;
+  }
+}

server/src/core/storage/interfaces/index.ts

@@ -0,0 +1,2 @@
+export * from './storage-driver.interface';
+export * from './storage.interface';

server/src/core/storage/interfaces/storage-driver.interface.ts

@@ -0,0 +1,19 @@
+export interface StorageDriver {
+  upload(filePath: string, file: Buffer): Promise<void>;
+
+  read(filePath: string): Promise<Buffer>;
+
+  exists(filePath: string): Promise<boolean>;
+
+  getUrl(filePath: string): string;
+
+  getSignedUrl(filePath: string, expireIn: number): Promise<string>;
+
+  delete(filePath: string): Promise<void>;
+
+  getDriver(): any;
+
+  getDriverName(): string;
+
+  getConfig(): Record<string, any>;
+}

server/src/core/storage/interfaces/storage.interface.ts

@@ -0,0 +1,33 @@
+import { S3ClientConfig } from '@aws-sdk/client-s3';
+
+export enum StorageOption {
+  LOCAL = 'local',
+  S3 = 's3',
+}
+
+export type StorageConfig =
+  | { driver: StorageOption.LOCAL; config: LocalStorageConfig }
+  | { driver: StorageOption.S3; config: S3StorageConfig };
+
+export interface LocalStorageConfig {
+  storagePath: string;
+}
+
+export interface S3StorageConfig
+  extends Omit<S3ClientConfig, 'endpoint' | 'bucket'> {
+  endpoint: string; // Enforce endpoint
+  bucket: string; // Enforce bucket
+  baseUrl?: string; // Optional CDN URL for assets
+}
+
+export interface StorageOptions {
+  disk: StorageConfig;
+}
+
+export interface StorageOptionsFactory {
+  createStorageOptions(): Promise<StorageConfig> | StorageConfig;
+}
+
+export interface StorageModuleOptions {
+  imports?: any[];
+}

server/src/core/storage/providers/storage.provider.ts

@@ -0,0 +1,66 @@
+import {
+  STORAGE_CONFIG_TOKEN,
+  STORAGE_DRIVER_TOKEN,
+} from '../constants/storage.constants';
+import { EnvironmentService } from '../../../environment/environment.service';
+import {
+  LocalStorageConfig,
+  S3StorageConfig,
+  StorageConfig,
+  StorageDriver,
+  StorageOption,
+} from '../interfaces';
+import { LocalDriver, S3Driver } from '../drivers';
+
+function createStorageDriver(disk: StorageConfig): StorageDriver {
+  switch (disk.driver) {
+    case StorageOption.LOCAL:
+      return new LocalDriver(disk.config as LocalStorageConfig);
+    case StorageOption.S3:
+      return new S3Driver(disk.config as S3StorageConfig);
+    default:
+      throw new Error(`Unknown storage driver`);
+  }
+}
+
+export const storageDriverConfigProvider = {
+  provide: STORAGE_CONFIG_TOKEN,
+  useFactory: async (environmentService: EnvironmentService) => {
+    const driver = environmentService.getStorageDriver();
+
+    if (driver === StorageOption.LOCAL) {
+      return {
+        driver,
+        config: {
+          storagePath:
+            process.cwd() + '/' + environmentService.getLocalStoragePath(),
+        },
+      };
+    }
+
+    if (driver === StorageOption.S3) {
+      return {
+        driver,
+        config: {
+          region: environmentService.getAwsS3Region(),
+          endpoint: environmentService.getAwsS3Endpoint(),
+          bucket: environmentService.getAwsS3Bucket(),
+          credentials: {
+            accessKeyId: environmentService.getAwsS3AccessKeyId(),
+            secretAccessKey: environmentService.getAwsS3SecretAccessKey(),
+          },
+        },
+      };
+    }
+
+    throw new Error(`Unknown storage driver: ${driver}`);
+  },
+
+  inject: [EnvironmentService],
+};
+
+export const storageDriverProvider = {
+  provide: STORAGE_DRIVER_TOKEN,
+  useFactory: (config) => createStorageDriver(config),
+  inject: [STORAGE_CONFIG_TOKEN],
+};

server/src/core/storage/storage.module.ts

@@ -0,0 +1,24 @@
+import { DynamicModule, Global, Module } from '@nestjs/common';
+import { StorageModuleOptions } from './interfaces';
+import { StorageService } from './storage.service';
+import {
+  storageDriverConfigProvider,
+  storageDriverProvider,
+} from './providers/storage.provider';
+
+@Global()
+@Module({})
+export class StorageModule {
+  static forRootAsync(options: StorageModuleOptions): DynamicModule {
+    return {
+      module: StorageModule,
+      imports: options.imports || [],
+      providers: [
+        storageDriverConfigProvider,
+        storageDriverProvider,
+        StorageService,
+      ],
+      exports: [StorageService],
+    };
+  }
+}

server/src/core/storage/storage.service.spec.ts

@@ -0,0 +1,18 @@
+import { Test, TestingModule } from '@nestjs/testing';
+import { StorageService } from './storage.service';
+
+describe('StorageService', () => {
+  let service: StorageService;
+
+  beforeEach(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      providers: [StorageService],
+    }).compile();
+
+    service = module.get<StorageService>(StorageService);
+  });
+
+  it('should be defined', () => {
+    expect(service).toBeDefined();
+  });
+});

server/src/core/storage/storage.service.ts

@@ -0,0 +1,34 @@
+import { Inject, Injectable } from '@nestjs/common';
+import { STORAGE_DRIVER_TOKEN } from './constants/storage.constants';
+import { StorageDriver } from './interfaces';
+
+@Injectable()
+export class StorageService {
+  constructor(
+    @Inject(STORAGE_DRIVER_TOKEN) private storageDriver: StorageDriver,
+  ) {}
+
+  async upload(filePath: string, fileContent: Buffer | any) {
+    await this.storageDriver.upload(filePath, fileContent);
+  }
+
+  async read(filePath: string): Promise<Buffer> {
+    return this.storageDriver.read(filePath);
+  }
+
+  async exists(filePath: string): Promise<boolean> {
+    return this.storageDriver.exists(filePath);
+  }
+
+  async signedUrl(path: string, expireIn: number): Promise<string> {
+    return this.storageDriver.getSignedUrl(path, expireIn);
+  }
+
+  url(filePath: string): string {
+    return this.storageDriver.getUrl(filePath);
+  }
+
+  async delete(filePath: string): Promise<void> {
+    await this.storageDriver.delete(filePath);
+  }
+}

server/src/core/storage/storage.utils.ts

@@ -0,0 +1,10 @@
+import { Readable } from 'stream';
+
+export function streamToBuffer(readableStream: Readable): Promise<Buffer> {
+  return new Promise((resolve, reject) => {
+    const chunks: Uint8Array[] = [];
+    readableStream.on('data', (chunk) => chunks.push(chunk));
+    readableStream.on('end', () => resolve(Buffer.concat(chunks)));
+    readableStream.on('error', reject);
+  });
+}

server/src/environment/environment.service.ts

@@ -23,4 +23,40 @@ export class EnvironmentService {
   getJwtTokenExpiresIn(): string {
     return this.configService.get<string>('JWT_TOKEN_EXPIRES_IN');
   }
+
+  getStorageDriver(): string {
+    return this.configService.get<string>('STORAGE_DRIVER');
+  }
+
+  getLocalStoragePath(): string {
+    return this.configService.get<string>('LOCAL_STORAGE_PATH');
+  }
+
+  getAwsS3AccessKeyId(): string {
+    return this.configService.get<string>('AWS_S3_ACCESS_KEY_ID');
+  }
+
+  getAwsS3SecretAccessKey(): string {
+    return this.configService.get<string>('AWS_S3_SECRET_ACCESS_KEY');
+  }
+
+  getAwsS3Region(): string {
+    return this.configService.get<string>('AWS_S3_REGION');
+  }
+
+  getAwsS3Bucket(): string {
+    return this.configService.get<string>('AWS_S3_BUCKET');
+  }
+
+  getAwsS3Endpoint(): string {
+    return this.configService.get<string>('AWS_S3_ENDPOINT');
+  }
+
+  getAwsS3Url(): string {
+    return this.configService.get<string>('AWS_S3_URL');
+  }
+
+  getAwsS3UsePathStyleEndpoint(): boolean {
+    return this.configService.get<boolean>('AWS_S3_USE_PATH_STYLE_ENDPOINT');
+  }
 }