Merge pull request #51 from tailor-cms/chore/fix-image-storage

Fix image storage

apps/backend/package.json

@@ -116,6 +116,7 @@
     "untildify": "^5.0.0",
     "url-join": "^5.0.0",
     "url-parse": "^1.5.10",
+    "uuid": "^8.3.2",
     "yn": "^5.0.0",
     "yup": "^0.32.11"
   },

apps/backend/shared/ai/ai.service.js

@@ -2,6 +2,7 @@ import isString from 'lodash/isString.js';
 import OpenAI from 'openai';
 import { schema as schemaConfig } from 'tailor-config-shared';
 import shuffle from 'lodash/shuffle.js';
+import StorageService from '../storage/storage.service.js';
 
 import { ai as aiConfig } from '../../config/server/index.js';
 import createLogger from '../logger.js';
@@ -249,11 +250,13 @@ class AIService {
       Return the prompt as JSON respecting the following format:
       { 'prompt': '' }`;
     const { prompt: dallePrompt } = await this.requestCompletion(promptQuery);
-    logger.info('DALL·E 3 prompt', dallePrompt);
-    const url = await this.generateImage(dallePrompt);
+    const imgUrl = await this.generateImage(dallePrompt);
+    const imgInternalUrl = await StorageService.downloadToStorage(imgUrl);
     const imageElement = {
       type: 'CE_IMAGE',
-      data: { url },
+      data: {
+        assets: { url: imgInternalUrl },
+      },
     };
     logger.info('Generated image element', imageElement);
     return imageElement;
@@ -268,7 +271,8 @@ class AIService {
       size: '1024x1024',
       style: 'natural',
     });
-    return data[0].url;
+    const url = new URL(data[0].url);
+    return url;
   }
 }
 

apps/backend/shared/storage/storage.controller.js

@@ -1,15 +1,9 @@
-import { readFile, sha256 } from './util.js';
-import { storage as config } from '../../config/server/index.js';
 import fecha from 'fecha';
-import fromPairs from 'lodash/fromPairs.js';
-import JSZip from 'jszip';
-import mime from 'mime-types';
 import path from 'node:path';
-import pickBy from 'lodash/pickBy.js';
 import Storage from '../../repository/storage.js';
+import StorageService from './storage.service.js';
 
-const { getFileUrl, getPath, saveFile } = Storage;
-const getStorageUrl = (key) => `${config.protocol}${key}`;
+const { getFileUrl } = Storage;
 
 function getUrl(req, res) {
   const {
@@ -24,41 +18,18 @@ async function upload({ file, body, user, repository }, res) {
   if (body.unpack) {
     const timestamp = fecha.format(new Date(), 'YYYY-MM-DDTHH:mm:ss');
     const root = `${timestamp}__${user.id}__${name}`;
-    const assets = await uploadArchiveContent(repositoryId, file, root);
+    const assets = await StorageService.uploadArchiveContent(
+      repositoryId,
+      file,
+      root,
+    );
     return res.json({ root, assets });
   }
-  const asset = await uploadFile(repositoryId, file, name);
+  const asset = await StorageService.uploadFile(repositoryId, file, name);
   return res.json(asset);
 }
 
 export default {
   getUrl,
   upload,
 };
-
-async function uploadFile(repositoryId, file, name) {
-  const buffer = await readFile(file);
-  const hash = sha256(file.originalname, buffer);
-  const extension = path.extname(file.originalname);
-  const fileName = `${hash}___${name}${extension}`;
-  const key = path.join(getPath(repositoryId), fileName);
-  await saveFile(key, buffer, { ContentType: file.mimetype });
-  const publicUrl = await getFileUrl(key);
-  return { key, publicUrl, url: getStorageUrl(key) };
-}
-
-async function uploadArchiveContent(repositoryId, archive, name) {
-  const buffer = await readFile(archive);
-  const content = await JSZip.loadAsync(buffer);
-  const files = pickBy(content.files, (it) => !it.dir);
-  const keys = await Promise.all(
-    Object.keys(files).map(async (src) => {
-      const key = path.join(getPath(repositoryId), name, src);
-      const file = await content.file(src).async('uint8array');
-      const mimeType = mime.lookup(src);
-      await saveFile(key, Buffer.from(file), { ContentType: mimeType });
-      return [key, getStorageUrl(key)];
-    }),
-  );
-  return fromPairs(keys);
-}

apps/backend/shared/storage/storage.service.js

@@ -0,0 +1,58 @@
+import { readFile, sha256 } from './util.js';
+import { storage as config } from '../../config/server/index.js';
+import fromPairs from 'lodash/fromPairs.js';
+import JSZip from 'jszip';
+import mime from 'mime-types';
+import path from 'node:path';
+import pickBy from 'lodash/pickBy.js';
+import request from 'axios';
+import { v4 as uuidv4 } from 'uuid';
+
+import Storage from '../../repository/storage.js';
+
+const { getFileUrl, getPath, saveFile } = Storage;
+const getStorageUrl = (key) => `${config.protocol}${key}`;
+
+class StorageService {
+  // Prefix key with custom protocol. e.g. storage://sample_key.ext
+  getStorageUrl = (key) => `${config.protocol}${key}`;
+
+  async uploadFile(repositoryId, file, name) {
+    const buffer = await readFile(file);
+    const hash = sha256(file.originalname, buffer);
+    const extension = path.extname(file.originalname);
+    const fileName = `${hash}___${name}${extension}`;
+    const key = path.join(getPath(repositoryId), fileName);
+    await saveFile(key, buffer, { ContentType: file.mimetype });
+    const publicUrl = await getFileUrl(key);
+    return { key, publicUrl, url: getStorageUrl(key) };
+  }
+
+  async uploadArchiveContent(repositoryId, archive, name) {
+    const buffer = await readFile(archive);
+    const content = await JSZip.loadAsync(buffer);
+    const files = pickBy(content.files, (it) => !it.dir);
+    const keys = await Promise.all(
+      Object.keys(files).map(async (src) => {
+        const key = path.join(getPath(repositoryId), name, src);
+        const file = await content.file(src).async('uint8array');
+        const mimeType = mime.lookup(src);
+        await saveFile(key, Buffer.from(file), { ContentType: mimeType });
+        return [key, getStorageUrl(key)];
+      }),
+    );
+    return fromPairs(keys);
+  }
+
+  async downloadToStorage(url) {
+    const res = await request.get(url, { responseType: 'arraybuffer' });
+    const filename = path.join(
+      getPath(),
+      `${uuidv4()}__${url.pathname.split('/').pop()}`,
+    );
+    await Storage.saveFile(filename, res.data);
+    return getStorageUrl(filename);
+  }
+}
+
+export default new StorageService();

apps/frontend/nuxt.config.ts

@@ -15,6 +15,9 @@ export default defineNuxtConfig({
     '/api/**': {
       proxy: 'http://localhost:3000/api/**',
     },
+    '/repository/assets/**': {
+      proxy: 'http://localhost:3000/repository/assets/**',
+    },
   },
   css: [
     '@mdi/font/css/materialdesignicons.min.css',