Video frame extraction

api/config.py

@@ -272,7 +272,7 @@ def __init__(self):
         self.SMTP_USERNAME = get_env('SMTP_USERNAME')
         self.SMTP_PASSWORD = get_env('SMTP_PASSWORD')
         self.SMTP_USE_TLS = get_bool_env('SMTP_USE_TLS')
-        
+
         # ------------------------
         # Workspace Configurations.
         # ------------------------

api/controllers/console/app/completion.py

@@ -78,7 +78,7 @@ def post(self, app_model):
         except ValueError as e:
             raise e
         except Exception as e:
-            logging.exception("internal server error.")
+            logging.exception(f"internal server error, {str(e)}")
             raise InternalServerError()
 
 

api/controllers/console/datasets/file.py

@@ -52,7 +52,7 @@ def post(self):
         if len(request.files) > 1:
             raise TooManyFilesError()
         try:
-            upload_file = FileService.upload_file(file, current_user)
+            upload_file = FileService.upload_file(file=file, user=current_user, tenant_id=current_user.current_tenant_id)
         except services.errors.file.FileTooLargeError as file_too_large_error:
             raise FileTooLargeError(file_too_large_error.description)
         except services.errors.file.UnsupportedFileTypeError:

api/controllers/files/__init__.py

@@ -5,5 +5,4 @@
 bp = Blueprint('files', __name__)
 api = ExternalApi(bp)
 
-
 from . import image_preview, tool_files

api/controllers/files/tool_files.py

@@ -20,10 +20,10 @@ def get(self, file_id, extension):
         args = parser.parse_args()
 
         if not ToolFileManager.verify_file(file_id=file_id,
-                                            timestamp=args['timestamp'],
-                                            nonce=args['nonce'],
-                                            sign=args['sign'],
-        ):
+                                           timestamp=args['timestamp'],
+                                           nonce=args['nonce'],
+                                           sign=args['sign'],
+                                           ):
             raise Forbidden('Invalid request.')
 
         try:

api/controllers/web/file.py

@@ -15,6 +15,7 @@ class FileApi(WebApiResource):
     def post(self, app_model, end_user):
         # get file from request
         file = request.files['file']
+        source = request.args.get('source')
 
         # check file
         if 'file' not in request.files:
@@ -23,7 +24,7 @@ def post(self, app_model, end_user):
         if len(request.files) > 1:
             raise TooManyFilesError()
         try:
-            upload_file = FileService.upload_file(file, end_user)
+            upload_file = FileService.upload_file(file=file, user=end_user, tenant_id=app_model.tenant_id, source=source)
         except services.errors.file.FileTooLargeError as file_too_large_error:
             raise FileTooLargeError(file_too_large_error.description)
         except services.errors.file.UnsupportedFileTypeError:

api/core/app/app_config/entities.py

@@ -188,6 +188,7 @@ class FileExtraConfig(BaseModel):
     File Upload Entity.
     """
     image_config: Optional[dict[str, Any]] = None
+    video_config: Optional[dict[str, Any]] = None
 
 
 class AppAdditionalFeatures(BaseModel):

api/core/app/app_config/features/file_upload/manager.py

@@ -12,6 +12,8 @@ def convert(cls, config: dict, is_vision: bool = True) -> Optional[FileExtraConf
         :param config: model config args
         :param is_vision: if True, the feature is vision feature
         """
+        video_config = None
+        image_config = None
         file_upload_dict = config.get('file_upload')
         if file_upload_dict:
             if 'image' in file_upload_dict and file_upload_dict['image']:
@@ -24,9 +26,21 @@ def convert(cls, config: dict, is_vision: bool = True) -> Optional[FileExtraConf
                     if is_vision:
                         image_config['detail'] = file_upload_dict['image']['detail']
 
-                    return FileExtraConfig(
-                        image_config=image_config
-                    )
+            if 'video' in file_upload_dict and file_upload_dict['video']:
+                video_config = dict()
+                if file_upload_dict['video']['extract_video'] == 'enabled':
+                    video_config.update({
+                        'extract_video': file_upload_dict['video']['extract_video'],
+                        'max_collect_frames': file_upload_dict['video']['max_collect_frames'],
+                        'similarity_threshold': file_upload_dict['video']['similarity_threshold'],
+                        'blur_threshold': file_upload_dict['video']['blur_threshold']
+                    })
+                if file_upload_dict['video']['extract_audio'] == 'enabled':
+                    video_config.update({
+                            'extract_audio': file_upload_dict['video']['extract_audio']
+                        })
+
+            return FileExtraConfig(image_config=image_config, video_config=video_config)
 
         return None
 

api/core/file/file_obj.py

@@ -1,18 +1,31 @@
+import base64
 import enum
+import logging
 from typing import Optional
 
+import click
 from pydantic import BaseModel
 
 from core.app.app_config.entities import FileExtraConfig
+from core.file.file_parser_cache import FileParserCache
 from core.file.tool_file_parser import ToolFileParser
 from core.file.upload_file_parser import UploadFileParser
-from core.model_runtime.entities.message_entities import ImagePromptMessageContent
+from core.model_runtime.entities.message_entities import (
+    ImagePromptMessageContent,
+    TextPromptMessageContent,
+    VideoPromptMessageContent,
+)
 from extensions.ext_database import db
-from models.model import UploadFile
+from models.account import Account
+from models.model import App, UploadFile
+from services.audio_service import AudioService
+from services.extract_video_frames import ExtractVideoFrames
+from services.file_service import FileService
 
 
 class FileType(enum.Enum):
     IMAGE = 'image'
+    VIDEO = 'video'
 
     @staticmethod
     def value_of(value):
@@ -34,6 +47,7 @@ def value_of(value):
                 return member
         raise ValueError(f"No matching enum found for value '{value}'")
 
+
 class FileBelongsTo(enum.Enum):
     USER = 'user'
     ASSISTANT = 'assistant'
@@ -57,6 +71,8 @@ class FileVar(BaseModel):
     filename: Optional[str] = None
     extension: Optional[str] = None
     mime_type: Optional[str] = None
+    app_id: Optional[str] = None
+    description: Optional[str] = None
 
     def to_dict(self) -> dict:
         return {
@@ -69,6 +85,7 @@ def to_dict(self) -> dict:
             'filename': self.filename,
             'extension': self.extension,
             'mime_type': self.mime_type,
+            'description': self.video_text,
         }
 
     def to_markdown(self) -> str:
@@ -93,6 +110,21 @@ def data(self) -> Optional[str]:
         """
         return self._get_data()
 
+    @property
+    def video_text(self) -> Optional[str]:
+        """
+        Get video data, file signed url or base64 data
+        depending on config MULTIMODAL_SEND_IMAGE_FORMAT
+        :return:
+        """
+        audio_text = self._get_video_text()
+        if isinstance(audio_text, bytes):
+            audio_data = audio_text.decode('utf-8')
+        else:
+            audio_data = audio_text
+        logging.info(click.style(f"video text: {audio_data}", fg='green'))
+        return audio_data
+
     @property
     def preview_url(self) -> Optional[str]:
         """
@@ -102,34 +134,110 @@ def preview_url(self) -> Optional[str]:
         return self._get_data(force_url=True)
 
     @property
-    def prompt_message_content(self) -> ImagePromptMessageContent:
-        if self.type == FileType.IMAGE:
-            image_config = self.extra_config.image_config
+    def prompt_message_content(
+            self) -> ImagePromptMessageContent | VideoPromptMessageContent | TextPromptMessageContent:
+        image_config = self.extra_config.image_config
+        video_config = self.extra_config.video_config
 
+        if self.type == FileType.IMAGE:
             return ImagePromptMessageContent(
                 data=self.data,
                 detail=ImagePromptMessageContent.DETAIL.HIGH
                 if image_config.get("detail") == "high" else ImagePromptMessageContent.DETAIL.LOW
             )
+        if self.type == FileType.VIDEO:
+            if video_config.get('extract_video') != 'enabled' and video_config.get('extract_audio') == 'enabled':
+                return TextPromptMessageContent(data=self.video_text)
+            elif video_config.get('extract_video') == 'enabled' and video_config.get('extract_audio') != 'enabled':
+                return ImagePromptMessageContent(
+                    data=self.data,
+                    detail=ImagePromptMessageContent.DETAIL.HIGH
+                    if image_config.get("detail") == "high" else ImagePromptMessageContent.DETAIL.LOW
+                )
+            elif video_config.get('extract_video') == 'enabled' and video_config.get('extract_audio') == 'enabled':
+                return VideoPromptMessageContent(
+                    data=self.data,
+                    detail=VideoPromptMessageContent.DETAIL.HIGH
+                    if image_config.get("detail") == "high" else VideoPromptMessageContent.DETAIL.LOW,
+                    description=self.video_text
+                )
+            else:
+                raise ValueError('Either video frame extraction or audio extraction one of them must be enabled!')
 
     def _get_data(self, force_url: bool = False) -> Optional[str]:
         if self.type == FileType.IMAGE:
             if self.transfer_method == FileTransferMethod.REMOTE_URL:
                 return self.url
             elif self.transfer_method == FileTransferMethod.LOCAL_FILE:
-                upload_file = (db.session.query(UploadFile)
-                               .filter(
-                    UploadFile.id == self.related_id,
-                    UploadFile.tenant_id == self.tenant_id
-                ).first())
-
-                return UploadFileParser.get_image_data(
-                    upload_file=upload_file,
-                    force_url=force_url
-                )
+                upload_file = db.session.query(UploadFile).filter(UploadFile.id == self.related_id,
+                                                                  UploadFile.tenant_id == self.tenant_id).first()
+
+                file_cache = FileParserCache(file_id=upload_file.id,
+                                             file_type=upload_file.extension,
+                                             separation_type='image')
+                if file_cache.get():
+                    image_data = file_cache.get()
+                else:
+                    image_data = UploadFileParser.get_image_data(upload_file=upload_file, force_url=force_url)
+                    file_cache.set(file_content=image_data, ttl=3600)
+                return image_data
             elif self.transfer_method == FileTransferMethod.TOOL_FILE:
                 extension = self.extension
                 # add sign url
-                return ToolFileParser.get_tool_file_manager().sign_file(tool_file_id=self.related_id, extension=extension)
+                return ToolFileParser.get_tool_file_manager().sign_file(tool_file_id=self.related_id,
+                                                                        extension=extension)
+        if self.type == FileType.VIDEO:
+            video_config = self.extra_config.video_config
+
+            upload_file = db.session.query(UploadFile).filter(UploadFile.id == self.related_id,
+                                                              UploadFile.tenant_id == self.tenant_id).first()
+
+            # Video frame extraction and audio extraction
+            if video_config.get('extract_video') == 'enabled':
+                file_cache = FileParserCache(file_id=upload_file.id,
+                                             file_type=upload_file.extension,
+                                             separation_type='video')
+                if file_cache.get():
+                    video_data = file_cache.get()
+                else:
+                    data = ExtractVideoFrames(max_collect_frames=video_config['max_collect_frames'],
+                                              similarity_threshold=video_config['similarity_threshold'],
+                                              blur_threshold=video_config['blur_threshold'],
+                                              file=upload_file).process_video()
+                    if force_url is True:
+                        image_upload_file = FileService.upload_file(file=data,
+                                                                    file_name=f'{upload_file.name.split(".")[0]}.jpg',
+                                                                    tenant_id=upload_file.tenant_id)
+                        video_data = UploadFileParser.get_signed_temp_image_url(upload_file_id=image_upload_file.id)
+                    else:
+                        encoded_string = base64.b64encode(data).decode('utf-8')
+                        video_data = f'data:image/jpeg;base64,{encoded_string}'
+                    file_cache.set(file_content=video_data, ttl=3600)
+                return video_data
+        return None
 
+    def _get_video_text(self) -> Optional[str]:
+        """
+        Get video text data
+        :return:
+        """
+        if self.type == FileType.VIDEO:
+            video_config = self.extra_config.video_config
+
+            if video_config.get('extract_audio') == 'enabled':
+                upload_file = db.session.query(UploadFile).filter(UploadFile.id == self.related_id,
+                                                                  UploadFile.tenant_id == self.tenant_id).first()
+
+                file_cache = FileParserCache(file_id=upload_file.id, file_type=upload_file.extension,
+                                             separation_type='audio')
+                if file_cache.get():
+                    return file_cache.get()
+                elif not file_cache.get() and self.app_id:
+                    user_info = db.session.query(Account).filter(Account.id == upload_file.created_by).first()
+                    app_info = db.session.query(App).filter(App.id == self.app_id).first()
+
+                    audio_text = AudioService.transcript_asr(app_model=app_info, file=upload_file, end_user=user_info)
+                    audio_data = audio_text.get('text').strip()
+                    file_cache.set(file_content=audio_data)
+                    return audio_data
         return None

api/core/file/file_parser_cache.py

@@ -0,0 +1,42 @@
+from typing import Optional
+
+from extensions.ext_redis import redis_client
+
+
+class FileParserCache:
+    def __init__(self, file_id: str, file_type: str, separation_type: str):
+        self.cache_key = f"media:{separation_type}:{file_id}.{file_type}"
+
+    def get(self) -> Optional[dict]:
+        """
+        Get cached model provider credentials.
+
+        :return:
+        """
+        cached_file_parser = redis_client.get(self.cache_key)
+        if cached_file_parser:
+            try:
+                cached_file_parser = cached_file_parser.decode('utf-8')
+            except:
+                pass
+            return cached_file_parser
+        else:
+            return None
+
+    def set(self, file_content: str, ttl: Optional[int] = 86400) -> None:
+        """
+        Cache model provider credentials.
+
+        :param file_content: file content
+        :param ttl: cache expiration time in seconds
+        :return:
+        """
+        redis_client.setex(name=self.cache_key, time=ttl, value=file_content)
+
+    def delete(self) -> None:
+        """
+        Delete cached model provider credentials.
+
+        :return:
+        """
+        redis_client.delete(self.cache_key)