media_attachment.rb 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393
  1. # frozen_string_literal: true
  2. # == Schema Information
  3. #
  4. # Table name: media_attachments
  5. #
  6. # id :bigint(8) not null, primary key
  7. # status_id :bigint(8)
  8. # file_file_name :string
  9. # file_content_type :string
  10. # file_file_size :integer
  11. # file_updated_at :datetime
  12. # remote_url :string default(""), not null
  13. # created_at :datetime not null
  14. # updated_at :datetime not null
  15. # shortcode :string
  16. # type :integer default("image"), not null
  17. # file_meta :json
  18. # account_id :bigint(8)
  19. # description :text
  20. # scheduled_status_id :bigint(8)
  21. # blurhash :string
  22. # processing :integer
  23. # file_storage_schema_version :integer
  24. # thumbnail_file_name :string
  25. # thumbnail_content_type :string
  26. # thumbnail_file_size :integer
  27. # thumbnail_updated_at :datetime
  28. # thumbnail_remote_url :string
  29. #
  30. class MediaAttachment < ApplicationRecord
  31. self.inheritance_column = nil
  32. include Attachmentable
  33. enum type: [:image, :gifv, :video, :unknown, :audio]
  34. enum processing: [:queued, :in_progress, :complete, :failed], _prefix: true
  35. MAX_DESCRIPTION_LENGTH = 1_500
  36. IMAGE_LIMIT = 10.megabytes
  37. VIDEO_LIMIT = 40.megabytes
  38. MAX_VIDEO_MATRIX_LIMIT = 2_304_000 # 1920x1200px
  39. MAX_VIDEO_FRAME_RATE = 60
  40. IMAGE_FILE_EXTENSIONS = %w(.jpg .jpeg .png .gif .webp).freeze
  41. VIDEO_FILE_EXTENSIONS = %w(.webm .mp4 .m4v .mov).freeze
  42. AUDIO_FILE_EXTENSIONS = %w(.ogg .oga .mp3 .wav .flac .opus .aac .m4a .3gp .wma).freeze
  43. META_KEYS = %i(
  44. focus
  45. colors
  46. original
  47. small
  48. ).freeze
  49. IMAGE_MIME_TYPES = %w(image/jpeg image/png image/gif image/webp).freeze
  50. VIDEO_MIME_TYPES = %w(video/webm video/mp4 video/quicktime video/ogg).freeze
  51. VIDEO_CONVERTIBLE_MIME_TYPES = %w(video/webm video/quicktime).freeze
  52. AUDIO_MIME_TYPES = %w(audio/wave audio/wav audio/x-wav audio/x-pn-wave audio/vnd.wave audio/ogg audio/vorbis audio/mpeg audio/mp3 audio/webm audio/flac audio/aac audio/m4a audio/x-m4a audio/mp4 audio/3gpp video/x-ms-asf).freeze
  53. BLURHASH_OPTIONS = {
  54. x_comp: 4,
  55. y_comp: 4,
  56. }.freeze
  57. IMAGE_STYLES = {
  58. original: {
  59. pixels: 2_073_600, # 1920x1080px
  60. file_geometry_parser: FastGeometryParser,
  61. }.freeze,
  62. small: {
  63. pixels: 160_000, # 400x400px
  64. file_geometry_parser: FastGeometryParser,
  65. blurhash: BLURHASH_OPTIONS,
  66. }.freeze,
  67. }.freeze
  68. VIDEO_FORMAT = {
  69. format: 'mp4',
  70. content_type: 'video/mp4',
  71. vfr_frame_rate_threshold: MAX_VIDEO_FRAME_RATE,
  72. convert_options: {
  73. output: {
  74. 'loglevel' => 'fatal',
  75. 'movflags' => 'faststart',
  76. 'pix_fmt' => 'yuv420p',
  77. 'vf' => 'scale=\'trunc(iw/2)*2:trunc(ih/2)*2\'',
  78. 'vsync' => 'cfr',
  79. 'c:v' => 'h264',
  80. 'maxrate' => '1300K',
  81. 'bufsize' => '1300K',
  82. 'frames:v' => 60 * 60 * 3,
  83. 'crf' => 18,
  84. 'map_metadata' => '-1',
  85. }.freeze,
  86. }.freeze,
  87. }.freeze
  88. VIDEO_PASSTHROUGH_OPTIONS = {
  89. video_codecs: ['h264'].freeze,
  90. audio_codecs: ['aac', nil].freeze,
  91. colorspaces: ['yuv420p'].freeze,
  92. options: {
  93. format: 'mp4',
  94. convert_options: {
  95. output: {
  96. 'loglevel' => 'fatal',
  97. 'map_metadata' => '-1',
  98. 'c:v' => 'copy',
  99. 'c:a' => 'copy',
  100. }.freeze,
  101. }.freeze,
  102. }.freeze,
  103. }.freeze
  104. VIDEO_STYLES = {
  105. small: {
  106. convert_options: {
  107. output: {
  108. 'loglevel' => 'fatal',
  109. vf: 'scale=\'min(400\, iw):min(400\, ih)\':force_original_aspect_ratio=decrease',
  110. }.freeze,
  111. }.freeze,
  112. format: 'png',
  113. time: 0,
  114. file_geometry_parser: FastGeometryParser,
  115. blurhash: BLURHASH_OPTIONS,
  116. }.freeze,
  117. original: VIDEO_FORMAT.merge(passthrough_options: VIDEO_PASSTHROUGH_OPTIONS).freeze,
  118. }.freeze
  119. AUDIO_STYLES = {
  120. original: {
  121. format: 'mp3',
  122. content_type: 'audio/mpeg',
  123. convert_options: {
  124. output: {
  125. 'loglevel' => 'fatal',
  126. 'q:a' => 2,
  127. }.freeze,
  128. }.freeze,
  129. }.freeze,
  130. }.freeze
  131. VIDEO_CONVERTED_STYLES = {
  132. small: VIDEO_STYLES[:small].freeze,
  133. original: VIDEO_FORMAT.freeze,
  134. }.freeze
  135. THUMBNAIL_STYLES = {
  136. original: IMAGE_STYLES[:small].freeze,
  137. }.freeze
  138. GLOBAL_CONVERT_OPTIONS = {
  139. all: '-quality 90 -strip +set modify-date +set create-date',
  140. }.freeze
  141. belongs_to :account, inverse_of: :media_attachments, optional: true
  142. belongs_to :status, inverse_of: :media_attachments, optional: true
  143. belongs_to :scheduled_status, inverse_of: :media_attachments, optional: true
  144. has_attached_file :file,
  145. styles: ->(f) { file_styles f },
  146. processors: ->(f) { file_processors f },
  147. convert_options: GLOBAL_CONVERT_OPTIONS
  148. before_file_validate :set_type_and_extension
  149. before_file_validate :check_video_dimensions
  150. validates_attachment_content_type :file, content_type: IMAGE_MIME_TYPES + VIDEO_MIME_TYPES + AUDIO_MIME_TYPES
  151. validates_attachment_size :file, less_than: ->(m) { m.larger_media_format? ? VIDEO_LIMIT : IMAGE_LIMIT }
  152. remotable_attachment :file, VIDEO_LIMIT, suppress_errors: false, download_on_assign: false, attribute_name: :remote_url
  153. has_attached_file :thumbnail,
  154. styles: THUMBNAIL_STYLES,
  155. processors: [:lazy_thumbnail, :blurhash_transcoder, :color_extractor],
  156. convert_options: GLOBAL_CONVERT_OPTIONS
  157. validates_attachment_content_type :thumbnail, content_type: IMAGE_MIME_TYPES
  158. validates_attachment_size :thumbnail, less_than: IMAGE_LIMIT
  159. remotable_attachment :thumbnail, IMAGE_LIMIT, suppress_errors: true, download_on_assign: false
  160. validates :account, presence: true
  161. validates :description, length: { maximum: MAX_DESCRIPTION_LENGTH }
  162. validates :file, presence: true, if: :local?
  163. validates :thumbnail, absence: true, if: -> { local? && !audio_or_video? }
  164. scope :attached, -> { where.not(status_id: nil).or(where.not(scheduled_status_id: nil)) }
  165. scope :unattached, -> { where(status_id: nil, scheduled_status_id: nil) }
  166. scope :local, -> { where(remote_url: '') }
  167. scope :remote, -> { where.not(remote_url: '') }
  168. scope :cached, -> { remote.where.not(file_file_name: nil) }
  169. default_scope { order(id: :asc) }
  170. def local?
  171. remote_url.blank?
  172. end
  173. def not_processed?
  174. processing.present? && !processing_complete?
  175. end
  176. def needs_redownload?
  177. file.blank? && remote_url.present?
  178. end
  179. def significantly_changed?
  180. description_previously_changed? || thumbnail_updated_at_previously_changed? || file_meta_previously_changed?
  181. end
  182. def larger_media_format?
  183. video? || gifv? || audio?
  184. end
  185. def audio_or_video?
  186. audio? || video?
  187. end
  188. def to_param
  189. shortcode.presence || id&.to_s
  190. end
  191. def focus=(point)
  192. return if point.blank?
  193. x, y = (point.is_a?(Enumerable) ? point : point.split(',')).map(&:to_f)
  194. meta = (file.instance_read(:meta) || {}).with_indifferent_access.slice(*META_KEYS)
  195. meta['focus'] = { 'x' => x, 'y' => y }
  196. file.instance_write(:meta, meta)
  197. end
  198. def focus
  199. x = file.meta&.dig('focus', 'x')
  200. y = file.meta&.dig('focus', 'y')
  201. return if x.nil? || y.nil?
  202. "#{x},#{y}"
  203. end
  204. attr_writer :delay_processing
  205. def delay_processing?
  206. @delay_processing
  207. end
  208. def delay_processing_for_attachment?(attachment_name)
  209. @delay_processing && attachment_name == :file
  210. end
  211. after_commit :enqueue_processing, on: :create
  212. after_commit :reset_parent_cache, on: :update
  213. before_create :set_unknown_type
  214. before_create :set_processing
  215. after_post_process :set_meta
  216. class << self
  217. def supported_mime_types
  218. IMAGE_MIME_TYPES + VIDEO_MIME_TYPES + AUDIO_MIME_TYPES
  219. end
  220. def supported_file_extensions
  221. IMAGE_FILE_EXTENSIONS + VIDEO_FILE_EXTENSIONS + AUDIO_FILE_EXTENSIONS
  222. end
  223. private
  224. def file_styles(attachment)
  225. if attachment.instance.file_content_type == 'image/gif' || VIDEO_CONVERTIBLE_MIME_TYPES.include?(attachment.instance.file_content_type)
  226. VIDEO_CONVERTED_STYLES
  227. elsif IMAGE_MIME_TYPES.include?(attachment.instance.file_content_type)
  228. IMAGE_STYLES
  229. elsif VIDEO_MIME_TYPES.include?(attachment.instance.file_content_type)
  230. VIDEO_STYLES
  231. else
  232. AUDIO_STYLES
  233. end
  234. end
  235. def file_processors(instance)
  236. if instance.file_content_type == 'image/gif'
  237. [:gif_transcoder, :blurhash_transcoder]
  238. elsif VIDEO_MIME_TYPES.include?(instance.file_content_type)
  239. [:transcoder, :blurhash_transcoder, :type_corrector]
  240. elsif AUDIO_MIME_TYPES.include?(instance.file_content_type)
  241. [:image_extractor, :transcoder, :type_corrector]
  242. else
  243. [:lazy_thumbnail, :blurhash_transcoder, :type_corrector]
  244. end
  245. end
  246. end
  247. private
  248. def set_unknown_type
  249. self.type = :unknown if file.blank? && !type_changed?
  250. end
  251. def set_type_and_extension
  252. self.type = begin
  253. if VIDEO_MIME_TYPES.include?(file_content_type)
  254. :video
  255. elsif AUDIO_MIME_TYPES.include?(file_content_type)
  256. :audio
  257. else
  258. :image
  259. end
  260. end
  261. end
  262. def set_processing
  263. self.processing = delay_processing? ? :queued : :complete
  264. end
  265. def check_video_dimensions
  266. return unless (video? || gifv?) && file.queued_for_write[:original].present?
  267. movie = ffmpeg_data(file.queued_for_write[:original].path)
  268. return unless movie.valid?
  269. raise Mastodon::StreamValidationError, 'Video has no video stream' if movie.width.nil? || movie.frame_rate.nil?
  270. raise Mastodon::DimensionsValidationError, "#{movie.width}x#{movie.height} videos are not supported" if movie.width * movie.height > MAX_VIDEO_MATRIX_LIMIT
  271. raise Mastodon::DimensionsValidationError, "#{movie.frame_rate.floor}fps videos are not supported" if movie.frame_rate.floor > MAX_VIDEO_FRAME_RATE
  272. end
  273. def set_meta
  274. file.instance_write :meta, populate_meta
  275. end
  276. def populate_meta
  277. meta = (file.instance_read(:meta) || {}).with_indifferent_access.slice(*META_KEYS)
  278. file.queued_for_write.each do |style, file|
  279. meta[style] = style == :small || image? ? image_geometry(file) : video_metadata(file)
  280. end
  281. meta[:small] = image_geometry(thumbnail.queued_for_write[:original]) if thumbnail.queued_for_write.key?(:original)
  282. meta
  283. end
  284. def image_geometry(file)
  285. width, height = FastImage.size(file.path)
  286. return {} if width.nil?
  287. {
  288. width: width,
  289. height: height,
  290. size: "#{width}x#{height}",
  291. aspect: width.to_f / height,
  292. }
  293. end
  294. def video_metadata(file)
  295. movie = ffmpeg_data(file.path)
  296. return {} unless movie.valid?
  297. {
  298. width: movie.width,
  299. height: movie.height,
  300. frame_rate: movie.frame_rate,
  301. duration: movie.duration,
  302. bitrate: movie.bitrate,
  303. }.compact
  304. end
  305. # We call this method about 3 different times on potentially different
  306. # paths but ultimately the same file, so it makes sense to memoize the
  307. # result while disregarding the path
  308. def ffmpeg_data(path = nil)
  309. @ffmpeg_data ||= VideoMetadataExtractor.new(path)
  310. end
  311. def enqueue_processing
  312. PostProcessMediaWorker.perform_async(id) if delay_processing?
  313. end
  314. def reset_parent_cache
  315. Rails.cache.delete("statuses/#{status_id}") if status_id.present?
  316. end
  317. end