backup_service.rb 5.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197
  1. # frozen_string_literal: true
  2. require 'zip'
  3. class BackupService < BaseService
  4. include Payloadable
  5. include ContextHelper
  6. attr_reader :account, :backup
  7. def call(backup)
  8. @backup = backup
  9. @account = backup.user.account
  10. build_archive!
  11. end
  12. private
  13. def build_outbox_json!(file)
  14. skeleton = serialize(collection_presenter, ActivityPub::CollectionSerializer)
  15. skeleton[:@context] = full_context
  16. skeleton[:orderedItems] = ['!PLACEHOLDER!']
  17. skeleton = Oj.dump(skeleton)
  18. prepend, append = skeleton.split('"!PLACEHOLDER!"')
  19. add_comma = false
  20. file.write(prepend)
  21. account.statuses.with_includes.reorder(nil).find_in_batches do |statuses|
  22. file.write(',') if add_comma
  23. add_comma = true
  24. file.write(statuses.map do |status|
  25. item = serialize_payload(ActivityPub::ActivityPresenter.from_status(status), ActivityPub::ActivitySerializer)
  26. item.delete('@context')
  27. unless item[:type] == 'Announce' || item[:object][:attachment].blank?
  28. item[:object][:attachment].each do |attachment|
  29. attachment[:url] = Addressable::URI.parse(attachment[:url]).path.delete_prefix('/system/')
  30. end
  31. end
  32. Oj.dump(item)
  33. end.join(','))
  34. GC.start
  35. end
  36. file.write(append)
  37. end
  38. def build_archive!
  39. tmp_file = Tempfile.new(%w(archive .zip))
  40. Zip::File.open(tmp_file, create: true) do |zipfile|
  41. dump_outbox!(zipfile)
  42. dump_media_attachments!(zipfile)
  43. dump_likes!(zipfile)
  44. dump_bookmarks!(zipfile)
  45. dump_actor!(zipfile)
  46. end
  47. archive_filename = "#{['archive', Time.now.utc.strftime('%Y%m%d%H%M%S'), SecureRandom.hex(16)].join('-')}.zip"
  48. @backup.dump = ActionDispatch::Http::UploadedFile.new(tempfile: tmp_file, filename: archive_filename)
  49. @backup.processed = true
  50. @backup.save!
  51. ensure
  52. tmp_file.close
  53. tmp_file.unlink
  54. end
  55. def dump_media_attachments!(zipfile)
  56. MediaAttachment.attached.where(account: account).find_in_batches do |media_attachments|
  57. media_attachments.each do |m|
  58. path = m.file&.path
  59. next unless path
  60. path = path.gsub(%r{\A.*/system/}, '')
  61. path = path.gsub(%r{\A/+}, '')
  62. download_to_zip(zipfile, m.file, path)
  63. end
  64. GC.start
  65. end
  66. end
  67. def dump_outbox!(zipfile)
  68. zipfile.get_output_stream('outbox.json') do |io|
  69. build_outbox_json!(io)
  70. end
  71. end
  72. def dump_actor!(zipfile)
  73. actor = serialize(account, ActivityPub::ActorSerializer)
  74. actor[:icon][:url] = "avatar#{File.extname(actor[:icon][:url])}" if actor[:icon]
  75. actor[:image][:url] = "header#{File.extname(actor[:image][:url])}" if actor[:image]
  76. actor[:outbox] = 'outbox.json'
  77. actor[:likes] = 'likes.json'
  78. actor[:bookmarks] = 'bookmarks.json'
  79. download_to_zip(zipfile, account.avatar, "avatar#{File.extname(account.avatar.path)}") if account.avatar.exists?
  80. download_to_zip(zipfile, account.header, "header#{File.extname(account.header.path)}") if account.header.exists?
  81. json = Oj.dump(actor)
  82. zipfile.get_output_stream('actor.json') do |io|
  83. io.write(json)
  84. end
  85. end
  86. def dump_likes!(zipfile)
  87. skeleton = serialize(ActivityPub::CollectionPresenter.new(id: 'likes.json', type: :ordered, size: 0, items: []), ActivityPub::CollectionSerializer)
  88. skeleton.delete(:totalItems)
  89. skeleton[:orderedItems] = ['!PLACEHOLDER!']
  90. skeleton = Oj.dump(skeleton)
  91. prepend, append = skeleton.split('"!PLACEHOLDER!"')
  92. zipfile.get_output_stream('likes.json') do |io|
  93. io.write(prepend)
  94. add_comma = false
  95. Status.reorder(nil).joins(:favourites).includes(:account).merge(account.favourites).find_in_batches do |statuses|
  96. io.write(',') if add_comma
  97. add_comma = true
  98. io.write(statuses.map do |status|
  99. Oj.dump(ActivityPub::TagManager.instance.uri_for(status))
  100. end.join(','))
  101. GC.start
  102. end
  103. io.write(append)
  104. end
  105. end
  106. def dump_bookmarks!(zipfile)
  107. skeleton = serialize(ActivityPub::CollectionPresenter.new(id: 'bookmarks.json', type: :ordered, size: 0, items: []), ActivityPub::CollectionSerializer)
  108. skeleton.delete(:totalItems)
  109. skeleton[:orderedItems] = ['!PLACEHOLDER!']
  110. skeleton = Oj.dump(skeleton)
  111. prepend, append = skeleton.split('"!PLACEHOLDER!"')
  112. zipfile.get_output_stream('bookmarks.json') do |io|
  113. io.write(prepend)
  114. add_comma = false
  115. Status.reorder(nil).joins(:bookmarks).includes(:account).merge(account.bookmarks).find_in_batches do |statuses|
  116. io.write(',') if add_comma
  117. add_comma = true
  118. io.write(statuses.map do |status|
  119. Oj.dump(ActivityPub::TagManager.instance.uri_for(status))
  120. end.join(','))
  121. GC.start
  122. end
  123. io.write(append)
  124. end
  125. end
  126. def collection_presenter
  127. ActivityPub::CollectionPresenter.new(
  128. id: 'outbox.json',
  129. type: :ordered,
  130. size: account.statuses_count,
  131. items: []
  132. )
  133. end
  134. def serialize(object, serializer)
  135. ActiveModelSerializers::SerializableResource.new(
  136. object,
  137. serializer: serializer,
  138. adapter: ActivityPub::Adapter
  139. ).as_json
  140. end
  141. CHUNK_SIZE = 1.megabyte
  142. def download_to_zip(zipfile, attachment, filename)
  143. adapter = Paperclip.io_adapters.for(attachment)
  144. zipfile.get_output_stream(filename) do |io|
  145. while (buffer = adapter.read(CHUNK_SIZE))
  146. io.write(buffer)
  147. end
  148. end
  149. rescue Errno::ENOENT, Seahorse::Client::NetworkingError => e
  150. Rails.logger.warn "Could not backup file #{filename}: #{e}"
  151. end
  152. end