fetch_link_card_service_spec.rb 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257
  1. # frozen_string_literal: true
  2. require 'rails_helper'
  3. RSpec.describe FetchLinkCardService, type: :service do
  4. subject { described_class.new }
  5. let(:html) { '<!doctype html><title>Hello world</title>' }
  6. let(:oembed_cache) { nil }
  7. before do
  8. stub_request(:get, 'http://example.com/html').to_return(headers: { 'Content-Type' => 'text/html' }, body: html)
  9. stub_request(:get, 'http://example.com/not-found').to_return(status: 404, headers: { 'Content-Type' => 'text/html' }, body: html)
  10. stub_request(:get, 'http://example.com/text').to_return(status: 404, headers: { 'Content-Type' => 'text/plain' }, body: 'Hello')
  11. stub_request(:get, 'http://example.com/redirect').to_return(status: 302, headers: { 'Location' => 'http://example.com/html' })
  12. stub_request(:get, 'http://example.com/redirect-to-404').to_return(status: 302, headers: { 'Location' => 'http://example.com/not-found' })
  13. stub_request(:get, 'http://example.com/oembed?url=http://example.com/html').to_return(headers: { 'Content-Type' => 'application/json' }, body: '{ "version": "1.0", "type": "link", "title": "oEmbed title" }')
  14. stub_request(:get, 'http://example.com/oembed?format=json&url=http://example.com/html').to_return(headers: { 'Content-Type' => 'application/json' }, body: '{ "version": "1.0", "type": "link", "title": "oEmbed title" }')
  15. stub_request(:get, 'http://example.xn--fiqs8s')
  16. stub_request(:get, 'http://example.com/日本語')
  17. stub_request(:get, 'http://example.com/test?data=file.gpx%5E1')
  18. stub_request(:get, 'http://example.com/test-')
  19. stub_request(:get, 'http://example.com/sjis').to_return(request_fixture('sjis.txt'))
  20. stub_request(:get, 'http://example.com/sjis_with_wrong_charset').to_return(request_fixture('sjis_with_wrong_charset.txt'))
  21. stub_request(:get, 'http://example.com/koi8-r').to_return(request_fixture('koi8-r.txt'))
  22. stub_request(:get, 'http://example.com/windows-1251').to_return(request_fixture('windows-1251.txt'))
  23. Rails.cache.write('oembed_endpoint:example.com', oembed_cache) if oembed_cache
  24. subject.call(status)
  25. end
  26. context 'with a local status' do
  27. context 'with URL of a regular HTML page' do
  28. let(:status) { Fabricate(:status, text: 'http://example.com/html') }
  29. it 'creates preview card' do
  30. expect(status.preview_card).to_not be_nil
  31. expect(status.preview_card.url).to eq 'http://example.com/html'
  32. expect(status.preview_card.title).to eq 'Hello world'
  33. end
  34. end
  35. context 'with URL of a page with no title' do
  36. let(:status) { Fabricate(:status, text: 'http://example.com/html') }
  37. let(:html) { '<!doctype html><title></title>' }
  38. it 'does not create a preview card' do
  39. expect(status.preview_card).to be_nil
  40. end
  41. end
  42. context 'with a URL of a plain-text page' do
  43. let(:status) { Fabricate(:status, text: 'http://example.com/text') }
  44. it 'does not create a preview card' do
  45. expect(status.preview_card).to be_nil
  46. end
  47. end
  48. context 'with multiple URLs' do
  49. let(:status) { Fabricate(:status, text: 'ftp://example.com http://example.com/html http://example.com/text') }
  50. it 'fetches the first valid URL' do
  51. expect(a_request(:get, 'http://example.com/html')).to have_been_made
  52. end
  53. it 'does not fetch the second valid URL' do
  54. expect(a_request(:get, 'http://example.com/text/')).to_not have_been_made
  55. end
  56. end
  57. context 'with a redirect URL' do
  58. let(:status) { Fabricate(:status, text: 'http://example.com/redirect') }
  59. it 'follows redirect' do
  60. expect(a_request(:get, 'http://example.com/redirect')).to have_been_made.once
  61. expect(a_request(:get, 'http://example.com/html')).to have_been_made.once
  62. end
  63. it 'creates preview card' do
  64. expect(status.preview_card).to_not be_nil
  65. expect(status.preview_card.url).to eq 'http://example.com/html'
  66. expect(status.preview_card.title).to eq 'Hello world'
  67. end
  68. end
  69. context 'with a broken redirect URL' do
  70. let(:status) { Fabricate(:status, text: 'http://example.com/redirect-to-404') }
  71. it 'follows redirect' do
  72. expect(a_request(:get, 'http://example.com/redirect-to-404')).to have_been_made.once
  73. expect(a_request(:get, 'http://example.com/not-found')).to have_been_made.once
  74. end
  75. it 'does not create a preview card' do
  76. expect(status.preview_card).to be_nil
  77. end
  78. end
  79. context 'with a 404 URL' do
  80. let(:status) { Fabricate(:status, text: 'http://example.com/not-found') }
  81. it 'does not create a preview card' do
  82. expect(status.preview_card).to be_nil
  83. end
  84. end
  85. context 'with an IDN URL' do
  86. let(:status) { Fabricate(:status, text: 'Check out http://example.中国') }
  87. it 'fetches the URL' do
  88. expect(a_request(:get, 'http://example.xn--fiqs8s/')).to have_been_made.once
  89. end
  90. end
  91. context 'with a URL of a page in Shift JIS encoding' do
  92. let(:status) { Fabricate(:status, text: 'Check out http://example.com/sjis') }
  93. it 'decodes the HTML' do
  94. expect(status.preview_cards.first.title).to eq('SJISのページ')
  95. end
  96. end
  97. context 'with a URL of a page in Shift JIS encoding labeled as UTF-8' do
  98. let(:status) { Fabricate(:status, text: 'Check out http://example.com/sjis_with_wrong_charset') }
  99. it 'decodes the HTML despite the wrong charset header' do
  100. expect(status.preview_cards.first.title).to eq('SJISのページ')
  101. end
  102. end
  103. context 'with a URL of a page in KOI8-R encoding' do
  104. let(:status) { Fabricate(:status, text: 'Check out http://example.com/koi8-r') }
  105. it 'decodes the HTML' do
  106. expect(status.preview_cards.first.title).to eq('Московя начинаетъ только въ XVI ст. привлекать внимане иностранцевъ.')
  107. end
  108. end
  109. context 'with a URL of a page in Windows-1251 encoding' do
  110. let(:status) { Fabricate(:status, text: 'Check out http://example.com/windows-1251') }
  111. it 'decodes the HTML' do
  112. expect(status.preview_cards.first.title).to eq('сэмпл текст')
  113. end
  114. end
  115. context 'with a Japanese path URL' do
  116. let(:status) { Fabricate(:status, text: 'テストhttp://example.com/日本語') }
  117. it 'fetches the URL' do
  118. expect(a_request(:get, 'http://example.com/日本語')).to have_been_made.once
  119. end
  120. end
  121. context 'with a hyphen-suffixed URL' do
  122. let(:status) { Fabricate(:status, text: 'test http://example.com/test-') }
  123. it 'fetches the URL' do
  124. expect(a_request(:get, 'http://example.com/test-')).to have_been_made.once
  125. end
  126. end
  127. context 'with a caret-suffixed URL' do
  128. let(:status) { Fabricate(:status, text: 'test http://example.com/test?data=file.gpx^1') }
  129. it 'fetches the URL' do
  130. expect(a_request(:get, 'http://example.com/test?data=file.gpx%5E1')).to have_been_made.once
  131. end
  132. it 'does not strip the caret before fetching' do
  133. expect(a_request(:get, 'http://example.com/test?data=file.gpx')).to_not have_been_made
  134. end
  135. end
  136. context 'with a non-isolated URL' do
  137. let(:status) { Fabricate(:status, text: 'testhttp://example.com/sjis') }
  138. it 'does not fetch URLs not isolated from their surroundings' do
  139. expect(a_request(:get, 'http://example.com/sjis')).to_not have_been_made
  140. end
  141. end
  142. context 'with a URL of a page with oEmbed support' do
  143. let(:html) { '<!doctype html><title>Hello world</title><link rel="alternate" type="application/json+oembed" href="http://example.com/oembed?url=http://example.com/html">' }
  144. let(:status) { Fabricate(:status, text: 'http://example.com/html') }
  145. it 'fetches the oEmbed URL' do
  146. expect(a_request(:get, 'http://example.com/oembed?url=http://example.com/html')).to have_been_made.once
  147. end
  148. it 'creates preview card' do
  149. expect(status.preview_card).to_not be_nil
  150. expect(status.preview_card.url).to eq 'http://example.com/html'
  151. expect(status.preview_card.title).to eq 'oEmbed title'
  152. end
  153. context 'when oEmbed endpoint cache populated' do
  154. let(:oembed_cache) { { endpoint: 'http://example.com/oembed?format=json&url={url}', format: :json } }
  155. it 'uses the cached oEmbed response' do
  156. expect(a_request(:get, 'http://example.com/oembed?url=http://example.com/html')).to_not have_been_made
  157. expect(a_request(:get, 'http://example.com/oembed?format=json&url=http://example.com/html')).to have_been_made
  158. end
  159. it 'creates preview card' do
  160. expect(status.preview_card).to_not be_nil
  161. expect(status.preview_card.url).to eq 'http://example.com/html'
  162. expect(status.preview_card.title).to eq 'oEmbed title'
  163. end
  164. end
  165. # If the original HTML URL for whatever reason (e.g. DOS protection) redirects to
  166. # an error page, we can still use the cached oEmbed but should not use the
  167. # redirect URL on the card.
  168. context 'when oEmbed endpoint cache populated but page returns 404' do
  169. let(:status) { Fabricate(:status, text: 'http://example.com/redirect-to-404') }
  170. let(:oembed_cache) { { endpoint: 'http://example.com/oembed?url=http://example.com/html', format: :json } }
  171. it 'uses the cached oEmbed response' do
  172. expect(a_request(:get, 'http://example.com/oembed?url=http://example.com/html')).to have_been_made
  173. end
  174. it 'creates preview card' do
  175. expect(status.preview_card).to_not be_nil
  176. expect(status.preview_card.title).to eq 'oEmbed title'
  177. end
  178. it 'uses the original URL' do
  179. expect(status.preview_card&.url).to eq 'http://example.com/redirect-to-404'
  180. end
  181. end
  182. end
  183. end
  184. context 'with a remote status' do
  185. let(:status) do
  186. Fabricate(:status, account: Fabricate(:account, domain: 'example.com'), text: <<-TEXT)
  187. Habt ihr ein paar gute Links zu <a>foo</a>
  188. #<span class="tag"><a href="https://quitter.se/tag/wannacry" target="_blank" rel="tag noopener noreferrer" title="https://quitter.se/tag/wannacry">Wannacry</a></span> herumfliegen?
  189. Ich will mal unter <br> <a href="http://example.com/not-found" target="_blank" rel="noopener noreferrer" title="http://example.com/not-found">http://example.com/not-found</a> was sammeln. !
  190. <a href="http://sn.jonkman.ca/group/416/id" target="_blank" rel="noopener noreferrer" title="http://sn.jonkman.ca/group/416/id">security</a>&nbsp;
  191. TEXT
  192. end
  193. it 'parses out URLs' do
  194. expect(a_request(:get, 'http://example.com/not-found')).to have_been_made.once
  195. end
  196. it 'ignores URLs to hashtags' do
  197. expect(a_request(:get, 'https://quitter.se/tag/wannacry')).to_not have_been_made
  198. end
  199. end
  200. end