1
0

request.rb 8.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300
  1. # frozen_string_literal: true
  2. require 'ipaddr'
  3. require 'socket'
  4. require 'resolv'
  5. # Monkey-patch the HTTP.rb timeout class to avoid using a timeout block
  6. # around the Socket#open method, since we use our own timeout blocks inside
  7. # that method
  8. class HTTP::Timeout::PerOperation
  9. def connect(socket_class, host, port, nodelay = false)
  10. @socket = socket_class.open(host, port)
  11. @socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1) if nodelay
  12. end
  13. end
  14. class Request
  15. REQUEST_TARGET = '(request-target)'
  16. # We enforce a 5s timeout on DNS resolving, 5s timeout on socket opening
  17. # and 5s timeout on the TLS handshake, meaning the worst case should take
  18. # about 15s in total
  19. TIMEOUT = { connect: 5, read: 10, write: 10 }.freeze
  20. include RoutingHelper
  21. def initialize(verb, url, **options)
  22. raise ArgumentError if url.blank?
  23. @verb = verb
  24. @url = Addressable::URI.parse(url).normalize
  25. @http_client = options.delete(:http_client)
  26. @allow_local = options.delete(:allow_local)
  27. @options = options.merge(socket_class: use_proxy? || @allow_local ? ProxySocket : Socket)
  28. @options = @options.merge(proxy_url) if use_proxy?
  29. @headers = {}
  30. raise Mastodon::HostValidationError, 'Instance does not support hidden service connections' if block_hidden_service?
  31. set_common_headers!
  32. set_digest! if options.key?(:body)
  33. end
  34. def on_behalf_of(actor, sign_with: nil)
  35. raise ArgumentError, 'actor must not be nil' if actor.nil?
  36. @actor = actor
  37. @keypair = sign_with.present? ? OpenSSL::PKey::RSA.new(sign_with) : @actor.keypair
  38. self
  39. end
  40. def add_headers(new_headers)
  41. @headers.merge!(new_headers)
  42. self
  43. end
  44. def perform
  45. begin
  46. response = http_client.public_send(@verb, @url.to_s, @options.merge(headers: headers))
  47. rescue => e
  48. raise e.class, "#{e.message} on #{@url}", e.backtrace[0]
  49. end
  50. begin
  51. # If we are using a persistent connection, we have to
  52. # read every response to be able to move forward at all.
  53. # However, simply calling #to_s or #flush may not be safe,
  54. # as the response body, if malicious, could be too big
  55. # for our memory. So we use the #body_with_limit method
  56. response.body_with_limit if http_client.persistent?
  57. yield response if block_given?
  58. ensure
  59. http_client.close unless http_client.persistent?
  60. end
  61. end
  62. def headers
  63. (@actor ? @headers.merge('Signature' => signature) : @headers).without(REQUEST_TARGET)
  64. end
  65. class << self
  66. def valid_url?(url)
  67. begin
  68. parsed_url = Addressable::URI.parse(url)
  69. rescue Addressable::URI::InvalidURIError
  70. return false
  71. end
  72. %w(http https).include?(parsed_url.scheme) && parsed_url.host.present?
  73. end
  74. def http_client
  75. HTTP.use(:auto_inflate).timeout(TIMEOUT.dup).follow(max_hops: 3)
  76. end
  77. end
  78. private
  79. def set_common_headers!
  80. @headers[REQUEST_TARGET] = "#{@verb} #{@url.path}"
  81. @headers['User-Agent'] = Mastodon::Version.user_agent
  82. @headers['Host'] = @url.host
  83. @headers['Date'] = Time.now.utc.httpdate
  84. @headers['Accept-Encoding'] = 'gzip' if @verb != :head
  85. end
  86. def set_digest!
  87. @headers['Digest'] = "SHA-256=#{Digest::SHA256.base64digest(@options[:body])}"
  88. end
  89. def signature
  90. algorithm = 'rsa-sha256'
  91. signature = Base64.strict_encode64(@keypair.sign(OpenSSL::Digest.new('SHA256'), signed_string))
  92. "keyId=\"#{key_id}\",algorithm=\"#{algorithm}\",headers=\"#{signed_headers.keys.join(' ').downcase}\",signature=\"#{signature}\""
  93. end
  94. def signed_string
  95. signed_headers.map { |key, value| "#{key.downcase}: #{value}" }.join("\n")
  96. end
  97. def signed_headers
  98. @headers.without('User-Agent', 'Accept-Encoding')
  99. end
  100. def key_id
  101. ActivityPub::TagManager.instance.key_uri_for(@actor)
  102. end
  103. def http_client
  104. @http_client ||= Request.http_client
  105. end
  106. def use_proxy?
  107. proxy_url.present?
  108. end
  109. def proxy_url
  110. if hidden_service? && Rails.configuration.x.http_client_hidden_proxy.present?
  111. Rails.configuration.x.http_client_hidden_proxy
  112. else
  113. Rails.configuration.x.http_client_proxy
  114. end
  115. end
  116. def block_hidden_service?
  117. !Rails.configuration.x.access_to_hidden_service && hidden_service?
  118. end
  119. def hidden_service?
  120. /\.(onion|i2p)$/.match?(@url.host)
  121. end
  122. module ClientLimit
  123. def truncated_body(limit = 1.megabyte)
  124. if charset.nil?
  125. encoding = Encoding::BINARY
  126. else
  127. begin
  128. encoding = Encoding.find(charset)
  129. rescue ArgumentError
  130. encoding = Encoding::BINARY
  131. end
  132. end
  133. contents = String.new(encoding: encoding)
  134. while (chunk = readpartial)
  135. contents << chunk
  136. chunk.clear
  137. break if contents.bytesize > limit
  138. end
  139. contents
  140. end
  141. def body_with_limit(limit = 1.megabyte)
  142. raise Mastodon::LengthValidationError if content_length.present? && content_length > limit
  143. contents = truncated_body(limit)
  144. raise Mastodon::LengthValidationError if contents.bytesize > limit
  145. contents
  146. end
  147. end
  148. if ::HTTP::Response.methods.include?(:body_with_limit) && !Rails.env.production?
  149. abort 'HTTP::Response#body_with_limit is already defined, the monkey patch will not be applied'
  150. else
  151. class ::HTTP::Response
  152. include Request::ClientLimit
  153. end
  154. end
  155. class Socket < TCPSocket
  156. class << self
  157. def open(host, *args)
  158. outer_e = nil
  159. port = args.first
  160. addresses = []
  161. begin
  162. addresses = [IPAddr.new(host)]
  163. rescue IPAddr::InvalidAddressError
  164. Resolv::DNS.open do |dns|
  165. dns.timeouts = 5
  166. addresses = dns.getaddresses(host)
  167. addresses = addresses.filter { |addr| addr.is_a?(Resolv::IPv6) }.take(2) + addresses.filter { |addr| !addr.is_a?(Resolv::IPv6) }.take(2)
  168. end
  169. end
  170. socks = []
  171. addr_by_socket = {}
  172. addresses.each do |address|
  173. begin
  174. check_private_address(address, host)
  175. sock = ::Socket.new(address.is_a?(Resolv::IPv6) ? ::Socket::AF_INET6 : ::Socket::AF_INET, ::Socket::SOCK_STREAM, 0)
  176. sockaddr = ::Socket.pack_sockaddr_in(port, address.to_s)
  177. sock.setsockopt(::Socket::IPPROTO_TCP, ::Socket::TCP_NODELAY, 1)
  178. sock.connect_nonblock(sockaddr)
  179. # If that hasn't raised an exception, we somehow managed to connect
  180. # immediately, close pending sockets and return immediately
  181. socks.each(&:close)
  182. return sock
  183. rescue IO::WaitWritable
  184. socks << sock
  185. addr_by_socket[sock] = sockaddr
  186. rescue => e
  187. outer_e = e
  188. end
  189. end
  190. until socks.empty?
  191. _, available_socks, = IO.select(nil, socks, nil, Request::TIMEOUT[:connect])
  192. if available_socks.nil?
  193. socks.each(&:close)
  194. raise HTTP::TimeoutError, "Connect timed out after #{Request::TIMEOUT[:connect]} seconds"
  195. end
  196. available_socks.each do |sock|
  197. socks.delete(sock)
  198. begin
  199. sock.connect_nonblock(addr_by_socket[sock])
  200. rescue Errno::EISCONN
  201. # Do nothing
  202. rescue => e
  203. sock.close
  204. outer_e = e
  205. next
  206. end
  207. socks.each(&:close)
  208. return sock
  209. end
  210. end
  211. if outer_e
  212. raise outer_e
  213. else
  214. raise SocketError, "No address for #{host}"
  215. end
  216. end
  217. alias new open
  218. def check_private_address(address, host)
  219. addr = IPAddr.new(address.to_s)
  220. return if private_address_exceptions.any? { |range| range.include?(addr) }
  221. raise Mastodon::PrivateNetworkAddressError, host if PrivateAddressCheck.private_address?(addr)
  222. end
  223. def private_address_exceptions
  224. @private_address_exceptions = begin
  225. (ENV['ALLOWED_PRIVATE_ADDRESSES'] || '').split(',').map { |addr| IPAddr.new(addr) }
  226. end
  227. end
  228. end
  229. end
  230. class ProxySocket < Socket
  231. class << self
  232. def check_private_address(_address, _host)
  233. # Accept connections to private addresses as HTTP proxies will usually
  234. # be on local addresses
  235. nil
  236. end
  237. end
  238. end
  239. private_constant :ClientLimit, :Socket, :ProxySocket
  240. end