1
0

request.rb 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346
  1. # frozen_string_literal: true
  2. require 'ipaddr'
  3. require 'socket'
  4. require 'resolv'
  5. # Use our own timeout class to avoid using HTTP.rb's timeout block
  6. # around the Socket#open method, since we use our own timeout blocks inside
  7. # that method
  8. #
  9. # Also changes how the read timeout behaves so that it is cumulative (closer
  10. # to HTTP::Timeout::Global, but still having distinct timeouts for other
  11. # operation types)
  12. class PerOperationWithDeadline < HTTP::Timeout::PerOperation
  13. READ_DEADLINE = 30
  14. def initialize(*args)
  15. super
  16. @read_deadline = options.fetch(:read_deadline, READ_DEADLINE)
  17. end
  18. def connect(socket_class, host, port, nodelay = false)
  19. @socket = socket_class.open(host, port)
  20. @socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1) if nodelay
  21. end
  22. # Reset deadline when the connection is re-used for different requests
  23. def reset_counter
  24. @deadline = nil
  25. end
  26. # Read data from the socket
  27. def readpartial(size, buffer = nil)
  28. @deadline ||= Process.clock_gettime(Process::CLOCK_MONOTONIC) + @read_deadline
  29. timeout = false
  30. loop do
  31. result = @socket.read_nonblock(size, buffer, exception: false)
  32. return :eof if result.nil?
  33. remaining_time = @deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
  34. raise HTTP::TimeoutError, "Read timed out after #{@read_timeout} seconds" if timeout
  35. raise HTTP::TimeoutError, "Read timed out after a total of #{@read_deadline} seconds" if remaining_time <= 0
  36. return result if result != :wait_readable
  37. # marking the socket for timeout. Why is this not being raised immediately?
  38. # it seems there is some race-condition on the network level between calling
  39. # #read_nonblock and #wait_readable, in which #read_nonblock signalizes waiting
  40. # for reads, and when waiting for x seconds, it returns nil suddenly without completing
  41. # the x seconds. In a normal case this would be a timeout on wait/read, but it can
  42. # also mean that the socket has been closed by the server. Therefore we "mark" the
  43. # socket for timeout and try to read more bytes. If it returns :eof, it's all good, no
  44. # timeout. Else, the first timeout was a proper timeout.
  45. # This hack has to be done because io/wait#wait_readable doesn't provide a value for when
  46. # the socket is closed by the server, and HTTP::Parser doesn't provide the limit for the chunks.
  47. timeout = true unless @socket.to_io.wait_readable([remaining_time, @read_timeout].min)
  48. end
  49. end
  50. end
  51. class Request
  52. REQUEST_TARGET = '(request-target)'
  53. # We enforce a 5s timeout on DNS resolving, 5s timeout on socket opening
  54. # and 5s timeout on the TLS handshake, meaning the worst case should take
  55. # about 15s in total
  56. TIMEOUT = { connect_timeout: 5, read_timeout: 10, write_timeout: 10, read_deadline: 30 }.freeze
  57. include RoutingHelper
  58. def initialize(verb, url, **options)
  59. raise ArgumentError if url.blank?
  60. @verb = verb
  61. @url = Addressable::URI.parse(url).normalize
  62. @http_client = options.delete(:http_client)
  63. @allow_local = options.delete(:allow_local)
  64. @options = options.merge(socket_class: use_proxy? || @allow_local ? ProxySocket : Socket)
  65. @options = @options.merge(timeout_class: PerOperationWithDeadline, timeout_options: TIMEOUT)
  66. @options = @options.merge(proxy_url) if use_proxy?
  67. @headers = {}
  68. raise Mastodon::HostValidationError, 'Instance does not support hidden service connections' if block_hidden_service?
  69. set_common_headers!
  70. set_digest! if options.key?(:body)
  71. end
  72. def on_behalf_of(actor, sign_with: nil)
  73. raise ArgumentError, 'actor must not be nil' if actor.nil?
  74. @actor = actor
  75. @keypair = sign_with.present? ? OpenSSL::PKey::RSA.new(sign_with) : @actor.keypair
  76. self
  77. end
  78. def add_headers(new_headers)
  79. @headers.merge!(new_headers)
  80. self
  81. end
  82. def perform
  83. begin
  84. response = http_client.request(@verb, @url.to_s, @options.merge(headers: headers))
  85. rescue => e
  86. raise e.class, "#{e.message} on #{@url}", e.backtrace[0]
  87. end
  88. begin
  89. # If we are using a persistent connection, we have to
  90. # read every response to be able to move forward at all.
  91. # However, simply calling #to_s or #flush may not be safe,
  92. # as the response body, if malicious, could be too big
  93. # for our memory. So we use the #body_with_limit method
  94. response.body_with_limit if http_client.persistent?
  95. yield response if block_given?
  96. ensure
  97. http_client.close unless http_client.persistent?
  98. end
  99. end
  100. def headers
  101. (@actor ? @headers.merge('Signature' => signature) : @headers).without(REQUEST_TARGET)
  102. end
  103. class << self
  104. def valid_url?(url)
  105. begin
  106. parsed_url = Addressable::URI.parse(url)
  107. rescue Addressable::URI::InvalidURIError
  108. return false
  109. end
  110. %w(http https).include?(parsed_url.scheme) && parsed_url.host.present?
  111. end
  112. def http_client
  113. HTTP.use(:auto_inflate).follow(max_hops: 3)
  114. end
  115. end
  116. private
  117. def set_common_headers!
  118. @headers[REQUEST_TARGET] = "#{@verb} #{@url.path}"
  119. @headers['User-Agent'] = Mastodon::Version.user_agent
  120. @headers['Host'] = @url.host
  121. @headers['Date'] = Time.now.utc.httpdate
  122. @headers['Accept-Encoding'] = 'gzip' if @verb != :head
  123. end
  124. def set_digest!
  125. @headers['Digest'] = "SHA-256=#{Digest::SHA256.base64digest(@options[:body])}"
  126. end
  127. def signature
  128. algorithm = 'rsa-sha256'
  129. signature = Base64.strict_encode64(@keypair.sign(OpenSSL::Digest.new('SHA256'), signed_string))
  130. "keyId=\"#{key_id}\",algorithm=\"#{algorithm}\",headers=\"#{signed_headers.keys.join(' ').downcase}\",signature=\"#{signature}\""
  131. end
  132. def signed_string
  133. signed_headers.map { |key, value| "#{key.downcase}: #{value}" }.join("\n")
  134. end
  135. def signed_headers
  136. @headers.without('User-Agent', 'Accept-Encoding')
  137. end
  138. def key_id
  139. ActivityPub::TagManager.instance.key_uri_for(@actor)
  140. end
  141. def http_client
  142. @http_client ||= Request.http_client
  143. end
  144. def use_proxy?
  145. proxy_url.present?
  146. end
  147. def proxy_url
  148. if hidden_service? && Rails.configuration.x.http_client_hidden_proxy.present?
  149. Rails.configuration.x.http_client_hidden_proxy
  150. else
  151. Rails.configuration.x.http_client_proxy
  152. end
  153. end
  154. def block_hidden_service?
  155. !Rails.configuration.x.access_to_hidden_service && hidden_service?
  156. end
  157. def hidden_service?
  158. /\.(onion|i2p)$/.match?(@url.host)
  159. end
  160. module ClientLimit
  161. def truncated_body(limit = 1.megabyte)
  162. if charset.nil?
  163. encoding = Encoding::BINARY
  164. else
  165. begin
  166. encoding = Encoding.find(charset)
  167. rescue ArgumentError
  168. encoding = Encoding::BINARY
  169. end
  170. end
  171. contents = String.new(encoding: encoding)
  172. while (chunk = readpartial)
  173. contents << chunk
  174. chunk.clear
  175. break if contents.bytesize > limit
  176. end
  177. contents
  178. end
  179. def body_with_limit(limit = 1.megabyte)
  180. raise Mastodon::LengthValidationError if content_length.present? && content_length > limit
  181. contents = truncated_body(limit)
  182. raise Mastodon::LengthValidationError if contents.bytesize > limit
  183. contents
  184. end
  185. end
  186. if ::HTTP::Response.methods.include?(:body_with_limit) && !Rails.env.production?
  187. abort 'HTTP::Response#body_with_limit is already defined, the monkey patch will not be applied'
  188. else
  189. class ::HTTP::Response
  190. include Request::ClientLimit
  191. end
  192. end
  193. class Socket < TCPSocket
  194. class << self
  195. def open(host, *args)
  196. outer_e = nil
  197. port = args.first
  198. addresses = []
  199. begin
  200. addresses = [IPAddr.new(host)]
  201. rescue IPAddr::InvalidAddressError
  202. Resolv::DNS.open do |dns|
  203. dns.timeouts = 5
  204. addresses = dns.getaddresses(host)
  205. addresses = addresses.filter { |addr| addr.is_a?(Resolv::IPv6) }.take(2) + addresses.filter { |addr| !addr.is_a?(Resolv::IPv6) }.take(2)
  206. end
  207. end
  208. socks = []
  209. addr_by_socket = {}
  210. addresses.each do |address|
  211. check_private_address(address, host)
  212. sock = ::Socket.new(address.is_a?(Resolv::IPv6) ? ::Socket::AF_INET6 : ::Socket::AF_INET, ::Socket::SOCK_STREAM, 0)
  213. sockaddr = ::Socket.pack_sockaddr_in(port, address.to_s)
  214. sock.setsockopt(::Socket::IPPROTO_TCP, ::Socket::TCP_NODELAY, 1)
  215. sock.connect_nonblock(sockaddr)
  216. # If that hasn't raised an exception, we somehow managed to connect
  217. # immediately, close pending sockets and return immediately
  218. socks.each(&:close)
  219. return sock
  220. rescue IO::WaitWritable
  221. socks << sock
  222. addr_by_socket[sock] = sockaddr
  223. rescue => e
  224. outer_e = e
  225. end
  226. until socks.empty?
  227. _, available_socks, = IO.select(nil, socks, nil, Request::TIMEOUT[:connect_timeout])
  228. if available_socks.nil?
  229. socks.each(&:close)
  230. raise HTTP::TimeoutError, "Connect timed out after #{Request::TIMEOUT[:connect_timeout]} seconds"
  231. end
  232. available_socks.each do |sock|
  233. socks.delete(sock)
  234. begin
  235. sock.connect_nonblock(addr_by_socket[sock])
  236. rescue Errno::EISCONN
  237. # Do nothing
  238. rescue => e
  239. sock.close
  240. outer_e = e
  241. next
  242. end
  243. socks.each(&:close)
  244. return sock
  245. end
  246. end
  247. if outer_e
  248. raise outer_e
  249. else
  250. raise SocketError, "No address for #{host}"
  251. end
  252. end
  253. alias new open
  254. def check_private_address(address, host)
  255. addr = IPAddr.new(address.to_s)
  256. return if Rails.env.development? || private_address_exceptions.any? { |range| range.include?(addr) }
  257. raise Mastodon::PrivateNetworkAddressError, host if PrivateAddressCheck.private_address?(addr)
  258. end
  259. def private_address_exceptions
  260. @private_address_exceptions = (ENV['ALLOWED_PRIVATE_ADDRESSES'] || '').split(/(?:\s*,\s*|\s+)/).map { |addr| IPAddr.new(addr) }
  261. end
  262. end
  263. end
  264. class ProxySocket < Socket
  265. class << self
  266. def check_private_address(_address, _host)
  267. # Accept connections to private addresses as HTTP proxies will usually
  268. # be on local addresses
  269. nil
  270. end
  271. end
  272. end
  273. private_constant :ClientLimit, :Socket, :ProxySocket
  274. end