fetch_link_card_service.rb 7.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180
  1. # frozen_string_literal: true
  2. class FetchLinkCardService < BaseService
  3. URL_PATTERN = %r{
  4. (#{Twitter::TwitterText::Regex[:valid_url_preceding_chars]}) # $1 preceeding chars
  5. ( # $2 URL
  6. (https?:\/\/) # $3 Protocol (required)
  7. (#{Twitter::TwitterText::Regex[:valid_domain]}) # $4 Domain(s)
  8. (?::(#{Twitter::TwitterText::Regex[:valid_port_number]}))? # $5 Port number (optional)
  9. (/#{Twitter::TwitterText::Regex[:valid_url_path]}*)? # $6 URL Path and anchor
  10. (\?#{Twitter::TwitterText::Regex[:valid_url_query_chars]}*#{Twitter::TwitterText::Regex[:valid_url_query_ending_chars]})? # $7 Query String
  11. )
  12. }iox
  13. def call(status)
  14. @status = status
  15. @url = parse_urls
  16. return if @url.nil? || @status.preview_cards.any?
  17. @url = @url.to_s
  18. RedisLock.acquire(lock_options) do |lock|
  19. if lock.acquired?
  20. @card = PreviewCard.find_by(url: @url)
  21. process_url if @card.nil? || @card.updated_at <= 2.weeks.ago || @card.missing_image?
  22. else
  23. raise Mastodon::RaceConditionError
  24. end
  25. end
  26. attach_card if @card&.persisted?
  27. rescue HTTP::Error, OpenSSL::SSL::SSLError, Addressable::URI::InvalidURIError, Mastodon::HostValidationError, Mastodon::LengthValidationError => e
  28. Rails.logger.debug "Error fetching link #{@url}: #{e}"
  29. nil
  30. end
  31. private
  32. def process_url
  33. @card ||= PreviewCard.new(url: @url)
  34. attempt_oembed || attempt_opengraph
  35. end
  36. def html
  37. return @html if defined?(@html)
  38. Request.new(:get, @url).add_headers('Accept' => 'text/html', 'User-Agent' => Mastodon::Version.user_agent + ' Bot').perform do |res|
  39. if res.code == 200 && res.mime_type == 'text/html'
  40. @html_charset = res.charset
  41. @html = res.body_with_limit
  42. else
  43. @html_charset = nil
  44. @html = nil
  45. end
  46. end
  47. end
  48. def attach_card
  49. @status.preview_cards << @card
  50. Rails.cache.delete(@status)
  51. end
  52. def parse_urls
  53. if @status.local?
  54. urls = @status.text.scan(URL_PATTERN).map { |array| Addressable::URI.parse(array[1]).normalize }
  55. else
  56. html = Nokogiri::HTML(@status.text)
  57. links = html.css('a')
  58. urls = links.filter_map { |a| Addressable::URI.parse(a['href']) unless skip_link?(a) }.filter_map(&:normalize)
  59. end
  60. urls.reject { |uri| bad_url?(uri) }.first
  61. end
  62. def bad_url?(uri)
  63. # Avoid local instance URLs and invalid URLs
  64. uri.host.blank? || TagManager.instance.local_url?(uri.to_s) || !%w(http https).include?(uri.scheme)
  65. end
  66. # rubocop:disable Naming/MethodParameterName
  67. def mention_link?(a)
  68. @status.mentions.any? do |mention|
  69. a['href'] == ActivityPub::TagManager.instance.url_for(mention.account)
  70. end
  71. end
  72. def skip_link?(a)
  73. # Avoid links for hashtags and mentions (microformats)
  74. a['rel']&.include?('tag') || a['class']&.match?(/u-url|h-card/) || mention_link?(a)
  75. end
  76. # rubocop:enable Naming/MethodParameterName
  77. def attempt_oembed
  78. service = FetchOEmbedService.new
  79. url_domain = Addressable::URI.parse(@url).normalized_host
  80. cached_endpoint = Rails.cache.read("oembed_endpoint:#{url_domain}")
  81. embed = service.call(@url, cached_endpoint: cached_endpoint) unless cached_endpoint.nil?
  82. embed ||= service.call(@url, html: html) unless html.nil?
  83. return false if embed.nil?
  84. url = Addressable::URI.parse(service.endpoint_url)
  85. @card.type = embed[:type]
  86. @card.title = embed[:title] || ''
  87. @card.author_name = embed[:author_name] || ''
  88. @card.author_url = embed[:author_url].present? ? (url + embed[:author_url]).to_s : ''
  89. @card.provider_name = embed[:provider_name] || ''
  90. @card.provider_url = embed[:provider_url].present? ? (url + embed[:provider_url]).to_s : ''
  91. @card.width = 0
  92. @card.height = 0
  93. case @card.type
  94. when 'link'
  95. @card.image_remote_url = (url + embed[:thumbnail_url]).to_s if embed[:thumbnail_url].present?
  96. when 'photo'
  97. return false if embed[:url].blank?
  98. @card.embed_url = (url + embed[:url]).to_s
  99. @card.image_remote_url = (url + embed[:url]).to_s
  100. @card.width = embed[:width].presence || 0
  101. @card.height = embed[:height].presence || 0
  102. when 'video'
  103. @card.width = embed[:width].presence || 0
  104. @card.height = embed[:height].presence || 0
  105. @card.html = Formatter.instance.sanitize(embed[:html], Sanitize::Config::MASTODON_OEMBED)
  106. @card.image_remote_url = (url + embed[:thumbnail_url]).to_s if embed[:thumbnail_url].present?
  107. when 'rich'
  108. # Most providers rely on <script> tags, which is a no-no
  109. return false
  110. end
  111. @card.save_with_optional_image!
  112. end
  113. def attempt_opengraph
  114. return if html.nil?
  115. detector = CharlockHolmes::EncodingDetector.new
  116. detector.strip_tags = true
  117. guess = detector.detect(@html, @html_charset)
  118. encoding = guess&.fetch(:confidence, 0).to_i > 60 ? guess&.fetch(:encoding, nil) : nil
  119. page = Nokogiri::HTML(@html, nil, encoding)
  120. player_url = meta_property(page, 'twitter:player')
  121. if player_url && !bad_url?(Addressable::URI.parse(player_url))
  122. @card.type = :video
  123. @card.width = meta_property(page, 'twitter:player:width') || 0
  124. @card.height = meta_property(page, 'twitter:player:height') || 0
  125. @card.html = content_tag(:iframe, nil, src: player_url,
  126. width: @card.width,
  127. height: @card.height,
  128. allowtransparency: 'true',
  129. scrolling: 'no',
  130. frameborder: '0')
  131. else
  132. @card.type = :link
  133. end
  134. @card.title = meta_property(page, 'og:title').presence || page.at_xpath('//title')&.content || ''
  135. @card.description = meta_property(page, 'og:description').presence || meta_property(page, 'description') || ''
  136. @card.image_remote_url = (Addressable::URI.parse(@url) + meta_property(page, 'og:image')).to_s if meta_property(page, 'og:image')
  137. return if @card.title.blank? && @card.html.blank?
  138. @card.save_with_optional_image!
  139. end
  140. def meta_property(page, property)
  141. page.at_xpath("//meta[contains(concat(' ', normalize-space(@property), ' '), ' #{property} ')]")&.attribute('content')&.value || page.at_xpath("//meta[@name=\"#{property}\"]")&.attribute('content')&.value
  142. end
  143. def lock_options
  144. { redis: Redis.current, key: "fetch:#{@url}" }
  145. end
  146. end