bida/bastodon
6
0
Fork 0
Code Issues 7 Pull requests Releases Wiki Activity

Add trending statuses (#17431)

Browse source 
* Add trending statuses

* Fix dangling items with stale scores in localized sets

* Various fixes and improvements

- Change approve_all/reject_all to approve_accounts/reject_accounts
- Change Trends::Query methods to not mutate the original query
- Change Trends::Query#skip to offset
- Change follow recommendations to be refreshed in a transaction

* Add tests for trending statuses filtering behaviour

* Fix not applying filtering scope in controller
This commit is contained in:
Eugen Rochko 2022-02-25 00:34:14 +01:00 committed by GitHub
parent a29a982eaa
commit 27965ce5ed
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
71 changed files with 1074 additions and 307 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
.rubocop.yml
View file

@ -32,10 +32,11 @@ Layout/EmptyLineAfterGuardClause:
Layout/EmptyLinesAroundAttributeAccessor:
Enabled: true
Layout/FirstHashElementIndentation:
EnforcedStyle: consistent
Layout/HashAlignment:
Enabled: false
# EnforcedHashRocketStyle: table
# EnforcedColonStyle: table
Layout/SpaceAroundMethodCallOperator:
Enabled: true

12
app/controllers/admin/trends/links/preview_card_providers_controller.rb
View file

@ -5,11 +5,11 @@ class Admin::Trends::Links::PreviewCardProvidersController < Admin::BaseControll
authorize :preview_card_provider, :index?
@preview_card_providers = filtered_preview_card_providers.page(params[:page])
@form = Form::PreviewCardProviderBatch.new
@form = Trends::PreviewCardProviderBatch.new
end
def batch
@form = Form::PreviewCardProviderBatch.new(form_preview_card_provider_batch_params.merge(current_account: current_account, action: action_from_button))
@form = Trends::PreviewCardProviderBatch.new(trends_preview_card_provider_batch_params.merge(current_account: current_account, action: action_from_button))
@form.save
rescue ActionController::ParameterMissing
flash[:alert] = I18n.t('admin.accounts.no_account_selected')
@ -20,15 +20,15 @@ class Admin::Trends::Links::PreviewCardProvidersController < Admin::BaseControll
private
def filtered_preview_card_providers
PreviewCardProviderFilter.new(filter_params).results
Trends::PreviewCardProviderFilter.new(filter_params).results
end
def filter_params
params.slice(:page, *PreviewCardProviderFilter::KEYS).permit(:page, *PreviewCardProviderFilter::KEYS)
params.slice(:page, *Trends::PreviewCardProviderFilter::KEYS).permit(:page, *Trends::PreviewCardProviderFilter::KEYS)
end
def form_preview_card_provider_batch_params
params.require(:form_preview_card_provider_batch).permit(:action, preview_card_provider_ids: [])
def trends_preview_card_provider_batch_params
params.require(:trends_preview_card_provider_batch).permit(:action, preview_card_provider_ids: [])
end
def action_from_button

20
app/controllers/admin/trends/links_controller.rb
View file

@ -5,11 +5,11 @@ class Admin::Trends::LinksController < Admin::BaseController
authorize :preview_card, :index?
@preview_cards = filtered_preview_cards.page(params[:page])
@form = Form::PreviewCardBatch.new
@form = Trends::PreviewCardBatch.new
end
def batch
@form = Form::PreviewCardBatch.new(form_preview_card_batch_params.merge(current_account: current_account, action: action_from_button))
@form = Trends::PreviewCardBatch.new(trends_preview_card_batch_params.merge(current_account: current_account, action: action_from_button))
@form.save
rescue ActionController::ParameterMissing
flash[:alert] = I18n.t('admin.accounts.no_account_selected')
@ -20,26 +20,26 @@ class Admin::Trends::LinksController < Admin::BaseController
private
def filtered_preview_cards
PreviewCardFilter.new(filter_params.with_defaults(trending: 'all')).results
Trends::PreviewCardFilter.new(filter_params.with_defaults(trending: 'all')).results
end
def filter_params
params.slice(:page, *PreviewCardFilter::KEYS).permit(:page, *PreviewCardFilter::KEYS)
params.slice(:page, *Trends::PreviewCardFilter::KEYS).permit(:page, *Trends::PreviewCardFilter::KEYS)
end
def form_preview_card_batch_params
params.require(:form_preview_card_batch).permit(:action, preview_card_ids: [])
def trends_preview_card_batch_params
params.require(:trends_preview_card_batch).permit(:action, preview_card_ids: [])
end
def action_from_button
if params[:approve]
'approve'
elsif params[:approve_all]
'approve_all'
elsif params[:approve_providers]
'approve_providers'
elsif params[:reject]
'reject'
elsif params[:reject_all]
'reject_all'
elsif params[:reject_providers]
'reject_providers'
end
end
end

45
app/controllers/admin/trends/statuses_controller.rb Normal file
View file

@ -0,0 +1,45 @@
# frozen_string_literal: true
class Admin::Trends::StatusesController < Admin::BaseController
def index
authorize :status, :index?
@statuses = filtered_statuses.page(params[:page])
@form = Trends::StatusBatch.new
end
def batch
@form = Trends::StatusBatch.new(trends_status_batch_params.merge(current_account: current_account, action: action_from_button))
@form.save
rescue ActionController::ParameterMissing
flash[:alert] = I18n.t('admin.accounts.no_account_selected')
ensure
redirect_to admin_trends_statuses_path(filter_params)
end
private
def filtered_statuses
Trends::StatusFilter.new(filter_params.with_defaults(trending: 'all')).results.includes(:account, :media_attachments, :active_mentions)
end
def filter_params
params.slice(:page, *Trends::StatusFilter::KEYS).permit(:page, *Trends::StatusFilter::KEYS)
end
def trends_status_batch_params
params.require(:trends_status_batch).permit(:action, status_ids: [])
end
def action_from_button
if params[:approve]
'approve'
elsif params[:approve_accounts]
'approve_accounts'
elsif params[:reject]
'reject'
elsif params[:reject_accounts]
'reject_accounts'
end
end
end

12
app/controllers/admin/trends/tags_controller.rb
View file

@ -5,11 +5,11 @@ class Admin::Trends::TagsController < Admin::BaseController
authorize :tag, :index?
@tags = filtered_tags.page(params[:page])
@form = Form::TagBatch.new
@form = Trends::TagBatch.new
end
def batch
@form = Form::TagBatch.new(form_tag_batch_params.merge(current_account: current_account, action: action_from_button))
@form = Trends::TagBatch.new(trends_tag_batch_params.merge(current_account: current_account, action: action_from_button))
@form.save
rescue ActionController::ParameterMissing
flash[:alert] = I18n.t('admin.accounts.no_account_selected')
@ -20,15 +20,15 @@ class Admin::Trends::TagsController < Admin::BaseController
private
def filtered_tags
TagFilter.new(filter_params).results
Trends::TagFilter.new(filter_params).results
end
def filter_params
params.slice(:page, *TagFilter::KEYS).permit(:page, *TagFilter::KEYS)
params.slice(:page, *Trends::TagFilter::KEYS).permit(:page, *Trends::TagFilter::KEYS)
end
def form_tag_batch_params
params.require(:form_tag_batch).permit(:action, tag_ids: [])
def trends_tag_batch_params
params.require(:trends_tag_batch).permit(:action, tag_ids: [])
end
def action_from_button

19
app/controllers/api/v1/admin/trends/links_controller.rb Normal file
View file

@ -0,0 +1,19 @@
# frozen_string_literal: true
class Api::V1::Admin::Trends::LinksController < Api::BaseController
protect_from_forgery with: :exception
before_action -> { authorize_if_got_token! :'admin:read' }
before_action :require_staff!
before_action :set_links
def index
render json: @links, each_serializer: REST::Trends::LinkSerializer
end
private
def set_links
@links = Trends.links.query.limit(limit_param(10))
end
end

19
app/controllers/api/v1/admin/trends/statuses_controller.rb Normal file
View file

@ -0,0 +1,19 @@
# frozen_string_literal: true
class Api::V1::Admin::Trends::StatusesController < Api::BaseController
protect_from_forgery with: :exception
before_action -> { authorize_if_got_token! :'admin:read' }
before_action :require_staff!
before_action :set_statuses
def index
render json: @statuses, each_serializer: REST::StatusSerializer
end
private
def set_statuses
@statuses = cache_collection(Trends.statuses.query.limit(limit_param(DEFAULT_STATUSES_LIMIT)), Status)
end
end

2
app/controllers/api/v1/admin/trends/tags_controller.rb
View file

@ -14,6 +14,6 @@ class Api::V1::Admin::Trends::TagsController < Api::BaseController
private
def set_tags
@tags = Trends.tags.get(false, limit_param(10))
@tags = Trends.tags.query.limit(limit_param(10))
end
end

6
app/controllers/api/v1/trends/links_controller.rb
View file

@ -12,10 +12,14 @@ class Api::V1::Trends::LinksController < Api::BaseController
def set_links
@links = begin
if Setting.trends
Trends.links.get(true, limit_param(10))
links_from_trends
else
[]
end
end
end
def links_from_trends
Trends.links.query.allowed.in_locale(content_locale).limit(limit_param(10))
end
end

27
app/controllers/api/v1/trends/statuses_controller.rb Normal file
View file

@ -0,0 +1,27 @@
# frozen_string_literal: true
class Api::V1::Trends::StatusesController < Api::BaseController
before_action :set_statuses
def index
render json: @statuses, each_serializer: REST::StatusSerializer
end
private
def set_statuses
@statuses = begin
if Setting.trends
cache_collection(statuses_from_trends, Status)
else
[]
end
end
end
def statuses_from_trends
scope = Trends.statuses.query.allowed.in_locale(content_locale)
scope = scope.filtered_for(current_account) if user_signed_in?
scope.limit(limit_param(DEFAULT_STATUSES_LIMIT))
end
end

2
app/controllers/api/v1/trends/tags_controller.rb
View file

@ -12,7 +12,7 @@ class Api::V1::Trends::TagsController < Api::BaseController
def set_tags
@tags = begin
if Setting.trends
Trends.tags.get(true, limit_param(10))
Trends.tags.query.allowed.limit(limit_param(10))
else
[]
end

4
app/controllers/concerns/localized.rb
View file

@ -27,4 +27,8 @@ module Localized
def available_locale_or_nil(locale_name)
locale_name.to_sym if locale_name.present? && I18n.available_locales.include?(locale_name.to_sym)
end
def content_locale
@content_locale ||= I18n.locale.to_s.split(/[_-]/).first
end
end

7
app/helpers/admin/filter_helper.rb
View file

@ -5,9 +5,10 @@ module Admin::FilterHelper
AccountFilter::KEYS,
CustomEmojiFilter::KEYS,
ReportFilter::KEYS,
TagFilter::KEYS,
PreviewCardProviderFilter::KEYS,
PreviewCardFilter::KEYS,
Trends::TagFilter::KEYS,
Trends::PreviewCardProviderFilter::KEYS,
Trends::PreviewCardFilter::KEYS,
Trends::StatusFilter::KEYS,
InstanceFilter::KEYS,
InviteFilter::KEYS,
RelationshipFilter::KEYS,

2
app/helpers/languages_helper.rb
View file

@ -242,6 +242,6 @@ module LanguagesHelper
end
def valid_locale?(locale)
SUPPORTED_LOCALES.key?(locale.to_sym)
locale.present? && SUPPORTED_LOCALES.key?(locale.to_sym)
end
end

10
app/javascript/styles/mastodon/accounts.scss
View file

@ -331,7 +331,8 @@
}
.batch-table__row--muted .pending-account__header,
.batch-table__row--muted .accounts-table {
.batch-table__row--muted .accounts-table,
.batch-table__row--muted .name-tag {
&,
a,
strong {
@ -339,6 +340,10 @@
}
}
.batch-table__row--muted .name-tag .avatar {
opacity: 0.5;
}
.batch-table__row--muted .accounts-table {
tbody td.accounts-table__extra,
&__count,
@ -352,7 +357,8 @@
}
.batch-table__row--attention .pending-account__header,
.batch-table__row--attention .accounts-table {
.batch-table__row--attention .accounts-table,
.batch-table__row--attention .name-tag {
&,
a,
strong {

7
app/javascript/styles/mastodon/tables.scss
View file

@ -210,6 +210,7 @@ a.table-action-link {
&__content {
padding-top: 12px;
padding-bottom: 16px;
overflow: hidden;
&--unpadded {
padding: 0;
@ -296,3 +297,9 @@ a.table-action-link {
}
}
}
.one-liner {
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
}

3
app/lib/activitypub/activity/announce.rb
View file

@ -23,8 +23,7 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity
visibility: visibility_from_audience
)
Trends.tags.register(@status)
Trends.links.register(@status)
Trends.register!(@status)
distribute
end

2
app/lib/activitypub/activity/like.rb
View file

@ -7,6 +7,8 @@ class ActivityPub::Activity::Like < ActivityPub::Activity
return if original_status.nil? || !original_status.account.local? || delete_arrived_first?(@json['id']) || @account.favourited?(original_status)
favourite = original_status.favourites.create!(account: @account)
NotifyService.new.call(original_status.account, :favourite, favourite)
Trends.statuses.register(original_status)
end
end

27
app/mailers/admin_mailer.rb
View file

@ -35,25 +35,18 @@ class AdminMailer < ApplicationMailer
end
end
def new_trending_tags(recipient, tags)
@tags = tags
@me = recipient
@instance = Rails.configuration.x.local_domain
@lowest_trending_tag = Trends.tags.get(true, Trends.tags.options[:review_threshold]).last
def new_trends(recipient, links, tags, statuses)
@links = links
@lowest_trending_link = Trends.links.query.allowed.limit(Trends.links.options[:review_threshold]).last
@tags = tags
@lowest_trending_tag = Trends.tags.query.allowed.limit(Trends.tags.options[:review_threshold]).last
@statuses = statuses
@lowest_trending_status = Trends.statuses.query.allowed.limit(Trends.statuses.options[:review_threshold]).last
@me = recipient
@instance = Rails.configuration.x.local_domain
locale_for_account(@me) do
mail to: @me.user_email, subject: I18n.t('admin_mailer.new_trending_tags.subject', instance: @instance)
end
end
def new_trending_links(recipient, links)
@links = links
@me = recipient
@instance = Rails.configuration.x.local_domain
@lowest_trending_link = Trends.links.get(true, Trends.links.options[:review_threshold]).last
locale_for_account(@me) do
mail to: @me.user_email, subject: I18n.t('admin_mailer.new_trending_links.subject', instance: @instance)
mail to: @me.user_email, subject: I18n.t('admin_mailer.new_trends.subject', instance: @instance)
end
end
end

30
app/models/account.rb
View file

@ -40,13 +40,15 @@
# also_known_as :string is an Array
# silenced_at :datetime
# suspended_at :datetime
# trust_level :integer
# hide_collections :boolean
# avatar_storage_schema_version :integer
# header_storage_schema_version :integer
# devices_url :string
# suspension_origin :integer
# sensitized_at :datetime
# trendable :boolean
# reviewed_at :datetime
# requested_review_at :datetime
#
class Account < ApplicationRecord
@ -56,6 +58,7 @@ class Account < ApplicationRecord
remote_url
salmon_url
hub_url
trust_level
)
USERNAME_RE = /[a-z0-9_]+([a-z0-9_\.-]+[a-z0-9_]+)?/i
@ -74,11 +77,6 @@ class Account < ApplicationRecord
include DomainMaterializable
include AccountMerging
TRUST_LEVELS = {
untrusted: 0,
trusted: 1,
}.freeze
enum protocol: [:ostatus, :activitypub]
enum suspension_origin: [:local, :remote], _prefix: true
@ -202,10 +200,6 @@ class Account < ApplicationRecord
last_webfingered_at.nil? || last_webfingered_at <= 1.day.ago
end
def trust_level
self[:trust_level] || 0
end
def refresh!
ResolveAccountService.new.call(acct) unless local?
end
@ -388,6 +382,22 @@ class Account < ApplicationRecord
@synchronization_uri_prefix ||= "#{uri[URL_PREFIX_RE]}/"
end
def requires_review?
reviewed_at.nil?
end
def reviewed?
reviewed_at.present?
end
def requested_review?
requested_review_at.present?
end
def requires_review_notification?
requires_review? && !requested_review?
end
class Field < ActiveModelSerializers::Model
attributes :name, :value, :verified_at, :account

12
app/models/status.rb
View file

@ -268,6 +268,18 @@ class Status < ApplicationRecord
update_status_stat!(key => [public_send(key) - 1, 0].max)
end
def trendable?
if attributes['trendable'].nil?
account.trendable?
else
attributes['trendable']
end
end
def requires_review_notification?
attributes['trendable'].nil? && account.requires_review_notification?
end
after_create_commit :increment_counter_caches
after_destroy_commit :decrement_counter_caches

26
app/models/trends.rb
View file

@ -13,15 +13,37 @@ module Trends
@tags ||= Trends::Tags.new
end
def self.statuses
@statuses ||= Trends::Statuses.new
end
def self.register!(status)
[links, tags, statuses].each { |trend_type| trend_type.register(status) }
end
def self.refresh!
[links, tags].each(&:refresh)
[links, tags, statuses].each(&:refresh)
end
def self.request_review!
[links, tags].each(&:request_review) if enabled?
return unless enabled?
links_requiring_review = links.request_review
tags_requiring_review = tags.request_review
statuses_requiring_review = statuses.request_review
return if links_requiring_review.empty? && tags_requiring_review.empty? && statuses_requiring_review.empty?
User.staff.includes(:account).find_each do |user|
AdminMailer.new_trends(user.account, links_requiring_review, tags_requiring_review, statuses_requiring_review).deliver_later! if user.allows_trends_review_emails?
end
end
def self.enabled?
Setting.trends
end
def self.available_locales
@available_locales ||= I18n.available_locales.map { |locale| locale.to_s.split(/[_-]/).first }.uniq
end
end

20
app/models/trends/base.rb
View file

@ -2,6 +2,7 @@
class Trends::Base
include Redisable
include LanguagesHelper
class_attribute :default_options
@ -32,8 +33,8 @@ class Trends::Base
raise NotImplementedError
end
def get(*)
raise NotImplementedError
def query
Trends::Query.new(key_prefix, klass)
end
def score(id)
@ -72,6 +73,21 @@ class Trends::Base
redis.zrevrange("#{key_prefix}:allowed", 0, rank, with_scores: true).last&.last || 0
end
# @param [Integer] id
# @param [Float] score
# @param [Hash<String, Boolean>] subsets
def add_to_and_remove_from_subsets(id, score, subsets = {})
subsets.each_key do |subset|
key = [key_prefix, subset].compact.join(':')
if score.positive? && subsets[subset]
redis.zadd(key, score, id)
else
redis.zrem(key, id)
end
end
end
private
def used_key(at_time)

52
app/models/trends/links.rb
View file

@ -4,8 +4,8 @@ class Trends::Links < Trends::Base
PREFIX = 'trending_links'
self.default_options = {
threshold: 15,
review_threshold: 10,
threshold: 5,
review_threshold: 3,
max_score_cooldown: 2.days.freeze,
max_score_halflife: 8.hours.freeze,
}
@ -27,12 +27,6 @@ class Trends::Links < Trends::Base
record_used_id(preview_card.id, at_time)
end
def get(allowed, limit)
preview_card_ids = currently_trending_ids(allowed, limit)
preview_cards = PreviewCard.where(id: preview_card_ids).index_by(&:id)
preview_card_ids.map { |id| preview_cards[id] }.compact
end
def refresh(at_time = Time.now.utc)
preview_cards = PreviewCard.where(id: (recently_used_ids(at_time) + currently_trending_ids(false, -1)).uniq)
calculate_scores(preview_cards, at_time)
@ -42,7 +36,7 @@ class Trends::Links < Trends::Base
def request_review
preview_cards = PreviewCard.where(id: currently_trending_ids(false, -1))
preview_cards_requiring_review = preview_cards.filter_map do |preview_card|
preview_cards.filter_map do |preview_card|
next unless would_be_trending?(preview_card.id) && !preview_card.trendable? && preview_card.requires_review_notification?
if preview_card.provider.nil?
@ -53,12 +47,6 @@ class Trends::Links < Trends::Base
preview_card
end
return if preview_cards_requiring_review.empty?
User.staff.includes(:account).find_each do |user|
AdminMailer.new_trending_links(user.account, preview_cards_requiring_review).deliver_later! if user.allows_trending_tag_emails?
end
end
protected
@ -67,6 +55,10 @@ class Trends::Links < Trends::Base
PREFIX
end
def klass
PreviewCard
end
private
def calculate_scores(preview_cards, at_time)
@ -96,17 +88,27 @@ class Trends::Links < Trends::Base
decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / options[:max_score_halflife].to_f))
if decaying_score.zero?
redis.zrem("#{PREFIX}:all", preview_card.id)
redis.zrem("#{PREFIX}:allowed", preview_card.id)
else
redis.zadd("#{PREFIX}:all", decaying_score, preview_card.id)
add_to_and_remove_from_subsets(preview_card.id, decaying_score, {
all: true,
allowed: preview_card.trendable?,
})
if preview_card.trendable?
redis.zadd("#{PREFIX}:allowed", decaying_score, preview_card.id)
else
redis.zrem("#{PREFIX}:allowed", preview_card.id)
end
next unless valid_locale?(preview_card.language)
add_to_and_remove_from_subsets(preview_card.id, decaying_score, {
"all:#{preview_card.language}" => true,
"allowed:#{preview_card.language}" => preview_card.trendable?,
})
end
# Clean up localized sets by calculating the intersection with the main
# set. We do this instead of just deleting the localized sets to avoid
# having moments where the API returns empty results
redis.pipelined do
Trends.available_locales.each do |locale|
redis.zinterstore("#{key_prefix}:all:#{locale}", ["#{key_prefix}:all:#{locale}", "#{key_prefix}:all"], aggregate: 'max')
redis.zinterstore("#{key_prefix}:allowed:#{locale}", ["#{key_prefix}:allowed:#{locale}", "#{key_prefix}:all"], aggregate: 'max')
end
end
end

22
app/models/form/preview_card_batch.rb → app/models/trends/preview_card_batch.rb
View file

@ -1,6 +1,6 @@
# frozen_string_literal: true
class Form::PreviewCardBatch
class Trends::PreviewCardBatch
include ActiveModel::Model
include Authorization
@ -10,12 +10,12 @@ class Form::PreviewCardBatch
case action
when 'approve'
approve!
when 'approve_all'
approve_all!
when 'approve_providers'
approve_providers!
when 'reject'
reject!
when 'reject_all'
reject_all!
when 'reject_providers'
reject_providers!
end
end
@ -30,13 +30,13 @@ class Form::PreviewCardBatch
end
def approve!
preview_cards.each { |preview_card| authorize(preview_card, :update?) }
preview_cards.each { |preview_card| authorize(preview_card, :review?) }
preview_cards.update_all(trendable: true)
end
def approve_all!
def approve_providers!
preview_card_providers.each do |provider|
authorize(provider, :update?)
authorize(provider, :review?)
provider.update(trendable: true, reviewed_at: action_time)
end
@ -45,13 +45,13 @@ class Form::PreviewCardBatch
end
def reject!
preview_cards.each { |preview_card| authorize(preview_card, :update?) }
preview_cards.each { |preview_card| authorize(preview_card, :review?) }
preview_cards.update_all(trendable: false)
end
def reject_all!
def reject_providers!
preview_card_providers.each do |provider|
authorize(provider, :update?)
authorize(provider, :review?)
provider.update(trendable: false, reviewed_at: action_time)
end

23
app/models/preview_card_filter.rb → app/models/trends/preview_card_filter.rb
View file

@ -1,8 +1,9 @@
# frozen_string_literal: true
class PreviewCardFilter
class Trends::PreviewCardFilter
KEYS = %i(
trending
locale
).freeze
attr_reader :params
@ -15,7 +16,7 @@ class PreviewCardFilter
scope = PreviewCard.unscoped
params.each do |key, value|
next if key.to_s == 'page'
next if %w(page locale).include?(key.to_s)
scope.merge!(scope_for(key, value.to_s.strip)) if value.present?
end
@ -35,19 +36,11 @@ class PreviewCardFilter
end
def trending_scope(value)
ids = begin
case value.to_s
when 'allowed'
Trends.links.currently_trending_ids(true, -1)
else
Trends.links.currently_trending_ids(false, -1)
end
end
scope = Trends.links.query
if ids.empty?
PreviewCard.none
else
PreviewCard.joins("join unnest(array[#{ids.map(&:to_i).join(',')}]::integer[]) with ordinality as x (id, ordering) on preview_cards.id = x.id").order('x.ordering')
end
scope = scope.in_locale(@params[:locale].to_s) if @params[:locale].present?
scope = scope.allowed if value == 'allowed'
scope.to_arel
end
end

6
app/models/form/preview_card_provider_batch.rb → app/models/trends/preview_card_provider_batch.rb
View file

@ -1,6 +1,6 @@
# frozen_string_literal: true
class Form::PreviewCardProviderBatch
class Trends::PreviewCardProviderBatch
include ActiveModel::Model
include Authorization
@ -22,12 +22,12 @@ class Form::PreviewCardProviderBatch
end
def approve!
preview_card_providers.each { |provider| authorize(provider, :update?) }
preview_card_providers.each { |provider| authorize(provider, :review?) }
preview_card_providers.update_all(trendable: true, reviewed_at: Time.now.utc)
end
def reject!
preview_card_providers.each { |provider| authorize(provider, :update?) }
preview_card_providers.each { |provider| authorize(provider, :review?) }
preview_card_providers.update_all(trendable: false, reviewed_at: Time.now.utc)
end
end

2
app/models/preview_card_provider_filter.rb → app/models/trends/preview_card_provider_filter.rb
View file

@ -1,6 +1,6 @@
# frozen_string_literal: true
class PreviewCardProviderFilter
class Trends::PreviewCardProviderFilter
KEYS = %i(
status
).freeze

106
app/models/trends/query.rb Normal file
View file

@ -0,0 +1,106 @@
# frozen_string_literal: true
class Trends::Query
include Redisable
include Enumerable
attr_reader :prefix, :klass, :loaded
alias loaded? loaded
def initialize(prefix, klass)
@prefix = prefix
@klass = klass
@records = []
@loaded = false
@allowed = false
@limit = -1
@offset = 0
end
def allowed!
@allowed = true
self
end
def allowed
clone.allowed!
end
def in_locale!(value)
@locale = value
self
end
def in_locale(value)
clone.in_locale!(value)
end
def offset!(value)
@offset = value
self
end
def offset(value)
clone.offset!(value)
end
def limit!(value)
@limit = value
self
end
def limit(value)
clone.limit!(value)
end
def records
load
@records
end
delegate :each, :empty?, :first, :last, to: :records
def to_ary
records.dup
end
alias to_a to_ary
def to_arel
tmp_ids = ids
if tmp_ids.empty?
klass.none
else
klass.joins("join unnest(array[#{tmp_ids.join(',')}]) with ordinality as x (id, ordering) on #{klass.table_name}.id = x.id").reorder('x.ordering')
end
end
private
def key
[@prefix, @allowed ? 'allowed' : 'all', @locale].compact.join(':')
end
def load
unless loaded?
@records = perform_queries
@loaded = true
end
self
end
def ids
redis.zrevrange(key, @offset, @limit.positive? ? @limit - 1 : @limit).map(&:to_i)
end
def perform_queries
apply_scopes(to_arel).to_a
end
def apply_scopes(scope)
scope
end
end

65
app/models/trends/status_batch.rb Normal file
View file

@ -0,0 +1,65 @@
# frozen_string_literal: true
class Trends::StatusBatch
include ActiveModel::Model
include Authorization
attr_accessor :status_ids, :action, :current_account
def save
case action
when 'approve'
approve!
when 'approve_accounts'
approve_accounts!
when 'reject'
reject!
when 'reject_accounts'
reject_accounts!
end
end
private
def statuses
@statuses ||= Status.where(id: status_ids)
end
def status_accounts
@status_accounts ||= Account.where(id: statuses.map(&:account_id).uniq)
end
def approve!
statuses.each { |status| authorize(status, :review?) }
statuses.update_all(trendable: true)
end
def approve_accounts!
status_accounts.each do |account|
authorize(account, :review?)
account.update(trendable: true, reviewed_at: action_time)
end
# Reset any individual overrides
statuses.update_all(trendable: nil)
end
def reject!
statuses.each { |status| authorize(status, :review?) }
statuses.update_all(trendable: false)
end
def reject_accounts!
status_accounts.each do |account|
authorize(account, :review?)
account.update(trendable: false, reviewed_at: action_time)
end
# Reset any individual overrides
statuses.update_all(trendable: nil)
end
def action_time
@action_time ||= Time.now.utc
end
end

46
app/models/trends/status_filter.rb Normal file
View file

@ -0,0 +1,46 @@
# frozen_string_literal: true
class Trends::StatusFilter
KEYS = %i(
trending
locale
).freeze
attr_reader :params
def initialize(params)
@params = params
end
def results
scope = Status.unscoped.kept
params.each do |key, value|
next if %w(page locale).include?(key.to_s)
scope.merge!(scope_for(key, value.to_s.strip)) if value.present?
end
scope
end
private
def scope_for(key, value)
case key.to_s
when 'trending'
trending_scope(value)
else
raise "Unknown filter: #{key}"
end
end
def trending_scope(value)
scope = Trends.statuses.query
scope = scope.in_locale(@params[:locale].to_s) if @params[:locale].present?
scope = scope.allowed if value == 'allowed'
scope.to_arel
end
end

142
app/models/trends/statuses.rb Normal file
View file

@ -0,0 +1,142 @@
# frozen_string_literal: true
class Trends::Statuses < Trends::Base
PREFIX = 'trending_statuses'
self.default_options = {
threshold: 5,
review_threshold: 3,
score_halflife: 2.hours.freeze,
}
class Query < Trends::Query
def filtered_for!(account)
@account = account
self
end
def filtered_for(account)
clone.filtered_for!(account)
end
private
def apply_scopes(scope)
scope.includes(:account)
end
def perform_queries
return super if @account.nil?
statuses = super
account_ids = statuses.map(&:account_id)
account_domains = statuses.map(&:account_domain)
preloaded_relations = {
blocking: Account.blocking_map(account_ids, @account.id),
blocked_by: Account.blocked_by_map(account_ids, @account.id),
muting: Account.muting_map(account_ids, @account.id),
following: Account.following_map(account_ids, @account.id),
domain_blocking_by_domain: Account.domain_blocking_map_by_domain(account_domains, @account.id),
}
statuses.reject { |status| StatusFilter.new(status, @account, preloaded_relations).filtered? }
end
end
def register(status, at_time = Time.now.utc)
add(status.proper, status.account_id, at_time) if eligible?(status)
end
def add(status, _account_id, at_time = Time.now.utc)
# We rely on the total reblogs and favourites count, so we
# don't record which account did the what and when here
record_used_id(status.id, at_time)
end
def query
Query.new(key_prefix, klass)
end
def refresh(at_time = Time.now.utc)
statuses = Status.where(id: (recently_used_ids(at_time) + currently_trending_ids(false, -1)).uniq).includes(:account, :media_attachments)
calculate_scores(statuses, at_time)
trim_older_items
end
def request_review
statuses = Status.where(id: currently_trending_ids(false, -1)).includes(:account)
statuses.filter_map do |status|
next unless would_be_trending?(status.id) && !status.trendable? && status.requires_review_notification?
status.account.touch(:requested_review_at)
status
end
end
protected
def key_prefix
PREFIX
end
def klass
Status
end
private
def eligible?(status)
original_status = status.proper
original_status.public_visibility? &&
original_status.account.discoverable? && !original_status.account.silenced? &&
original_status.spoiler_text.blank? && !original_status.sensitive? && !original_status.reply?
end
def calculate_scores(statuses, at_time)
redis.pipelined do
statuses.each do |status|
expected = 1.0
observed = (status.reblogs_count + status.favourites_count).to_f
score = begin
if expected > observed || observed < options[:threshold]
0
else
((observed - expected)**2) / expected
end
end
decaying_score = score * (0.5**((at_time.to_f - status.created_at.to_f) / options[:score_halflife].to_f))
add_to_and_remove_from_subsets(status.id, decaying_score, {
all: true,
allowed: status.trendable? && status.account.discoverable?,
})
next unless valid_locale?(status.language)
add_to_and_remove_from_subsets(status.id, decaying_score, {
"all:#{status.language}" => true,
"allowed:#{status.language}" => status.trendable? && status.account.discoverable?,
})
end
# Clean up localized sets by calculating the intersection with the main
# set. We do this instead of just deleting the localized sets to avoid
# having moments where the API returns empty results
Trends.available_locales.each do |locale|
redis.zinterstore("#{key_prefix}:all:#{locale}", ["#{key_prefix}:all:#{locale}", "#{key_prefix}:all"], aggregate: 'max')
redis.zinterstore("#{key_prefix}:allowed:#{locale}", ["#{key_prefix}:allowed:#{locale}", "#{key_prefix}:all"], aggregate: 'max')
end
end
end
def would_be_trending?(id)
score(id) > score_at_rank(options[:review_threshold] - 1)
end
end

6
app/models/form/tag_batch.rb → app/models/trends/tag_batch.rb
View file

@ -1,6 +1,6 @@
# frozen_string_literal: true
class Form::TagBatch
class Trends::TagBatch
include ActiveModel::Model
include Authorization
@ -22,12 +22,12 @@ class Form::TagBatch
end
def approve!
tags.each { |tag| authorize(tag, :update?) }
tags.each { |tag| authorize(tag, :review?) }
tags.update_all(trendable: true, reviewed_at: action_time)
end
def reject!
tags.each { |tag| authorize(tag, :update?) }
tags.each { |tag| authorize(tag, :review?) }
tags.update_all(trendable: false, reviewed_at: action_time)
end

10
app/models/tag_filter.rb → app/models/trends/tag_filter.rb
View file

@ -1,6 +1,6 @@
# frozen_string_literal: true
class TagFilter
class Trends::TagFilter
KEYS = %i(
trending
status
@ -42,13 +42,7 @@ class TagFilter
end
def trending_scope
ids = Trends.tags.currently_trending_ids(false, -1)
if ids.empty?
Tag.none
else
Tag.joins("join unnest(array[#{ids.map(&:to_i).join(',')}]::integer[]) with ordinality as x (id, ordering) on tags.id = x.id").order('x.ordering')
end
Trends.tags.query.to_arel
end
def status_scope(value)

36
app/models/trends/tags.rb
View file

@ -5,7 +5,7 @@ class Trends::Tags < Trends::Base
self.default_options = {
threshold: 5,
review_threshold: 10,
review_threshold: 3,
max_score_cooldown: 2.days.freeze,
max_score_halflife: 4.hours.freeze,
}
@ -29,27 +29,15 @@ class Trends::Tags < Trends::Base
trim_older_items
end
def get(allowed, limit)
tag_ids = currently_trending_ids(allowed, limit)
tags = Tag.where(id: tag_ids).index_by(&:id)
tag_ids.map { |id| tags[id] }.compact
end
def request_review
tags = Tag.where(id: currently_trending_ids(false, -1))
tags_requiring_review = tags.filter_map do |tag|
tags.filter_map do |tag|
next unless would_be_trending?(tag.id) && !tag.trendable? && tag.requires_review_notification?
tag.touch(:requested_review_at)
tag
end
return if tags_requiring_review.empty?
User.staff.includes(:account).find_each do |user|
AdminMailer.new_trending_tags(user.account, tags_requiring_review).deliver_later! if user.allows_trending_tag_emails?
end
end
protected
@ -58,6 +46,10 @@ class Trends::Tags < Trends::Base
PREFIX
end
def klass
Tag
end
private
def calculate_scores(tags, at_time)
@ -87,18 +79,10 @@ class Trends::Tags < Trends::Base
decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / options[:max_score_halflife].to_f))
if decaying_score.zero?
redis.zrem("#{PREFIX}:all", tag.id)
redis.zrem("#{PREFIX}:allowed", tag.id)
else
redis.zadd("#{PREFIX}:all", decaying_score, tag.id)
if tag.trendable?
redis.zadd("#{PREFIX}:allowed", decaying_score, tag.id)
else
redis.zrem("#{PREFIX}:allowed", tag.id)
end
end
add_to_and_remove_from_subsets(tag.id, decaying_score, {
all: true,
allowed: tag.trendable?,
})
end
end

2
app/models/user.rb
View file

@ -269,7 +269,7 @@ class User < ApplicationRecord
settings.notification_emails['appeal']
end
def allows_trending_tag_emails?
def allows_trends_review_emails?
settings.notification_emails['trending_tag']
end

4
app/policies/account_policy.rb
View file

@ -68,4 +68,8 @@ class AccountPolicy < ApplicationPolicy
def unblock_email?
staff?
end
def review?
staff?
end
end

2
app/policies/preview_card_policy.rb
View file

@ -5,7 +5,7 @@ class PreviewCardPolicy < ApplicationPolicy
staff?
end
def update?
def review?
staff?
end
end

2
app/policies/preview_card_provider_policy.rb
View file

@ -5,7 +5,7 @@ class PreviewCardProviderPolicy < ApplicationPolicy
staff?
end
def update?
def review?
staff?
end
end

4
app/policies/status_policy.rb
View file

@ -41,6 +41,10 @@ class StatusPolicy < ApplicationPolicy
staff? || owned?
end
def review?
staff?
end
private
def requires_mention?

4
app/policies/tag_policy.rb
View file

@ -12,4 +12,8 @@ class TagPolicy < ApplicationPolicy
def update?
staff?
end
def review?
staff?
end
end

32
app/services/delete_account_service.rb
View file

@ -220,21 +220,23 @@ class DeleteAccountService < BaseService
return unless keep_account_record?
@account.silenced_at = nil
@account.suspended_at = @options[:suspended_at] || Time.now.utc
@account.suspension_origin = :local
@account.locked = false
@account.memorial = false
@account.discoverable = false
@account.display_name = ''
@account.note = ''
@account.fields = []
@account.statuses_count = 0
@account.followers_count = 0
@account.following_count = 0
@account.moved_to_account = nil
@account.also_known_as = []
@account.trust_level = :untrusted
@account.silenced_at = nil
@account.suspended_at = @options[:suspended_at] || Time.now.utc
@account.suspension_origin = :local
@account.locked = false
@account.memorial = false
@account.discoverable = false
@account.trendable = false
@account.display_name = ''
@account.note = ''
@account.fields = []
@account.statuses_count = 0
@account.followers_count = 0
@account.following_count = 0
@account.moved_to_account = nil
@account.reviewed_at = nil
@account.requested_review_at = nil
@account.also_known_as = []
@account.avatar.destroy
@account.header.destroy
@account.save!

2
app/services/favourite_service.rb
View file

@ -17,6 +17,8 @@ class FavouriteService < BaseService
favourite = Favourite.create!(account: account, status: status)
Trends.statuses.register(status)
create_notification(favourite)
bump_potential_friendship(account, status)

3
app/services/reblog_service.rb
View file

@ -30,8 +30,7 @@ class ReblogService < BaseService
reblog = account.statuses.create!(reblog: reblogged_status, text: '', visibility: visibility, rate_limit: options[:with_rate_limit])
Trends.tags.register(reblog)
Trends.links.register(reblog)
Trends.register!(reblog)
DistributionWorker.perform_async(reblog.id)
ActivityPub::DistributionWorker.perform_async(reblog.id)

2
app/views/admin/custom_emojis/_custom_emoji.html.haml
View file

@ -3,7 +3,7 @@
= f.check_box :custom_emoji_ids, { multiple: true, include_hidden: false }, custom_emoji.id
.batch-table__row__content.batch-table__row__content--with-image
.batch-table__row__content__image
= custom_emoji_tag(custom_emoji, animate = current_account&.user&.setting_auto_play_gif)
= custom_emoji_tag(custom_emoji, current_account&.user&.setting_auto_play_gif)
.batch-table__row__content__text
%samp= ":#{custom_emoji.shortcode}:"

6
app/views/admin/follow_recommendations/show.html.haml
View file

@ -9,12 +9,14 @@
%hr.spacer/
= form_tag admin_follow_recommendations_path, method: 'GET', class: 'simple_form' do
- RelationshipFilter::KEYS.each do |key|
= hidden_field_tag key, params[key] if params[key].present?
.filters
.filter-subset.filter-subset--with-select
%strong= t('admin.follow_recommendations.language')
.input.select.optional
= select_tag :language, options_for_select(I18n.available_locales.map { |key| key.to_s.split(/[_-]/).first.to_sym }.uniq.map { |key| [standard_locale_name(key), key]}, @language)
= select_tag :language, options_for_select(Trends.available_locales.map { |key| [standard_locale_name(key), key] }, @language)
.filter-subset
%strong= t('admin.follow_recommendations.status')
%ul

34
app/views/admin/trends/links/index.html.haml
View file

@ -4,23 +4,29 @@
- content_for :header_tags do
= javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
.filters
.filter-subset
%strong= t('admin.trends.trending')
%ul
%li= filter_link_to t('generic.all'), trending: nil
%li= filter_link_to t('admin.trends.only_allowed'), trending: 'allowed'
.back-link
= link_to admin_trends_links_preview_card_providers_path do
= t('admin.trends.preview_card_providers.title')
= fa_icon 'chevron-right fw'
= form_tag admin_trends_links_path, method: 'GET', class: 'simple_form' do
- Trends::PreviewCardFilter::KEYS.each do |key|
= hidden_field_tag key, params[key] if params[key].present?
%hr.spacer/
.filters
.filter-subset.filter-subset--with-select
%strong= t('admin.follow_recommendations.language')
.input.select.optional
= select_tag :locale, options_for_select(Trends.available_locales.map { |key| [standard_locale_name(key), key] }, params[:locale]), include_blank: true
.filter-subset
%strong= t('admin.trends.trending')
%ul
%li= filter_link_to t('generic.all'), trending: nil
%li= filter_link_to t('admin.trends.only_allowed'), trending: 'allowed'
.back-link
= link_to admin_trends_links_preview_card_providers_path do
= t('admin.trends.preview_card_providers.title')
= fa_icon 'chevron-right fw'
= form_for(@form, url: batch_admin_trends_links_path) do |f|
= hidden_field_tag :page, params[:page] || 1
- PreviewCardFilter::KEYS.each do |key|
- Trends::PreviewCardFilter::KEYS.each do |key|
= hidden_field_tag key, params[key] if params[key].present?
.batch-table
@ -29,9 +35,9 @@
= check_box_tag :batch_checkbox_all, nil, false
.batch-table__toolbar__actions
= f.button safe_join([fa_icon('check'), t('admin.trends.links.allow')]), name: :approve, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
= f.button safe_join([fa_icon('check'), t('admin.trends.links.allow_provider')]), name: :approve_all, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
= f.button safe_join([fa_icon('check'), t('admin.trends.links.allow_provider')]), name: :approve_providers, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
= f.button safe_join([fa_icon('times'), t('admin.trends.links.disallow')]), name: :reject, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
= f.button safe_join([fa_icon('times'), t('admin.trends.links.disallow_provider')]), name: :reject_all, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
= f.button safe_join([fa_icon('times'), t('admin.trends.links.disallow_provider')]), name: :reject_providers, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
.batch-table__body
- if @preview_cards.empty?
= nothing_here 'nothing-here--under-tabs'

2
app/views/admin/trends/links/preview_card_providers/index.html.haml
View file

@ -23,7 +23,7 @@
= form_for(@form, url: batch_admin_trends_links_preview_card_providers_path) do |f|
= hidden_field_tag :page, params[:page] || 1
- PreviewCardProviderFilter::KEYS.each do |key|
- Trends::PreviewCardProviderFilter::KEYS.each do |key|
= hidden_field_tag key, params[key] if params[key].present?
.batch-table.optional

30
app/views/admin/trends/statuses/_status.html.haml Normal file
View file

@ -0,0 +1,30 @@
.batch-table__row{ class: [status.account.requires_review? && 'batch-table__row--attention', !status.account.requires_review? && !status.trendable? && 'batch-table__row--muted'] }
%label.batch-table__row__select.batch-table__row__select--aligned.batch-checkbox
= f.check_box :status_ids, { multiple: true, include_hidden: false }, status.id
.batch-table__row__content.pending-account__header
.one-liner
= admin_account_link_to status.account
= link_to ActivityPub::TagManager.instance.url_for(status), target: '_blank', class: 'emojify', rel: 'noopener noreferrer' do
= one_line_preview(status)
- status.media_attachments.each do |media_attachment|
%abbr{ title: media_attachment.description }
= fa_icon 'link'
= media_attachment.file_file_name
= t('admin.trends.statuses.shared_by', count: status.reblogs_count + status.favourites_count, friendly_count: friendly_number_to_human(status.reblogs_count + status.favourites_count))
- if status.account.domain.present?
•
= status.account.domain
- if status.language.present?
•
= standard_locale_name(status.language)
- if status.trendable? && (rank = Trends.statuses.rank(status.id))
•
%abbr{ title: t('admin.trends.tags.current_score', score: Trends.statuses.score(status.id)) }= t('admin.trends.tags.trending_rank', rank: rank + 1)
- elsif status.account.requires_review?
•
= t('admin.trends.pending_review')

43
app/views/admin/trends/statuses/index.html.haml Normal file
View file

@ -0,0 +1,43 @@
- content_for :page_title do
= t('admin.trends.statuses.title')
- content_for :header_tags do
= javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
= form_tag admin_trends_statuses_path, method: 'GET', class: 'simple_form' do
- Trends::StatusFilter::KEYS.each do |key|
= hidden_field_tag key, params[key] if params[key].present?
.filters
.filter-subset.filter-subset--with-select
%strong= t('admin.follow_recommendations.language')
.input.select.optional
= select_tag :locale, options_for_select(Trends.available_locales.map { |key| [standard_locale_name(key), key]}, params[:locale]), include_blank: true
.filter-subset
%strong= t('admin.trends.trending')
%ul
%li= filter_link_to t('generic.all'), trending: nil
%li= filter_link_to t('admin.trends.only_allowed'), trending: 'allowed'
= form_for(@form, url: batch_admin_trends_statuses_path) do |f|
= hidden_field_tag :page, params[:page] || 1
- Trends::StatusFilter::KEYS.each do |key|
= hidden_field_tag key, params[key] if params[key].present?
.batch-table
.batch-table__toolbar
%label.batch-table__toolbar__select.batch-checkbox-all
= check_box_tag :batch_checkbox_all, nil, false
.batch-table__toolbar__actions
= f.button safe_join([fa_icon('check'), t('admin.trends.statuses.allow')]), name: :approve, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
= f.button safe_join([fa_icon('check'), t('admin.trends.statuses.allow_account')]), name: :approve_accounts, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
= f.button safe_join([fa_icon('times'), t('admin.trends.statuses.disallow')]), name: :reject, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
= f.button safe_join([fa_icon('times'), t('admin.trends.statuses.disallow_account')]), name: :reject_accounts, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
.batch-table__body
- if @statuses.empty?
= nothing_here 'nothing-here--under-tabs'
- else
= render partial: 'status', collection: @statuses, locals: { f: f }
= paginate @statuses

4
app/views/admin/trends/tags/index.html.haml
View file

@ -13,12 +13,10 @@
%li= filter_link_to t('admin.trends.rejected'), status: 'rejected'
%li= filter_link_to safe_join([t('admin.accounts.moderation.pending'), "(#{Tag.pending_review.count})"], ' '), status: 'pending_review'
%hr.spacer/
= form_for(@form, url: batch_admin_trends_tags_path) do |f|
= hidden_field_tag :page, params[:page] || 1
- TagFilter::KEYS.each do |key|
- Trends::TagFilter::KEYS.each do |key|
= hidden_field_tag key, params[key] if params[key].present?
.batch-table.optional

14
app/views/admin_mailer/_new_trending_links.text.erb Normal file
View file

@ -0,0 +1,14 @@
<%= raw t('admin_mailer.new_trends.new_trending_links.title') %>
<% @links.each do |link| %>
- <%= link.title %> • <%= link.url %>
<%= raw t('admin.trends.links.usage_comparison', today: link.history.get(Time.now.utc).accounts, yesterday: link.history.get(Time.now.utc - 1.day).accounts) %> • <%= t('admin.trends.tags.current_score', score: Trends.links.score(link.id).round(2)) %>
<% end %>
<% if @lowest_trending_link %>
<%= raw t('admin_mailer.new_trends.new_trending_links.requirements', lowest_link_title: @lowest_trending_link.title, lowest_link_score: Trends.links.score(@lowest_trending_link.id).round(2), rank: Trends.links.options[:review_threshold]) %>
<% else %>
<%= raw t('admin_mailer.new_trends.new_trending_links.no_approved_links') %>
<% end %>
<%= raw t('application_mailer.view')%> <%= admin_trends_links_url %>

14
app/views/admin_mailer/_new_trending_statuses.text.erb Normal file
View file

@ -0,0 +1,14 @@
<%= raw t('admin_mailer.new_trends.new_trending_statuses.title') %>
<% @statuses.each do |status| %>
- <%= ActivityPub::TagManager.instance.url_for(status) %>
<%= raw t('admin.trends.tags.current_score', score: Trends.statuses.score(status.id).round(2)) %>
<% end %>
<% if @lowest_trending_status %>
<%= raw t('admin_mailer.new_trends.new_trending_statuses.requirements', lowest_status_url: ActivityPub::TagManager.instance.url_for(@lowest_trending_status), lowest_status_score: Trends.statuses.score(@lowest_trending_status.id).round(2), rank: Trends.statuses.options[:review_threshold]) %>
<% else %>
<%= raw t('admin_mailer.new_trends.new_trending_statuses.no_approved_statuses') %>
<% end %>
<%= raw t('application_mailer.view')%> <%= admin_trends_statuses_url %>

14
app/views/admin_mailer/_new_trending_tags.text.erb Normal file
View file

@ -0,0 +1,14 @@
<%= raw t('admin_mailer.new_trends.new_trending_tags.title') %>
<% @tags.each do |tag| %>
- #<%= tag.name %>
<%= raw t('admin.trends.tags.usage_comparison', today: tag.history.get(Time.now.utc).accounts, yesterday: tag.history.get(Time.now.utc - 1.day).accounts) %> • <%= t('admin.trends.tags.current_score', score: Trends.tags.score(tag.id).round(2)) %>
<% end %>
<% if @lowest_trending_tag %>
<%= raw t('admin_mailer.new_trends.new_trending_tags.requirements', lowest_tag_name: @lowest_trending_tag.name, lowest_tag_score: Trends.tags.score(@lowest_trending_tag.id).round(2), rank: Trends.tags.options[:review_threshold]) %>
<% else %>
<%= raw t('admin_mailer.new_trends.new_trending_tags.no_approved_tags') %>
<% end %>
<%= raw t('application_mailer.view')%> <%= admin_trends_tags_url(pending_review: '1') %>

16
app/views/admin_mailer/new_trending_links.text.erb
View file

@ -1,16 +0,0 @@
<%= raw t('application_mailer.salutation', name: display_name(@me)) %>
<%= raw t('admin_mailer.new_trending_links.body') %>
<% @links.each do |link| %>
- <%= link.title %> • <%= link.url %>
<%= t('admin.trends.links.usage_comparison', today: link.history.get(Time.now.utc).accounts, yesterday: link.history.get(Time.now.utc - 1.day).accounts) %> • <%= t('admin.trends.tags.current_score', score: Trends.links.score(link.id).round(2)) %>
<% end %>
<% if @lowest_trending_link %>
<%= t('admin_mailer.new_trending_links.requirements', lowest_link_title: @lowest_trending_link.title, lowest_link_score: Trends.links.score(@lowest_trending_link.id).round(2)) %>
<% else %>
<%= t('admin_mailer.new_trending_links.no_approved_links') %>
<% end %>
<%= raw t('application_mailer.view')%> <%= admin_trends_links_url %>

16
app/views/admin_mailer/new_trending_tags.text.erb
View file

@ -1,16 +0,0 @@
<%= raw t('application_mailer.salutation', name: display_name(@me)) %>
<%= raw t('admin_mailer.new_trending_tags.body') %>
<% @tags.each do |tag| %>
- #<%= tag.name %>
<%= t('admin.trends.tags.usage_comparison', today: tag.history.get(Time.now.utc).accounts, yesterday: tag.history.get(Time.now.utc - 1.day).accounts) %> • <%= t('admin.trends.tags.current_score', score: Trends.tags.score(tag.id).round(2)) %>
<% end %>
<% if @lowest_trending_tag %>
<%= t('admin_mailer.new_trending_tags.requirements', lowest_tag_name: @lowest_trending_tag.name, lowest_tag_score: Trends.tags.score(@lowest_trending_tag.id).round(2)) %>
<% else %>
<%= t('admin_mailer.new_trending_tags.no_approved_tags') %>
<% end %>
<%= raw t('application_mailer.view')%> <%= admin_trends_tags_url(status: 'pending_review') %>

13
app/views/admin_mailer/new_trends.text.erb Normal file
View file

@ -0,0 +1,13 @@
<%= raw t('application_mailer.salutation', name: display_name(@me)) %>
<%= raw t('admin_mailer.new_trends.body') %>
<% unless @links.empty? %>
<%= render 'new_trending_links' %>
<% end %>
<% unless @tags.empty? %>
<%= render 'new_trending_tags' unless @tags.empty? %>
<% end %>
<% unless @statuses.empty? %>
<%= render 'new_trending_statuses' unless @statuses.empty? %>
<% end %>

2
app/views/application/_sidebar.html.haml
View file

@ -6,7 +6,7 @@
%p= @instance_presenter.site_short_description.html_safe.presence || t('about.about_mastodon_html')
- if Setting.trends && !(user_signed_in? && !current_user.setting_trends)
- trends = Trends.tags.get(true, 3)
- trends = Trends.tags.query.allowed.limit(3)
- unless trends.empty?
.endorsements-widget.trends-widget

8
app/workers/scheduler/follow_recommendations_scheduler.rb
View file

@ -18,7 +18,7 @@ class Scheduler::FollowRecommendationsScheduler
fallback_recommendations = FollowRecommendation.order(rank: :desc).limit(SET_SIZE)
I18n.available_locales.map { |locale| locale.to_s.split(/[_-]/).first }.uniq.each do |locale|
Trends.available_locales.each do |locale|
recommendations = begin
if AccountSummary.safe.filtered.localized(locale).exists? # We can skip the work if no accounts with that language exist
FollowRecommendation.localized(locale).order(rank: :desc).limit(SET_SIZE).map { |recommendation| [recommendation.account_id, recommendation.rank] }
@ -49,11 +49,11 @@ class Scheduler::FollowRecommendationsScheduler
end
end
redis.pipelined do
redis.del(key(locale))
redis.multi do |multi|
multi.del(key(locale))
recommendations.each do |(account_id, rank)|
redis.zadd(key(locale), rank, account_id)
multi.zadd(key(locale), rank, account_id)
end
end
end

68
config/brakeman.ignore
View file

@ -7,7 +7,7 @@
"check_name": "SQL",
"message": "Possible SQL injection",
"file": "app/models/status.rb",
"line": 104,
"line": 105,
"link": "https://brakemanscanner.org/docs/warning_types/sql_injection/",
"code": "result.joins(\"INNER JOIN statuses_tags t#{id} ON t#{id}.status_id = statuses.id AND t#{id}.tag_id = #{id}\")",
"render_path": null,
@ -20,6 +20,26 @@
"confidence": "Weak",
"note": ""
},
{
"warning_type": "SQL Injection",
"warning_code": 0,
"fingerprint": "30dfe36e87fe1b8f239df9a33d576e44a9863f73b680198d4713be6540ae61d3",
"check_name": "SQL",
"message": "Possible SQL injection",
"file": "app/models/trends/query.rb",
"line": 60,
"link": "https://brakemanscanner.org/docs/warning_types/sql_injection/",
"code": "klass.joins(\"join unnest(array[#{ids.join(\",\")}]) with ordinality as x (id, ordering) on #{klass.table_name}.id = x.id\")",
"render_path": null,
"location": {
"type": "method",
"class": "Trends::Query",
"method": "to_arel"
},
"user_input": "ids.join(\",\")",
"confidence": "Weak",
"note": ""
},
{
"warning_type": "Redirect",
"warning_code": 18,
@ -100,26 +120,6 @@
"confidence": "High",
"note": ""
},
{
"warning_type": "SQL Injection",
"warning_code": 0,
"fingerprint": "8c1d8c4b76c1cd3960e90dff999f854a6ff742fcfd8de6c7184ac5a1b1a4d7dd",
"check_name": "SQL",
"message": "Possible SQL injection",
"file": "app/models/preview_card_filter.rb",
"line": 50,
"link": "https://brakemanscanner.org/docs/warning_types/sql_injection/",
"code": "PreviewCard.joins(\"join unnest(array[#{(Trends.links.currently_trending_ids(true, -1) or Trends.links.currently_trending_ids(false, -1)).map(&:to_i).join(\",\")}]::integer[]) with ordinality as x (id, ordering) on preview_cards.id = x.id\")",
"render_path": null,
"location": {
"type": "method",
"class": "PreviewCardFilter",
"method": "trending_scope"
},
"user_input": "(Trends.links.currently_trending_ids(true, -1) or Trends.links.currently_trending_ids(false, -1)).map(&:to_i).join(\",\")",
"confidence": "Medium",
"note": ""
},
{
"warning_type": "Cross-Site Scripting",
"warning_code": 2,
@ -134,7 +134,7 @@
{
"type": "template",
"name": "admin/disputes/appeals/index",
"line": 16,
"line": 20,
"file": "app/views/admin/disputes/appeals/index.html.haml",
"rendered": {
"name": "admin/disputes/appeals/_appeal",
@ -170,26 +170,6 @@
"confidence": "High",
"note": ""
},
{
"warning_type": "SQL Injection",
"warning_code": 0,
"fingerprint": "c32a484ccd9da46abd3bc93d08b72029d7dbc0576ccf4e878a9627e9a83cad2e",
"check_name": "SQL",
"message": "Possible SQL injection",
"file": "app/models/tag_filter.rb",
"line": 50,
"link": "https://brakemanscanner.org/docs/warning_types/sql_injection/",
"code": "Tag.joins(\"join unnest(array[#{Trends.tags.currently_trending_ids(false, -1).map(&:to_i).join(\",\")}]::integer[]) with ordinality as x (id, ordering) on tags.id = x.id\")",
"render_path": null,
"location": {
"type": "method",
"class": "TagFilter",
"method": "trending_scope"
},
"user_input": "Trends.tags.currently_trending_ids(false, -1).map(&:to_i).join(\",\")",
"confidence": "Medium",
"note": ""
},
{
"warning_type": "Cross-Site Scripting",
"warning_code": 4,
@ -204,7 +184,7 @@
{
"type": "template",
"name": "admin/trends/links/index",
"line": 39,
"line": 45,
"file": "app/views/admin/trends/links/index.html.haml",
"rendered": {
"name": "admin/trends/links/_preview_card",
@ -241,6 +221,6 @@
"note": ""
}
],
"updated": "2022-02-13 02:24:12 +0100",
"updated": "2022-02-15 03:48:53 +0100",
"brakeman_version": "5.2.1"
}

34
config/locales/en.yml
View file

@ -787,6 +787,15 @@ en:
rejected: Links from this publisher won't trend
title: Publishers
rejected: Rejected
statuses:
allow: Allow post
allow_account: Allow author
disallow: Disallow post
disallow_account: Disallow author
shared_by:
one: Shared or favourited one time
other: Shared and favourited %{friendly_count} times
title: Trending posts
tags:
current_score: Current score %{score}
dashboard:
@ -835,16 +844,21 @@ en:
body: "%{reporter} has reported %{target}"
body_remote: Someone from %{domain} has reported %{target}
subject: New report for %{instance} (#%{id})
new_trending_links:
body: The following links are trending today, but their publishers have not been previously reviewed. They will not be displayed publicly unless you approve them. Further notifications from the same publishers will not be generated.
no_approved_links: There are currently no approved trending links.
requirements: The lowest approved trending link is currently "%{lowest_link_title}" with a score of %{lowest_link_score}.
subject: New trending links up for review on %{instance}
new_trending_tags:
body: 'The following hashtags are trending today, but they have not been previously reviewed. They will not be displayed publicly unless you approve them:'
no_approved_tags: There are currently no approved trending hashtags.
requirements: 'The lowest approved trending hashtag is currently #%{lowest_tag_name} with a score of %{lowest_tag_score}.'
subject: New trending hashtags up for review on %{instance}
new_trends:
body: 'The following items need a review before they can be displayed publicly:'
new_trending_links:
no_approved_links: There are currently no approved trending links.
requirements: 'Any of these candidates could surpass the #%{rank} approved trending link, which is currently "%{lowest_link_title}" with a score of %{lowest_link_score}.'
title: Trending links
new_trending_statuses:
no_approved_statuses: There are currently no approved trending posts.
requirements: 'Any of these candidates could surpass the #%{rank} approved trending post, which is currently %{lowest_status_url} with a score of %{lowest_status_score}.'
title: Trending posts
new_trending_tags:
no_approved_tags: There are currently no approved trending hashtags.
requirements: 'Any of these candidates could surpass the #%{rank} approved trending hashtag, which is currently #%{lowest_tag_name} with a score of %{lowest_tag_score}.'
title: Trending hashtags
subject: New trends up for review on %{instance}
aliases:
add_new: Create alias
created_msg: Successfully created a new alias. You can now initiate the move from the old account.

1
config/navigation.rb
View file

@ -34,6 +34,7 @@ SimpleNavigation::Configuration.run do |navigation|
n.item :development, safe_join([fa_icon('code fw'), t('settings.development')]), settings_applications_url, if: -> { current_user.functional? }
n.item :trends, safe_join([fa_icon('fire fw'), t('admin.trends.title')]), admin_trends_tags_path, if: proc { current_user.staff? } do |s|
s.item :statuses, safe_join([fa_icon('comments-o fw'), t('admin.trends.statuses.title')]), admin_trends_statuses_path, highlights_on: %r{/admin/trends/statuses}
s.item :tags, safe_join([fa_icon('hashtag fw'), t('admin.trends.tags.title')]), admin_trends_tags_path, highlights_on: %r{/admin/tags|/admin/trends/tags}
s.item :links, safe_join([fa_icon('newspaper-o fw'), t('admin.trends.links.title')]), admin_trends_links_path, highlights_on: %r{/admin/trends/links}
end

9
config/routes.rb
View file

@ -327,6 +327,12 @@ Rails.application.routes.draw do
end
end
resources :statuses, only: [:index] do
collection do
post :batch
end
end
namespace :links do
resources :preview_card_providers, only: [:index], path: :publishers do
collection do
@ -448,6 +454,7 @@ Rails.application.routes.draw do
namespace :trends do
resources :links, only: [:index]
resources :tags, only: [:index]
resources :statuses, only: [:index]
end
namespace :emails do
@ -554,6 +561,8 @@ Rails.application.routes.draw do
namespace :trends do
resources :tags, only: [:index]
resources :links, only: [:index]
resources :statuses, only: [:index]
end
post :measures, to: 'measures#create'

7
db/migrate/20220202200743_add_trendable_to_accounts.rb Normal file
View file

@ -0,0 +1,7 @@
class AddTrendableToAccounts < ActiveRecord::Migration[6.1]
def change
add_column :accounts, :trendable, :boolean
add_column :accounts, :reviewed_at, :datetime
add_column :accounts, :requested_review_at, :datetime
end
end

5
db/migrate/20220202200926_add_trendable_to_statuses.rb Normal file
View file

@ -0,0 +1,5 @@
class AddTrendableToStatuses < ActiveRecord::Migration[6.1]
def change
add_column :statuses, :trendable, :boolean
end
end

9
db/post_migrate/20220202201015_remove_trust_level_from_accounts.rb Normal file
View file

@ -0,0 +1,9 @@
# frozen_string_literal: true
class RemoveTrustLevelFromAccounts < ActiveRecord::Migration[5.2]
disable_ddl_transaction!
def change
safety_assured { remove_column :accounts, :trust_level, :integer }
end
end

6
db/schema.rb
View file

@ -177,13 +177,15 @@ ActiveRecord::Schema.define(version: 2022_02_24_010024) do
t.string "also_known_as", array: true
t.datetime "silenced_at"
t.datetime "suspended_at"
t.integer "trust_level"
t.boolean "hide_collections"
t.integer "avatar_storage_schema_version"
t.integer "header_storage_schema_version"
t.string "devices_url"
t.integer "suspension_origin"
t.datetime "sensitized_at"
t.boolean "trendable"
t.datetime "reviewed_at"
t.datetime "requested_review_at"
t.index "(((setweight(to_tsvector('simple'::regconfig, (display_name)::text), 'A'::\"char\") || setweight(to_tsvector('simple'::regconfig, (username)::text), 'B'::\"char\")) || setweight(to_tsvector('simple'::regconfig, (COALESCE(domain, ''::character varying))::text), 'C'::\"char\")))", name: "search_index", using: :gin
t.index "lower((username)::text), COALESCE(lower((domain)::text), ''::text)", name: "index_accounts_on_username_and_domain_lower", unique: true
t.index ["moved_to_account_id"], name: "index_accounts_on_moved_to_account_id"
@ -887,6 +889,7 @@ ActiveRecord::Schema.define(version: 2022_02_24_010024) do
t.bigint "poll_id"
t.datetime "deleted_at"
t.datetime "edited_at"
t.boolean "trendable"
t.index ["account_id", "id", "visibility", "updated_at"], name: "index_statuses_20190820", order: { id: :desc }, where: "(deleted_at IS NULL)"
t.index ["deleted_at"], name: "index_statuses_on_deleted_at", where: "(deleted_at IS NOT NULL)"
t.index ["id", "account_id"], name: "index_statuses_local_20190824", order: { id: :desc }, where: "((local OR (uri IS NULL)) AND (deleted_at IS NULL) AND (visibility = 0) AND (reblog_of_id IS NULL) AND ((NOT reply) OR (in_reply_to_account_id = account_id)))"
@ -1228,5 +1231,4 @@ ActiveRecord::Schema.define(version: 2022_02_24_010024) do
ORDER BY (sum(t0.rank)) DESC;
SQL
add_index "follow_recommendations", ["account_id"], name: "index_follow_recommendations_on_account_id", unique: true
end

7
spec/controllers/api/v1/trends/tags_controller_spec.rb
View file

@ -7,10 +7,9 @@ RSpec.describe Api::V1::Trends::TagsController, type: :controller do
describe 'GET #index' do
before do
trending_tags = double()
allow(trending_tags).to receive(:get).and_return(Fabricate.times(10, :tag))
allow(Trends).to receive(:tags).and_return(trending_tags)
Fabricate.times(10, :tag).each do |tag|
10.times { |i| Trends.tags.add(tag, i) }
end
get :index
end

11
spec/mailers/previews/admin_mailer_preview.rb
View file

@ -6,14 +6,9 @@ class AdminMailerPreview < ActionMailer::Preview
AdminMailer.new_pending_account(Account.first, User.pending.first)
end
# Preview this email at http://localhost:3000/rails/mailers/admin_mailer/new_trending_tags
def new_trending_tags
AdminMailer.new_trending_tags(Account.first, Tag.limit(3))
end
# Preview this email at http://localhost:3000/rails/mailers/admin_mailer/new_trending_links
def new_trending_links
AdminMailer.new_trending_links(Account.first, PreviewCard.limit(3))
# Preview this email at http://localhost:3000/rails/mailers/admin_mailer/new_trends
def new_trends
AdminMailer.new_trends(Account.first, PreviewCard.limit(3), Tag.limit(3), Status.where(reblog_of_id: nil).limit(3))
end
# Preview this email at http://localhost:3000/rails/mailers/admin_mailer/new_appeal

110
spec/models/trends/statuses_spec.rb Normal file
View file

@ -0,0 +1,110 @@
require 'rails_helper'
RSpec.describe Trends::Statuses do
subject! { described_class.new(threshold: 5, review_threshold: 10, score_halflife: 8.hours) }
let!(:at_time) { DateTime.new(2021, 11, 14, 10, 15, 0) }
describe 'Trends::Statuses::Query' do
let!(:query) { subject.query }
let!(:today) { at_time }
let!(:status1) { Fabricate(:status, text: 'Foo', trendable: true, created_at: today) }
let!(:status2) { Fabricate(:status, text: 'Bar', trendable: true, created_at: today) }
before do
15.times { reblog(status1, today) }
12.times { reblog(status2, today) }
subject.refresh(today)
end
describe '#filtered_for' do
let(:account) { Fabricate(:account) }
it 'returns a composable query scope' do
expect(query.filtered_for(account)).to be_a Trends::Query
end
it 'filters out blocked accounts' do
account.block!(status1.account)
expect(query.filtered_for(account).to_a).to eq [status2]
end
it 'filters out muted accounts' do
account.mute!(status2.account)
expect(query.filtered_for(account).to_a).to eq [status1]
end
it 'filters out blocked-by accounts' do
status1.account.block!(account)
expect(query.filtered_for(account).to_a).to eq [status2]
end
end
end
describe '#add' do
let(:status) { Fabricate(:status) }
before do
subject.add(status, 1, at_time)
end
it 'records use' do
expect(subject.send(:recently_used_ids, at_time)).to eq [status.id]
end
end
describe '#query' do
it 'returns a composable query scope' do
expect(subject.query).to be_a Trends::Query
end
it 'responds to filtered_for' do
expect(subject.query).to respond_to(:filtered_for)
end
end
describe '#refresh' do
let!(:today) { at_time }
let!(:yesterday) { today - 1.day }
let!(:status1) { Fabricate(:status, text: 'Foo', trendable: true, created_at: yesterday) }
let!(:status2) { Fabricate(:status, text: 'Bar', trendable: true, created_at: today) }
let!(:status3) { Fabricate(:status, text: 'Baz', trendable: true, created_at: today) }
before do
13.times { reblog(status1, today) }
13.times { reblog(status2, today) }
4.times { reblog(status3, today) }
end
context do
before do
subject.refresh(today)
end
it 'calculates and re-calculates scores' do
expect(subject.query.limit(10).to_a).to eq [status2, status1]
end
it 'omits statuses below threshold' do
expect(subject.query.limit(10).to_a).to_not include(status3)
end
end
it 'decays scores' do
subject.refresh(today)
original_score = subject.score(status2.id)
expect(original_score).to be_a Float
subject.refresh(today + subject.options[:score_halflife])
decayed_score = subject.score(status2.id)
expect(decayed_score).to be <= original_score / 2
end
end
def reblog(status, at_time)
reblog = Fabricate(:status, reblog: status, created_at: at_time)
subject.add(status, reblog.account_id, at_time)
end
end

6
spec/models/trends/tags_spec.rb
View file

@ -21,7 +21,7 @@ RSpec.describe Trends::Tags do
end
end
describe '#get' do
describe '#query' do
pending
end
@ -47,11 +47,11 @@ RSpec.describe Trends::Tags do
end
it 'calculates and re-calculates scores' do
expect(subject.get(false, 10)).to eq [tag1, tag3]
expect(subject.query.limit(10).to_a).to eq [tag1, tag3]
end
it 'omits hashtags below threshold' do
expect(subject.get(false, 10)).to_not include(tag2)
expect(subject.query.limit(10).to_a).to_not include(tag2)
end
end