Merge tag 'v3.4.5' into bida

This commit is contained in:
jops 2022-05-11 17:42:06 +02:00
commit d6d422642a
19 changed files with 378 additions and 50 deletions

View file

@ -167,8 +167,45 @@ jobs:
name: Create database
command: ./bin/rails db:create
- run:
name: Run migrations
command: ./bin/rails db:migrate VERSION=20171010025614
name: Run migrations up to v2.0.0
- run:
command: ./bin/rails tests:migrations:populate_v2
name: Populate database with test data
- run:
command: ./bin/rails db:migrate
name: Run all remaining migrations
test-two-step-migrations:
<<: *defaults
docker:
- image: circleci/ruby:2.7-buster-node
environment: *ruby_environment
- image: circleci/postgres:12.2
environment:
POSTGRES_USER: root
POSTGRES_HOST_AUTH_METHOD: trust
- image: circleci/redis:5-alpine
steps:
- *attach_workspace
- *install_system_dependencies
- run:
command: ./bin/rails db:create
name: Create database
- run:
command: ./bin/rails db:migrate VERSION=20171010025614
name: Run migrations up to v2.0.0
- run:
command: ./bin/rails tests:migrations:populate_v2
name: Populate database with test data
- run:
command: ./bin/rails db:migrate
name: Run all pre-deployment migrations
evironment:
SKIP_POST_DEPLOYMENT_MIGRATIONS: true
- run:
command: ./bin/rails db:migrate
name: Run all post-deployment remaining migrations
test-ruby2.7:
<<: *defaults
@ -238,6 +275,9 @@ workflows:
- test-migrations:
requires:
- install-ruby2.7
- test-two-step-migrations:
requires:
- install-ruby2.7
- test-ruby2.7:
requires:
- install-ruby2.7

34
.github/workflows/build-image.yml vendored Normal file
View file

@ -0,0 +1,34 @@
name: Build container image
on:
workflow_dispatch:
push:
branches:
- "main"
tags:
- "*"
jobs:
build-image:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: docker/setup-buildx-action@v1
- uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- uses: docker/metadata-action@v3
id: meta
with:
images: tootsuite/mastodon
flavor: |
latest=auto
tags: |
type=edge,branch=main
type=semver,pattern={{ raw }}
- uses: docker/build-push-action@v2
with:
context: .
push: true
tags: ${{ steps.meta.outputs.tags }}
cache-from: type=registry,ref=tootsuite/mastodon:latest
cache-to: type=inline

View file

@ -3,6 +3,18 @@ Changelog
All notable changes to this project will be documented in this file.
## [3.4.5] - 2022-01-31
### Added
- Add more advanced migration tests ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17393))
- Add github workflow to build Docker images ([unasuke](https://github.com/mastodon/mastodon/pull/16973), [Gargron](https://github.com/mastodon/mastodon/pull/16980), [Gargron](https://github.com/mastodon/mastodon/pull/17000))
### Fixed
- Fix some old migrations failing when skipping releases ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17394))
- Fix migrations script failing in certain edge cases ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17398))
- Fix Docker build ([tribela](https://github.com/mastodon/mastodon/pull/17188))
- Fix Ruby 3.0 dependencies ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16723))
- Fix followers synchronization mechanism ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16510))
## [3.4.4] - 2021-11-26
### Fixed

View file

@ -54,8 +54,8 @@ RUN npm install -g yarn && \
COPY Gemfile* package.json yarn.lock /opt/mastodon/
RUN cd /opt/mastodon && \
bundle config set deployment 'true' && \
bundle config set without 'development test' && \
bundle config set --local deployment 'true' && \
bundle config set --local without 'development test' && \
bundle install -j"$(nproc)" && \
yarn install --pure-lockfile

View file

@ -545,8 +545,9 @@ GEM
rack (>= 1.1)
rubocop (>= 1.7.0, < 2.0)
ruby-progressbar (1.11.0)
ruby-saml (1.11.0)
nokogiri (>= 1.5.10)
ruby-saml (1.13.0)
nokogiri (>= 1.10.5)
rexml
ruby2_keywords (0.0.4)
rufus-scheduler (3.6.0)
fugit (~> 1.1, >= 1.1.6)

View file

@ -60,6 +60,7 @@ class Account < ApplicationRecord
USERNAME_RE = /[a-z0-9_]+([a-z0-9_\.-]+[a-z0-9_]+)?/i
MENTION_RE = /(?<=^|[^\/[:word:]])@((#{USERNAME_RE})(?:@[[:word:]\.\-]+[[:word:]]+)?)/i
URL_PREFIX_RE = /\Ahttp(s?):\/\/[^\/]+/
include AccountAssociations
include AccountAvatar
@ -380,7 +381,7 @@ class Account < ApplicationRecord
def synchronization_uri_prefix
return 'local' if local?
@synchronization_uri_prefix ||= uri[/http(s?):\/\/[^\/]+\//]
@synchronization_uri_prefix ||= "#{uri[URL_PREFIX_RE]}/"
end
class Field < ActiveModelSerializers::Model

View file

@ -251,10 +251,13 @@ module AccountInteractions
.where('users.current_sign_in_at > ?', User::ACTIVE_DURATION.ago)
end
def remote_followers_hash(url_prefix)
Rails.cache.fetch("followers_hash:#{id}:#{url_prefix}") do
def remote_followers_hash(url)
url_prefix = url[Account::URL_PREFIX_RE]
return if url_prefix.blank?
Rails.cache.fetch("followers_hash:#{id}:#{url_prefix}/") do
digest = "\x00" * 32
followers.where(Account.arel_table[:uri].matches(url_prefix + '%', false, true)).pluck_each(:uri) do |uri|
followers.where(Account.arel_table[:uri].matches("#{Account.sanitize_sql_like(url_prefix)}/%", false, true)).or(followers.where(uri: url_prefix)).pluck_each(:uri) do |uri|
Xorcist.xor!(digest, Digest::SHA256.digest(uri))
end
digest.unpack('H*')[0]

View file

@ -44,11 +44,7 @@ class ActivityPub::DeliveryWorker
end
def synchronization_header
"collectionId=\"#{account_followers_url(@source_account)}\", digest=\"#{@source_account.remote_followers_hash(inbox_url_prefix)}\", url=\"#{account_followers_synchronization_url(@source_account)}\""
end
def inbox_url_prefix
@inbox_url[/http(s?):\/\/[^\/]+\//]
"collectionId=\"#{account_followers_url(@source_account)}\", digest=\"#{@source_account.remote_followers_hash(@inbox_url)}\", url=\"#{account_followers_synchronization_url(@source_account)}\""
end
def perform_request

View file

@ -1,6 +1,46 @@
class RemoveFauxRemoteAccountDuplicates < ActiveRecord::Migration[5.2]
disable_ddl_transaction!
class StreamEntry < ApplicationRecord
# Dummy class, to make migration possible across version changes
belongs_to :account, inverse_of: :stream_entries
end
class Status < ApplicationRecord
# Dummy class, to make migration possible across version changes
belongs_to :account, inverse_of: :statuses
has_many :favourites, inverse_of: :status, dependent: :destroy
has_many :mentions, dependent: :destroy, inverse_of: :status
end
class Favourite < ApplicationRecord
# Dummy class, to make migration possible across version changes
belongs_to :account, inverse_of: :favourites
belongs_to :status, inverse_of: :favourites
end
class Mention < ApplicationRecord
# Dummy class, to make migration possible across version changes
belongs_to :account, inverse_of: :mentions
belongs_to :status
end
class Notification < ApplicationRecord
# Dummy class, to make migration possible across version changes
belongs_to :account, optional: true
belongs_to :from_account, class_name: 'Account', optional: true
belongs_to :activity, polymorphic: true, optional: true
end
class Account < ApplicationRecord
# Dummy class, to make migration possible across version changes
has_many :stream_entries, inverse_of: :account, dependent: :destroy
has_many :statuses, inverse_of: :account, dependent: :destroy
has_many :favourites, inverse_of: :account, dependent: :destroy
has_many :mentions, inverse_of: :account, dependent: :destroy
has_many :notifications, inverse_of: :account, dependent: :destroy
end
def up
local_domain = Rails.configuration.x.local_domain

View file

@ -1,4 +1,9 @@
class AddInstanceActor < ActiveRecord::Migration[5.2]
class Account < ApplicationRecord
# Dummy class, to make migration possible across version changes
validates :username, uniqueness: { scope: :domain, case_sensitive: false }
end
def up
Account.create!(id: -99, actor_type: 'Application', locked: true, username: Rails.configuration.x.local_domain)
end

View file

@ -1,4 +1,8 @@
class UpdatePtLocales < ActiveRecord::Migration[5.2]
class User < ApplicationRecord
# Dummy class, to make migration possible across version changes
end
disable_ddl_transaction!
def up

View file

@ -20,7 +20,7 @@ FROM (
HAVING count(follows.id) >= 5
UNION ALL
SELECT accounts.id AS account_id,
sum(reblogs_count + favourites_count) / (1.0 + sum(reblogs_count + favourites_count)) AS rank,
sum(status_stats.reblogs_count + status_stats.favourites_count) / (1.0 + sum(status_stats.reblogs_count + status_stats.favourites_count)) AS rank,
'most_interactions' AS reason
FROM status_stats
INNER JOIN statuses ON statuses.id = status_stats.status_id
@ -32,7 +32,7 @@ FROM (
AND accounts.locked = 'f'
AND accounts.discoverable = 't'
GROUP BY accounts.id
HAVING sum(reblogs_count + favourites_count) >= 5
HAVING sum(status_stats.reblogs_count + status_stats.favourites_count) >= 5
) t0
GROUP BY account_id
ORDER BY rank DESC

View file

@ -18,7 +18,7 @@ FROM (
HAVING count(follows.id) >= 5
UNION ALL
SELECT account_summaries.account_id AS account_id,
sum(reblogs_count + favourites_count) / (1.0 + sum(reblogs_count + favourites_count)) AS rank,
sum(status_stats.reblogs_count + status_stats.favourites_count) / (1.0 + sum(status_stats.reblogs_count + status_stats.favourites_count)) AS rank,
'most_interactions' AS reason
FROM status_stats
INNER JOIN statuses ON statuses.id = status_stats.status_id
@ -28,7 +28,7 @@ FROM (
AND account_summaries.sensitive = 'f'
AND follow_recommendation_suppressions.id IS NULL
GROUP BY account_summaries.account_id
HAVING sum(reblogs_count + favourites_count) >= 5
HAVING sum(status_stats.reblogs_count + status_stats.favourites_count) >= 5
) t0
GROUP BY account_id
ORDER BY rank DESC

View file

@ -43,7 +43,7 @@ services:
web:
build: .
image: tootsuite/mastodon
image: tootsuite/mastodon:v3.4.5
restart: always
env_file: .env.production
command: bash -c "rm -f /mastodon/tmp/pids/server.pid; bundle exec rails s -p 3000"
@ -63,7 +63,7 @@ services:
streaming:
build: .
image: tootsuite/mastodon
image: tootsuite/mastodon:v3.4.5
restart: always
env_file: .env.production
command: node ./streaming
@ -80,7 +80,7 @@ services:
sidekiq:
build: .
image: tootsuite/mastodon
image: tootsuite/mastodon:v3.4.5
restart: always
env_file: .env.production
command: bundle exec sidekiq

View file

@ -295,7 +295,7 @@ module Mastodon
table = Arel::Table.new(table_name)
total = estimate_rows_in_table(table_name).to_i
if total == 0
if total < 1
count_arel = table.project(Arel.star.count.as('count'))
count_arel = yield table, count_arel if block_given?

View file

@ -13,7 +13,7 @@ module Mastodon
end
def patch
4
5
end
def flags

181
lib/tasks/tests.rake Normal file
View file

@ -0,0 +1,181 @@
# frozen_string_literal: true
namespace :tests do
namespace :migrations do
desc 'Populate the database with test data for 2.0.0'
task populate_v2: :environment do
admin_key = OpenSSL::PKey::RSA.new(2048)
user_key = OpenSSL::PKey::RSA.new(2048)
remote_key = OpenSSL::PKey::RSA.new(2048)
remote_key2 = OpenSSL::PKey::RSA.new(2048)
remote_key3 = OpenSSL::PKey::RSA.new(2048)
admin_private_key = ActiveRecord::Base.connection.quote(admin_key.to_pem)
admin_public_key = ActiveRecord::Base.connection.quote(admin_key.public_key.to_pem)
user_private_key = ActiveRecord::Base.connection.quote(user_key.to_pem)
user_public_key = ActiveRecord::Base.connection.quote(user_key.public_key.to_pem)
remote_public_key = ActiveRecord::Base.connection.quote(remote_key.public_key.to_pem)
remote_public_key2 = ActiveRecord::Base.connection.quote(remote_key2.public_key.to_pem)
remote_public_key_ap = ActiveRecord::Base.connection.quote(remote_key3.public_key.to_pem)
local_domain = ActiveRecord::Base.connection.quote(Rails.configuration.x.local_domain)
ActiveRecord::Base.connection.execute(<<~SQL)
-- accounts
INSERT INTO "accounts"
(id, username, domain, private_key, public_key, created_at, updated_at)
VALUES
(1, 'admin', NULL, #{admin_private_key}, #{admin_public_key}, now(), now()),
(2, 'user', NULL, #{user_private_key}, #{user_public_key}, now(), now());
INSERT INTO "accounts"
(id, username, domain, private_key, public_key, created_at, updated_at, remote_url, salmon_url)
VALUES
(3, 'remote', 'remote.com', NULL, #{remote_public_key}, now(), now(),
'https://remote.com/@remote', 'https://remote.com/salmon/1'),
(4, 'Remote', 'remote.com', NULL, #{remote_public_key}, now(), now(),
'https://remote.com/@Remote', 'https://remote.com/salmon/1'),
(5, 'REMOTE', 'Remote.com', NULL, #{remote_public_key2}, now(), now(),
'https://remote.com/stale/@REMOTE', 'https://remote.com/stale/salmon/1');
INSERT INTO "accounts"
(id, username, domain, private_key, public_key, created_at, updated_at, protocol, inbox_url, outbox_url, followers_url)
VALUES
(6, 'bob', 'activitypub.com', NULL, #{remote_public_key_ap}, now(), now(),
1, 'https://activitypub.com/users/bob/inbox', 'https://activitypub.com/users/bob/outbox', 'https://activitypub.com/users/bob/followers');
INSERT INTO "accounts"
(id, username, domain, private_key, public_key, created_at, updated_at)
VALUES
(7, 'user', #{local_domain}, #{user_private_key}, #{user_public_key}, now(), now()),
(8, 'pt_user', NULL, #{user_private_key}, #{user_public_key}, now(), now());
-- users
INSERT INTO "users"
(id, account_id, email, created_at, updated_at, admin)
VALUES
(1, 1, 'admin@localhost', now(), now(), true),
(2, 2, 'user@localhost', now(), now(), false);
INSERT INTO "users"
(id, account_id, email, created_at, updated_at, admin, locale)
VALUES
(3, 7, 'ptuser@localhost', now(), now(), false, 'pt');
-- statuses
INSERT INTO "statuses"
(id, account_id, text, created_at, updated_at)
VALUES
(1, 1, 'test', now(), now()),
(2, 1, '@remote@remote.com hello', now(), now()),
(3, 1, '@Remote@remote.com hello', now(), now()),
(4, 1, '@REMOTE@remote.com hello', now(), now());
INSERT INTO "statuses"
(id, account_id, text, created_at, updated_at, uri, local)
VALUES
(5, 1, 'activitypub status', now(), now(), 'https://localhost/users/admin/statuses/4', true);
INSERT INTO "statuses"
(id, account_id, text, created_at, updated_at)
VALUES
(6, 3, 'test', now(), now());
INSERT INTO "statuses"
(id, account_id, text, created_at, updated_at, in_reply_to_id, in_reply_to_account_id)
VALUES
(7, 4, '@admin hello', now(), now(), 3, 1);
INSERT INTO "statuses"
(id, account_id, text, created_at, updated_at)
VALUES
(8, 5, 'test', now(), now());
INSERT INTO "statuses"
(id, account_id, reblog_of_id, created_at, updated_at)
VALUES
(9, 1, 2, now(), now());
-- mentions (from previous statuses)
INSERT INTO "mentions"
(status_id, account_id, created_at, updated_at)
VALUES
(2, 3, now(), now()),
(3, 4, now(), now()),
(4, 5, now(), now());
-- stream entries
INSERT INTO "stream_entries"
(activity_id, account_id, activity_type, created_at, updated_at)
VALUES
(1, 1, 'status', now(), now()),
(2, 1, 'status', now(), now()),
(3, 1, 'status', now(), now()),
(4, 1, 'status', now(), now()),
(5, 1, 'status', now(), now()),
(6, 3, 'status', now(), now()),
(7, 4, 'status', now(), now()),
(8, 5, 'status', now(), now()),
(9, 1, 'status', now(), now());
-- custom emoji
INSERT INTO "custom_emojis"
(shortcode, created_at, updated_at)
VALUES
('test', now(), now()),
('Test', now(), now()),
('blobcat', now(), now());
INSERT INTO "custom_emojis"
(shortcode, domain, uri, created_at, updated_at)
VALUES
('blobcat', 'remote.org', 'https://remote.org/emoji/blobcat', now(), now()),
('blobcat', 'Remote.org', 'https://remote.org/emoji/blobcat', now(), now()),
('Blobcat', 'remote.org', 'https://remote.org/emoji/Blobcat', now(), now());
-- favourites
INSERT INTO "favourites"
(account_id, status_id, created_at, updated_at)
VALUES
(1, 1, now(), now()),
(1, 7, now(), now()),
(4, 1, now(), now()),
(3, 1, now(), now()),
(5, 1, now(), now());
-- pinned statuses
INSERT INTO "status_pins"
(account_id, status_id, created_at, updated_at)
VALUES
(1, 1, now(), now()),
(3, 6, now(), now()),
(4, 7, now(), now());
-- follows
INSERT INTO "follows"
(account_id, target_account_id, created_at, updated_at)
VALUES
(1, 5, now(), now()),
(6, 2, now(), now()),
(5, 2, now(), now()),
(6, 1, now(), now());
-- follow requests
INSERT INTO "follow_requests"
(account_id, target_account_id, created_at, updated_at)
VALUES
(2, 5, now(), now()),
(5, 1, now(), now());
SQL
end
end
end

View file

@ -539,46 +539,57 @@ describe AccountInteractions do
end
end
describe '#followers_hash' do
describe '#remote_followers_hash' do
let(:me) { Fabricate(:account, username: 'Me') }
let(:remote_1) { Fabricate(:account, username: 'alice', domain: 'example.org', uri: 'https://example.org/users/alice') }
let(:remote_2) { Fabricate(:account, username: 'bob', domain: 'example.org', uri: 'https://example.org/users/bob') }
let(:remote_3) { Fabricate(:account, username: 'eve', domain: 'foo.org', uri: 'https://foo.org/users/eve') }
let(:remote_3) { Fabricate(:account, username: 'instance-actor', domain: 'example.org', uri: 'https://example.org') }
let(:remote_4) { Fabricate(:account, username: 'eve', domain: 'foo.org', uri: 'https://foo.org/users/eve') }
before do
remote_1.follow!(me)
remote_2.follow!(me)
remote_3.follow!(me)
remote_4.follow!(me)
me.follow!(remote_1)
end
context 'on a local user' do
it 'returns correct hash for remote domains' do
expect(me.remote_followers_hash('https://example.org/')).to eq '707962e297b7bd94468a21bc8e506a1bcea607a9142cd64e27c9b106b2a5f6ec'
expect(me.remote_followers_hash('https://foo.org/')).to eq 'ccb9c18a67134cfff9d62c7f7e7eb88e6b803446c244b84265565f4eba29df0e'
end
it 'invalidates cache as needed when removing or adding followers' do
expect(me.remote_followers_hash('https://example.org/')).to eq '707962e297b7bd94468a21bc8e506a1bcea607a9142cd64e27c9b106b2a5f6ec'
remote_1.unfollow!(me)
expect(me.remote_followers_hash('https://example.org/')).to eq '241b00794ce9b46aa864f3220afadef128318da2659782985bac5ed5bd436bff'
remote_1.follow!(me)
expect(me.remote_followers_hash('https://example.org/')).to eq '707962e297b7bd94468a21bc8e506a1bcea607a9142cd64e27c9b106b2a5f6ec'
end
it 'returns correct hash for remote domains' do
expect(me.remote_followers_hash('https://example.org/')).to eq '20aecbe774b3d61c25094370baf370012b9271c5b172ecedb05caff8d79ef0c7'
expect(me.remote_followers_hash('https://foo.org/')).to eq 'ccb9c18a67134cfff9d62c7f7e7eb88e6b803446c244b84265565f4eba29df0e'
expect(me.remote_followers_hash('https://foo.org.evil.com/')).to eq '0000000000000000000000000000000000000000000000000000000000000000'
expect(me.remote_followers_hash('https://foo')).to eq '0000000000000000000000000000000000000000000000000000000000000000'
end
context 'on a remote user' do
it 'returns correct hash for remote domains' do
expect(remote_1.local_followers_hash).to eq Digest::SHA256.hexdigest(ActivityPub::TagManager.instance.uri_for(me))
end
it 'invalidates cache as needed when removing or adding followers' do
expect(me.remote_followers_hash('https://example.org/')).to eq '20aecbe774b3d61c25094370baf370012b9271c5b172ecedb05caff8d79ef0c7'
remote_3.unfollow!(me)
expect(me.remote_followers_hash('https://example.org/')).to eq '707962e297b7bd94468a21bc8e506a1bcea607a9142cd64e27c9b106b2a5f6ec'
remote_1.unfollow!(me)
expect(me.remote_followers_hash('https://example.org/')).to eq '241b00794ce9b46aa864f3220afadef128318da2659782985bac5ed5bd436bff'
remote_1.follow!(me)
expect(me.remote_followers_hash('https://example.org/')).to eq '707962e297b7bd94468a21bc8e506a1bcea607a9142cd64e27c9b106b2a5f6ec'
end
end
it 'invalidates cache as needed when removing or adding followers' do
expect(remote_1.local_followers_hash).to eq Digest::SHA256.hexdigest(ActivityPub::TagManager.instance.uri_for(me))
me.unfollow!(remote_1)
expect(remote_1.local_followers_hash).to eq '0000000000000000000000000000000000000000000000000000000000000000'
me.follow!(remote_1)
expect(remote_1.local_followers_hash).to eq Digest::SHA256.hexdigest(ActivityPub::TagManager.instance.uri_for(me))
end
describe '#local_followers_hash' do
let(:me) { Fabricate(:account, username: 'Me') }
let(:remote_1) { Fabricate(:account, username: 'alice', domain: 'example.org', uri: 'https://example.org/users/alice') }
before do
me.follow!(remote_1)
end
it 'returns correct hash for local users' do
expect(remote_1.local_followers_hash).to eq Digest::SHA256.hexdigest(ActivityPub::TagManager.instance.uri_for(me))
end
it 'invalidates cache as needed when removing or adding followers' do
expect(remote_1.local_followers_hash).to eq Digest::SHA256.hexdigest(ActivityPub::TagManager.instance.uri_for(me))
me.unfollow!(remote_1)
expect(remote_1.local_followers_hash).to eq '0000000000000000000000000000000000000000000000000000000000000000'
me.follow!(remote_1)
expect(remote_1.local_followers_hash).to eq Digest::SHA256.hexdigest(ActivityPub::TagManager.instance.uri_for(me))
end
end

View file

@ -11,7 +11,7 @@ describe ActivityPub::DeliveryWorker do
let(:payload) { 'test' }
before do
allow_any_instance_of(Account).to receive(:remote_followers_hash).with('https://example.com/').and_return('somehash')
allow_any_instance_of(Account).to receive(:remote_followers_hash).with('https://example.com/api').and_return('somehash')
end
describe 'perform' do