Merge remote-tracking branch 'origin/release-v1.12.0' into dinsic-release-v1.12.x

* origin/release-v1.12.0:
  Missing bits
  Use r0 instead of 'unstable' for joined_members|rooms
  Add test to optionally keep other sessions when modifying a user's password. (#832)
  Remove tests that call query_auth endpoint. (#827)
  Add tests for validating alt_aliases of a canonical alias event. (#824)
  Update the haproxy config to route fallback auth and register to the same worker. (#828)
  Add tests for a user with sufficient power level removing an alias. (#818)
  Add a test for an event with an invalid signature (#821)
  Fix race in self signing key tests. (#822)
  Use retry_until_success instead of repeat_until_true.
  Attempt to deflake a test.
  Fix frontend proxy worker binding to wrong port. (#823)
  Stop checking for aliases in the public room list.
  Attempt to deflake a sytest (#816)
  Fix formatting from review.
  Support running Dendrite in more than 1 sqlite db file
  Add tests for moving alt_aliases to an upgraded room.
  Support running dendrite in sqlite

Author: anoadragon453

docker/base.Dockerfile

@@ -60,6 +60,3 @@ ENV PGHOST=/var/run/postgresql
 ENV PGDATA=$PGHOST/data
 ENV PGUSER=postgres
 RUN for ver in `ls /usr/lib/postgresql | head -n 1`; do su -c '/usr/lib/postgresql/'$ver'/bin/initdb -E "UTF-8" --lc-collate="C" --lc-ctype="C" --username=postgres' postgres; done
-
-# Turn off all the fsync stuff for postgres
-RUN for ver in /etc/postgresql/*/main; do mkdir -p $ver/conf.d/; echo -e "fsync=off\nfull_page_writes=off" >> $ver/conf.d/fsync.conf; done

lib/SyTest/Homeserver/Dendrite.pm

@@ -136,7 +136,10 @@ sub _get_config
       },
 
       database => {
-         map { $_ => $db_uri } qw(
+         # POSTGRES not set or is 0, use sqlite, which has separate .db files for each server
+         map { ( ! defined $ENV{'POSTGRES'} || $ENV{'POSTGRES'} == '0') ?
+         ($_ => "file:$self->{hs_dir}/" . $_ . ".db") :
+         ($_ => $db_uri) } qw(
             account device media_api sync_api room_server server_key
             federation_sender public_rooms_api naffka appservice
          ),

lib/SyTest/Homeserver/Synapse.pm

@@ -712,6 +712,7 @@ sub wrap_synapse_command
          "worker_log_config"       => $self->configure_logger("pusher"),
          "worker_replication_host" => "$bind_host",
          "worker_replication_port" => $self->{ports}{synapse_replication_tcp},
+         "worker_replication_http_port" => $self->{ports}{synapse_unsecure},
          "worker_listeners"        => [
             {
                type      => "http",
@@ -737,6 +738,7 @@ sub wrap_synapse_command
          "worker_log_config"       => $self->configure_logger("appservice"),
          "worker_replication_host" => "$bind_host",
          "worker_replication_port" => $self->{ports}{synapse_replication_tcp},
+         "worker_replication_http_port" => $self->{ports}{synapse_unsecure},
          "worker_listeners"        => [
             {
                type => "manhole",
@@ -762,6 +764,7 @@ sub wrap_synapse_command
          "worker_log_config"       => $self->configure_logger("federation_sender"),
          "worker_replication_host" => "$bind_host",
          "worker_replication_port" => $self->{ports}{synapse_replication_tcp},
+         "worker_replication_http_port" => $self->{ports}{synapse_unsecure},
          "worker_listeners"        => [
             {
                type => "manhole",
@@ -787,6 +790,7 @@ sub wrap_synapse_command
          "worker_log_config"       => $self->configure_logger("synchrotron"),
          "worker_replication_host" => "$bind_host",
          "worker_replication_port" => $self->{ports}{synapse_replication_tcp},
+         "worker_replication_http_port" => $self->{ports}{synapse_unsecure},
          "worker_listeners"        => [
             {
                type      => "http",
@@ -854,6 +858,7 @@ sub wrap_synapse_command
          "worker_log_config"       => $self->configure_logger("media_repository"),
          "worker_replication_host" => "$bind_host",
          "worker_replication_port" => $self->{ports}{synapse_replication_tcp},
+         "worker_replication_http_port" => $self->{ports}{synapse_unsecure},
          "worker_listeners"        => [
             {
                type      => "http",
@@ -921,6 +926,7 @@ sub wrap_synapse_command
          "worker_log_config"       => $self->configure_logger("user_dir"),
          "worker_replication_host" => "$bind_host",
          "worker_replication_port" => $self->{ports}{synapse_replication_tcp},
+         "worker_replication_http_port" => $self->{ports}{synapse_unsecure},
          "worker_listeners"        => [
             {
                type      => "http",
@@ -999,7 +1005,7 @@ sub wrap_synapse_command
             },
             {
                type => "manhole",
-               port => $self->{ports}{frontend_proxy},
+               port => $self->{ports}{frontend_proxy_manhole},
                bind_address => $bind_host,
             },
             {
@@ -1207,6 +1213,7 @@ sub generate_haproxy_map
 ^/_matrix/client/versions$                                        client_reader
 ^/_matrix/client/(api/v1|r0|unstable)/voip/turnServer$            client_reader
 ^/_matrix/client/(r0|unstable)/register$                          client_reader
+^/_matrix/client/(r0|unstable)/auth/.*/fallback/web$              client_reader
 ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/messages$          client_reader
 ^/_matrix/client/(api/v1|r0|unstable)/get_groups_publicised$      client_reader
 ^/_matrix/client/(api/v1|r0|unstable)/joined_groups$              client_reader

scripts/synapse_sytest.sh

@@ -16,15 +16,26 @@ cd "$(dirname $0)/.."
 mkdir /work
 
 # PostgreSQL setup
-if [ -n "$MULTI_POSTGRES" ]; then
-    # In this mode we want to run synapse against multiple split out databases.
-
+if [ -n "$MULTI_POSTGRES" ] || [ -n "$POSTGRES" ]; then
     # We increase the max connections as we have more databases.
-    echo -e "max_connections=1000" >> /var/run/postgresql/data/postgresql.conf
+    sed -i -r "s/^max_connections.*$/max_connections = 500/" /var/run/postgresql/data/postgresql.conf
+
+    echo -e "fsync = off" >> /var/run/postgresql/data/postgresql.conf
+    echo -e "full_page_writes = off" >> /var/run/postgresql/data/postgresql.conf
 
     # Start the database
     su -c 'eatmydata /usr/lib/postgresql/*/bin/pg_ctl -w -D $PGDATA start' postgres
 
+    su -c psql postgres <<< "show config_file"
+    su -c psql postgres <<< "show max_connections"
+    su -c psql postgres <<< "show full_page_writes"
+    su -c psql postgres <<< "show fsync"
+fi
+
+# Now create the databases
+if [ -n "$MULTI_POSTGRES" ]; then
+    # In this mode we want to run synapse against multiple split out databases.
+
     # Make the test databases for the two Synapse servers that will be spun up
     su -c psql postgres <<EOF
 CREATE DATABASE pg1_main;
@@ -85,13 +96,11 @@ state_db:
 EOF
 
 elif [ -n "$POSTGRES" ]; then
+    # Env vars used by prep_sytest_for_postgres script.
     export PGUSER=postgres
     export POSTGRES_DB_1=pg1
     export POSTGRES_DB_2=pg2
 
-    # Start the database
-    su -c 'eatmydata /usr/lib/postgresql/*/bin/pg_ctl -w -D $PGDATA start' postgres
-
     # Write out the configuration for a PostgreSQL using Synapse
     ./scripts/prep_sytest_for_postgres.sh
 

tests/10apidoc/31room-state.pl

@@ -112,7 +112,7 @@
 
       do_request_json_for( $user,
          method => "GET",
-         uri    => "/unstable/rooms/$room_id/joined_members",
+         uri    => "/r0/rooms/$room_id/joined_members",
       )->then( sub {
          my ( $body ) = @_;
 
@@ -220,7 +220,7 @@
 
       do_request_json_for( $user,
          method => "GET",
-         uri    => "/unstable/joined_rooms",
+         uri    => "/r0/joined_rooms",
       )->then( sub {
          my ( $body ) = @_;
 

tests/14account/01change-password.pl

@@ -82,7 +82,7 @@ sub matrix_set_password
       })->then_done(1);
    };
 
-test "After changing password, a different session no longer works",
+test "After changing password, a different session no longer works by default",
    requires => [ local_user_fixture( password => $password ) ],
 
    check => sub {
@@ -107,6 +107,38 @@ sub matrix_set_password
       })->then_done(1);
    };
 
+test "After changing password, different sessions can optionally be kept",
+   requires => [ local_user_fixture( password => $password ) ],
+
+   check => sub {
+      my ( $user ) = @_;
+
+      my $other_login;
+
+      matrix_login_again_with_user( $user )->then( sub {
+         ( $other_login ) = @_;
+         # ensure other login works to start with
+         matrix_sync( $other_login );
+      })->then( sub {
+         do_request_json_for( $user,
+            method => "POST",
+            uri    => "/r0/account/password",
+            content => {
+               auth => {
+                  type           => "m.login.password",
+                  user           => $user->user_id,
+                  password       => $password,
+               },
+               new_password => "my new password",
+               logout_devices => JSON::false,
+            },
+         );
+      })->then( sub {
+         # The access token should still be valid.
+         matrix_sync( $other_login );
+      })->then_done(1);
+   };
+
 test "Pushers created with a different access token are deleted on password change",
    requires => [ local_user_fixture( password => $password ) ],
 

tests/30rooms/05aliases.pl

@@ -90,6 +90,7 @@
             }
          )->SyTest::pass_on_done( "m.room.canonical_alias accepts present aliases" );
       })->then( sub {
+         # Create an unknown, but valid alias name.
          my $bad_alias = $room_alias =~ s/^#/#NOT-/r;
 
          matrix_put_room_state( $user, $room_id,
@@ -99,6 +100,84 @@
             }
          )->main::expect_http_4xx
             ->SyTest::pass_on_done( "m.room.canonical_alias rejects missing aliases" );
+      })->then( sub {
+         # Create an invalid alias name (starts with % instead of #).
+         my $bad_alias = $room_alias =~ s/^#/%/r;
+
+         matrix_put_room_state( $user, $room_id,
+            type    => "m.room.canonical_alias",
+            content => {
+               alias => $bad_alias,
+            }
+         )->main::expect_http_4xx
+            ->SyTest::pass_on_done( "m.room.canonical_alias rejects invalid aliases" );
+      });
+   };
+
+multi_test "Canonical alias can include alt_aliases",
+   requires => [ local_user_fixture(), room_alias_name_fixture() ],
+
+   do => sub {
+      my ( $user, $room_alias_name ) = @_;
+
+      my ( $room_id, $room_alias );
+
+      matrix_create_room( $user,
+         room_alias_name => $room_alias_name,
+      )->then( sub {
+         ( $room_id, $room_alias ) = @_;
+
+         matrix_put_room_state( $user, $room_id,
+            type    => "m.room.canonical_alias",
+            content => {
+               alias       => $room_alias,
+               alt_aliases => [ $room_alias ],
+            }
+         )->SyTest::pass_on_done( "m.room.canonical_alias accepts present aliases" );
+      })->then( sub {
+         # Create an unknown, but valid alias name.
+         my $bad_alias = $room_alias =~ s/^#/#NOT-/r;
+
+         matrix_put_room_state( $user, $room_id,
+            type    => "m.room.canonical_alias",
+            content => {
+               alias => $room_alias,
+               alt_aliases => [ $bad_alias ],
+            }
+         )->main::expect_matrix_error( 404, "M_NOT_FOUND" )
+            ->SyTest::pass_on_done( "m.room.canonical_alias rejects missing aliases" );
+      })->then( sub {
+         # Create an invalid alias name (starts with % instead of #).
+         my $bad_alias = $room_alias =~ s/^#/%/r;
+
+         matrix_put_room_state( $user, $room_id,
+            type    => "m.room.canonical_alias",
+            content => {
+               alias => $room_alias,
+               alt_aliases => [ $bad_alias ],
+            }
+         )->main::expect_matrix_error( 400, "M_INVALID_PARAM" )
+            ->SyTest::pass_on_done( "m.room.canonical_alias rejects invalid aliases" );
+      })->then( sub {
+         # Create a second room with an alias on it.
+         my $other_alias_name = $room_alias_name . "2";
+
+         matrix_create_room( $user,
+            room_alias_name => $other_alias_name,
+         )->then( sub {
+            my ( $other_room_id, $other_room_alias ) = @_;
+
+            # Attempt to set a canonical alias for the original room using an
+            # alias from the second room.
+            matrix_put_room_state( $user, $room_id,
+               type    => "m.room.canonical_alias",
+               content => {
+                  alias => $room_alias,
+                  alt_aliases => [ $other_room_alias ],
+               }
+            )->main::expect_matrix_error( 400, "M_BAD_ALIAS" )
+               ->SyTest::pass_on_done( "m.room.canonical_alias rejects alias pointing to different room" );
+            })
       });
    };
 
@@ -135,8 +214,6 @@
    do => sub {
       my ( $creator, $other_user, $room_id, $alias ) = @_;
 
-      my $server_name = $other_user->http->server_name;
-
       matrix_change_room_power_levels(
          $creator, $room_id, sub {
             $_[0]->{events}->{'m.room.aliases'} = 50;
@@ -169,8 +246,6 @@
 sub _test_can_create_and_delete_alias {
    my ( $room_id, $user, $alias ) = @_;
 
-   my $server_name = $user->http->server_name;
-
    do_request_json_for( $user,
       method => "PUT",
       uri    => "/r0/directory/room/$alias",
@@ -209,7 +284,6 @@ sub _test_can_create_and_delete_alias {
 
    do => sub {
       my ( $user, $room_id, $other_user, $room_alias ) = @_;
-      my $server_name = $user->http->server_name;
 
       do_request_json_for( $user,
          method => "PUT",
@@ -226,6 +300,50 @@ sub _test_can_create_and_delete_alias {
       })
    };
 
+test "Users with sufficient power-level can delete other's aliases",
+   requires => [ $creator_fixture, local_user_fixture(), room_alias_fixture(),
+                 qw( can_create_room_alias )],
+
+   do => sub {
+      my ( $user, $other_user, $room_alias ) = @_;
+      my $room_id;
+
+      matrix_create_and_join_room(
+         [ $user, $other_user ],
+      )->then( sub {
+         ( $room_id ) = @_;
+
+         matrix_change_room_power_levels(
+            $user, $room_id, sub {
+               $_[0]->{users}->{$other_user->user_id} = 100;
+            },
+         )
+      })->then( sub {
+         do_request_json_for( $user,
+           method => "PUT",
+           uri    => "/r0/directory/room/$room_alias",
+
+           content => { room_id => $room_id },
+         )
+      })->then( sub {
+         do_request_json_for( $other_user,
+           method => "DELETE",
+           uri    => "/r0/directory/room/$room_alias",
+
+           content => {},
+         )
+      })->then( sub {
+         my ( $res ) = @_;
+         log_if_fail "Unable to delete alias", $res;
+
+         do_request_json_for(
+            $user,
+            method => "GET",
+            uri  => "/r0/directory/room/$room_alias",
+         );
+      })->main::expect_http_404;
+   };
+
 test "Can delete canonical alias",
    requires => [ local_user_fixture( with_events => 0 ), room_alias_fixture(),
                  qw( can_create_room_alias )],
@@ -275,7 +393,6 @@ sub _test_can_create_and_delete_alias {
 
    do => sub {
       my ( $creator, $other_user, $room_alias ) = @_;
-      my $server_name = $creator->http->server_name;
       my $room_id;
 
       matrix_create_and_join_room( [ $creator, $other_user ] )
@@ -305,7 +422,6 @@ sub _test_can_create_and_delete_alias {
 
    do => sub {
       my ( $creator, $other_user, $room_alias ) = @_;
-      my $server_name = $creator->http->server_name;
       my $room_id;
 
       matrix_create_and_join_room( [ $creator, $other_user ] )