Document a scalable installation with multiple videobridges (#5783)

doc/README.md

 * [Enabling TURN](https://github.com/jitsi/jitsi-meet/blob/master/doc/turn.md)
 * [Networking FAQ](https://github.com/jitsi/jitsi-meet/blob/master/doc/faq.md)
 * [Cloud APIs](https://github.com/jitsi/jitsi-meet/blob/master/doc/cloud-api.md)
+* [Manual Installation](https://github.com/jitsi/jitsi-meet/blob/master/doc/manual-install.md)
+* [Scalable Installation](https://github.com/jitsi/jitsi-meet/blob/master/doc/scalable-installation.md)

doc/example-config-files/scalable/jicofo_config.scalable.example

+# Jitsi Conference Focus settings
+# sets the host name of the XMPP server
+JICOFO_HOST=localhost
+
+# sets the XMPP domain (default: none)
+JICOFO_HOSTNAME=meet.example.com
+
+# sets the secret used to authenticate as an XMPP component
+JICOFO_SECRET=$JICOFO_SECRET
+
+# sets the port to use for the XMPP component connection
+JICOFO_PORT=5347
+
+# sets the XMPP domain name to use for XMPP user logins
+JICOFO_AUTH_DOMAIN=auth.meet.example.com
+
+# sets the username to use for XMPP user logins
+JICOFO_AUTH_USER=focus
+
+# sets the password to use for XMPP user logins
+JICOFO_AUTH_PASSWORD=$JICOFO_PASSWORD
+
+# extra options to pass to the jicofo daemon
+JICOFO_OPTS=""
+
+# adds java system props that are passed to jicofo (default are for home and logging config file)
+JAVA_SYS_PROPS="-Dnet.java.sip.communicator.SC_HOME_DIR_LOCATION=/etc/jitsi -Dnet.java.sip.communicator.SC_HOME_DIR_NAME=jicofo -Dnet.java.sip.communicator.SC_LOG_DIR_LOCATION=/var/log/jitsi -Djava.util.logging.config.file=/etc/jitsi/jicofo/logging.properties"

doc/example-config-files/scalable/jicofo_sip-communicator.properties.scalable.example

+
+org.jitsi.jicofo.BRIDGE_MUC=JvbBrewery@internal.auth.meet.example.com
+org.jitsi.jicofo.ALWAYS_TRUST_MODE_ENABLED=true
+
+org.jitsi.jicofo.jibri.BREWERY=JibriBrewery@internal.auth.meet.example.com
+org.jitsi.jicofo.jibri.PENDING_TIMEOUT=90

doc/example-config-files/scalable/meet.example.com.cfg.lua.scalable.example

+plugin_paths = { "/usr/share/jitsi-meet/prosody-plugins/" }
+
+-- domain mapper options, must at least have domain base set to use the mapper
+muc_mapper_domain_base = "meet.example.com";
+
+turncredentials_secret = "turncredentials_secret_test";
+
+turncredentials = {
+  { type = "stun", host = "meet.example.com", port = "443" },
+  { type = "turn", host = "meet.example.com", port = "443", transport = "udp" },
+  { type = "turns", host = "meet.example.com", port = "443", transport = "tcp" }
+};
+
+cross_domain_bosh = false;
+consider_bosh_secure = true;
+
+VirtualHost "meet.example.com"
+        -- enabled = false -- Remove this line to enable this host
+        authentication = "anonymous"
+        -- Properties below are modified by jitsi-meet-tokens package config
+        -- and authentication above is switched to "token"
+        --app_id="example_app_id"
+        --app_secret="example_app_secret"
+        -- Assign this host a certificate for TLS, otherwise it would use the one
+        -- set in the global section (if any).
+        -- Note that old-style SSL on port 5223 only supports one certificate, and will always
+        -- use the global one.
+        ssl = {
+                key = "/etc/prosody/certs/meet.example.com.key";
+                certificate = "/etc/prosody/certs/meet.example.com.crt";
+        }
+        speakerstats_component = "speakerstats.meet.example.com"
+        conference_duration_component = "conferenceduration.meet.example.com"
+        -- we need bosh
+        modules_enabled = {
+            "bosh";
+            "pubsub";
+            "ping"; -- Enable mod_ping
+            "speakerstats";
+            "turncredentials";
+            "conference_duration";
+        }
+        c2s_require_encryption = false
+
+Component "conference.meet.example.com" "muc"
+    storage = "memory"
+    modules_enabled = {
+        "muc_meeting_id";
+        "muc_domain_mapper";
+        -- "token_verification";
+    }
+    admins = { "focus@auth.meet.example.com" }
+    muc_room_locking = false
+    muc_room_default_public_jids = true
+
+-- internal muc component
+-- Note: This is also used from jibris
+Component "internal.auth.meet.example.com" "muc"
+    storage = "memory"
+    modules_enabled = {
+      "ping";
+    }
+    admins = { "focus@auth.meet.example.com", "jvb@auth.meet.example.com" }
+
+VirtualHost "auth.meet.example.com"
+    ssl = {
+        key = "/etc/prosody/certs/auth.meet.example.com.key";
+        certificate = "/etc/prosody/certs/auth.meet.example.com.crt";
+    }
+    authentication = "internal_plain"
+
+Component "focus.meet.example.com"
+    component_secret = "jicofo_secret_test"
+
+Component "speakerstats.meet.example.com" "speakerstats_component"
+    muc_component = "conference.meet.example.com"
+
+
+Component "conferenceduration.meet.example.com" "conference_duration_component"
+    muc_component = "conference.meet.example.com"
+
+-- for Jibri
+VirtualHost "recorder.meet.example.com"
+    modules_enabled = {
+        "ping";
+    }
+    authentication = "internal_plain"
+    c2s_require_encryption = false

doc/example-config-files/scalable/prosody.cfg.lua.scalable.example

+-- Prosody XMPP Server Configuration
+
+---------- Server-wide settings ----------
+-- Settings in this section apply to the whole server and are the default settings
+-- for any virtual hosts
+
+admins = { }
+
+network_backend = "epoll"
+
+-- This is the list of modules Prosody will load on startup.
+-- It looks for mod_modulename.lua in the plugins folder, so make sure that exists too.
+-- Documentation for bundled modules can be found at: https://prosody.im/doc/modules
+modules_enabled = {
+
+	-- Generally required
+		"roster"; -- Allow users to have a roster. Recommended ;)
+		"saslauth"; -- Authentication for clients and servers. Recommended if you want to log in.
+		"tls"; -- Add support for secure TLS on c2s/s2s connections
+		"dialback"; -- s2s dialback support
+		"disco"; -- Service discovery
+
+	-- Not essential, but recommended
+		"carbons"; -- Keep multiple clients in sync
+		"pep"; -- Enables users to publish their avatar, mood, activity, playing music and more
+		"private"; -- Private XML storage (for room bookmarks, etc.)
+		"blocklist"; -- Allow users to block communications with other users
+		"vcard4"; -- User profiles (stored in PEP)
+		"vcard_legacy"; -- Conversion between legacy vCard and PEP Avatar, vcard
+
+	-- Nice to have
+		"version"; -- Replies to server version requests
+		"uptime"; -- Report how long server has been running
+		"time"; -- Let others know the time here on this server
+		"ping"; -- Replies to XMPP pings with pongs
+		"register"; -- Allow users to register on this server using a client and change passwords
+		--"mam"; -- Store messages in an archive and allow users to access it
+		--"csi_simple"; -- Simple Mobile optimizations
+
+	-- Admin interfaces
+		"admin_adhoc"; -- Allows administration via an XMPP client that supports ad-hoc commands
+		--"admin_telnet"; -- Opens telnet console interface on localhost port 5582
+
+	-- HTTP modules
+		--"bosh"; -- Enable BOSH clients, aka "Jabber over HTTP"
+		--"websocket"; -- XMPP over WebSockets
+		--"http_files"; -- Serve static files from a directory over HTTP
+
+	-- Other specific functionality
+		--"limits"; -- Enable bandwidth limiting for XMPP connections
+		--"groups"; -- Shared roster support
+		--"server_contact_info"; -- Publish contact information for this service
+		--"announce"; -- Send announcement to all online users
+		--"welcome"; -- Welcome users who register accounts
+		--"watchregistrations"; -- Alert admins of registrations
+		--"motd"; -- Send a message to users when they log in
+		--"legacyauth"; -- Legacy authentication. Only used by some old clients and bots.
+		--"proxy65"; -- Enables a file transfer proxy service which clients behind NAT can use
+}
+
+-- These modules are auto-loaded, but should you want
+-- to disable them then uncomment them here:
+modules_disabled = {
+	-- "offline"; -- Store offline messages
+	-- "c2s"; -- Handle client connections
+	-- "s2s"; -- Handle server-to-server connections
+	-- "posix"; -- POSIX functionality, sends server to background, enables syslog, etc.
+}
+
+-- Disable account creation by default, for security
+-- For more information see https://prosody.im/doc/creating_accounts
+allow_registration = false
+
+-- Force clients to use encrypted connections? This option will
+-- prevent clients from authenticating unless they are using encryption.
+
+c2s_require_encryption = true
+
+-- Force servers to use encrypted connections? This option will
+-- prevent servers from authenticating unless they are using encryption.
+
+s2s_require_encryption = true
+
+-- Force certificate authentication for server-to-server connections?
+
+s2s_secure_auth = false
+
+
+-- Required for init scripts and prosodyctl
+pidfile = "/var/run/prosody/prosody.pid"
+
+-- Select the authentication backend to use. The 'internal' providers
+-- use Prosody's configured data storage to store the authentication data.
+
+authentication = "internal_hashed"
+
+archive_expires_after = "1w" -- Remove archived messages after 1 week
+
+-- Logging configuration
+-- For advanced logging see https://prosody.im/doc/logging
+log = {
+	info = "/var/log/prosody/prosody.log"; -- Change 'info' to 'debug' for verbose logging
+	error = "/var/log/prosody/prosody.err";
+	-- "*syslog"; -- Uncomment this for logging to syslog
+	-- "*console"; -- Log to the console, useful for debugging with daemonize=false
+}
+
+
+-- Location of directory to find certificates in (relative to main config file):
+certificates = "certs"
+
+VirtualHost "localhost"
+
+Include "conf.d/*.cfg.lua"

doc/example-config-files/scalable/videobridge_config.scalable.example

+# Jitsi Videobridge settings
+
+# sets the XMPP domain (default: none)
+JVB_HOSTNAME=meet.example.com
+
+# sets the hostname of the XMPP server (default: domain if set, localhost otherwise)
+JVB_HOST=
+
+# sets the port of the XMPP server (default: 5275)
+JVB_PORT=5347
+
+# sets the shared secret used to authenticate to the XMPP server
+JVB_SECRET=$VP_SECRET
+
+# extra options to pass to the JVB daemon
+JVB_OPTS="--apis=rest,"
+
+
+# adds java system props that are passed to jvb (default are for home and logging config file)
+JAVA_SYS_PROPS="-Dnet.java.sip.communicator.SC_HOME_DIR_LOCATION=/etc/jitsi -Dnet.java.sip.communicator.SC_HOME_DIR_NAME=videobridge -Dnet.java.sip.communicator.SC_LOG_DIR_LOCATION=/var/log/jitsi -Djava.util.logging.config.file=/etc/jitsi/videobridge/logging.properties"

doc/example-config-files/scalable/videobridge_sip-communicator.properties.scalable.example

+org.ice4j.ice.harvest.DISABLE_AWS_HARVESTER=true
+org.ice4j.ice.harvest.STUN_MAPPING_HARVESTER_ADDRESSES=meet-jit-si-turnrelay.jitsi.net:443
+
+org.jitsi.jicofo.ALWAYS_TRUST_MODE_ENABLED=true
+
+org.jitsi.videobridge.ENABLE_REST_SHUTDOWN=true
+
+# Enable broadcasting stats/presence in a MUC
+org.jitsi.videobridge.ENABLE_STATISTICS=true
+org.jitsi.videobridge.STATISTICS_TRANSPORT=muc,colibri,rest
+org.jitsi.videobridge.STATISTICS_INTERVAL=5000
+
+org.jitsi.videobridge.xmpp.user.shard-1.HOSTNAME=meet.example.com
+org.jitsi.videobridge.xmpp.user.shard-1.DOMAIN=auth.meet.example.com
+org.jitsi.videobridge.xmpp.user.shard-1.USERNAME=jvb
+org.jitsi.videobridge.xmpp.user.shard-1.PASSWORD=$VB_PASSWORD
+org.jitsi.videobridge.xmpp.user.shard-1.MUC_JIDS=JvbBrewery@internal.auth.meet.example.com
+org.jitsi.videobridge.xmpp.user.shard-1.MUC_NICKNAME=$NICKNAME_OF_VB
+org.jitsi.videobridge.xmpp.user.shard-1.DISABLE_CERTIFICATE_VERIFICATION=true

doc/manual-install.md

 
 There are also some complete [example config files](https://github.com/jitsi/jitsi-meet/tree/master/doc/example-config-files/) available, mentioned in each section.
 
+There are additional configurations to be done for a [scalable installation](https://github.com/jitsi/jitsi-meet/tree/master/doc/scalable-installation.md)
+
 ## Network description
 
 This is how the network looks:

doc/scalable-installation.md

+# Scalable Jitsi installation
+
+A single server Jitsi installation is good for a limited size of concurrent conferences.
+The first limiting factor is the videobridge component, that handles the actual video and audio traffic.
+It is easy to scale the video bridges horizontally by adding as many as needed.
+In a cloud based environment, additionally the bridges can be scaled up or down as needed.
+
+*NB*: The [Youtube Tutorial on Scaling](https://www.youtube.com/watch?v=LyGV4uW8km8) is outdated and describes an old configuration method.
+
+*NB*: Building a scalable infrastructure is not a task for beginning Jitsi Administrators.
+The instructions assume that you have installed a single node version successfully, and that
+you are comfortable installing, configuring and debugging Linux software.
+This is not a step-by-step guide, but will show you, which packages to install and which
+configurations to change. Use the [manual install](https://github.com/jitsi/jitsi-meet/blob/master/doc/manual-install.md) for
+details on how to setup Jitsi on a single host.
+It is highly recommended to use configuration management tools like Ansible or Puppet to manage the
+installation and configuration.
+
+## Architecture (Single Jitsi-Meet, multiple videobridges)
+
+A first step is to split the functions of the central jitsi-meet instance (with nginx, prosody and jicofo) and
+videobridges.
+
+A simplified diagram (with open network ports) of an installation with one Jitsi-Meet instance and three
+videobridges that are load balanced looks as follows. Each box is a server/VM.
+
+```
+               +                                       +
+               |                                       |
+               |                                       |
+               v                                       v
+          80, 443 TCP                          443 TCP, 10000 UDP
+       +--------------+                     +---------------------+
+       |  nginx       |  5222, 5347 TCP     |                     |
+       |  jitsi-meet  |<-------------------+|  jitsi-videobridge  |
+       |  prosody     |         |           |                     |
+       |  jicofo      |         |           +---------------------+
+       +--------------+         |
+                                |           +---------------------+
+                                |           |                     |
+                                +----------+|  jitsi-videobridge  |
+                                |           |                     |
+                                |           +---------------------+
+                                |
+                                |           +---------------------+
+                                |           |                     |
+                                +----------+|  jitsi-videobridge  |
+                                            |                     |
+                                            +---------------------+
+```
+
+## Machine Sizing
+
+The Jitsi-Meet server will generally not have that much load (unless you have many) conferences
+going at the same time. A 4 CPU, 8 GB machine will probably be fine.
+
+The videobridges will have more load. 4 or 8 CPU with 8 GB RAM seems to be a good configuration.
+
+
+### Installation of Jitsi-Meet
+
+Assuming that the installation will run under the following FQDN: `meet.example.com` and you have
+SSL cert and key in `/etc/ssl/meet.example.com.{crt,key}`
+
+Set the following DebConf variables prior to installing the packages.
+(We are not installing the `jitsi-meet` package which would handle that for us)
+
+Install the `debconf-utils` package
+
+```
+$ cat << EOF | sudo debconf-set-selections
+jitsi-videobridge	jitsi-videobridge/jvb-hostname	string	meet.example.com
+jitsi-meet	jitsi-meet/jvb-serve	boolean	false
+jitsi-meet-prosody	jitsi-videobridge/jvb-hostname	string	meet.example.com
+jitsi-meet-web-config	jitsi-meet/cert-choice	select	I want to use my own certificate
+jitsi-meet-web-config	jitsi-meet/cert-path-crt	string	/etc/ssl/meet.example.com.crt
+jitsi-meet-web-config	jitsi-meet/cert-path-key	string	/etc/ssl/meet.example.com.key
+EOF
+```
+
+On the jitsi-meet server, install the following packages:
+
+* `nginx`
+* `prosody`
+* `jicofo`
+* `jitsi-meet-web`
+* `jitsi-meet-prosody`
+* `jitsi-meet-web-config`
+
+### Installation of Videobridge(s)
+
+For simplicities sake, set the same `debconf` variables as above and install
+
+* `jitsi-videobridge2`
+
+### Configuration of jitsi-meet
+
+#### Firewall
+
+Open the following ports:
+
+Open to world:
+
+* 80 TCP
+* 443 TCP
+
+Open to the videobridges only
+
+* 5222 TCP (for Prosody)
+* 5437 TCP (for Jicofo)
+
+
+#### NGINX
+
+Create the `/etc/nginx/sites-available/meet.example.com.conf` as usual
+
+#### Prosody
+
+Follow the steps in the [manual install](https://github.com/jitsi/jitsi-meet/blob/master/doc/manual-install.md) for setup tasks
+
+You will need to adapt the following files (see the files in `example-config-files/scalable`)
+
+* `/etc/prosody/prosody.cfg.lua`
+* `/etc/prosody/conf.avail/meet.example.com.cfg.lua`
+
+#### Jitsi-Meet
+
+Adapt `/usr/share/jitsi-meet/config.js` and `/usr/share/jitsi-meet/interface-config.js` to your specific needs
+
+#### Jicofo
+
+You will need to adapt the following files (see the files in `example-config-files/scalable`)
+
+* `/etc/jitsi/jicofo/config` (hostname, jicofo_secret, jicofo_password)
+* `/etc/jitsi/jicofo/sip-communicator.properties` (hostname)
+
+### Configuration of the Videobridge
+
+#### Firewall
+
+Open the following ports:
+
+Open to world:
+
+* 443 TCP
+* 10000 UDP
+
+#### jitsi-videobridge2
+
+You will need to adapt the following files (see the files in `example-config-files/scalable`)
+
+Each videobridge will have to have it's own, unique nickname
+
+* `/etc/jitsi/videobridge/config` (hostname, password)
+* `/etc/jitsi/jicofo/sip-communicator.properties` (hostname of jitsi-meet, nickname of videobridge, vb_password)
+
+With the latest stable (April 2020) videobridge, it is no longer necessary to set public and private IP
+adresses in the `sip-communicator.properties` as the bridge will figure out the correct configuration by itself.
+
+## Testing
+
+After restarting all services (`prosody`, `jicofo` and all the `jitsi-videobridge2`) you can see in
+`/var/log/prosody/prosody.log` and
+`/var/log/jitsi/jicofo.log` that the videobridges connect to Prososy and that Jicofo picks them up.
+
+When a new conference starts, Jicofo picks a videobridge and schedules the conference on it.