Adds support for listening on and connecting to I2P and Onion services securely (#3293)

This PR adds 2 `dendrite-demo` main's, each designed expressly to serve
a Hidden Service/Overlay network.

The first, `dendrite-demo-i2p` add self-configuration for use of
dendrite as an I2P hidden service(eepsite) and to connect to I2P
services(federate) as an I2P client. It further disables the `dendrite`
server from communicating with non-anonymous servers by
federation(because I2P does not canonically have the ability to exit, we
rely on donors for exit traffic), and enables the use of self-signed TLS
certificates([because I2P services are self-authenticating but TLS is
still required for other aspects of the system to work
reliably](https://tor.stackexchange.com/questions/13887/registering-onion-with-certificate-authority)).
This demo turns the system into an "pseudonymous" homeserver which
people can connect to using an I2P-enabled Matrix client(I like `cinny`
and it's what I tested with).

The second, `dendrite-demo-tor` adds self-configuration for the use of
dendrite as an Onion service and to connect to other onion services and
non-anonymous web sites using Tor to obfuscate it's physical location
and providing, optionally, pseudonymity. It also enables the use of
self-signed TLS certificates, for the same reason as with I2P, because
onion services aren't typically eligible for TLS certificates. It has
also been tested with `cinny`.

These services are both pseudonymous like myself, not anonymous. I will
be meeting members of the element team at the CCC assembly shortly to
discuss contributing under my pseudonym.

As none of the other `dendrite-demo` have unit tests I did not add them
to these checkins.

* [*] I have added Go unit tests or [Complement integration
tests](https://github.com/matrix-org/complement) for this PR _or_ I have
justified why this PR doesn't need tests

---------

Co-authored-by: eyedeekay <idk@mulder>
Co-authored-by: Till Faelligen <[email protected]>

Author: 3 people

contrib/dendrite-demo-i2p/main.go

@@ -0,0 +1,185 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"flag"
+	"os"
+	"time"
+
+	"github.com/getsentry/sentry-go"
+	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/internal/caching"
+	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/setup/jetstream"
+	"github.com/matrix-org/dendrite/setup/process"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/sirupsen/logrus"
+
+	"github.com/matrix-org/dendrite/appservice"
+	"github.com/matrix-org/dendrite/federationapi"
+	"github.com/matrix-org/dendrite/roomserver"
+	"github.com/matrix-org/dendrite/setup"
+	basepkg "github.com/matrix-org/dendrite/setup/base"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/mscs"
+	"github.com/matrix-org/dendrite/userapi"
+)
+
+var (
+	samAddr = flag.String("samaddr", "127.0.0.1:7656", "Address to connect to the I2P SAMv3 API")
+	_, skip = os.LookupEnv("CI")
+)
+
+func main() {
+	cfg := setup.ParseFlags(true)
+	if skip {
+		return
+	}
+
+	configErrors := &config.ConfigErrors{}
+	cfg.Verify(configErrors)
+	if len(*configErrors) > 0 {
+		for _, err := range *configErrors {
+			logrus.Errorf("Configuration error: %s", err)
+		}
+		logrus.Fatalf("Failed to start due to configuration errors")
+	}
+	processCtx := process.NewProcessContext()
+
+	internal.SetupStdLogging()
+	internal.SetupHookLogging(cfg.Logging)
+	internal.SetupPprof()
+
+	basepkg.PlatformSanityChecks()
+
+	logrus.Infof("Dendrite version %s", internal.VersionString())
+	if !cfg.ClientAPI.RegistrationDisabled && cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled {
+		logrus.Warn("Open registration is enabled")
+	}
+
+	// create DNS cache
+	var dnsCache *fclient.DNSCache
+	if cfg.Global.DNSCache.Enabled {
+		dnsCache = fclient.NewDNSCache(
+			cfg.Global.DNSCache.CacheSize,
+			cfg.Global.DNSCache.CacheLifetime,
+		)
+		logrus.Infof(
+			"DNS cache enabled (size %d, lifetime %s)",
+			cfg.Global.DNSCache.CacheSize,
+			cfg.Global.DNSCache.CacheLifetime,
+		)
+	}
+
+	// setup tracing
+	closer, err := cfg.SetupTracing()
+	if err != nil {
+		logrus.WithError(err).Panicf("failed to start opentracing")
+	}
+	defer closer.Close() // nolint: errcheck
+
+	// setup sentry
+	if cfg.Global.Sentry.Enabled {
+		logrus.Info("Setting up Sentry for debugging...")
+		err = sentry.Init(sentry.ClientOptions{
+			Dsn:              cfg.Global.Sentry.DSN,
+			Environment:      cfg.Global.Sentry.Environment,
+			Debug:            true,
+			ServerName:       string(cfg.Global.ServerName),
+			Release:          "dendrite@" + internal.VersionString(),
+			AttachStacktrace: true,
+		})
+		if err != nil {
+			logrus.WithError(err).Panic("failed to start Sentry")
+		}
+		go func() {
+			processCtx.ComponentStarted()
+			<-processCtx.WaitForShutdown()
+			if !sentry.Flush(time.Second * 5) {
+				logrus.Warnf("failed to flush all Sentry events!")
+			}
+			processCtx.ComponentFinished()
+		}()
+	}
+
+	federationClient := basepkg.CreateFederationClient(cfg, dnsCache)
+	httpClient := basepkg.CreateClient(cfg, dnsCache)
+
+	// prepare required dependencies
+	cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+	routers := httputil.NewRouters()
+
+	caches := caching.NewRistrettoCache(cfg.Global.Cache.EstimatedMaxSize, cfg.Global.Cache.MaxAge, caching.EnableMetrics)
+	natsInstance := jetstream.NATSInstance{}
+	rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.EnableMetrics)
+	fsAPI := federationapi.NewInternalAPI(
+		processCtx, cfg, cm, &natsInstance, federationClient, rsAPI, caches, nil, false,
+	)
+
+	keyRing := fsAPI.KeyRing()
+
+	// The underlying roomserver implementation needs to be able to call the fedsender.
+	// This is different to rsAPI which can be the http client which doesn't need this
+	// dependency. Other components also need updating after their dependencies are up.
+	rsAPI.SetFederationAPI(fsAPI, keyRing)
+
+	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federationClient, caching.EnableMetrics, fsAPI.IsBlacklistedOrBackingOff)
+	asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI)
+
+	rsAPI.SetAppserviceAPI(asAPI)
+	rsAPI.SetUserAPI(userAPI)
+
+	monolith := setup.Monolith{
+		Config:    cfg,
+		Client:    httpClient,
+		FedClient: federationClient,
+		KeyRing:   keyRing,
+
+		AppserviceAPI: asAPI,
+		// always use the concrete impl here even in -http mode because adding public routes
+		// must be done on the concrete impl not an HTTP client else fedapi will call itself
+		FederationAPI: fsAPI,
+		RoomserverAPI: rsAPI,
+		UserAPI:       userAPI,
+	}
+	monolith.AddAllPublicRoutes(processCtx, cfg, routers, cm, &natsInstance, caches, caching.EnableMetrics)
+
+	if len(cfg.MSCs.MSCs) > 0 {
+		if err := mscs.Enable(cfg, cm, routers, &monolith, caches); err != nil {
+			logrus.WithError(err).Fatalf("Failed to enable MSCs")
+		}
+	}
+
+	upCounter := prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: "dendrite",
+		Name:      "up",
+		ConstLabels: map[string]string{
+			"version": internal.VersionString(),
+		},
+	})
+	upCounter.Add(1)
+	prometheus.MustRegister(upCounter)
+
+	// Expose the matrix APIs directly rather than putting them under a /api path.
+	go func() {
+		SetupAndServeHTTPS(processCtx, cfg, routers) //, httpsAddr, nil, nil)
+	}()
+
+	// We want to block forever to let the HTTP and HTTPS handler serve the APIs
+	basepkg.WaitForShutdown(processCtx)
+}