Replace publicroomsapi with a combination of clientapi/roomserver/currentstateserver (#1174)

* Use content_value instead of membership

* Fix build

* Replace publicroomsapi with a combination of clientapi/roomserver/currentstateserver

- All public rooms paths are now handled by clientapi
- Requests to (un)publish rooms are sent to the roomserver via `PerformPublish`
  which are stored in a new `published_table.go`
- Requests for public rooms are handled in clientapi by:
    * Fetch all room IDs which are published using `QueryPublishedRooms` on the roomserver.
    * Apply pagination parameters to the slice.
    * Do a `QueryBulkStateContent` request to the currentstateserver to pull out
      required state event *content* (not entire events).
    * Aggregate and return the chunk.

Mostly but not fully implemented (DB queries on currentstateserver are missing)

* Fix pq query

* Make postgres work

* Make sqlite work

* Fix tests

* Unbreak pagination tests

* Linting

currentstateserver/inthttp/client.go

@@ -26,8 +26,9 @@ import (
 
 // HTTP paths for the internal HTTP APIs
 const (
-	QueryCurrentStatePath = "/currentstateserver/queryCurrentState"
-	QueryRoomsForUserPath = "/currentstateserver/queryRoomsForUser"
+	QueryCurrentStatePath     = "/currentstateserver/queryCurrentState"
+	QueryRoomsForUserPath     = "/currentstateserver/queryRoomsForUser"
+	QueryBulkStateContentPath = "/currentstateserver/queryBulkStateContent"
 )
 
 // NewCurrentStateAPIClient creates a CurrentStateInternalAPI implemented by talking to a HTTP POST API.
@@ -73,3 +74,15 @@ func (h *httpCurrentStateInternalAPI) QueryRoomsForUser(
 	apiURL := h.apiURL + QueryRoomsForUserPath
 	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
 }
+
+func (h *httpCurrentStateInternalAPI) QueryBulkStateContent(
+	ctx context.Context,
+	request *api.QueryBulkStateContentRequest,
+	response *api.QueryBulkStateContentResponse,
+) error {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryBulkStateContent")
+	defer span.Finish()
+
+	apiURL := h.apiURL + QueryBulkStateContentPath
+	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+}

currentstateserver/inthttp/server.go

@@ -51,4 +51,17 @@ func AddRoutes(internalAPIMux *mux.Router, intAPI api.CurrentStateInternalAPI) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
+	internalAPIMux.Handle(QueryBulkStateContentPath,
+		httputil.MakeInternalAPI("queryBulkStateContent", func(req *http.Request) util.JSONResponse {
+			request := api.QueryBulkStateContentRequest{}
+			response := api.QueryBulkStateContentResponse{}
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.MessageResponse(http.StatusBadRequest, err.Error())
+			}
+			if err := intAPI.QueryBulkStateContent(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
 }