Search (#36)

First implementation of search functionality for remote account and status lookups.

PROGRESS.md

@@ -100,7 +100,7 @@
   * [ ] Timelines
     * [ ] /api/v1/timelines/public GET                      (See the public/federated timeline)
     * [ ] /api/v1/timelines/tag/:hashtag GET                (Get public statuses that use hashtag)
-    * [ ] /api/v1/timelines/home GET                        (View statuses from followed users)
+    * [x] /api/v1/timelines/home GET                        (View statuses from followed users)
     * [ ] /api/v1/timelines/list/:list_id GET               (Get statuses in given list)
   * [ ] Conversations
     * [ ] /api/v1/conversations GET                         (Get a list of direct message convos)
@@ -121,8 +121,8 @@
   * [ ] Streaming
     * [ ] /api/v1/streaming WEBSOCKETS                      (Stream live events to user via websockets)
   * [ ] Notifications
-    * [ ] /api/v1/notifications GET                         (Get list of notifications)
+    * [x] /api/v1/notifications GET                         (Get list of notifications)
-    * [ ] /api/v1/notifications/:id GET                     (Get a single notification)
+    * [x] /api/v1/notifications/:id GET                     (Get a single notification)
     * [ ] /api/v1/notifications/clear POST                  (Clear all notifications)
     * [ ] /api/v1/notifications/:id POST                    (Clear a single notification)
   * [ ] Push
@@ -130,8 +130,8 @@
     * [ ] /api/v1/push/subscription GET                     (Get current subscription)
     * [ ] /api/v1/push/subscription PUT                     (Change notification types)
     * [ ] /api/v1/push/subscription DELETE                  (Delete current subscription)
-  * [ ] Search
+  * [x] Search
-    * [ ] /api/v2/search GET                                (Get search query results)
+    * [x] /api/v2/search GET                                (Get search query results)
   * [ ] Instance
     * [x] /api/v1/instance GET                              (Get instance information)
     * [ ] /api/v1/instance PATCH                            (Update instance information)
@@ -174,7 +174,7 @@
   * [ ] Federation modes
     * [ ] 'Slow' federation
       * [ ] Reputation scoring system for instances
-    * [ ] 'Greedy' federation
+    * [x] 'Greedy' federation
     * [ ] No federation (insulate this instance from the Fediverse)
       * [ ] Allowlist
   * [x] Secure HTTP signatures (creation and validation)

internal/api/client/search/search.go

@@ -0,0 +1,88 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package search
+
+import (
+	"net/http"
+
+	"github.com/sirupsen/logrus"
+	"github.com/superseriousbusiness/gotosocial/internal/api"
+	"github.com/superseriousbusiness/gotosocial/internal/config"
+	"github.com/superseriousbusiness/gotosocial/internal/message"
+	"github.com/superseriousbusiness/gotosocial/internal/router"
+)
+
+const (
+	// BasePath is the base path for serving v1 of the search API
+	BasePathV1 = "/api/v1/search"
+
+	// BasePathV2 is the base path for serving v2 of the search API
+	BasePathV2 = "/api/v2/search"
+
+	// AccountIDKey -- If provided, statuses returned will be authored only by this account
+	AccountIDKey = "account_id"
+	// MaxIDKey -- Return results older than this id
+	MaxIDKey = "max_id"
+	// MinIDKey -- Return results immediately newer than this id
+	MinIDKey = "min_id"
+	// TypeKey -- Enum(accounts, hashtags, statuses)
+	TypeKey = "type"
+	// ExcludeUnreviewedKey -- Filter out unreviewed tags? Defaults to false. Use true when trying to find trending tags.
+	ExcludeUnreviewedKey = "exclude_unreviewed"
+	// QueryKey -- The search query
+	QueryKey = "q"
+	// ResolveKey -- Attempt WebFinger lookup. Defaults to false.
+	ResolveKey = "resolve"
+	// LimitKey -- Maximum number of results to load, per type. Defaults to 20. Max 40.
+	LimitKey = "limit"
+	// OffsetKey -- Offset in search results. Used for pagination. Defaults to 0.
+	OffsetKey = "offset"
+	// FollowingKey -- Only include accounts that the user is following. Defaults to false.
+	FollowingKey = "following"
+
+	// TypeAccounts --
+	TypeAccounts = "accounts"
+	// TypeHashtags --
+	TypeHashtags = "hashtags"
+	// TypeStatuses --
+	TypeStatuses = "statuses"
+)
+
+// Module implements the ClientAPIModule interface for everything related to searching
+type Module struct {
+	config    *config.Config
+	processor message.Processor
+	log       *logrus.Logger
+}
+
+// New returns a new search module
+func New(config *config.Config, processor message.Processor, log *logrus.Logger) api.ClientModule {
+	return &Module{
+		config:    config,
+		processor: processor,
+		log:       log,
+	}
+}
+
+// Route attaches all routes from this module to the given router
+func (m *Module) Route(r router.Router) error {
+	r.AttachHandler(http.MethodGet, BasePathV1, m.SearchGETHandler)
+	r.AttachHandler(http.MethodGet, BasePathV2, m.SearchGETHandler)
+	return nil
+}

internal/api/client/search/searchget.go

@@ -0,0 +1,151 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package search
+
+import (
+	"fmt"
+	"net/http"
+	"strconv"
+
+	"github.com/gin-gonic/gin"
+	"github.com/sirupsen/logrus"
+	"github.com/superseriousbusiness/gotosocial/internal/api/model"
+	"github.com/superseriousbusiness/gotosocial/internal/oauth"
+)
+
+// SearchGETHandler handles searches for local and remote accounts, statuses, and hashtags.
+// It corresponds to the mastodon endpoint described here: https://docs.joinmastodon.org/methods/search/
+func (m *Module) SearchGETHandler(c *gin.Context) {
+	l := m.log.WithFields(logrus.Fields{
+		"func":        "SearchGETHandler",
+		"request_uri": c.Request.RequestURI,
+		"user_agent":  c.Request.UserAgent(),
+		"origin_ip":   c.ClientIP(),
+	})
+	l.Debugf("entering function")
+
+	authed, err := oauth.Authed(c, true, true, true, true) // we don't really need an app here but we want everything else
+	if err != nil {
+		l.Errorf("error authing search request: %s", err)
+		c.JSON(http.StatusBadRequest, gin.H{"error": "not authed"})
+		return
+	}
+
+	accountID := c.Query(AccountIDKey)
+	maxID := c.Query(MaxIDKey)
+	minID := c.Query(MinIDKey)
+	searchType := c.Query(TypeKey)
+
+	excludeUnreviewed := false
+	excludeUnreviewedString := c.Query(ExcludeUnreviewedKey)
+	if excludeUnreviewedString != "" {
+		var err error
+		excludeUnreviewed, err = strconv.ParseBool(excludeUnreviewedString)
+		if err != nil {
+			c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("couldn't parse param %s: %s", excludeUnreviewedString, err)})
+			return
+		}
+	}
+
+	query := c.Query(QueryKey)
+	if query == "" {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "query parameter q was empty"})
+		return
+	}
+
+	resolve := false
+	resolveString := c.Query(ResolveKey)
+	if resolveString != "" {
+		var err error
+		resolve, err = strconv.ParseBool(resolveString)
+		if err != nil {
+			c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("couldn't parse param %s: %s", resolveString, err)})
+			return
+		}
+	}
+
+	limit := 20
+	limitString := c.Query(LimitKey)
+	if limitString != "" {
+		i, err := strconv.ParseInt(limitString, 10, 64)
+		if err != nil {
+			l.Debugf("error parsing limit string: %s", err)
+			c.JSON(http.StatusBadRequest, gin.H{"error": "couldn't parse limit query param"})
+			return
+		}
+		limit = int(i)
+	}
+	if limit > 40 {
+		limit = 40
+	}
+	if limit < 1 {
+		limit = 1
+	}
+
+	offset := 0
+	offsetString := c.Query(OffsetKey)
+	if offsetString != "" {
+		i, err := strconv.ParseInt(offsetString, 10, 64)
+		if err != nil {
+			l.Debugf("error parsing offset string: %s", err)
+			c.JSON(http.StatusBadRequest, gin.H{"error": "couldn't parse offset query param"})
+			return
+		}
+		offset = int(i)
+	}
+	if limit > 40 {
+		limit = 40
+	}
+	if limit < 1 {
+		limit = 1
+	}
+
+	following := false
+	followingString := c.Query(FollowingKey)
+	if followingString != "" {
+		var err error
+		following, err = strconv.ParseBool(followingString)
+		if err != nil {
+			c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("couldn't parse param %s: %s", followingString, err)})
+			return
+		}
+	}
+
+	searchQuery := &model.SearchQuery{
+		AccountID:         accountID,
+		MaxID:             maxID,
+		MinID:             minID,
+		Type:              searchType,
+		ExcludeUnreviewed: excludeUnreviewed,
+		Query:             query,
+		Resolve:           resolve,
+		Limit:             limit,
+		Offset:            offset,
+		Following:         following,
+	}
+
+	results, errWithCode := m.processor.SearchGet(authed, searchQuery)
+	if errWithCode != nil {
+		l.Debugf("error searching: %s", errWithCode.Error())
+		c.JSON(errWithCode.Code(), gin.H{"error": errWithCode.Safe()})
+		return
+	}
+
+	c.JSON(http.StatusOK, results)
+}

internal/api/model/search.go

@@ -0,0 +1,52 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package model
+
+// SearchQuery corresponds to search parameters as submitted through the client API.
+// See https://docs.joinmastodon.org/methods/search/
+type SearchQuery struct {
+	// If provided, statuses returned will be authored only by this account
+	AccountID string
+	// Return results older than this id
+	MaxID string
+	// Return results immediately newer than this id
+	MinID string
+	// Enum(accounts, hashtags, statuses)
+	Type string
+	// Filter out unreviewed tags? Defaults to false. Use true when trying to find trending tags.
+	ExcludeUnreviewed bool
+	// The search query
+	Query string
+	// Attempt WebFinger lookup. Defaults to false.
+	Resolve bool
+	// Maximum number of results to load, per type. Defaults to 20. Max 40.
+	Limit int
+	// Offset in search results. Used for pagination. Defaults to 0.
+	Offset int
+	// Only include accounts that the user is following. Defaults to false.
+	Following bool
+}
+
+// SearchResult corresponds to a search result, containing accounts, statuses, and hashtags.
+// See https://docs.joinmastodon.org/methods/search/
+type SearchResult struct {
+	Accounts []Account `json:"accounts"`
+	Statuses []Status  `json:"statuses"`
+	Hashtags []Tag     `json:"hashtags"`
+}

internal/db/db.go

@@ -45,8 +45,9 @@ func (e ErrAlreadyExists) Error() string {
 }
 
 type Where struct {
-	Key   string
+	Key             string
-	Value interface{}
+	Value           interface{}
+	CaseInsensitive bool
 }
 
 // DB provides methods for interacting with an underlying database or other storage mechanism (for now, just postgres).

internal/db/pg/pg.go

@@ -223,7 +223,12 @@ func (ps *postgresService) GetWhere(where []db.Where, i interface{}) error {
 
 	q := ps.conn.Model(i)
 	for _, w := range where {
-		q = q.Where("? = ?", pg.Safe(w.Key), w.Value)
+		if w.CaseInsensitive {
+			q = q.Where("LOWER(?) = LOWER(?)", pg.Safe(w.Key), w.Value)
+		} else {
+			q = q.Where("? = ?", pg.Safe(w.Key), w.Value)
+		}
+
 	}
 
 	if err := q.Select(); err != nil {
@@ -1143,7 +1148,6 @@ func (ps *postgresService) GetNotificationsForAccount(accountID string, limit in
 
 	q := ps.conn.Model(&notifications).Where("target_account_id = ?", accountID)
 
-
 	if maxID != "" {
 		n := &gtsmodel.Notification{}
 		if err := ps.conn.Model(n).Where("id = ?", maxID).Select(); err != nil {

internal/federation/commonbehavior.go

@@ -25,6 +25,7 @@
 	"net/url"
 
 	"github.com/go-fed/activity/pub"
+	"github.com/go-fed/activity/streams"
 	"github.com/go-fed/activity/streams/vocab"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/util"
@@ -59,7 +60,7 @@
 func (f *federator) AuthenticateGetInbox(ctx context.Context, w http.ResponseWriter, r *http.Request) (context.Context, bool, error) {
 	// IMPLEMENTATION NOTE: For GoToSocial, we serve GETS to outboxes and inboxes through
 	// the CLIENT API, not through the federation API, so we just do nothing here.
-	return nil, false, nil
+	return ctx, false, nil
 }
 
 // AuthenticateGetOutbox delegates the authentication of a GET to an
@@ -84,7 +85,7 @@ func (f *federator) AuthenticateGetInbox(ctx context.Context, w http.ResponseWri
 func (f *federator) AuthenticateGetOutbox(ctx context.Context, w http.ResponseWriter, r *http.Request) (context.Context, bool, error) {
 	// IMPLEMENTATION NOTE: For GoToSocial, we serve GETS to outboxes and inboxes through
 	// the CLIENT API, not through the federation API, so we just do nothing here.
-	return nil, false, nil
+	return ctx, false, nil
 }
 
 // GetOutbox returns the OrderedCollection inbox of the actor for this
@@ -98,7 +99,7 @@ func (f *federator) AuthenticateGetOutbox(ctx context.Context, w http.ResponseWr
 func (f *federator) GetOutbox(ctx context.Context, r *http.Request) (vocab.ActivityStreamsOrderedCollectionPage, error) {
 	// IMPLEMENTATION NOTE: For GoToSocial, we serve GETS to outboxes and inboxes through
 	// the CLIENT API, not through the federation API, so we just do nothing here.
-	return nil, nil
+	return streams.NewActivityStreamsOrderedCollectionPage(), nil
 }
 
 // NewTransport returns a new Transport on behalf of a specific actor.

internal/federation/federatingdb/followers.go

@@ -10,6 +10,7 @@
 	"github.com/sirupsen/logrus"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/util"
 )
 
 // Followers obtains the Followers Collection for an actor with the
@@ -28,8 +29,17 @@ func (f *federatingDB) Followers(c context.Context, actorIRI *url.URL) (follower
 	l.Debugf("entering FOLLOWERS function with actorIRI %s", actorIRI.String())
 
 	acct := &gtsmodel.Account{}
-	if err := f.db.GetWhere([]db.Where{{Key: "uri", Value: actorIRI.String()}}, acct); err != nil {
+
-		return nil, fmt.Errorf("db error getting account with uri %s: %s", actorIRI.String(), err)
+	if util.IsUserPath(actorIRI) {
+		if err := f.db.GetWhere([]db.Where{{Key: "uri", Value: actorIRI.String()}}, acct); err != nil {
+			return nil, fmt.Errorf("db error getting account with uri %s: %s", actorIRI.String(), err)
+		}
+	} else if util.IsFollowersPath(actorIRI) {
+		if err := f.db.GetWhere([]db.Where{{Key: "followers_uri", Value: actorIRI.String()}}, acct); err != nil {
+			return nil, fmt.Errorf("db error getting account with followers uri %s: %s", actorIRI.String(), err)
+		}
+	} else {
+		return nil, fmt.Errorf("could not parse actor IRI %s as users or followers path", actorIRI.String())
 	}
 
 	acctFollowers := []gtsmodel.Follow{}

internal/federation/federatingdb/following.go

@@ -10,6 +10,7 @@
 	"github.com/sirupsen/logrus"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/util"
 )
 
 // Following obtains the Following Collection for an actor with the
@@ -28,8 +29,16 @@ func (f *federatingDB) Following(c context.Context, actorIRI *url.URL) (followin
 	l.Debugf("entering FOLLOWING function with actorIRI %s", actorIRI.String())
 
 	acct := &gtsmodel.Account{}
-	if err := f.db.GetWhere([]db.Where{{Key: "uri", Value: actorIRI.String()}}, acct); err != nil {
+	if util.IsUserPath(actorIRI) {
-		return nil, fmt.Errorf("db error getting account with uri %s: %s", actorIRI.String(), err)
+		if err := f.db.GetWhere([]db.Where{{Key: "uri", Value: actorIRI.String()}}, acct); err != nil {
+			return nil, fmt.Errorf("db error getting account with uri %s: %s", actorIRI.String(), err)
+		}
+	} else if util.IsFollowingPath(actorIRI) {
+		if err := f.db.GetWhere([]db.Where{{Key: "following_uri", Value: actorIRI.String()}}, acct); err != nil {
+			return nil, fmt.Errorf("db error getting account with following uri %s: %s", actorIRI.String(), err)
+		}
+	} else {
+		return nil, fmt.Errorf("could not parse actor IRI %s as users or following path", actorIRI.String())
 	}
 
 	acctFollowing := []gtsmodel.Follow{}

internal/federation/federatingdb/lock.go

@@ -42,6 +42,10 @@ func (f *federatingDB) Lock(c context.Context, id *url.URL) error {
 
 	// Strategy: create a new lock, if stored, continue. Otherwise, lock the
 	// existing mutex.
+	if id == nil {
+		return errors.New("Lock: id was nil")
+	}
+
 	mu := &sync.Mutex{}
 	mu.Lock() // Optimistically lock if we do store it.
 	i, loaded := f.locks.LoadOrStore(id.String(), mu)
@@ -59,6 +63,9 @@ func (f *federatingDB) Lock(c context.Context, id *url.URL) error {
 func (f *federatingDB) Unlock(c context.Context, id *url.URL) error {
 	// Once Go-Fed is done calling Database methods, the relevant `id`
 	// entries are unlocked.
+	if id == nil {
+		return errors.New("Unlock: id was nil")
+	}
 
 	i, ok := f.locks.Load(id.String())
 	if !ok {

internal/federation/federatingprotocol.go

@@ -26,6 +26,7 @@
 	"net/url"
 
 	"github.com/go-fed/activity/pub"
+	"github.com/go-fed/activity/streams"
 	"github.com/go-fed/activity/streams/vocab"
 	"github.com/sirupsen/logrus"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
@@ -310,7 +311,7 @@ func (f *federator) MaxDeliveryRecursionDepth(ctx context.Context) int {
 // logic to be used, but the implementation must not modify it.
 func (f *federator) FilterForwarding(ctx context.Context, potentialRecipients []*url.URL, a pub.Activity) ([]*url.URL, error) {
 	// TODO
-	return nil, nil
+	return []*url.URL{}, nil
 }
 
 // GetInbox returns the OrderedCollection inbox of the actor for this
@@ -324,5 +325,5 @@ func (f *federator) FilterForwarding(ctx context.Context, potentialRecipients []
 func (f *federator) GetInbox(ctx context.Context, r *http.Request) (vocab.ActivityStreamsOrderedCollectionPage, error) {
 	// IMPLEMENTATION NOTE: For GoToSocial, we serve GETS to outboxes and inboxes through
 	// the CLIENT API, not through the federation API, so we just do nothing here.
-	return nil, nil
+	return streams.NewActivityStreamsOrderedCollectionPage(), nil
 }

internal/federation/federator.go

@@ -40,6 +40,9 @@ type Federator interface {
 	// AuthenticateFederatedRequest can be used to check the authenticity of incoming http-signed requests for federating resources.
 	// The given username will be used to create a transport for making outgoing requests. See the implementation for more detailed comments.
 	AuthenticateFederatedRequest(username string, r *http.Request) (*url.URL, error)
+	// FingerRemoteAccount performs a webfinger lookup for a remote account, using the .well-known path. It will return the ActivityPub URI for that
+	// account, or an error if it doesn't exist or can't be retrieved.
+	FingerRemoteAccount(requestingUsername string, targetUsername string, targetDomain string) (*url.URL, error)
 	// DereferenceRemoteAccount can be used to get the representation of a remote account, based on the account ID (which is a URI).
 	// The given username will be used to create a transport for making outgoing requests. See the implementation for more detailed comments.
 	DereferenceRemoteAccount(username string, remoteAccountID *url.URL) (typeutils.Accountable, error)

internal/federation/finger.go

@@ -0,0 +1,69 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package federation
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/url"
+	"strings"
+
+	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
+)
+
+func (f *federator) FingerRemoteAccount(requestingUsername string, targetUsername string, targetDomain string) (*url.URL, error) {
+
+	t, err := f.GetTransportForUser(requestingUsername)
+	if err != nil {
+		return nil, fmt.Errorf("FingerRemoteAccount: error getting transport for username %s while dereferencing @%s@%s: %s", requestingUsername, targetUsername, targetDomain, err)
+	}
+
+	b, err := t.Finger(context.Background(), targetUsername, targetDomain)
+	if err != nil {
+		return nil, fmt.Errorf("FingerRemoteAccount: error doing request on behalf of username %s while dereferencing @%s@%s: %s", requestingUsername, targetUsername, targetDomain, err)
+	}
+
+	resp := &apimodel.WebfingerAccountResponse{}
+	if err := json.Unmarshal(b, resp); err != nil {
+		return nil, fmt.Errorf("FingerRemoteAccount: could not unmarshal server response as WebfingerAccountResponse on behalf of username %s while dereferencing @%s@%s: %s", requestingUsername, targetUsername, targetDomain, err)
+	}
+
+	if len(resp.Links) == 0 {
+		return nil, fmt.Errorf("FingerRemoteAccount: no links found in webfinger response %s", string(b))
+	}
+
+	// look through the links for the first one that matches "application/activity+json", this is what we need
+	for _, l := range resp.Links {
+		if strings.EqualFold(l.Type, "application/activity+json") {
+			if l.Href == "" || l.Rel != "self" {
+				continue
+			}
+			accountURI, err := url.Parse(l.Href)
+			if err != nil {
+				return nil, fmt.Errorf("FingerRemoteAccount: couldn't parse url %s: %s", l.Href, err)
+			}
+			// found it!
+			return accountURI, nil
+		}
+	}
+
+	return nil, errors.New("FingerRemoteAccount: no match found in webfinger response")
+}

internal/gotosocial/actions.go

@@ -38,6 +38,7 @@
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/instance"
 	mediaModule "github.com/superseriousbusiness/gotosocial/internal/api/client/media"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/notification"
+	"github.com/superseriousbusiness/gotosocial/internal/api/client/search"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/status"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/timeline"
 	"github.com/superseriousbusiness/gotosocial/internal/api/s2s/user"
@@ -122,6 +123,7 @@
 	usersModule := user.New(c, processor, log)
 	timelineModule := timeline.New(c, processor, log)
 	notificationModule := notification.New(c, processor, log)
+	searchModule := search.New(c, processor, log)
 	mm := mediaModule.New(c, processor, log)
 	fileServerModule := fileserver.New(c, processor, log)
 	adminModule := admin.New(c, processor, log)
@@ -146,6 +148,7 @@
 		usersModule,
 		timelineModule,
 		notificationModule,
+		searchModule,
 	}
 
 	for _, m := range apis {

internal/message/fromfederatorprocess.go

@@ -105,7 +105,9 @@ func (p *processor) processFromFederator(federatorMsg gtsmodel.FromFederator) er
 			}
 
 			if err := p.db.Put(incomingAnnounce); err != nil {
-				return fmt.Errorf("error adding dereferenced announce to the db: %s", err)
+				if _, ok := err.(db.ErrAlreadyExists); !ok {
+					return fmt.Errorf("error adding dereferenced announce to the db: %s", err)
+				}
 			}
 
 			if err := p.notifyAnnounce(incomingAnnounce); err != nil {
@@ -407,7 +409,7 @@ func (p *processor) dereferenceAnnounce(announce *gtsmodel.Status, requestingUse
 	}
 
 	// now dereference additional fields straight away (we're already async here so we have time)
-	if err := p.dereferenceStatusFields(boostedStatus, requestingUsername); err !=  nil {
+	if err := p.dereferenceStatusFields(boostedStatus, requestingUsername); err != nil {
 		return fmt.Errorf("dereferenceAnnounce: error dereferencing status fields for status with id %s: %s", announce.GTSBoostedStatus.URI, err)
 	}
 

internal/message/processor.go

@@ -109,6 +109,9 @@ type Processor interface {
 	// NotificationsGet
 	NotificationsGet(authed *oauth.Auth, limit int, maxID string) ([]*apimodel.Notification, ErrorWithCode)
 
+	// SearchGet performs a search with the given params, resolving/dereferencing remotely as desired
+	SearchGet(authed *oauth.Auth, searchQuery *apimodel.SearchQuery) (*apimodel.SearchResult, ErrorWithCode)
+
 	// StatusCreate processes the given form to create a new status, returning the api model representation of that status if it's OK.
 	StatusCreate(authed *oauth.Auth, form *apimodel.AdvancedStatusCreateForm) (*apimodel.Status, error)
 	// StatusDelete processes the delete of a given status, returning the deleted status if the delete goes through.

internal/message/searchprocess.go

@@ -0,0 +1,292 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package message
+
+import (
+	"errors"
+	"net/url"
+	"strings"
+
+	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/oauth"
+	"github.com/superseriousbusiness/gotosocial/internal/util"
+)
+
+func (p *processor) SearchGet(authed *oauth.Auth, searchQuery *apimodel.SearchQuery) (*apimodel.SearchResult, ErrorWithCode) {
+	results := &apimodel.SearchResult{
+		Accounts: []apimodel.Account{},
+		Statuses: []apimodel.Status{},
+		Hashtags: []apimodel.Tag{},
+	}
+	foundAccounts := []*gtsmodel.Account{}
+	foundStatuses := []*gtsmodel.Status{}
+	// foundHashtags := []*gtsmodel.Tag{}
+
+	// convert the query to lowercase and trim leading/trailing spaces
+	query := strings.ToLower(strings.TrimSpace(searchQuery.Query))
+
+	// check if the query is a URI and just do a lookup for that, straight up
+	if uri, err := url.Parse(query); err == nil {
+		// 1. check if it's a status
+		foundStatus, err := p.searchStatusByURI(authed, uri, searchQuery.Resolve)
+		if err != nil {
+			return nil, NewErrorInternalError(err)
+		}
+		if foundStatus != nil {
+			foundStatuses = append(foundStatuses, foundStatus)
+		}
+
+		// 2. check if it's an account
+		foundAccount, err := p.searchAccountByURI(authed, uri, searchQuery.Resolve)
+		if err != nil {
+			return nil, NewErrorInternalError(err)
+		}
+		if foundAccount != nil {
+			foundAccounts = append(foundAccounts, foundAccount)
+		}
+	}
+
+	// check if the query is something like @whatever_username@example.org -- this means it's a remote account
+	if util.IsMention(searchQuery.Query) {
+		foundAccount, err := p.searchAccountByMention(authed, searchQuery.Query, searchQuery.Resolve)
+		if err != nil {
+			return nil, NewErrorInternalError(err)
+		}
+		if foundAccount != nil {
+			foundAccounts = append(foundAccounts, foundAccount)
+		}
+	}
+
+	/*
+		FROM HERE ON we have our search results, it's just a matter of filtering them according to what this user is allowed to see,
+		and then converting them into our frontend format.
+	*/
+	for _, foundAccount := range foundAccounts {
+		// make sure there's no block in either direction between the account and the requester
+		if blocked, err := p.db.Blocked(authed.Account.ID, foundAccount.ID); err == nil && !blocked {
+			// all good, convert it and add it to the results
+			acctMasto, err := p.tc.AccountToMastoPublic(foundAccount)
+			if err != nil {
+				return nil, NewErrorInternalError(err)
+			}
+			results.Accounts = append(results.Accounts, *acctMasto)
+		}
+	}
+
+	for _, foundStatus := range foundStatuses {
+		statusOwner := &gtsmodel.Account{}
+		if err := p.db.GetByID(foundStatus.AccountID, statusOwner); err != nil {
+			continue
+		}
+
+		relevantAccounts, err := p.db.PullRelevantAccountsFromStatus(foundStatus)
+		if err != nil {
+			continue
+		}
+		if visible, err := p.db.StatusVisible(foundStatus, statusOwner, authed.Account, relevantAccounts); !visible || err != nil {
+			continue
+		}
+
+		statusMasto, err := p.tc.StatusToMasto(foundStatus, statusOwner, authed.Account, relevantAccounts.BoostedAccount, relevantAccounts.ReplyToAccount, nil)
+		if err != nil {
+			continue
+		}
+
+		results.Statuses = append(results.Statuses, *statusMasto)
+	}
+
+	return results, nil
+}
+
+func (p *processor) searchStatusByURI(authed *oauth.Auth, uri *url.URL, resolve bool) (foundStatus *gtsmodel.Status, err error) {
+	// 1. check if it's a status
+	maybeStatus := &gtsmodel.Status{}
+	if err = p.db.GetWhere([]db.Where{{Key: "uri", Value: uri.String(), CaseInsensitive: true}}, maybeStatus); err == nil {
+		// we have it and it's a status
+		foundStatus = maybeStatus
+		return
+	} else if err = p.db.GetWhere([]db.Where{{Key: "url", Value: uri.String(), CaseInsensitive: true}}, maybeStatus); err == nil {
+		// we have it and it's a status
+		foundStatus = maybeStatus
+		return
+	}
+
+	// we don't have it locally so dereference it if we're allowed to
+	if resolve {
+		statusable, err := p.federator.DereferenceRemoteStatus(authed.Account.Username, uri)
+		if err == nil {
+			// it IS a status!
+
+			// extract the status owner's IRI from the statusable
+			var statusOwnerURI *url.URL
+			statusAttributedTo := statusable.GetActivityStreamsAttributedTo()
+			for i := statusAttributedTo.Begin(); i != statusAttributedTo.End(); i = i.Next() {
+				if i.IsIRI() {
+					statusOwnerURI = i.GetIRI()
+					break
+				}
+			}
+			if statusOwnerURI == nil {
+				return nil, NewErrorInternalError(errors.New("couldn't extract ownerAccountURI from statusable"))
+			}
+
+			// make sure the status owner exists in the db by searching for it
+			_, err := p.searchAccountByURI(authed, statusOwnerURI, resolve)
+			if err != nil {
+				return nil, err
+			}
+
+			// we have the status owner, we have the dereferenced status, so now we should finish dereferencing the status properly
+
+			// first turn it into a gtsmodel.Status
+			status, err := p.tc.ASStatusToStatus(statusable)
+			if err != nil {
+				return nil, NewErrorInternalError(err)
+			}
+
+			// put it in the DB so it gets a UUID
+			if err := p.db.Put(status); err != nil {
+				return nil, NewErrorInternalError(err)
+			}
+
+			// properly dereference everything in the status (media attachments etc)
+			if err := p.dereferenceStatusFields(status, authed.Account.Username); err != nil {
+				return nil, NewErrorInternalError(err)
+			}
+
+			// update with the nicely dereferenced status
+			if err := p.db.UpdateByID(status.ID, status); err != nil {
+				return nil, NewErrorInternalError(err)
+			}
+
+			foundStatus = status
+		}
+	}
+	return
+}
+
+func (p *processor) searchAccountByURI(authed *oauth.Auth, uri *url.URL, resolve bool) (foundAccount *gtsmodel.Account, err error) {
+	maybeAccount := &gtsmodel.Account{}
+	if err = p.db.GetWhere([]db.Where{{Key: "uri", Value: uri.String(), CaseInsensitive: true}}, maybeAccount); err == nil {
+		// we have it and it's an account
+		foundAccount = maybeAccount
+		return
+	} else if err = p.db.GetWhere([]db.Where{{Key: "url", Value: uri.String(), CaseInsensitive: true}}, maybeAccount); err == nil {
+		// we have it and it's an account
+		foundAccount = maybeAccount
+		return
+	}
+	if resolve {
+		// we don't have it locally so try and dereference it
+		accountable, err := p.federator.DereferenceRemoteAccount(authed.Account.Username, uri)
+		if err == nil {
+			// it IS an account!
+			account, err := p.tc.ASRepresentationToAccount(accountable, false)
+			if err != nil {
+				return nil, NewErrorInternalError(err)
+			}
+
+			if err := p.db.Put(account); err != nil {
+				return nil, NewErrorInternalError(err)
+			}
+
+			if err := p.dereferenceAccountFields(account, authed.Account.Username, false); err != nil {
+				return nil, NewErrorInternalError(err)
+			}
+
+			foundAccount = account
+		}
+	}
+	return
+}
+
+func (p *processor) searchAccountByMention(authed *oauth.Auth, mention string, resolve bool) (foundAccount *gtsmodel.Account, err error) {
+	// query is for a remote account
+	username, domain, err := util.ExtractMentionParts(mention)
+	if err != nil {
+		return nil, NewErrorBadRequest(err)
+	}
+
+	// if it's a local account we can skip a whole bunch of stuff
+	maybeAcct := &gtsmodel.Account{}
+	if domain == p.config.Host {
+		if err = p.db.GetLocalAccountByUsername(username, maybeAcct); err != nil {
+			return
+		}
+		foundAccount = maybeAcct
+		return
+	}
+
+	// it's not a local account so first we'll check if it's in the database already...
+	where := []db.Where{
+		{Key: "username", Value: username, CaseInsensitive: true},
+		{Key: "domain", Value: domain, CaseInsensitive: true},
+	}
+	err = p.db.GetWhere(where, maybeAcct)
+	if err == nil {
+		// we've got it stored locally already!
+		foundAccount = maybeAcct
+		return
+	}
+
+	if _, ok := err.(db.ErrNoEntries); !ok {
+		// if it's  not errNoEntries there's been a real database error so bail at this point
+		return nil, NewErrorInternalError(err)
+	}
+
+	// we got a db.ErrNoEntries, so we just don't have the account locally stored -- check if we can dereference it
+	if resolve {
+		// we're allowed to resolve it so let's try
+
+		// first we need to webfinger the remote account to convert the username and domain into the activitypub URI for the account
+		acctURI, err := p.federator.FingerRemoteAccount(authed.Account.Username, username, domain)
+		if err != nil {
+			// something went wrong doing the webfinger lookup so we can't process the request
+			return nil, NewErrorInternalError(err)
+		}
+
+		// dereference the account based on the URI we retrieved from the webfinger lookup
+		accountable, err := p.federator.DereferenceRemoteAccount(authed.Account.Username, acctURI)
+		if err != nil {
+			// something went wrong doing the dereferencing so we can't process the request
+			return nil, NewErrorInternalError(err)
+		}
+
+		// convert the dereferenced account to the gts model of that account
+		foundAccount, err = p.tc.ASRepresentationToAccount(accountable, false)
+		if err != nil {
+			// something went wrong doing the conversion to a gtsmodel.Account so we can't process the request
+			return nil, NewErrorInternalError(err)
+		}
+
+		// put this new account in our database
+		if err := p.db.Put(foundAccount); err != nil {
+			return nil, NewErrorInternalError(err)
+		}
+
+		// properly dereference all the fields on the account immediately
+		if err := p.dereferenceAccountFields(foundAccount, authed.Account.Username, true); err != nil {
+			return nil, NewErrorInternalError(err)
+		}
+	}
+
+	return
+}

internal/transport/transport.go

@@ -19,6 +19,8 @@
 type Transport interface {
 	pub.Transport
 	DereferenceMedia(c context.Context, iri *url.URL, expectedContentType string) ([]byte, error)
+	// Finger performs a webfinger request with the given username and domain, and returns the bytes from the response body.
+	Finger(c context.Context, targetUsername string, targetDomains string) ([]byte, error)
 }
 
 // transport implements the Transport interface
@@ -83,3 +85,42 @@ func (t *transport) DereferenceMedia(c context.Context, iri *url.URL, expectedCo
 	}
 	return ioutil.ReadAll(resp.Body)
 }
+
+func (t *transport) Finger(c context.Context, targetUsername string, targetDomain string) ([]byte, error) {
+	l := t.log.WithField("func", "Finger")
+	urlString := fmt.Sprintf("https://%s/.well-known/webfinger?resource=acct:%s@%s", targetDomain, targetUsername, targetDomain)
+	l.Debugf("performing GET to %s", urlString)
+
+	iri, err := url.Parse(urlString)
+	if err != nil {
+		return nil, fmt.Errorf("Finger: error parsing url %s: %s", urlString, err)
+	}
+
+	l.Debugf("performing GET to %s", iri.String())
+
+	req, err := http.NewRequest("GET", iri.String(), nil)
+	if err != nil {
+		return nil, err
+	}
+	req = req.WithContext(c)
+	req.Header.Add("Accept", "application/json")
+	req.Header.Add("Accept", "application/jrd+json")
+	req.Header.Add("Date", t.clock.Now().UTC().Format("Mon, 02 Jan 2006 15:04:05")+" GMT")
+	req.Header.Add("User-Agent", fmt.Sprintf("%s %s", t.appAgent, t.gofedAgent))
+	req.Header.Set("Host", iri.Host)
+	t.getSignerMu.Lock()
+	err = t.getSigner.SignRequest(t.privkey, t.pubKeyID, req, nil)
+	t.getSignerMu.Unlock()
+	if err != nil {
+		return nil, err
+	}
+	resp, err := t.client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("GET request to %s failed (%d): %s", iri.String(), resp.StatusCode, resp.Status)
+	}
+	return ioutil.ReadAll(resp.Body)
+}

internal/typeutils/converter.go

@@ -145,7 +145,7 @@ type TypeConverter interface {
 	/*
 		WRAPPER CONVENIENCE FUNCTIONS
 	*/
-	
+
 	// WrapPersonInUpdate
 	WrapPersonInUpdate(person vocab.ActivityStreamsPerson, originAccount *gtsmodel.Account) (vocab.ActivityStreamsUpdate, error)
 }

internal/typeutils/internal.go

@@ -40,10 +40,10 @@ func (c *converter) StatusToBoost(s *gtsmodel.Status, boostingAccount *gtsmodel.
 		URL: boostWrapperStatusURL,
 
 		// the boosted status is not created now, but the boost certainly is
-		CreatedAt:                time.Now(),
+		CreatedAt: time.Now(),
-		UpdatedAt:                time.Now(),
+		UpdatedAt: time.Now(),
-		Local:                    local,
+		Local:     local,
-		AccountID:                boostingAccount.ID,
+		AccountID: boostingAccount.ID,
 
 		// replies can be boosted, but boosts are never replies
 		InReplyToID:        "",

internal/typeutils/internaltoas.go

@@ -252,6 +252,7 @@ func (c *converter) AccountToAS(a *gtsmodel.Account) (vocab.ActivityStreamsPerso
 		headerImage.SetActivityStreamsUrl(headerURLProperty)
 
 		headerProperty.AppendActivityStreamsImage(headerImage)
+		person.SetActivityStreamsImage(headerProperty)
 	}
 
 	return person, nil

internal/util/statustools.go

@@ -77,6 +77,11 @@ func ExtractMentionParts(mention string) (username, domain string, err error) {
 	return
 }
 
+// IsMention returns true if the passed string looks like @whatever@example.org
+func IsMention(mention string) bool {
+	return mentionNameRegex.MatchString(strings.ToLower(mention))
+}
+
 // unique returns a deduplicated version of a given string slice.
 func unique(s []string) []string {
 	keys := make(map[string]bool)