diff --git a/docs/api/swagger.yaml b/docs/api/swagger.yaml index fa1bd7499..ec0963190 100644 --- a/docs/api/swagger.yaml +++ b/docs/api/swagger.yaml @@ -784,6 +784,35 @@ definitions: type: object x-go-name: Card x-go-package: github.com/superseriousbusiness/gotosocial/internal/api/model + domain: + description: Domain represents a remote domain + properties: + domain: + description: The hostname of the domain. + example: example.org + type: string + x-go-name: Domain + public_comment: + description: If the domain is blocked, what's the publicly-stated reason for + the block. + example: they smell + type: string + x-go-name: PublicComment + silenced_at: + description: Time at which this domain was silenced. Key will not be present + on open domains. + example: "2021-07-30T09:20:25+00:00" + type: string + x-go-name: SilencedAt + suspended_at: + description: Time at which this domain was suspended. Key will not be present + on open domains. + example: "2021-07-30T09:20:25+00:00" + type: string + x-go-name: SuspendedAt + type: object + x-go-name: Domain + x-go-package: github.com/superseriousbusiness/gotosocial/internal/api/model domainBlock: description: DomainBlock represents a block on one domain properties: @@ -798,7 +827,7 @@ definitions: type: string x-go-name: CreatedBy domain: - description: The hostname of the blocked domain. + description: The hostname of the domain. example: example.org type: string x-go-name: Domain @@ -822,16 +851,28 @@ definitions: type: string x-go-name: PrivateComment public_comment: - description: Public comment for this block, visible if domain blocks are served - publicly. + description: If the domain is blocked, what's the publicly-stated reason for + the block. example: they smell type: string x-go-name: PublicComment + silenced_at: + description: Time at which this domain was silenced. Key will not be present + on open domains. + example: "2021-07-30T09:20:25+00:00" + type: string + x-go-name: SilencedAt subscription_id: description: The ID of the subscription that created/caused this domain block. example: 01FBW25TF5J67JW3HFHZCSD23K type: string x-go-name: SubscriptionID + suspended_at: + description: Time at which this domain was suspended. Key will not be present + on open domains. + example: "2021-07-30T09:20:25+00:00" + type: string + x-go-name: SuspendedAt type: object x-go-name: DomainBlock x-go-package: github.com/superseriousbusiness/gotosocial/internal/api/model @@ -934,6 +975,8 @@ definitions: description: New account registrations require admin approval. type: boolean x-go-name: ApprovalRequired + configuration: + $ref: '#/definitions/instanceConfiguration' contact_account: $ref: '#/definitions/account' description: @@ -1021,6 +1064,126 @@ definitions: type: object x-go-name: Instance x-go-package: github.com/superseriousbusiness/gotosocial/internal/api/model + instanceConfiguration: + properties: + media_attachments: + $ref: '#/definitions/instanceConfigurationMediaAttachments' + polls: + $ref: '#/definitions/instanceConfigurationPolls' + statuses: + $ref: '#/definitions/instanceConfigurationStatuses' + title: InstanceConfiguration models instance configuration parameters. + type: object + x-go-name: InstanceConfiguration + x-go-package: github.com/superseriousbusiness/gotosocial/internal/api/model + instanceConfigurationMediaAttachments: + properties: + image_matrix_limit: + description: |- + Max allowed image size in pixels as height*width. + + GtS doesn't set a limit on this, but for compatibility + we give Mastodon's 4096x4096px value here. + example: 16777216 + format: int64 + type: integer + x-go-name: ImageMatrixLimit + image_size_limit: + description: Max allowed image size in bytes + example: 2097152 + format: int64 + type: integer + x-go-name: ImageSizeLimit + supported_mime_types: + description: List of mime types that it's possible to upload to this instance. + example: + - image/jpeg + - image/gif + items: + type: string + type: array + x-go-name: SupportedMimeTypes + video_frame_rate_limit: + description: Max allowed video frame rate. + example: 60 + format: int64 + type: integer + x-go-name: VideoFrameRateLimit + video_matrix_limit: + description: |- + Max allowed video size in pixels as height*width. + + GtS doesn't set a limit on this, but for compatibility + we give Mastodon's 4096x4096px value here. + example: 16777216 + format: int64 + type: integer + x-go-name: VideoMatrixLimit + video_size_limit: + description: Max allowed video size in bytes + example: 10485760 + format: int64 + type: integer + x-go-name: VideoSizeLimit + title: InstanceConfigurationMediaAttachments models instance media attachment + config parameters. + type: object + x-go-name: InstanceConfigurationMediaAttachments + x-go-package: github.com/superseriousbusiness/gotosocial/internal/api/model + instanceConfigurationPolls: + properties: + max_characters_per_option: + description: Number of characters allowed per option in the poll. + example: 50 + format: int64 + type: integer + x-go-name: MaxCharactersPerOption + max_expiration: + description: Maximum expiration time of the poll in seconds. + example: 2629746 + format: int64 + type: integer + x-go-name: MaxExpiration + max_options: + description: Number of options permitted in a poll on this instance. + example: 4 + format: int64 + type: integer + x-go-name: MaxOptions + min_expiration: + description: Minimum expiration time of the poll in seconds. + example: 300 + format: int64 + type: integer + x-go-name: MinExpiration + title: InstanceConfigurationPolls models instance poll config parameters. + type: object + x-go-name: InstanceConfigurationPolls + x-go-package: github.com/superseriousbusiness/gotosocial/internal/api/model + instanceConfigurationStatuses: + properties: + characters_reserved_per_url: + description: Amount of characters that a URL will be compressed to. + example: 999 + format: int64 + type: integer + x-go-name: CharactersReservedPerURL + max_characters: + description: Maximum allowed length of a post on this instance, in characters. + example: 5000 + format: int64 + type: integer + x-go-name: MaxCharacters + max_media_attachments: + description: Max number of attachments allowed on a status. + example: 4 + format: int64 + type: integer + x-go-name: MaxMediaAttachments + title: InstanceConfigurationStatuses models instance status config parameters. + type: object + x-go-name: InstanceConfigurationStatuses + x-go-package: github.com/superseriousbusiness/gotosocial/internal/api/model instanceURLs: properties: streaming_api: @@ -3038,6 +3201,60 @@ paths: for the instance. tags: - instance + /api/v1/instance/peers: + get: + operationId: instancePeersGet + parameters: + - description: |- + Comma-separated list of filters to apply to results. Recognized values are: + 'open' -- include peers that are not suspended or silenced + 'suspended' -- include peers that have been suspended. + If filter is 'open', only instances that haven't been suspended or silenced will be returned. + If filter is 'suspended', only suspended instances will be shown. + If filter is 'open,suspended', then all known instances will be returned. + If filter is an empty string or not set, then 'open' will be assumed as the default. + in: query + name: filter + type: string + produces: + - application/json + responses: + "200": + description: |- + If no filter parameter is provided, or filter is empty, then a legacy, + Mastodon-API compatible response will be returned. This will consist of + just a 'flat' array of strings like `["example.com", "example.org"]`. + + If a filter parameter is provided, then an array of objects with at least + a `domain` key set on each object will be returned. + + Domains that are silenced or suspended will also have a key + 'suspended_at' or 'silenced_at' that contains an iso8601 date string. + If one of these keys is not present on the domain object, it is open. + Suspended instances may in some cases be obfuscated, which means they + will have some letters replaced by '*' to make it more difficult for + bad actors to target instances with harassment. + + Whether a flat response or a more detailed response is returned, domains + will be sorted alphabetically by hostname. + schema: + items: + $ref: '#/definitions/domain' + type: array + "400": + description: bad request + "401": + description: unauthorized + "403": + description: forbidden + "404": + description: not found + "406": + description: not acceptable + "500": + description: internal server error + tags: + - instance /api/v1/media: post: consumes: diff --git a/docs/configuration/instance.md b/docs/configuration/instance.md new file mode 100644 index 000000000..2451b9903 --- /dev/null +++ b/docs/configuration/instance.md @@ -0,0 +1,26 @@ +# Instance + +## Settings + +```yaml +########################### +##### INSTANCE CONFIG ##### +########################### + +# Config pertaining to instance federation settings, pages to hide/expose, etc. + +# Bool. Allow unauthenticated users to make queries to /api/v1/instance/peers?filter=open in order +# to see a list of instances that this instance 'peers' with. Even if set to 'false', then authenticated +# users (members of the instance) will still be able to query the endpoint. +# Options: [true, false] +# Default: false +instance-expose-peers: false + +# Bool. Allow unauthenticated users to make queries to /api/v1/instance/peers?filter=suspended in order +# to see a list of instances that this instance blocks/suspends. This will also allow unauthenticated +# users to see the list through the web UI. Even if set to 'false', then authenticated users (members +# of the instance) will still be able to query the endpoint. +# Options: [true, false] +# Default: false +instance-expose-suspended: false +``` diff --git a/example/config.yaml b/example/config.yaml index f442b62bf..2d1537166 100644 --- a/example/config.yaml +++ b/example/config.yaml @@ -165,6 +165,27 @@ web-template-base-dir: "./web/template/" # Default: "./web/assets/" web-asset-base-dir: "./web/assets/" +########################### +##### INSTANCE CONFIG ##### +########################### + +# Config pertaining to instance federation settings, pages to hide/expose, etc. + +# Bool. Allow unauthenticated users to make queries to /api/v1/instance/peers?filter=open in order +# to see a list of instances that this instance 'peers' with. Even if set to 'false', then authenticated +# users (members of the instance) will still be able to query the endpoint. +# Options: [true, false] +# Default: false +instance-expose-peers: false + +# Bool. Allow unauthenticated users to make queries to /api/v1/instance/peers?filter=suspended in order +# to see a list of instances that this instance blocks/suspends. This will also allow unauthenticated +# users to see the list through the web UI. Even if set to 'false', then authenticated users (members +# of the instance) will still be able to query the endpoint. +# Options: [true, false] +# Default: false +instance-expose-suspended: false + ########################### ##### ACCOUNTS CONFIG ##### ########################### diff --git a/go.mod b/go.mod index b374ade8f..530416164 100644 --- a/go.mod +++ b/go.mod @@ -25,6 +25,7 @@ require ( github.com/jackc/pgconn v1.11.0 github.com/jackc/pgx/v4 v4.15.0 github.com/microcosm-cc/bluemonday v1.0.18 + github.com/miekg/dns v1.1.49 github.com/mitchellh/mapstructure v1.5.0 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 github.com/oklog/ulid v1.3.1 @@ -35,7 +36,7 @@ require ( github.com/spf13/viper v1.11.0 github.com/stretchr/testify v1.7.1 github.com/superseriousbusiness/activity v1.1.0-gts - github.com/superseriousbusiness/exif-terminator v0.2.0 + github.com/superseriousbusiness/exif-terminator v0.3.0 github.com/superseriousbusiness/oauth2/v4 v4.3.2-SSB github.com/tdewolff/minify/v2 v2.11.2 github.com/uptrace/bun v1.1.3 @@ -94,7 +95,6 @@ require ( github.com/leodido/go-urn v1.2.1 // indirect github.com/magiconair/properties v1.8.6 // indirect github.com/mattn/go-isatty v0.0.14 // indirect - github.com/miekg/dns v1.1.49 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml v1.9.5 // indirect diff --git a/go.sum b/go.sum index 83182261f..85ca4acfe 100644 --- a/go.sum +++ b/go.sum @@ -482,8 +482,8 @@ github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/superseriousbusiness/activity v1.1.0-gts h1:BSnMzs/84s0Zme7BngE9iJAHV7g1Bv1nhLCP0aJtU3I= github.com/superseriousbusiness/activity v1.1.0-gts/go.mod h1:AZw0Xb4Oju8rmaJCZ21gc5CPg47MmNgyac+Hx5jo8VM= -github.com/superseriousbusiness/exif-terminator v0.2.0 h1:C21KOUr54E37qTqYS7WJX0J83sNzzCwBEy0KXyDprqU= -github.com/superseriousbusiness/exif-terminator v0.2.0/go.mod h1:DHJuKguXqyOVqB/oyOylutEDIZCbkYsn2GZFNSUDT9E= +github.com/superseriousbusiness/exif-terminator v0.3.0 h1:ej7YePEB2UnAGPal5s7CnoN8eMFmDFESEAEJmbFoHh0= +github.com/superseriousbusiness/exif-terminator v0.3.0/go.mod h1:OPfOSEDWjXaW3BILJBN89j0VLD8bglmHwHHwwwSLb5A= github.com/superseriousbusiness/go-jpeg-image-structure/v2 v2.0.0-20220321154430-d89a106fdabe h1:ksl2oCx/Qo8sNDc3Grb8WGKBM9nkvhCm25uvlT86azE= github.com/superseriousbusiness/go-jpeg-image-structure/v2 v2.0.0-20220321154430-d89a106fdabe/go.mod h1:gH4P6gN1V+wmIw5o97KGaa1RgXB/tVpC2UNzijhg3E4= github.com/superseriousbusiness/oauth2/v4 v4.3.2-SSB h1:PtW2w6budTvRV2J5QAoSvThTHBuvh8t/+BXIZFAaBSc= @@ -693,6 +693,7 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/internal/api/client/account/block_test.go b/internal/api/client/account/block_test.go new file mode 100644 index 000000000..fcbe1ec39 --- /dev/null +++ b/internal/api/client/account/block_test.go @@ -0,0 +1,73 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package account_test + +import ( + "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/api/client/account" + "github.com/superseriousbusiness/gotosocial/internal/oauth" +) + +type BlockTestSuite struct { + AccountStandardTestSuite +} + +func (suite *BlockTestSuite) TestBlockSelf() { + testAcct := suite.testAccounts["local_account_1"] + recorder := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(recorder) + ctx.Set(oauth.SessionAuthorizedAccount, testAcct) + ctx.Set(oauth.SessionAuthorizedToken, oauth.DBTokenToToken(suite.testTokens["local_account_1"])) + ctx.Set(oauth.SessionAuthorizedApplication, suite.testApplications["application_1"]) + ctx.Set(oauth.SessionAuthorizedUser, suite.testUsers["local_account_1"]) + ctx.Request = httptest.NewRequest(http.MethodPost, fmt.Sprintf("http://localhost:8080%s", strings.Replace(account.BlockPath, ":id", testAcct.ID, 1)), nil) + + ctx.Params = gin.Params{ + gin.Param{ + Key: account.IDKey, + Value: testAcct.ID, + }, + } + + suite.accountModule.AccountBlockPOSTHandler(ctx) + + // 1. status should be Not Acceptable due to attempted self-block + suite.Equal(http.StatusNotAcceptable, recorder.Code) + + result := recorder.Result() + defer result.Body.Close() + + // check the response + b, err := ioutil.ReadAll(result.Body) + _ = b + assert.NoError(suite.T(), err) +} + +func TestBlockTestSuite(t *testing.T) { + suite.Run(t, new(BlockTestSuite)) +} diff --git a/internal/api/client/account/follow_test.go b/internal/api/client/account/follow_test.go new file mode 100644 index 000000000..478330c83 --- /dev/null +++ b/internal/api/client/account/follow_test.go @@ -0,0 +1,74 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package account_test + +import ( + "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/api/client/account" + "github.com/superseriousbusiness/gotosocial/internal/oauth" +) + +type FollowTestSuite struct { + AccountStandardTestSuite +} + +func (suite *FollowTestSuite) TestFollowSelf() { + testAcct := suite.testAccounts["local_account_1"] + recorder := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(recorder) + ctx.Set(oauth.SessionAuthorizedAccount, testAcct) + ctx.Set(oauth.SessionAuthorizedToken, oauth.DBTokenToToken(suite.testTokens["local_account_1"])) + ctx.Set(oauth.SessionAuthorizedApplication, suite.testApplications["application_1"]) + ctx.Set(oauth.SessionAuthorizedUser, suite.testUsers["local_account_1"]) + ctx.Request = httptest.NewRequest(http.MethodPost, fmt.Sprintf("http://localhost:8080%s", strings.Replace(account.FollowPath, ":id", testAcct.ID, 1)), nil) + + ctx.Params = gin.Params{ + gin.Param{ + Key: account.IDKey, + Value: testAcct.ID, + }, + } + + // call the handler + suite.accountModule.AccountFollowPOSTHandler(ctx) + + // 1. status should be Not Acceptable due to self-follow attempt + suite.Equal(http.StatusNotAcceptable, recorder.Code) + + result := recorder.Result() + defer result.Body.Close() + + // check the response + b, err := ioutil.ReadAll(result.Body) + _ = b + assert.NoError(suite.T(), err) +} + +func TestFollowTestSuite(t *testing.T) { + suite.Run(t, new(FollowTestSuite)) +} diff --git a/internal/api/client/instance/instance.go b/internal/api/client/instance/instance.go index 758cce376..16ff7c9f9 100644 --- a/internal/api/client/instance/instance.go +++ b/internal/api/client/instance/instance.go @@ -1,3 +1,21 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + package instance import ( @@ -11,6 +29,10 @@ const ( // InstanceInformationPath is for serving instance info requests InstanceInformationPath = "api/v1/instance" + // InstancePeersPath is for serving instance peers requests. + InstancePeersPath = InstanceInformationPath + "/peers" + // PeersFilterKey is used to provide filters to /api/v1/instance/peers + PeersFilterKey = "filter" ) // Module implements the ClientModule interface @@ -29,5 +51,6 @@ func New(processor processing.Processor) api.ClientModule { func (m *Module) Route(s router.Router) error { s.AttachHandler(http.MethodGet, InstanceInformationPath, m.InstanceInformationGETHandler) s.AttachHandler(http.MethodPatch, InstanceInformationPath, m.InstanceUpdatePATCHHandler) + s.AttachHandler(http.MethodGet, InstancePeersPath, m.InstancePeersGETHandler) return nil } diff --git a/internal/api/client/instance/instance_test.go b/internal/api/client/instance/instance_test.go index 248b87761..1e5c59c97 100644 --- a/internal/api/client/instance/instance_test.go +++ b/internal/api/client/instance/instance_test.go @@ -21,7 +21,6 @@ import ( "bytes" "fmt" - "net/http" "net/http/httptest" "codeberg.org/gruf/go-store/kv" @@ -113,7 +112,7 @@ func (suite *InstanceStandardTestSuite) newContext(recorder *httptest.ResponseRe baseURI := fmt.Sprintf("%s://%s", protocol, host) requestURI := fmt.Sprintf("%s/%s", baseURI, requestPath) - ctx.Request = httptest.NewRequest(http.MethodPatch, requestURI, bytes.NewReader(requestBody)) // the endpoint we're hitting + ctx.Request = httptest.NewRequest(requestMethod, requestURI, bytes.NewReader(requestBody)) // the endpoint we're hitting if bodyContentType != "" { ctx.Request.Header.Set("Content-Type", bodyContentType) diff --git a/internal/api/client/instance/instanceget.go b/internal/api/client/instance/instanceget.go index 35e842102..5250b3b46 100644 --- a/internal/api/client/instance/instanceget.go +++ b/internal/api/client/instance/instanceget.go @@ -1,3 +1,21 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + package instance import ( diff --git a/internal/api/client/instance/instancepatch.go b/internal/api/client/instance/instancepatch.go index 4e3f1e454..6f1b3586b 100644 --- a/internal/api/client/instance/instancepatch.go +++ b/internal/api/client/instance/instancepatch.go @@ -1,3 +1,21 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + package instance import ( diff --git a/internal/api/client/instance/instancepatch_test.go b/internal/api/client/instance/instancepatch_test.go index 5ca4f2b7a..d8d7da8c9 100644 --- a/internal/api/client/instance/instancepatch_test.go +++ b/internal/api/client/instance/instancepatch_test.go @@ -63,7 +63,7 @@ func (suite *InstancePatchTestSuite) TestInstancePatch1() { b, err := io.ReadAll(result.Body) suite.NoError(err) - suite.Equal(`{"uri":"http://localhost:8080","title":"Example Instance","description":"","short_description":"","email":"someone@example.org","version":"","registrations":true,"approval_required":true,"invites_enabled":false,"urls":{"streaming_api":"wss://localhost:8080"},"stats":{"domain_count":0,"status_count":16,"user_count":4},"thumbnail":"","contact_account":{"id":"01F8MH17FWEB39HZJ76B6VXSKF","username":"admin","acct":"admin","display_name":"","locked":false,"bot":false,"created_at":"2022-05-17T13:10:59.000Z","note":"","url":"http://localhost:8080/@admin","avatar":"","avatar_static":"","header":"","header_static":"","followers_count":1,"following_count":1,"statuses_count":4,"last_status_at":"2021-10-20T10:41:37.000Z","emojis":[],"fields":[]},"max_toot_chars":5000}`, string(b)) + suite.Equal(`{"uri":"http://localhost:8080","title":"Example Instance","description":"","short_description":"","email":"someone@example.org","version":"","registrations":true,"approval_required":true,"invites_enabled":false,"configuration":{"statuses":{"max_characters":5000,"max_media_attachments":6,"characters_reserved_per_url":999},"media_attachments":{"supported_mime_types":["image/jpeg","image/gif","image/png"],"image_size_limit":1048576,"image_matrix_limit":16777216,"video_size_limit":5242880,"video_frame_rate_limit":60,"video_matrix_limit":16777216},"polls":{"max_options":6,"max_characters_per_option":50,"min_expiration":300,"max_expiration":2629746}},"urls":{"streaming_api":"wss://localhost:8080"},"stats":{"domain_count":2,"status_count":16,"user_count":4},"thumbnail":"http://localhost:8080/assets/logo.png","contact_account":{"id":"01F8MH17FWEB39HZJ76B6VXSKF","username":"admin","acct":"admin","display_name":"","locked":false,"bot":false,"created_at":"2022-05-17T13:10:59.000Z","note":"","url":"http://localhost:8080/@admin","avatar":"","avatar_static":"","header":"","header_static":"","followers_count":1,"following_count":1,"statuses_count":4,"last_status_at":"2021-10-20T10:41:37.000Z","emojis":[],"fields":[]},"max_toot_chars":5000}`, string(b)) } func (suite *InstancePatchTestSuite) TestInstancePatch2() { @@ -93,7 +93,7 @@ func (suite *InstancePatchTestSuite) TestInstancePatch2() { b, err := io.ReadAll(result.Body) suite.NoError(err) - suite.Equal(`{"uri":"http://localhost:8080","title":"Geoff's Instance","description":"","short_description":"","email":"","version":"","registrations":true,"approval_required":true,"invites_enabled":false,"urls":{"streaming_api":"wss://localhost:8080"},"stats":{"domain_count":0,"status_count":16,"user_count":4},"thumbnail":"","max_toot_chars":5000}`, string(b)) + suite.Equal(`{"uri":"http://localhost:8080","title":"Geoff's Instance","description":"","short_description":"","email":"","version":"","registrations":true,"approval_required":true,"invites_enabled":false,"configuration":{"statuses":{"max_characters":5000,"max_media_attachments":6,"characters_reserved_per_url":999},"media_attachments":{"supported_mime_types":["image/jpeg","image/gif","image/png"],"image_size_limit":1048576,"image_matrix_limit":16777216,"video_size_limit":5242880,"video_frame_rate_limit":60,"video_matrix_limit":16777216},"polls":{"max_options":6,"max_characters_per_option":50,"min_expiration":300,"max_expiration":2629746}},"urls":{"streaming_api":"wss://localhost:8080"},"stats":{"domain_count":2,"status_count":16,"user_count":4},"thumbnail":"http://localhost:8080/assets/logo.png","max_toot_chars":5000}`, string(b)) } func (suite *InstancePatchTestSuite) TestInstancePatch3() { @@ -123,7 +123,7 @@ func (suite *InstancePatchTestSuite) TestInstancePatch3() { b, err := io.ReadAll(result.Body) suite.NoError(err) - suite.Equal(`{"uri":"http://localhost:8080","title":"localhost:8080","description":"","short_description":"\u003cp\u003eThis is some html, which is \u003cem\u003eallowed\u003c/em\u003e in short descriptions.\u003c/p\u003e","email":"","version":"","registrations":true,"approval_required":true,"invites_enabled":false,"urls":{"streaming_api":"wss://localhost:8080"},"stats":{"domain_count":0,"status_count":16,"user_count":4},"thumbnail":"","max_toot_chars":5000}`, string(b)) + suite.Equal(`{"uri":"http://localhost:8080","title":"localhost:8080","description":"","short_description":"\u003cp\u003eThis is some html, which is \u003cem\u003eallowed\u003c/em\u003e in short descriptions.\u003c/p\u003e","email":"","version":"","registrations":true,"approval_required":true,"invites_enabled":false,"configuration":{"statuses":{"max_characters":5000,"max_media_attachments":6,"characters_reserved_per_url":999},"media_attachments":{"supported_mime_types":["image/jpeg","image/gif","image/png"],"image_size_limit":1048576,"image_matrix_limit":16777216,"video_size_limit":5242880,"video_frame_rate_limit":60,"video_matrix_limit":16777216},"polls":{"max_options":6,"max_characters_per_option":50,"min_expiration":300,"max_expiration":2629746}},"urls":{"streaming_api":"wss://localhost:8080"},"stats":{"domain_count":2,"status_count":16,"user_count":4},"thumbnail":"http://localhost:8080/assets/logo.png","max_toot_chars":5000}`, string(b)) } func (suite *InstancePatchTestSuite) TestInstancePatch4() { @@ -187,6 +187,65 @@ func (suite *InstancePatchTestSuite) TestInstancePatch5() { suite.Equal(`{"error":"Forbidden: user is not an admin so cannot update instance settings"}`, string(b)) } +func (suite *InstancePatchTestSuite) TestInstancePatch6() { + requestBody, w, err := testrig.CreateMultipartFormData( + "", "", + map[string]string{ + "contact_email": "", + }) + if err != nil { + panic(err) + } + bodyBytes := requestBody.Bytes() + + // set up the request + recorder := httptest.NewRecorder() + ctx := suite.newContext(recorder, http.MethodPatch, bodyBytes, instance.InstanceInformationPath, w.FormDataContentType()) + + // call the handler + suite.instanceModule.InstanceUpdatePATCHHandler(ctx) + + // we should have OK because our request was valid + suite.Equal(http.StatusOK, recorder.Code) + + result := recorder.Result() + defer result.Body.Close() + + b, err := io.ReadAll(result.Body) + suite.NoError(err) + + suite.Equal(`{"uri":"http://localhost:8080","title":"localhost:8080","description":"","short_description":"","email":"","version":"","registrations":true,"approval_required":true,"invites_enabled":false,"configuration":{"statuses":{"max_characters":5000,"max_media_attachments":6,"characters_reserved_per_url":999},"media_attachments":{"supported_mime_types":["image/jpeg","image/gif","image/png"],"image_size_limit":1048576,"image_matrix_limit":16777216,"video_size_limit":5242880,"video_frame_rate_limit":60,"video_matrix_limit":16777216},"polls":{"max_options":6,"max_characters_per_option":50,"min_expiration":300,"max_expiration":2629746}},"urls":{"streaming_api":"wss://localhost:8080"},"stats":{"domain_count":2,"status_count":16,"user_count":4},"thumbnail":"http://localhost:8080/assets/logo.png","max_toot_chars":5000}`, string(b)) +} + +func (suite *InstancePatchTestSuite) TestInstancePatch7() { + requestBody, w, err := testrig.CreateMultipartFormData( + "", "", + map[string]string{ + "contact_email": "not.an.email.address", + }) + if err != nil { + panic(err) + } + bodyBytes := requestBody.Bytes() + + // set up the request + recorder := httptest.NewRecorder() + ctx := suite.newContext(recorder, http.MethodPatch, bodyBytes, instance.InstanceInformationPath, w.FormDataContentType()) + + // call the handler + suite.instanceModule.InstanceUpdatePATCHHandler(ctx) + + suite.Equal(http.StatusBadRequest, recorder.Code) + + result := recorder.Result() + defer result.Body.Close() + + b, err := io.ReadAll(result.Body) + suite.NoError(err) + + suite.Equal(`{"error":"Bad Request: mail: missing '@' or angle-addr"}`, string(b)) +} + func TestInstancePatchTestSuite(t *testing.T) { suite.Run(t, &InstancePatchTestSuite{}) } diff --git a/internal/api/client/instance/instancepeersget.go b/internal/api/client/instance/instancepeersget.go new file mode 100644 index 000000000..d4d33d5bf --- /dev/null +++ b/internal/api/client/instance/instancepeersget.go @@ -0,0 +1,136 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package instance + +import ( + "fmt" + "net/http" + "strings" + + "github.com/superseriousbusiness/gotosocial/internal/api" + "github.com/superseriousbusiness/gotosocial/internal/gtserror" + "github.com/superseriousbusiness/gotosocial/internal/oauth" + + "github.com/gin-gonic/gin" +) + +// InstancePeersGETHandler swagger:operation GET /api/v1/instance/peers instancePeersGet +// +// --- +// tags: +// - instance +// +// produces: +// - application/json +// +// parameters: +// - name: filter +// type: string +// description: |- +// Comma-separated list of filters to apply to results. Recognized values are: +// 'open' -- include peers that are not suspended or silenced +// 'suspended' -- include peers that have been suspended. +// If filter is 'open', only instances that haven't been suspended or silenced will be returned. +// If filter is 'suspended', only suspended instances will be shown. +// If filter is 'open,suspended', then all known instances will be returned. +// If filter is an empty string or not set, then 'open' will be assumed as the default. +// in: query +// required: false +// +// responses: +// '200': +// description: |- +// If no filter parameter is provided, or filter is empty, then a legacy, +// Mastodon-API compatible response will be returned. This will consist of +// just a 'flat' array of strings like `["example.com", "example.org"]`. +// +// If a filter parameter is provided, then an array of objects with at least +// a `domain` key set on each object will be returned. +// +// Domains that are silenced or suspended will also have a key +// 'suspended_at' or 'silenced_at' that contains an iso8601 date string. +// If one of these keys is not present on the domain object, it is open. +// Suspended instances may in some cases be obfuscated, which means they +// will have some letters replaced by '*' to make it more difficult for +// bad actors to target instances with harassment. +// +// Whether a flat response or a more detailed response is returned, domains +// will be sorted alphabetically by hostname. +// schema: +// type: array +// items: +// "$ref": "#/definitions/domain" +// '400': +// description: bad request +// '401': +// description: unauthorized +// '403': +// description: forbidden +// '404': +// description: not found +// '406': +// description: not acceptable +// '500': +// description: internal server error +func (m *Module) InstancePeersGETHandler(c *gin.Context) { + authed, err := oauth.Authed(c, false, false, false, false) + if err != nil { + api.ErrorHandler(c, gtserror.NewErrorUnauthorized(err, err.Error()), m.processor.InstanceGet) + return + } + + if _, err := api.NegotiateAccept(c, api.JSONAcceptHeaders...); err != nil { + api.ErrorHandler(c, gtserror.NewErrorNotAcceptable(err, err.Error()), m.processor.InstanceGet) + return + } + + var includeSuspended bool + var includeOpen bool + var flat bool + if filterParam := c.Query(PeersFilterKey); filterParam != "" { + filters := strings.Split(filterParam, ",") + for _, f := range filters { + trimmed := strings.TrimSpace(f) + switch { + case strings.EqualFold(trimmed, "suspended"): + includeSuspended = true + case strings.EqualFold(trimmed, "open"): + includeOpen = true + default: + err := fmt.Errorf("filter %s not recognized; accepted values are 'open', 'suspended'", trimmed) + api.ErrorHandler(c, gtserror.NewErrorBadRequest(err, err.Error()), m.processor.InstanceGet) + return + } + } + } else { + // default is to only include open domains, and present + // them in a 'flat' manner (just an array of strings), + // to maintain compatibility with mastodon API + includeOpen = true + flat = true + } + + data, errWithCode := m.processor.InstancePeersGet(c.Request.Context(), authed, includeSuspended, includeOpen, flat) + if errWithCode != nil { + api.ErrorHandler(c, errWithCode, m.processor.InstanceGet) + return + } + + c.JSON(http.StatusOK, data) +} diff --git a/internal/api/client/instance/instancepeersget_test.go b/internal/api/client/instance/instancepeersget_test.go new file mode 100644 index 000000000..cb35a9e50 --- /dev/null +++ b/internal/api/client/instance/instancepeersget_test.go @@ -0,0 +1,247 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package instance_test + +import ( + "context" + "fmt" + "io" + "net/http" + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/api/client/instance" + "github.com/superseriousbusiness/gotosocial/internal/config" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/testrig" +) + +type InstancePeersGetTestSuite struct { + InstanceStandardTestSuite +} + +func (suite *InstancePeersGetTestSuite) TestInstancePeersGetNoParams() { + recorder := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(recorder) + + baseURI := fmt.Sprintf("%s://%s", config.GetProtocol(), config.GetHost()) + requestURI := fmt.Sprintf("%s/%s", baseURI, instance.InstancePeersPath) + ctx.Request = httptest.NewRequest(http.MethodGet, requestURI, nil) + + suite.instanceModule.InstancePeersGETHandler(ctx) + + suite.Equal(http.StatusOK, recorder.Code) + + result := recorder.Result() + defer result.Body.Close() + + b, err := io.ReadAll(result.Body) + suite.NoError(err) + + suite.Equal(`["example.org","fossbros-anonymous.io"]`, string(b)) +} + +func (suite *InstancePeersGetTestSuite) TestInstancePeersGetNoParamsUnauthorized() { + config.SetInstanceExposePeers(false) + + recorder := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(recorder) + + baseURI := fmt.Sprintf("%s://%s", config.GetProtocol(), config.GetHost()) + requestURI := fmt.Sprintf("%s/%s", baseURI, instance.InstancePeersPath) + ctx.Request = httptest.NewRequest(http.MethodGet, requestURI, nil) + + suite.instanceModule.InstancePeersGETHandler(ctx) + + suite.Equal(http.StatusUnauthorized, recorder.Code) + + result := recorder.Result() + defer result.Body.Close() + + b, err := io.ReadAll(result.Body) + suite.NoError(err) + + suite.Equal(`{"error":"Unauthorized: peers open query requires an authenticated account/user"}`, string(b)) +} + +func (suite *InstancePeersGetTestSuite) TestInstancePeersGetNoParamsAuthorized() { + config.SetInstanceExposePeers(false) + + recorder := httptest.NewRecorder() + baseURI := fmt.Sprintf("%s://%s", config.GetProtocol(), config.GetHost()) + requestURI := fmt.Sprintf("%s/%s", baseURI, instance.InstancePeersPath) + ctx := suite.newContext(recorder, http.MethodGet, []byte{}, requestURI, "") + + suite.instanceModule.InstancePeersGETHandler(ctx) + + suite.Equal(http.StatusOK, recorder.Code) + + result := recorder.Result() + defer result.Body.Close() + + b, err := io.ReadAll(result.Body) + suite.NoError(err) + + suite.Equal(`["example.org","fossbros-anonymous.io"]`, string(b)) +} + +func (suite *InstancePeersGetTestSuite) TestInstancePeersGetOnlySuspended() { + recorder := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(recorder) + + baseURI := fmt.Sprintf("%s://%s", config.GetProtocol(), config.GetHost()) + requestURI := fmt.Sprintf("%s/%s?filter=suspended", baseURI, instance.InstancePeersPath) + ctx.Request = httptest.NewRequest(http.MethodGet, requestURI, nil) + + suite.instanceModule.InstancePeersGETHandler(ctx) + + suite.Equal(http.StatusOK, recorder.Code) + + result := recorder.Result() + defer result.Body.Close() + + b, err := io.ReadAll(result.Body) + suite.NoError(err) + + suite.Equal(`[{"domain":"replyguys.com","suspended_at":"2020-05-13T13:29:12.000Z","public_comment":"reply-guying to tech posts"}]`, string(b)) +} + +func (suite *InstancePeersGetTestSuite) TestInstancePeersGetOnlySuspendedUnauthorized() { + config.SetInstanceExposeSuspended(false) + + recorder := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(recorder) + + baseURI := fmt.Sprintf("%s://%s", config.GetProtocol(), config.GetHost()) + requestURI := fmt.Sprintf("%s/%s?filter=suspended", baseURI, instance.InstancePeersPath) + ctx.Request = httptest.NewRequest(http.MethodGet, requestURI, nil) + + suite.instanceModule.InstancePeersGETHandler(ctx) + + suite.Equal(http.StatusUnauthorized, recorder.Code) + + result := recorder.Result() + defer result.Body.Close() + + b, err := io.ReadAll(result.Body) + suite.NoError(err) + + suite.Equal(`{"error":"Unauthorized: peers suspended query requires an authenticated account/user"}`, string(b)) +} + +func (suite *InstancePeersGetTestSuite) TestInstancePeersGetOnlySuspendedAuthorized() { + config.SetInstanceExposeSuspended(false) + + recorder := httptest.NewRecorder() + baseURI := fmt.Sprintf("%s://%s", config.GetProtocol(), config.GetHost()) + requestURI := fmt.Sprintf("%s/%s?filter=suspended", baseURI, instance.InstancePeersPath) + ctx := suite.newContext(recorder, http.MethodGet, []byte{}, requestURI, "") + + suite.instanceModule.InstancePeersGETHandler(ctx) + + suite.Equal(http.StatusOK, recorder.Code) + + result := recorder.Result() + defer result.Body.Close() + + b, err := io.ReadAll(result.Body) + suite.NoError(err) + + suite.Equal(`[{"domain":"replyguys.com","suspended_at":"2020-05-13T13:29:12.000Z","public_comment":"reply-guying to tech posts"}]`, string(b)) +} + +func (suite *InstancePeersGetTestSuite) TestInstancePeersGetAll() { + recorder := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(recorder) + + baseURI := fmt.Sprintf("%s://%s", config.GetProtocol(), config.GetHost()) + requestURI := fmt.Sprintf("%s/%s?filter=suspended,open", baseURI, instance.InstancePeersPath) + ctx.Request = httptest.NewRequest(http.MethodGet, requestURI, nil) + + suite.instanceModule.InstancePeersGETHandler(ctx) + + suite.Equal(http.StatusOK, recorder.Code) + + result := recorder.Result() + defer result.Body.Close() + + b, err := io.ReadAll(result.Body) + suite.NoError(err) + + suite.Equal(`[{"domain":"example.org"},{"domain":"fossbros-anonymous.io"},{"domain":"replyguys.com","suspended_at":"2020-05-13T13:29:12.000Z","public_comment":"reply-guying to tech posts"}]`, string(b)) +} + +func (suite *InstancePeersGetTestSuite) TestInstancePeersGetAllWithObfuscated() { + err := suite.db.Put(context.Background(), >smodel.DomainBlock{ + ID: "01G633XTNK51GBADQZFZQDP6WR", + CreatedAt: testrig.TimeMustParse("2021-06-09T12:34:55+02:00"), + UpdatedAt: testrig.TimeMustParse("2021-06-09T12:34:55+02:00"), + Domain: "omg.just.the.worst.org.ever", + CreatedByAccountID: "01F8MH17FWEB39HZJ76B6VXSKF", + PublicComment: "just absolutely the worst, wowza", + Obfuscate: true, + }) + suite.NoError(err) + + recorder := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(recorder) + + baseURI := fmt.Sprintf("%s://%s", config.GetProtocol(), config.GetHost()) + requestURI := fmt.Sprintf("%s/%s?filter=suspended,open", baseURI, instance.InstancePeersPath) + ctx.Request = httptest.NewRequest(http.MethodGet, requestURI, nil) + + suite.instanceModule.InstancePeersGETHandler(ctx) + + suite.Equal(http.StatusOK, recorder.Code) + + result := recorder.Result() + defer result.Body.Close() + + b, err := io.ReadAll(result.Body) + suite.NoError(err) + + suite.Equal(`[{"domain":"example.org"},{"domain":"fossbros-anonymous.io"},{"domain":"o*g.*u**.t**.*or*t.*r**ev**","suspended_at":"2021-06-09T10:34:55.000Z","public_comment":"just absolutely the worst, wowza"},{"domain":"replyguys.com","suspended_at":"2020-05-13T13:29:12.000Z","public_comment":"reply-guying to tech posts"}]`, string(b)) +} + +func (suite *InstancePeersGetTestSuite) TestInstancePeersGetFunkyParams() { + recorder := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(recorder) + + baseURI := fmt.Sprintf("%s://%s", config.GetProtocol(), config.GetHost()) + requestURI := fmt.Sprintf("%s/%s?filter=aaaaaaaaaaaaaaaaa,open", baseURI, instance.InstancePeersPath) + ctx.Request = httptest.NewRequest(http.MethodGet, requestURI, nil) + + suite.instanceModule.InstancePeersGETHandler(ctx) + + suite.Equal(http.StatusBadRequest, recorder.Code) + + result := recorder.Result() + defer result.Body.Close() + + b, err := io.ReadAll(result.Body) + suite.NoError(err) + + suite.Equal(`{"error":"Bad Request: filter aaaaaaaaaaaaaaaaa not recognized; accepted values are 'open', 'suspended'"}`, string(b)) +} + +func TestInstancePeersGetTestSuite(t *testing.T) { + suite.Run(t, &InstancePeersGetTestSuite{}) +} diff --git a/internal/api/model/domainblock.go b/internal/api/model/domain.go similarity index 81% rename from internal/api/model/domainblock.go rename to internal/api/model/domain.go index 2911dfbaa..90c08fa6f 100644 --- a/internal/api/model/domainblock.go +++ b/internal/api/model/domain.go @@ -20,17 +20,33 @@ import "mime/multipart" +// Domain represents a remote domain +// +// swagger:model domain +type Domain struct { + // The hostname of the domain. + // example: example.org + Domain string `form:"domain" json:"domain" validate:"required"` + // Time at which this domain was suspended. Key will not be present on open domains. + // example: 2021-07-30T09:20:25+00:00 + SuspendedAt string `json:"suspended_at,omitempty"` + // Time at which this domain was silenced. Key will not be present on open domains. + // example: 2021-07-30T09:20:25+00:00 + SilencedAt string `json:"silenced_at,omitempty"` + // If the domain is blocked, what's the publicly-stated reason for the block. + // example: they smell + PublicComment string `form:"public_comment" json:"public_comment,omitempty"` +} + // DomainBlock represents a block on one domain // // swagger:model domainBlock type DomainBlock struct { + Domain // The ID of the domain block. // example: 01FBW21XJA09XYX51KV5JVBW0F // readonly: true ID string `json:"id,omitempty"` - // The hostname of the blocked domain. - // example: example.org - Domain string `form:"domain" json:"domain" validation:"required"` // Obfuscate the domain name when serving this domain block publicly. // A useful anti-harassment tool. // example: false @@ -38,9 +54,6 @@ type DomainBlock struct { // Private comment for this block, visible to our instance admins only. // example: they are poopoo PrivateComment string `json:"private_comment,omitempty"` - // Public comment for this block, visible if domain blocks are served publicly. - // example: they smell - PublicComment string `form:"public_comment" json:"public_comment,omitempty"` // The ID of the subscription that created/caused this domain block. // example: 01FBW25TF5J67JW3HFHZCSD23K SubscriptionID string `json:"subscription_id,omitempty"` diff --git a/internal/api/model/instance.go b/internal/api/model/instance.go index f36713cdc..e01710e34 100644 --- a/internal/api/model/instance.go +++ b/internal/api/model/instance.go @@ -62,6 +62,9 @@ type Instance struct { ApprovalRequired bool `json:"approval_required"` // Invites are enabled on this instance. InvitesEnabled bool `json:"invites_enabled"` + // Configuration object containing values about status limits etc. + // This key/value will be omitted for remote instances. + Configuration *InstanceConfiguration `json:"configuration,omitempty"` // URLs of interest for client applications. URLS *InstanceURLs `json:"urls,omitempty"` // Statistics about the instance: number of posts, accounts, etc. @@ -79,6 +82,94 @@ type Instance struct { MaxTootChars uint `json:"max_toot_chars"` } +// InstanceConfiguration models instance configuration parameters. +// +// swagger:model instanceConfiguration +type InstanceConfiguration struct { + // Instance configuration pertaining to status limits. + Statuses *InstanceConfigurationStatuses `json:"statuses"` + // Instance configuration pertaining to media attachment types + size limits. + MediaAttachments *InstanceConfigurationMediaAttachments `json:"media_attachments"` + // Instance configuration pertaining to poll limits. + Polls *InstanceConfigurationPolls `json:"polls"` +} + +// InstanceConfigurationStatuses models instance status config parameters. +// +// swagger:model instanceConfigurationStatuses +type InstanceConfigurationStatuses struct { + // Maximum allowed length of a post on this instance, in characters. + // + // example: 5000 + MaxCharacters int `json:"max_characters"` + // Max number of attachments allowed on a status. + // + // example: 4 + MaxMediaAttachments int `json:"max_media_attachments"` + // Amount of characters that a URL will be compressed to. + // + // example: 999 + CharactersReservedPerURL int `json:"characters_reserved_per_url"` +} + +// InstanceConfigurationMediaAttachments models instance media attachment config parameters. +// +// swagger:model instanceConfigurationMediaAttachments +type InstanceConfigurationMediaAttachments struct { + // List of mime types that it's possible to upload to this instance. + // + // example: ["image/jpeg","image/gif"] + SupportedMimeTypes []string `json:"supported_mime_types"` + // Max allowed image size in bytes + // + // example: 2097152 + ImageSizeLimit int `json:"image_size_limit"` + // Max allowed image size in pixels as height*width. + // + // GtS doesn't set a limit on this, but for compatibility + // we give Mastodon's 4096x4096px value here. + // + // example: 16777216 + ImageMatrixLimit int `json:"image_matrix_limit"` + // Max allowed video size in bytes + // + // example: 10485760 + VideoSizeLimit int `json:"video_size_limit"` + // Max allowed video frame rate. + // + // example: 60 + VideoFrameRateLimit int `json:"video_frame_rate_limit"` + // Max allowed video size in pixels as height*width. + // + // GtS doesn't set a limit on this, but for compatibility + // we give Mastodon's 4096x4096px value here. + // + // example: 16777216 + VideoMatrixLimit int `json:"video_matrix_limit"` +} + +// InstanceConfigurationPolls models instance poll config parameters. +// +// swagger:model instanceConfigurationPolls +type InstanceConfigurationPolls struct { + // Number of options permitted in a poll on this instance. + // + // example: 4 + MaxOptions int `json:"max_options"` + // Number of characters allowed per option in the poll. + // + // example: 50 + MaxCharactersPerOption int `json:"max_characters_per_option"` + // Minimum expiration time of the poll in seconds. + // + // example: 300 + MinExpiration int `json:"min_expiration"` + // Maximum expiration time of the poll in seconds. + // + // example: 2629746 + MaxExpiration int `json:"max_expiration"` +} + // InstanceURLs models instance-relevant URLs for client application consumption. // // swagger:model instanceURLs diff --git a/internal/config/config.go b/internal/config/config.go index 3a1775778..0b7fc3d31 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -67,6 +67,9 @@ type Configuration struct { WebTemplateBaseDir string `name:"web-template-base-dir" usage:"Basedir for html templating files for rendering pages and composing emails."` WebAssetBaseDir string `name:"web-asset-base-dir" usage:"Directory to serve static assets from, accessible at example.org/assets/"` + InstanceExposePeers bool `name:"instance-expose-peers" usage:"Allow unauthenticated users to query /api/v1/instance/peers?filter=open"` + InstanceExposeSuspended bool `name:"instance-expose-suspended" usage:"Expose suspended instances via web UI, and allow unauthenticated users to query /api/v1/instance/peers?filter=suspended"` + AccountsRegistrationOpen bool `name:"accounts-registration-open" usage:"Allow anyone to submit an account signup request. If false, server will be invite-only."` AccountsApprovalRequired bool `name:"accounts-approval-required" usage:"Do account signups require approval by an admin or moderator before user can log in? If false, new registrations will be automatically approved."` AccountsReasonRequired bool `name:"accounts-reason-required" usage:"Do new account signups require a reason to be submitted on registration?"` diff --git a/internal/config/defaults.go b/internal/config/defaults.go index f9f0bdba9..5bfd7aa53 100644 --- a/internal/config/defaults.go +++ b/internal/config/defaults.go @@ -46,6 +46,9 @@ WebTemplateBaseDir: "./web/template/", WebAssetBaseDir: "./web/assets/", + InstanceExposePeers: false, + InstanceExposeSuspended: false, + AccountsRegistrationOpen: true, AccountsApprovalRequired: true, AccountsReasonRequired: true, diff --git a/internal/config/flags.go b/internal/config/flags.go index 6f946f1d6..891449934 100644 --- a/internal/config/flags.go +++ b/internal/config/flags.go @@ -60,6 +60,10 @@ func AddServerFlags(cmd *cobra.Command) { cmd.Flags().String(WebTemplateBaseDirFlag(), cfg.WebTemplateBaseDir, fieldtag("WebTemplateBaseDir", "usage")) cmd.Flags().String(WebAssetBaseDirFlag(), cfg.WebAssetBaseDir, fieldtag("WebAssetBaseDir", "usage")) + // Instance + cmd.Flags().Bool(InstanceExposePeersFlag(), cfg.InstanceExposePeers, fieldtag("InstanceExposePeers", "usage")) + cmd.Flags().Bool(InstanceExposeSuspendedFlag(), cfg.InstanceExposeSuspended, fieldtag("InstanceExposeSuspended", "usage")) + // Accounts cmd.Flags().Bool(AccountsRegistrationOpenFlag(), cfg.AccountsRegistrationOpen, fieldtag("AccountsRegistrationOpen", "usage")) cmd.Flags().Bool(AccountsApprovalRequiredFlag(), cfg.AccountsApprovalRequired, fieldtag("AccountsApprovalRequired", "usage")) diff --git a/internal/config/helpers.gen.go b/internal/config/helpers.gen.go index a48828059..a4c2adebf 100644 --- a/internal/config/helpers.gen.go +++ b/internal/config/helpers.gen.go @@ -543,6 +543,56 @@ func GetWebAssetBaseDir() string { return global.GetWebAssetBaseDir() } // SetWebAssetBaseDir safely sets the value for global configuration 'WebAssetBaseDir' field func SetWebAssetBaseDir(v string) { global.SetWebAssetBaseDir(v) } +// GetInstanceExposePeers safely fetches the Configuration value for state's 'InstanceExposePeers' field +func (st *ConfigState) GetInstanceExposePeers() (v bool) { + st.mutex.Lock() + v = st.config.InstanceExposePeers + st.mutex.Unlock() + return +} + +// SetInstanceExposePeers safely sets the Configuration value for state's 'InstanceExposePeers' field +func (st *ConfigState) SetInstanceExposePeers(v bool) { + st.mutex.Lock() + defer st.mutex.Unlock() + st.config.InstanceExposePeers = v + st.reloadToViper() +} + +// InstanceExposePeersFlag returns the flag name for the 'InstanceExposePeers' field +func InstanceExposePeersFlag() string { return "instance-expose-peers" } + +// GetInstanceExposePeers safely fetches the value for global configuration 'InstanceExposePeers' field +func GetInstanceExposePeers() bool { return global.GetInstanceExposePeers() } + +// SetInstanceExposePeers safely sets the value for global configuration 'InstanceExposePeers' field +func SetInstanceExposePeers(v bool) { global.SetInstanceExposePeers(v) } + +// GetInstanceExposeSuspended safely fetches the Configuration value for state's 'InstanceExposeSuspended' field +func (st *ConfigState) GetInstanceExposeSuspended() (v bool) { + st.mutex.Lock() + v = st.config.InstanceExposeSuspended + st.mutex.Unlock() + return +} + +// SetInstanceExposeSuspended safely sets the Configuration value for state's 'InstanceExposeSuspended' field +func (st *ConfigState) SetInstanceExposeSuspended(v bool) { + st.mutex.Lock() + defer st.mutex.Unlock() + st.config.InstanceExposeSuspended = v + st.reloadToViper() +} + +// InstanceExposeSuspendedFlag returns the flag name for the 'InstanceExposeSuspended' field +func InstanceExposeSuspendedFlag() string { return "instance-expose-suspended" } + +// GetInstanceExposeSuspended safely fetches the value for global configuration 'InstanceExposeSuspended' field +func GetInstanceExposeSuspended() bool { return global.GetInstanceExposeSuspended() } + +// SetInstanceExposeSuspended safely sets the value for global configuration 'InstanceExposeSuspended' field +func SetInstanceExposeSuspended(v bool) { global.SetInstanceExposeSuspended(v) } + // GetAccountsRegistrationOpen safely fetches the Configuration value for state's 'AccountsRegistrationOpen' field func (st *ConfigState) GetAccountsRegistrationOpen() (v bool) { st.mutex.Lock() diff --git a/internal/db/bundb/instance.go b/internal/db/bundb/instance.go index d16fac90b..c8a9c5776 100644 --- a/internal/db/bundb/instance.go +++ b/internal/db/bundb/instance.go @@ -98,6 +98,25 @@ func (i *instanceDB) CountInstanceDomains(ctx context.Context, domain string) (i return count, nil } +func (i *instanceDB) GetInstancePeers(ctx context.Context, includeSuspended bool) ([]*gtsmodel.Instance, db.Error) { + instances := []*gtsmodel.Instance{} + + q := i.conn. + NewSelect(). + Model(&instances). + Where("domain != ?", config.GetHost()) + + if !includeSuspended { + q = q.Where("? IS NULL", bun.Ident("suspended_at")) + } + + if err := q.Scan(ctx); err != nil { + return nil, i.conn.ProcessError(err) + } + + return instances, nil +} + func (i *instanceDB) GetInstanceAccounts(ctx context.Context, domain string, maxID string, limit int) ([]*gtsmodel.Account, db.Error) { logrus.Debug("GetAccountsForInstance") diff --git a/internal/db/bundb/media.go b/internal/db/bundb/media.go index fc3280ddf..71433b901 100644 --- a/internal/db/bundb/media.go +++ b/internal/db/bundb/media.go @@ -98,3 +98,29 @@ func (m *mediaDB) GetAvatarsAndHeaders(ctx context.Context, maxID string, limit return attachments, nil } + +func (m *mediaDB) GetLocalUnattachedOlderThan(ctx context.Context, olderThan time.Time, maxID string, limit int) ([]*gtsmodel.MediaAttachment, db.Error) { + attachments := []*gtsmodel.MediaAttachment{} + + q := m.newMediaQ(&attachments). + Where("media_attachment.cached = true"). + Where("media_attachment.avatar = false"). + Where("media_attachment.header = false"). + Where("media_attachment.created_at < ?", olderThan). + Where("media_attachment.remote_url IS NULL"). + Where("media_attachment.status_id IS NULL") + + if maxID != "" { + q = q.Where("media_attachment.id < ?", maxID) + } + + if limit != 0 { + q = q.Limit(limit) + } + + if err := q.Scan(ctx); err != nil { + return nil, m.conn.ProcessError(err) + } + + return attachments, nil +} diff --git a/internal/db/bundb/media_test.go b/internal/db/bundb/media_test.go index f1809b3fb..d6a4981f8 100644 --- a/internal/db/bundb/media_test.go +++ b/internal/db/bundb/media_test.go @@ -24,6 +24,7 @@ "time" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/testrig" ) type MediaTestSuite struct { @@ -51,6 +52,14 @@ func (suite *MediaTestSuite) TestGetAvisAndHeaders() { suite.Len(attachments, 2) } +func (suite *MediaTestSuite) TestGetLocalUnattachedOlderThan() { + ctx := context.Background() + + attachments, err := suite.db.GetLocalUnattachedOlderThan(ctx, testrig.TimeMustParse("2090-06-04T13:12:00Z"), "", 10) + suite.NoError(err) + suite.Len(attachments, 1) +} + func TestMediaTestSuite(t *testing.T) { suite.Run(t, new(MediaTestSuite)) } diff --git a/internal/db/instance.go b/internal/db/instance.go index 45ff7d7b4..9e69d1b9b 100644 --- a/internal/db/instance.go +++ b/internal/db/instance.go @@ -37,4 +37,7 @@ type Instance interface { // GetInstanceAccounts returns a slice of accounts from the given instance, arranged by ID. GetInstanceAccounts(ctx context.Context, domain string, maxID string, limit int) ([]*gtsmodel.Account, Error) + + // GetInstancePeers returns a slice of instances that the host instance knows about. + GetInstancePeers(ctx context.Context, includeSuspended bool) ([]*gtsmodel.Instance, Error) } diff --git a/internal/db/media.go b/internal/db/media.go index 636fc61f2..2f9ed79dc 100644 --- a/internal/db/media.go +++ b/internal/db/media.go @@ -38,4 +38,9 @@ type Media interface { // GetAvatarsAndHeaders fetches limit n avatars and headers with an id < maxID. These headers // and avis may be in use or not; the caller should check this if it's important. GetAvatarsAndHeaders(ctx context.Context, maxID string, limit int) ([]*gtsmodel.MediaAttachment, Error) + // GetLocalUnattachedOlderThan fetches limit n local media attachments, older than the given time, which + // aren't header or avatars, and aren't attached to a status. In other words, attachments which were uploaded + // but never used for whatever reason, or attachments that were attached to a status which was subsequently + // deleted. + GetLocalUnattachedOlderThan(ctx context.Context, olderThan time.Time, maxID string, limit int) ([]*gtsmodel.MediaAttachment, Error) } diff --git a/internal/media/manager.go b/internal/media/manager.go index 663f74123..aacf607cc 100644 --- a/internal/media/manager.go +++ b/internal/media/manager.go @@ -34,6 +34,10 @@ // selectPruneLimit is the amount of media entries to select at a time from the db when pruning const selectPruneLimit = 20 +// UnusedLocalAttachmentCacheDays is the amount of days to keep local media in storage if it +// is not attached to a status, or was never attached to a status. +const UnusedLocalAttachmentCacheDays = 3 + // Manager provides an interface for managing media: parsing, storing, and retrieving media objects like photos, videos, and gifs. type Manager interface { // ProcessMedia begins the process of decoding and storing the given data as an attachment. @@ -75,11 +79,16 @@ type Manager interface { // // The returned int is the amount of media that was pruned by this function. PruneAllRemote(ctx context.Context, olderThanDays int) (int, error) - // PruneAllMeta prunes unused meta media -- currently, this means unused avatars + headers, but can also be extended - // to include things like attachments that were uploaded on this server but left unused, etc. + // PruneAllMeta prunes unused/out of date headers and avatars cached on this instance. // // The returned int is the amount of media that was pruned by this function. PruneAllMeta(ctx context.Context) (int, error) + // PruneUnusedLocalAttachments prunes unused media attachments that were uploaded by + // a user on this instance, but never actually attached to a status, or attached but + // later detached. + // + // The returned int is the amount of media that was pruned by this function. + PruneUnusedLocalAttachments(ctx context.Context) (int, error) // Stop stops the underlying worker pool of the manager. It should be called // when closing GoToSocial in order to cleanly finish any in-progress jobs. @@ -210,6 +219,19 @@ func scheduleCleanupJobs(m *manager) error { return fmt.Errorf("error starting media manager meta cleanup job: %s", err) } + if _, err := c.AddFunc("@midnight", func() { + begin := time.Now() + pruned, err := m.PruneUnusedLocalAttachments(pruneCtx) + if err != nil { + logrus.Errorf("media manager: error pruning unused local attachments: %s", err) + return + } + logrus.Infof("media manager: pruned %d unused local attachments in %s", pruned, time.Since(begin)) + }); err != nil { + pruneCancel() + return fmt.Errorf("error starting media manager unused local attachments cleanup job: %s", err) + } + // start remote cache cleanup cronjob if configured if mediaRemoteCacheDays := config.GetMediaRemoteCacheDays(); mediaRemoteCacheDays > 0 { if _, err := c.AddFunc("@midnight", func() { diff --git a/internal/media/pruneremote.go b/internal/media/pruneremote.go index f7b77d32e..a01995740 100644 --- a/internal/media/pruneremote.go +++ b/internal/media/pruneremote.go @@ -21,7 +21,6 @@ import ( "context" "fmt" - "time" "codeberg.org/gruf/go-store/storage" "github.com/sirupsen/logrus" @@ -32,15 +31,10 @@ func (m *manager) PruneAllRemote(ctx context.Context, olderThanDays int) (int, error) { var totalPruned int - // convert days into a duration string - olderThanHoursString := fmt.Sprintf("%dh", olderThanDays*24) - // parse the duration string into a duration - olderThanHours, err := time.ParseDuration(olderThanHoursString) + olderThan, err := parseOlderThan(olderThanDays) if err != nil { - return totalPruned, fmt.Errorf("PruneAllRemote: %d", err) + return totalPruned, fmt.Errorf("PruneAllRemote: error parsing olderThanDays %d: %s", olderThanDays, err) } - // 'subtract' that from the time now to give our threshold - olderThan := time.Now().Add(-olderThanHours) logrus.Infof("PruneAllRemote: pruning media older than %s", olderThan) // select 20 attachments at a time and prune them diff --git a/internal/media/pruneunusedlocal.go b/internal/media/pruneunusedlocal.go new file mode 100644 index 000000000..0c464e857 --- /dev/null +++ b/internal/media/pruneunusedlocal.go @@ -0,0 +1,86 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package media + +import ( + "context" + "fmt" + + "codeberg.org/gruf/go-store/storage" + "github.com/sirupsen/logrus" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" +) + +func (m *manager) PruneUnusedLocalAttachments(ctx context.Context) (int, error) { + var totalPruned int + var maxID string + var attachments []*gtsmodel.MediaAttachment + var err error + + olderThan, err := parseOlderThan(UnusedLocalAttachmentCacheDays) + if err != nil { + return totalPruned, fmt.Errorf("PruneUnusedLocalAttachments: error parsing olderThanDays %d: %s", UnusedLocalAttachmentCacheDays, err) + } + logrus.Infof("PruneUnusedLocalAttachments: pruning unused local attachments older than %s", olderThan) + + // select 20 attachments at a time and prune them + for attachments, err = m.db.GetLocalUnattachedOlderThan(ctx, olderThan, maxID, selectPruneLimit); err == nil && len(attachments) != 0; attachments, err = m.db.GetLocalUnattachedOlderThan(ctx, olderThan, maxID, selectPruneLimit) { + // use the id of the last attachment in the slice as the next 'maxID' value + l := len(attachments) + maxID = attachments[l-1].ID + logrus.Tracef("PruneUnusedLocalAttachments: got %d unused local attachments older than %s with maxID < %s", l, olderThan, maxID) + + for _, attachment := range attachments { + if err := m.pruneOneLocal(ctx, attachment); err != nil { + return totalPruned, err + } + totalPruned++ + } + } + + // make sure we don't have a real error when we leave the loop + if err != nil && err != db.ErrNoEntries { + return totalPruned, err + } + + logrus.Infof("PruneUnusedLocalAttachments: finished pruning: pruned %d entries", totalPruned) + return totalPruned, nil +} + +func (m *manager) pruneOneLocal(ctx context.Context, attachment *gtsmodel.MediaAttachment) error { + if attachment.File.Path != "" { + // delete the full size attachment from storage + logrus.Tracef("pruneOneLocal: deleting %s", attachment.File.Path) + if err := m.storage.Delete(attachment.File.Path); err != nil && err != storage.ErrNotFound { + return err + } + } + + if attachment.Thumbnail.Path != "" { + // delete the thumbnail from storage + logrus.Tracef("pruneOneLocal: deleting %s", attachment.Thumbnail.Path) + if err := m.storage.Delete(attachment.Thumbnail.Path); err != nil && err != storage.ErrNotFound { + return err + } + } + + // delete the attachment completely + return m.db.DeleteByID(ctx, attachment.ID, attachment) +} diff --git a/internal/media/pruneunusedlocal_test.go b/internal/media/pruneunusedlocal_test.go new file mode 100644 index 000000000..5f6bfbfba --- /dev/null +++ b/internal/media/pruneunusedlocal_test.go @@ -0,0 +1,75 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package media_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/db" +) + +type PruneUnusedLocalTestSuite struct { + MediaStandardTestSuite +} + +func (suite *PruneUnusedLocalTestSuite) TestPruneUnusedLocal() { + testAttachment := suite.testAttachments["local_account_1_unattached_1"] + suite.True(testAttachment.Cached) + + totalPruned, err := suite.manager.PruneUnusedLocalAttachments(context.Background()) + suite.NoError(err) + suite.Equal(1, totalPruned) + + _, err = suite.db.GetAttachmentByID(context.Background(), testAttachment.ID) + suite.ErrorIs(err, db.ErrNoEntries) +} + +func (suite *PruneUnusedLocalTestSuite) TestPruneRemoteTwice() { + totalPruned, err := suite.manager.PruneUnusedLocalAttachments(context.Background()) + suite.NoError(err) + suite.Equal(1, totalPruned) + + // final prune should prune nothing, since the first prune already happened + totalPrunedAgain, err := suite.manager.PruneUnusedLocalAttachments(context.Background()) + suite.NoError(err) + suite.Equal(0, totalPrunedAgain) +} + +func (suite *PruneUnusedLocalTestSuite) TestPruneOneNonExistent() { + ctx := context.Background() + testAttachment := suite.testAttachments["local_account_1_unattached_1"] + + // Delete this attachment cached on disk + media, err := suite.db.GetAttachmentByID(ctx, testAttachment.ID) + suite.NoError(err) + suite.True(media.Cached) + err = suite.storage.Delete(media.File.Path) + suite.NoError(err) + + // Now attempt to prune for item with db entry no file + totalPruned, err := suite.manager.PruneUnusedLocalAttachments(ctx) + suite.NoError(err) + suite.Equal(1, totalPruned) +} + +func TestPruneUnusedLocalTestSuite(t *testing.T) { + suite.Run(t, &PruneUnusedLocalTestSuite{}) +} diff --git a/internal/media/util.go b/internal/media/util.go index f3cd1b986..9d62619f5 100644 --- a/internal/media/util.go +++ b/internal/media/util.go @@ -21,11 +21,22 @@ import ( "errors" "fmt" + "time" "github.com/h2non/filetype" "github.com/sirupsen/logrus" ) +// AllSupportedMIMETypes just returns all media +// MIME types supported by this instance. +func AllSupportedMIMETypes() []string { + return []string{ + mimeImageJpeg, + mimeImageGif, + mimeImagePng, + } +} + // parseContentType parses the MIME content type from a file, returning it as a string in the form (eg., "image/jpeg"). // Returns an error if the content type is not something we can process. // @@ -118,3 +129,19 @@ func (l *logrusWrapper) Info(msg string, keysAndValues ...interface{}) { func (l *logrusWrapper) Error(err error, msg string, keysAndValues ...interface{}) { logrus.Error("media manager cron logger: ", err, msg, keysAndValues) } + +func parseOlderThan(olderThanDays int) (time.Time, error) { + // convert days into a duration string + olderThanHoursString := fmt.Sprintf("%dh", olderThanDays*24) + + // parse the duration string into a duration + olderThanHours, err := time.ParseDuration(olderThanHoursString) + if err != nil { + return time.Time{}, err + } + + // 'subtract' that from the time now to give our threshold + olderThan := time.Now().Add(-olderThanHours) + + return olderThan, nil +} diff --git a/internal/processing/account/createblock.go b/internal/processing/account/createblock.go index e1bad0d38..dfe1475cb 100644 --- a/internal/processing/account/createblock.go +++ b/internal/processing/account/createblock.go @@ -46,6 +46,11 @@ func (p *processor) BlockCreate(ctx context.Context, requestingAccount *gtsmodel return p.RelationshipGet(ctx, requestingAccount, targetAccountID) } + // don't block yourself, silly + if requestingAccount.ID == targetAccountID { + return nil, gtserror.NewErrorNotAcceptable(fmt.Errorf("BlockCreate: account %s cannot block itself", requestingAccount.ID)) + } + // make the block block := >smodel.Block{} newBlockID, err := id.NewULID() diff --git a/internal/processing/account/createfollow.go b/internal/processing/account/createfollow.go index cd2dcbd8c..49430b9fb 100644 --- a/internal/processing/account/createfollow.go +++ b/internal/processing/account/createfollow.go @@ -65,6 +65,11 @@ func (p *processor) FollowCreate(ctx context.Context, requestingAccount *gtsmode return p.RelationshipGet(ctx, requestingAccount, form.ID) } + // check for attempt to follow self + if requestingAccount.ID == targetAcct.ID { + return nil, gtserror.NewErrorNotAcceptable(fmt.Errorf("accountfollowcreate: account %s cannot follow itself", requestingAccount.ID)) + } + // make the follow request newFollowID, err := id.NewRandomULID() if err != nil { diff --git a/internal/processing/admin/importdomainblocks.go b/internal/processing/admin/importdomainblocks.go index 64f02128a..b78589b8a 100644 --- a/internal/processing/admin/importdomainblocks.go +++ b/internal/processing/admin/importdomainblocks.go @@ -55,7 +55,7 @@ func (p *processor) DomainBlocksImport(ctx context.Context, account *gtsmodel.Ac blocks := []*apimodel.DomainBlock{} for _, d := range d { - block, err := p.DomainBlockCreate(ctx, account, d.Domain, false, d.PublicComment, "", "") + block, err := p.DomainBlockCreate(ctx, account, d.Domain.Domain, false, d.PublicComment, "", "") if err != nil { return nil, err diff --git a/internal/processing/admin/mediaprune.go b/internal/processing/admin/mediaprune.go index 0e6abe028..1c3398b78 100644 --- a/internal/processing/admin/mediaprune.go +++ b/internal/processing/admin/mediaprune.go @@ -41,6 +41,15 @@ func (p *processor) MediaPrune(ctx context.Context, mediaRemoteCacheDays int) gt } }() + go func() { + pruned, err := p.mediaManager.PruneUnusedLocalAttachments(ctx) + if err != nil { + logrus.Errorf("MediaPrune: error pruning unused local cache: %s", err) + } else { + logrus.Infof("MediaPrune: pruned %d unused local cache entries", pruned) + } + }() + go func() { pruned, err := p.mediaManager.PruneAllMeta(ctx) if err != nil { diff --git a/internal/processing/fromclientapi.go b/internal/processing/fromclientapi.go index 13be84305..24465a059 100644 --- a/internal/processing/fromclientapi.go +++ b/internal/processing/fromclientapi.go @@ -284,7 +284,12 @@ func (p *processor) processDeleteStatusFromClientAPI(ctx context.Context, client statusToDelete.Account = clientMsg.OriginAccount } - if err := p.wipeStatus(ctx, statusToDelete); err != nil { + // don't delete attachments, just unattach them; + // since this request comes from the client API + // and the poster might want to use the attachments + // again in a new post + deleteAttachments := false + if err := p.wipeStatus(ctx, statusToDelete, deleteAttachments); err != nil { return err } diff --git a/internal/processing/fromcommon.go b/internal/processing/fromcommon.go index e9a2e4994..2cac20193 100644 --- a/internal/processing/fromcommon.go +++ b/internal/processing/fromcommon.go @@ -444,11 +444,23 @@ func (p *processor) deleteStatusFromTimelines(ctx context.Context, status *gtsmo // wipeStatus contains common logic used to totally delete a status // + all its attachments, notifications, boosts, and timeline entries. -func (p *processor) wipeStatus(ctx context.Context, statusToDelete *gtsmodel.Status) error { - // delete all attachments for this status - for _, a := range statusToDelete.AttachmentIDs { - if err := p.mediaProcessor.Delete(ctx, a); err != nil { - return err +func (p *processor) wipeStatus(ctx context.Context, statusToDelete *gtsmodel.Status, deleteAttachments bool) error { + // either delete all attachments for this status, or simply + // unattach all attachments for this status, so they'll be + // cleaned later by a separate process; reason to unattach rather + // than delete is that the poster might want to reattach them + // to another status immediately (in case of delete + redraft) + if deleteAttachments { + for _, a := range statusToDelete.AttachmentIDs { + if err := p.mediaProcessor.Delete(ctx, a); err != nil { + return err + } + } + } else { + for _, a := range statusToDelete.AttachmentIDs { + if _, err := p.mediaProcessor.Unattach(ctx, statusToDelete.Account, a); err != nil { + return err + } } } diff --git a/internal/processing/fromfederator.go b/internal/processing/fromfederator.go index 60f5cc787..e39a6b4e8 100644 --- a/internal/processing/fromfederator.go +++ b/internal/processing/fromfederator.go @@ -367,7 +367,11 @@ func (p *processor) processDeleteStatusFromFederator(ctx context.Context, federa return errors.New("note was not parseable as *gtsmodel.Status") } - return p.wipeStatus(ctx, statusToDelete) + // delete attachments from this status since this request + // comes from the federating API, and there's no way the + // poster can do a delete + redraft for it on our instance + deleteAttachments := true + return p.wipeStatus(ctx, statusToDelete, deleteAttachments) } // processDeleteAccountFromFederator handles Activity Delete and Object Profile diff --git a/internal/processing/instance.go b/internal/processing/instance.go index ab601d3b3..4d1e8b8fd 100644 --- a/internal/processing/instance.go +++ b/internal/processing/instance.go @@ -21,13 +21,16 @@ import ( "context" "fmt" + "sort" apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/text" + "github.com/superseriousbusiness/gotosocial/internal/util" "github.com/superseriousbusiness/gotosocial/internal/validate" ) @@ -45,6 +48,67 @@ func (p *processor) InstanceGet(ctx context.Context, domain string) (*apimodel.I return ai, nil } +func (p *processor) InstancePeersGet(ctx context.Context, authed *oauth.Auth, includeSuspended bool, includeOpen bool, flat bool) (interface{}, gtserror.WithCode) { + domains := []*apimodel.Domain{} + + if includeOpen { + if !config.GetInstanceExposePeers() && (authed.Account == nil || authed.User == nil) { + err := fmt.Errorf("peers open query requires an authenticated account/user") + return nil, gtserror.NewErrorUnauthorized(err, err.Error()) + } + + instances, err := p.db.GetInstancePeers(ctx, false) + if err != nil && err != db.ErrNoEntries { + err = fmt.Errorf("error selecting instance peers: %s", err) + return nil, gtserror.NewErrorInternalError(err) + } + + for _, i := range instances { + domain := &apimodel.Domain{Domain: i.Domain} + domains = append(domains, domain) + } + } + + if includeSuspended { + if !config.GetInstanceExposeSuspended() && (authed.Account == nil || authed.User == nil) { + err := fmt.Errorf("peers suspended query requires an authenticated account/user") + return nil, gtserror.NewErrorUnauthorized(err, err.Error()) + } + + domainBlocks := []*gtsmodel.DomainBlock{} + if err := p.db.GetAll(ctx, &domainBlocks); err != nil && err != db.ErrNoEntries { + return nil, gtserror.NewErrorInternalError(err) + } + + for _, d := range domainBlocks { + if d.Obfuscate { + d.Domain = obfuscate(d.Domain) + } + + domain := &apimodel.Domain{ + Domain: d.Domain, + SuspendedAt: util.FormatISO8601(d.CreatedAt), + PublicComment: d.PublicComment, + } + domains = append(domains, domain) + } + } + + sort.Slice(domains, func(i, j int) bool { + return domains[i].Domain < domains[j].Domain + }) + + if flat { + flattened := []string{} + for _, d := range domains { + flattened = append(flattened, d.Domain) + } + return flattened, nil + } + + return domains, nil +} + func (p *processor) InstancePatch(ctx context.Context, form *apimodel.InstanceSettingsUpdateRequest) (*apimodel.Instance, gtserror.WithCode) { // fetch the instance entry from the db for processing i := >smodel.Instance{} @@ -103,10 +167,13 @@ func (p *processor) InstancePatch(ctx context.Context, form *apimodel.InstanceSe // validate & update site contact email if it's set on the form if form.ContactEmail != nil { - if err := validate.Email(*form.ContactEmail); err != nil { - return nil, gtserror.NewErrorBadRequest(err, err.Error()) + contactEmail := *form.ContactEmail + if contactEmail != "" { + if err := validate.Email(contactEmail); err != nil { + return nil, gtserror.NewErrorBadRequest(err, err.Error()) + } } - i.ContactEmail = *form.ContactEmail + i.ContactEmail = contactEmail } // validate & update site short description if it's set on the form @@ -160,3 +227,15 @@ func (p *processor) InstancePatch(ctx context.Context, form *apimodel.InstanceSe return ai, nil } + +func obfuscate(domain string) string { + obfuscated := make([]rune, len(domain)) + for i, r := range domain { + if i%3 == 1 || i%5 == 1 { + obfuscated[i] = '*' + } else { + obfuscated[i] = r + } + } + return string(obfuscated) +} diff --git a/internal/processing/media/media.go b/internal/processing/media/media.go index 05bea615f..50cbc1b3c 100644 --- a/internal/processing/media/media.go +++ b/internal/processing/media/media.go @@ -37,6 +37,9 @@ type Processor interface { Create(ctx context.Context, account *gtsmodel.Account, form *apimodel.AttachmentRequest) (*apimodel.Attachment, gtserror.WithCode) // Delete deletes the media attachment with the given ID, including all files pertaining to that attachment. Delete(ctx context.Context, mediaAttachmentID string) gtserror.WithCode + // Unattach unattaches the media attachment with the given ID from any statuses it was attached to, making it available + // for reattachment again. + Unattach(ctx context.Context, account *gtsmodel.Account, mediaAttachmentID string) (*apimodel.Attachment, gtserror.WithCode) // GetFile retrieves a file from storage and streams it back to the caller via an io.reader embedded in *apimodel.Content. GetFile(ctx context.Context, account *gtsmodel.Account, form *apimodel.GetContentRequestForm) (*apimodel.Content, gtserror.WithCode) GetCustomEmojis(ctx context.Context) ([]*apimodel.Emoji, gtserror.WithCode) diff --git a/internal/processing/media/unattach.go b/internal/processing/media/unattach.go new file mode 100644 index 000000000..bb09525fe --- /dev/null +++ b/internal/processing/media/unattach.go @@ -0,0 +1,59 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package media + +import ( + "context" + "errors" + "fmt" + "time" + + apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtserror" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" +) + +func (p *processor) Unattach(ctx context.Context, account *gtsmodel.Account, mediaAttachmentID string) (*apimodel.Attachment, gtserror.WithCode) { + attachment, err := p.db.GetAttachmentByID(ctx, mediaAttachmentID) + if err != nil { + if err == db.ErrNoEntries { + return nil, gtserror.NewErrorNotFound(errors.New("attachment doesn't exist in the db")) + } + return nil, gtserror.NewErrorNotFound(fmt.Errorf("db error getting attachment: %s", err)) + } + + if attachment.AccountID != account.ID { + return nil, gtserror.NewErrorNotFound(errors.New("attachment not owned by requesting account")) + } + + attachment.UpdatedAt = time.Now() + attachment.StatusID = "" + + if err := p.db.UpdateByPrimaryKey(ctx, attachment); err != nil { + return nil, gtserror.NewErrorNotFound(fmt.Errorf("db error updating attachment: %s", err)) + } + + a, err := p.tc.AttachmentToAPIAttachment(ctx, attachment) + if err != nil { + return nil, gtserror.NewErrorNotFound(fmt.Errorf("error converting attachment: %s", err)) + } + + return &a, nil +} diff --git a/internal/processing/media/unattach_test.go b/internal/processing/media/unattach_test.go new file mode 100644 index 000000000..60efc2688 --- /dev/null +++ b/internal/processing/media/unattach_test.go @@ -0,0 +1,53 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package media_test + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/suite" +) + +type UnattachTestSuite struct { + MediaStandardTestSuite +} + +func (suite *GetFileTestSuite) TestUnattachMedia() { + ctx := context.Background() + + testAttachment := suite.testAttachments["admin_account_status_1_attachment_1"] + testAccount := suite.testAccounts["admin_account"] + suite.NotEmpty(testAttachment.StatusID) + + a, err := suite.mediaProcessor.Unattach(ctx, testAccount, testAttachment.ID) + suite.NoError(err) + suite.NotNil(a) + + dbAttachment, errWithCode := suite.db.GetAttachmentByID(ctx, a.ID) + suite.NoError(errWithCode) + + suite.WithinDuration(dbAttachment.UpdatedAt, time.Now(), 1*time.Minute) + suite.Empty(dbAttachment.StatusID) +} + +func TestUnattachTestSuite(t *testing.T) { + suite.Run(t, &UnattachTestSuite{}) +} diff --git a/internal/processing/processor.go b/internal/processing/processor.go index 3afc25196..8572b583c 100644 --- a/internal/processing/processor.go +++ b/internal/processing/processor.go @@ -137,6 +137,7 @@ type Processor interface { // InstanceGet retrieves instance information for serving at api/v1/instance InstanceGet(ctx context.Context, domain string) (*apimodel.Instance, gtserror.WithCode) + InstancePeersGet(ctx context.Context, authed *oauth.Auth, includeSuspended bool, includeOpen bool, flat bool) (interface{}, gtserror.WithCode) // InstancePatch updates this instance according to the given form. // // It should already be ascertained that the requesting account is authenticated and an admin. diff --git a/internal/typeutils/internaltofrontend.go b/internal/typeutils/internaltofrontend.go index 364e7152c..068ba700c 100644 --- a/internal/typeutils/internaltofrontend.go +++ b/internal/typeutils/internaltofrontend.go @@ -29,6 +29,7 @@ "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/media" "github.com/superseriousbusiness/gotosocial/internal/util" ) @@ -575,6 +576,16 @@ func (c *converter) InstanceToAPIInstance(ctx context.Context, i *gtsmodel.Insta // if the requested instance is *this* instance, we can add some extra information if host := config.GetHost(); i.Domain == host { + if ia, err := c.db.GetInstanceAccount(ctx, ""); err == nil { + if ia.HeaderMediaAttachment != nil { + // take instance account header as instance thumbnail + mi.Thumbnail = ia.HeaderMediaAttachment.URL + } else { + // or just use a default + mi.Thumbnail = config.GetProtocol() + "://" + host + "/assets/logo.png" + } + } + userCount, err := c.db.CountInstanceUsers(ctx, host) if err == nil { mi.Stats["user_count"] = userCount @@ -595,16 +606,31 @@ func (c *converter) InstanceToAPIInstance(ctx context.Context, i *gtsmodel.Insta mi.InvitesEnabled = false // TODO mi.MaxTootChars = uint(config.GetStatusesMaxChars()) mi.URLS = &model.InstanceURLs{ - StreamingAPI: fmt.Sprintf("wss://%s", host), + StreamingAPI: "wss://" + host, } mi.Version = config.GetSoftwareVersion() - } - // get the instance account if it exists and just skip if it doesn't - ia, err := c.db.GetInstanceAccount(ctx, "") - if err == nil { - if ia.HeaderMediaAttachment != nil { - mi.Thumbnail = ia.HeaderMediaAttachment.URL + // todo: remove hardcoded values and put them in config somewhere + mi.Configuration = &model.InstanceConfiguration{ + Statuses: &model.InstanceConfigurationStatuses{ + MaxCharacters: config.GetStatusesMaxChars(), + MaxMediaAttachments: config.GetStatusesMediaMaxFiles(), + CharactersReservedPerURL: 999, + }, + MediaAttachments: &model.InstanceConfigurationMediaAttachments{ + SupportedMimeTypes: media.AllSupportedMIMETypes(), + ImageSizeLimit: config.GetMediaImageMaxSize(), + ImageMatrixLimit: 16777216, // height*width + VideoSizeLimit: config.GetMediaVideoMaxSize(), + VideoFrameRateLimit: 60, + VideoMatrixLimit: 16777216, // height*width + }, + Polls: &model.InstanceConfigurationPolls{ + MaxOptions: config.GetStatusesPollMaxOptions(), + MaxCharactersPerOption: config.GetStatusesPollOptionMaxChars(), + MinExpiration: 300, // seconds + MaxExpiration: 2629746, // seconds + }, } } @@ -701,8 +727,10 @@ func (c *converter) NotificationToAPINotification(ctx context.Context, n *gtsmod func (c *converter) DomainBlockToAPIDomainBlock(ctx context.Context, b *gtsmodel.DomainBlock, export bool) (*model.DomainBlock, error) { domainBlock := &model.DomainBlock{ - Domain: b.Domain, - PublicComment: b.PublicComment, + Domain: model.Domain{ + Domain: b.Domain, + PublicComment: b.PublicComment, + }, } // if we're exporting a domain block, return it with minimal information attached diff --git a/mkdocs.yml b/mkdocs.yml index 2ed4edcb7..074381f15 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -27,6 +27,7 @@ nav: - "configuration/general.md" - "configuration/database.md" - "configuration/web.md" + - "configuration/instance.md" - "configuration/accounts.md" - "configuration/media.md" - "configuration/storage.md" diff --git a/test/cliparsing.sh b/test/cliparsing.sh index 0cf83a82d..f6def6650 100755 --- a/test/cliparsing.sh +++ b/test/cliparsing.sh @@ -5,7 +5,7 @@ set -e echo "STARTING CLI TESTS" echo "TEST_1 Make sure defaults are set correctly." -TEST_1_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"","db-address":"","db-database":"gotosocial","db-password":"","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"","email":"","host":"","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":false,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-remote-cache-days":30,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","profile","email","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"GoToSocial","smtp-host":"","smtp-password":"","smtp-port":0,"smtp-username":"","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}' +TEST_1_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"","db-address":"","db-database":"gotosocial","db-password":"","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"","email":"","host":"","instance-expose-peers":false,"instance-expose-suspended":false,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":false,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-remote-cache-days":30,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","profile","email","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"GoToSocial","smtp-host":"","smtp-password":"","smtp-port":0,"smtp-username":"","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}' TEST_1="$(go run ./cmd/gotosocial/... debug config)" if [ "${TEST_1}" != "${TEST_1_EXPECTED}" ]; then echo "TEST_1 not equal TEST_1_EXPECTED" @@ -15,7 +15,7 @@ else fi echo "TEST_2 Override db-address from default using cli flag." -TEST_2_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"","db-address":"some.db.address","db-database":"gotosocial","db-password":"","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"","email":"","host":"","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":false,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-remote-cache-days":30,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","profile","email","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"GoToSocial","smtp-host":"","smtp-password":"","smtp-port":0,"smtp-username":"","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}' +TEST_2_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"","db-address":"some.db.address","db-database":"gotosocial","db-password":"","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"","email":"","host":"","instance-expose-peers":false,"instance-expose-suspended":false,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":false,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-remote-cache-days":30,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","profile","email","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"GoToSocial","smtp-host":"","smtp-password":"","smtp-port":0,"smtp-username":"","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}' TEST_2="$(go run ./cmd/gotosocial/... --db-address some.db.address debug config)" if [ "${TEST_2}" != "${TEST_2_EXPECTED}" ]; then echo "TEST_2 not equal TEST_2_EXPECTED" @@ -25,7 +25,7 @@ else fi echo "TEST_3 Override db-address from default using env var." -TEST_3_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"","db-address":"some.db.address","db-database":"gotosocial","db-password":"","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"","email":"","host":"","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":false,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-remote-cache-days":30,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","profile","email","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"GoToSocial","smtp-host":"","smtp-password":"","smtp-port":0,"smtp-username":"","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}' +TEST_3_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"","db-address":"some.db.address","db-database":"gotosocial","db-password":"","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"","email":"","host":"","instance-expose-peers":false,"instance-expose-suspended":false,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":false,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-remote-cache-days":30,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","profile","email","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"GoToSocial","smtp-host":"","smtp-password":"","smtp-port":0,"smtp-username":"","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}' TEST_3="$(GTS_DB_ADDRESS=some.db.address go run ./cmd/gotosocial/... debug config)" if [ "${TEST_3}" != "${TEST_3_EXPECTED}" ]; then echo "TEST_3 not equal TEST_3_EXPECTED" @@ -35,7 +35,7 @@ else fi echo "TEST_4 Override db-address from default using both env var and cli flag. The cli flag should take priority." -TEST_4_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"","db-address":"some.other.db.address","db-database":"gotosocial","db-password":"","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"","email":"","host":"","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":false,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-remote-cache-days":30,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","profile","email","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"GoToSocial","smtp-host":"","smtp-password":"","smtp-port":0,"smtp-username":"","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}' +TEST_4_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"","db-address":"some.other.db.address","db-database":"gotosocial","db-password":"","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"","email":"","host":"","instance-expose-peers":false,"instance-expose-suspended":false,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":false,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-remote-cache-days":30,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","profile","email","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"GoToSocial","smtp-host":"","smtp-password":"","smtp-port":0,"smtp-username":"","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}' TEST_4="$(GTS_DB_ADDRESS=some.db.address go run ./cmd/gotosocial/... --db-address some.other.db.address debug config)" if [ "${TEST_4}" != "${TEST_4_EXPECTED}" ]; then echo "TEST_4 not equal TEST_4_EXPECTED" @@ -45,7 +45,7 @@ else fi echo "TEST_5 Test loading a config file by passing an env var." -TEST_5_EXPECTED='{"account-domain":"example.org","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.yaml","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","email":"","host":"gts.example.org","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-remote-cache-days":30,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}' +TEST_5_EXPECTED='{"account-domain":"example.org","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.yaml","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","email":"","host":"gts.example.org","instance-expose-peers":false,"instance-expose-suspended":false,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-remote-cache-days":30,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}' TEST_5="$(GTS_CONFIG_PATH=./test/test.yaml go run ./cmd/gotosocial/... debug config)" if [ "${TEST_5}" != "${TEST_5_EXPECTED}" ]; then echo "TEST_5 not equal TEST_5_EXPECTED" @@ -55,7 +55,7 @@ else fi echo "TEST_6 Test loading a config file by passing cli flag." -TEST_6_EXPECTED='{"account-domain":"example.org","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.yaml","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","email":"","host":"gts.example.org","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-remote-cache-days":30,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}' +TEST_6_EXPECTED='{"account-domain":"example.org","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.yaml","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","email":"","host":"gts.example.org","instance-expose-peers":false,"instance-expose-suspended":false,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-remote-cache-days":30,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}' TEST_6="$(go run ./cmd/gotosocial/... --config-path ./test/test.yaml debug config)" if [ "${TEST_6}" != "${TEST_6_EXPECTED}" ]; then echo "TEST_6 not equal TEST_6_EXPECTED" @@ -65,7 +65,7 @@ else fi echo "TEST_7 Test loading a config file and overriding one of the variables with a cli flag." -TEST_7_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.yaml","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","email":"","host":"gts.example.org","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-remote-cache-days":30,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}' +TEST_7_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.yaml","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","email":"","host":"gts.example.org","instance-expose-peers":false,"instance-expose-suspended":false,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-remote-cache-days":30,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}' TEST_7="$(go run ./cmd/gotosocial/... --config-path ./test/test.yaml --account-domain '' debug config)" if [ "${TEST_7}" != "${TEST_7_EXPECTED}" ]; then echo "TEST_7 not equal TEST_7_EXPECTED" @@ -75,7 +75,7 @@ else fi echo "TEST_8 Test loading a config file and overriding one of the variables with an env var." -TEST_8_EXPECTED='{"account-domain":"peepee","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.yaml","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","email":"","host":"gts.example.org","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-remote-cache-days":30,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}' +TEST_8_EXPECTED='{"account-domain":"peepee","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.yaml","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","email":"","host":"gts.example.org","instance-expose-peers":false,"instance-expose-suspended":false,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-remote-cache-days":30,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}' TEST_8="$(GTS_ACCOUNT_DOMAIN='peepee' go run ./cmd/gotosocial/... --config-path ./test/test.yaml debug config)" if [ "${TEST_8}" != "${TEST_8_EXPECTED}" ]; then echo "TEST_8 not equal TEST_8_EXPECTED" @@ -85,7 +85,7 @@ else fi echo "TEST_9 Test loading a config file and overriding one of the variables with both an env var and a cli flag. The cli flag should have priority." -TEST_9_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.yaml","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","email":"","host":"gts.example.org","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-remote-cache-days":30,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}' +TEST_9_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.yaml","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","email":"","host":"gts.example.org","instance-expose-peers":false,"instance-expose-suspended":false,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-remote-cache-days":30,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}' TEST_9="$(GTS_ACCOUNT_DOMAIN='peepee' go run ./cmd/gotosocial/... --config-path ./test/test.yaml --account-domain '' debug config)" if [ "${TEST_9}" != "${TEST_9_EXPECTED}" ]; then echo "TEST_9 not equal TEST_9_EXPECTED" @@ -95,7 +95,7 @@ else fi echo "TEST_10 Test loading a config file from json." -TEST_10_EXPECTED='{"account-domain":"example.org","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.json","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","email":"","host":"gts.example.org","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-remote-cache-days":30,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}' +TEST_10_EXPECTED='{"account-domain":"example.org","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.json","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","email":"","host":"gts.example.org","instance-expose-peers":false,"instance-expose-suspended":false,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-remote-cache-days":30,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}' TEST_10="$(go run ./cmd/gotosocial/... --config-path ./test/test.json debug config)" if [ "${TEST_10}" != "${TEST_10_EXPECTED}" ]; then echo "TEST_10 not equal TEST_10_EXPECTED" @@ -105,7 +105,7 @@ else fi echo "TEST_11 Test loading a partial config file. Default values should be used apart from those set in the config file." -TEST_11_EXPECTED='{"account-domain":"peepee.poopoo","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test2.yaml","db-address":"","db-database":"gotosocial","db-password":"","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"","email":"","host":"","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":false,"letsencrypt-port":80,"log-db-queries":false,"log-level":"trace","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-remote-cache-days":30,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","profile","email","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"GoToSocial","smtp-host":"","smtp-password":"","smtp-port":0,"smtp-username":"","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}' +TEST_11_EXPECTED='{"account-domain":"peepee.poopoo","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test2.yaml","db-address":"","db-database":"gotosocial","db-password":"","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"","email":"","host":"","instance-expose-peers":false,"instance-expose-suspended":false,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":false,"letsencrypt-port":80,"log-db-queries":false,"log-level":"trace","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-remote-cache-days":30,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","profile","email","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"GoToSocial","smtp-host":"","smtp-password":"","smtp-port":0,"smtp-username":"","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}' TEST_11="$(go run ./cmd/gotosocial/... --config-path ./test/test2.yaml debug config)" if [ "${TEST_11}" != "${TEST_11_EXPECTED}" ]; then echo "TEST_11 not equal TEST_11_EXPECTED" diff --git a/test/envparsing.sh b/test/envparsing.sh index 8bced72e9..9c7bdb341 100755 --- a/test/envparsing.sh +++ b/test/envparsing.sh @@ -2,7 +2,7 @@ set -eu -EXPECTED='{"account-domain":"peepee","accounts-approval-required":false,"accounts-reason-required":false,"accounts-registration-open":true,"advanced-cookies-samesite":"strict","application-name":"gts","bind-address":"127.0.0.1","config-path":"./test/test.yaml","db-address":":memory:","db-database":"gotosocial_prod","db-password":"hunter2","db-port":6969,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"sqlite","db-user":"sex-haver","email":"","host":"example.com","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":true,"log-level":"info","media-description-max-chars":5000,"media-description-min-chars":69,"media-image-max-size":420,"media-remote-cache-days":30,"media-video-max-size":420,"oidc-client-id":"1234","oidc-client-secret":"shhhh its a secret","oidc-enabled":true,"oidc-idp-name":"sex-haver","oidc-issuer":"whoknows","oidc-scopes":["read","write"],"oidc-skip-verification":true,"password":"","path":"","port":6969,"protocol":"http","smtp-from":"queen@terfisland.org","smtp-host":"example.com","smtp-password":"hunter2","smtp-port":4269,"smtp-username":"sex-haver","software-version":"","statuses-cw-max-chars":420,"statuses-max-chars":69,"statuses-media-max-files":1,"statuses-poll-max-options":1,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/root/store","syslog-address":"127.0.0.1:6969","syslog-enabled":true,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"username":"","web-asset-base-dir":"/root","web-template-base-dir":"/root"}' +EXPECTED='{"account-domain":"peepee","accounts-approval-required":false,"accounts-reason-required":false,"accounts-registration-open":true,"advanced-cookies-samesite":"strict","application-name":"gts","bind-address":"127.0.0.1","config-path":"./test/test.yaml","db-address":":memory:","db-database":"gotosocial_prod","db-password":"hunter2","db-port":6969,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"sqlite","db-user":"sex-haver","email":"","host":"example.com","instance-expose-peers":true,"instance-expose-suspended":true,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":true,"log-level":"info","media-description-max-chars":5000,"media-description-min-chars":69,"media-image-max-size":420,"media-remote-cache-days":30,"media-video-max-size":420,"oidc-client-id":"1234","oidc-client-secret":"shhhh its a secret","oidc-enabled":true,"oidc-idp-name":"sex-haver","oidc-issuer":"whoknows","oidc-scopes":["read","write"],"oidc-skip-verification":true,"password":"","path":"","port":6969,"protocol":"http","smtp-from":"queen@terfisland.org","smtp-host":"example.com","smtp-password":"hunter2","smtp-port":4269,"smtp-username":"sex-haver","software-version":"","statuses-cw-max-chars":420,"statuses-max-chars":69,"statuses-media-max-files":1,"statuses-poll-max-options":1,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/root/store","syslog-address":"127.0.0.1:6969","syslog-enabled":true,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"username":"","web-asset-base-dir":"/root","web-template-base-dir":"/root"}' # Set all the environment variables to # ensure that these are parsed without panic @@ -25,6 +25,8 @@ GTS_TLS_MODE='' \ GTS_DB_TLS_CA_CERT='' \ GTS_WEB_TEMPLATE_BASE_DIR='/root' \ GTS_WEB_ASSET_BASE_DIR='/root' \ +GTS_INSTANCE_EXPOSE_PEERS=true \ +GTS_INSTANCE_EXPOSE_SUSPENDED=true \ GTS_ACCOUNTS_REGISTRATION_OPEN=true \ GTS_ACCOUNTS_APPROVAL_REQUIRED=false \ GTS_ACCOUNTS_REASON_REQUIRED=false \ diff --git a/testrig/config.go b/testrig/config.go index 92d04c453..e0e5d1029 100644 --- a/testrig/config.go +++ b/testrig/config.go @@ -56,6 +56,9 @@ func InitTestConfig() { WebTemplateBaseDir: "./web/template/", WebAssetBaseDir: "./web/assets/", + InstanceExposePeers: true, + InstanceExposeSuspended: true, + AccountsRegistrationOpen: true, AccountsApprovalRequired: true, AccountsReasonRequired: true, diff --git a/testrig/db.go b/testrig/db.go index 67bc681c6..b4c05d727 100644 --- a/testrig/db.go +++ b/testrig/db.go @@ -153,6 +153,12 @@ func StandardDBSetup(db db.DB, accounts map[string]*gtsmodel.Account) { } } + for _, v := range NewTestInstances() { + if err := db.Put(ctx, v); err != nil { + logrus.Panic(err) + } + } + for _, v := range NewTestUsers() { if err := db.Put(ctx, v); err != nil { logrus.Panic(err) diff --git a/testrig/testmodels.go b/testrig/testmodels.go index e734ff255..bca894266 100644 --- a/testrig/testmodels.go +++ b/testrig/testmodels.go @@ -936,10 +936,31 @@ func NewTestEmojis() map[string]*gtsmodel.Emoji { } } +func NewTestInstances() map[string]*gtsmodel.Instance { + return map[string]*gtsmodel.Instance{ + "fossbros-anonymous.io": { + ID: "01G5H6YMJQKR86QZKXXQ2S95FZ", + CreatedAt: TimeMustParse("2021-09-20T12:40:37+02:00"), + UpdatedAt: TimeMustParse("2021-09-20T12:40:37+02:00"), + Domain: "fossbros-anonymous.io", + URI: "http://fossbros-anonymous.io", + }, + "example.org": { + ID: "01G5H71G52DJKVBYKXPNPNDN1G", + CreatedAt: TimeMustParse("2020-05-13T15:29:12+02:00"), + UpdatedAt: TimeMustParse("2020-05-13T15:29:12+02:00"), + Domain: "example.org", + URI: "http://example.org", + }, + } +} + func NewTestDomainBlocks() map[string]*gtsmodel.DomainBlock { return map[string]*gtsmodel.DomainBlock{ "replyguys.com": { ID: "01FF22EQM7X8E3RX1XGPN7S87D", + CreatedAt: TimeMustParse("2020-05-13T15:29:12+02:00"), + UpdatedAt: TimeMustParse("2020-05-13T15:29:12+02:00"), Domain: "replyguys.com", CreatedByAccountID: "01F8MH17FWEB39HZJ76B6VXSKF", PrivateComment: "i blocked this domain because they keep replying with pushy + unwarranted linux advice", diff --git a/vendor/github.com/superseriousbusiness/exif-terminator/README.md b/vendor/github.com/superseriousbusiness/exif-terminator/README.md index 8866202d5..7f7209259 100644 --- a/vendor/github.com/superseriousbusiness/exif-terminator/README.md +++ b/vendor/github.com/superseriousbusiness/exif-terminator/README.md @@ -53,10 +53,14 @@ Exif removal is a pain in the arse. Most other libraries seem to parse the whole `exif-terminator` differs in that it removes exif data *while scanning through the image bytes*, and it doesn't do any reencoding of the image. Bytes of exif data are simply all set to 0, and the image data is piped back out again into the returned reader. +The only exception is orientation data: if an image contains orientation data, this and only this data will be preserved since it's *actually useful*. + ## Example +You can run the following example with `go run ./example/main.go`: + ```go -package test +package main import ( "io" @@ -71,6 +75,7 @@ func main() { if err != nil { panic(err) } + defer sloth.Close() // get the length of the file stat, err := sloth.Stat() @@ -103,6 +108,7 @@ func main() { `exif-terminator` borrows heavily from the two [`dsoprea`](https://github.com/dsoprea) libraries credited below. In fact, it's basically a hack on top of those libraries. Thanks `dsoprea`! +- [dsoprea/go-exif](https://github.com/dsoprea/go-exif): exif header reconstruction. [MIT License](https://spdx.org/licenses/MIT.html). - [dsoprea/go-jpeg-image-structure](https://github.com/dsoprea/go-jpeg-image-structure): jpeg structure parsing. [MIT License](https://spdx.org/licenses/MIT.html). - [dsoprea/go-png-image-structure](https://github.com/dsoprea/go-png-image-structure): png structure parsing. [MIT License](https://spdx.org/licenses/MIT.html). - [stretchr/testify](https://github.com/stretchr/testify); test framework. [MIT License](https://spdx.org/licenses/MIT.html). diff --git a/vendor/github.com/superseriousbusiness/exif-terminator/jpeg.go b/vendor/github.com/superseriousbusiness/exif-terminator/jpeg.go index e3da53d54..9538d4146 100644 --- a/vendor/github.com/superseriousbusiness/exif-terminator/jpeg.go +++ b/vendor/github.com/superseriousbusiness/exif-terminator/jpeg.go @@ -19,10 +19,12 @@ package terminator import ( + "bytes" "encoding/binary" "fmt" "io" + exif "github.com/dsoprea/go-exif/v3" jpegstructure "github.com/superseriousbusiness/go-jpeg-image-structure/v2" ) @@ -121,18 +123,129 @@ func (v *jpegVisitor) writeSegment(s *jpegstructure.Segment) error { } } - if s.IsExif() { - // if this segment is exif data, write blank bytes - blank := make([]byte, len(s.Data)) - if _, err := w.Write(blank); err != nil { + if !s.IsExif() { + // if this isn't exif data just copy it over and bail + _, err := w.Write(s.Data) + return err + } + + ifd, _, err := s.Exif() + if err != nil { + return err + } + + // amount of bytes we've written into the exif body + var written int + + if orientationEntries, err := ifd.FindTagWithName("Orientation"); err == nil && len(orientationEntries) == 1 { + // If we have an orientation entry, we don't want to completely obliterate the exif data. + // Instead, we want to surgically obliterate everything *except* the orientation tag, so + // that the image will still be rotated correctly when shown in client applications etc. + // + // To accomplish this, we're going to extract just the bytes that we need and write them + // in according to the exif specification, then fill in the rest of the space with empty + // bytes. + // + // First we need to write the exif prefix for this segment. + // + // Then we write the exif header which contains the byte order and offset of the first ifd. + // + // Then we write the ifd0 entry which contains the orientation data. + // + // After that we just fill fill fill. + + newData := &bytes.Buffer{} + + // 1. Write exif prefix. + // https://www.ozhiker.com/electronics/pjmt/jpeg_info/app_segments.html + prefix := []byte{'E', 'x', 'i', 'f', 0, 0} + if err := binary.Write(newData, ifd.ByteOrder(), &prefix); err != nil { return err } - } else { - // otherwise write the data - if _, err := w.Write(s.Data); err != nil { + written += 6 + + // 2. Write exif header, taking the existing byte order. + exifHeader, err := exif.BuildExifHeader(ifd.ByteOrder(), exif.ExifDefaultFirstIfdOffset) + if err != nil { + return err + } + hWritten, err := newData.Write(exifHeader) + if err != nil { + return err + } + written += hWritten + + // https://web.archive.org/web/20190624045241if_/http://www.cipa.jp:80/std/documents/e/DC-008-Translation-2019-E.pdf + // + // An ifd with one orientation entry is structured like this: + // 2 bytes: the number of entries in the ifd uint16(1) + // 2 bytes: the tag id uint16(274) + // 2 bytes: the tag type uint16(3) + // 4 bytes: the tag count uint32(1) + // 4 bytes: the tag value offset: uint32(one of the below with padding on the end) + // 1 = Horizontal (normal) + // 2 = Mirror horizontal + // 3 = Rotate 180 + // 4 = Mirror vertical + // 5 = Mirror horizontal and rotate 270 CW + // 6 = Rotate 90 CW + // 7 = Mirror horizontal and rotate 90 CW + // 8 = Rotate 270 CW + orientationEntry := orientationEntries[0] + + ifdCount := uint16(1) // we're only adding one entry into the ifd + if err := binary.Write(newData, ifd.ByteOrder(), &ifdCount); err != nil { + return err + } + written += 2 + + tagID := orientationEntry.TagId() + if err := binary.Write(newData, ifd.ByteOrder(), &tagID); err != nil { + return err + } + written += 2 + + tagType := orientationEntry.TagType() + if err := binary.Write(newData, ifd.ByteOrder(), &tagType); err != nil { + return err + } + written += 2 + + tagCount := orientationEntry.UnitCount() + if err := binary.Write(newData, ifd.ByteOrder(), &tagCount); err != nil { + return err + } + written += 4 + + valueOffset, err := orientationEntry.GetRawBytes() + if err != nil { + return err + } + + vWritten, err := newData.Write(valueOffset) + if err != nil { + return err + } + written += vWritten + + valuePad := make([]byte, 4-vWritten) + pWritten, err := newData.Write(valuePad) + if err != nil { + return err + } + written += pWritten + + // write everything in + if _, err := io.Copy(w, newData); err != nil { return err } } + // fill in the (remaining) exif body with blank bytes + blank := make([]byte, len(s.Data)-written) + if _, err := w.Write(blank); err != nil { + return err + } + return nil } diff --git a/vendor/github.com/superseriousbusiness/exif-terminator/terminator.go b/vendor/github.com/superseriousbusiness/exif-terminator/terminator.go index 36f2d8b3a..0eb46ea9f 100644 --- a/vendor/github.com/superseriousbusiness/exif-terminator/terminator.go +++ b/vendor/github.com/superseriousbusiness/exif-terminator/terminator.go @@ -25,8 +25,8 @@ "fmt" "io" - jpegstructure "github.com/superseriousbusiness/go-jpeg-image-structure/v2" pngstructure "github.com/dsoprea/go-png-image-structure/v2" + jpegstructure "github.com/superseriousbusiness/go-jpeg-image-structure/v2" ) func Terminate(in io.Reader, fileSize int, mediaType string) (io.Reader, error) { diff --git a/vendor/modules.txt b/vendor/modules.txt index c80c0d0b9..c2bc24c81 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -487,7 +487,7 @@ github.com/superseriousbusiness/activity/streams/values/rfc2045 github.com/superseriousbusiness/activity/streams/values/rfc5988 github.com/superseriousbusiness/activity/streams/values/string github.com/superseriousbusiness/activity/streams/vocab -# github.com/superseriousbusiness/exif-terminator v0.2.0 +# github.com/superseriousbusiness/exif-terminator v0.3.0 ## explicit; go 1.17 github.com/superseriousbusiness/exif-terminator # github.com/superseriousbusiness/go-jpeg-image-structure/v2 v2.0.0-20220321154430-d89a106fdabe