switch to a safer crontab fork; update deps

This commit is contained in:
Aine 2025-11-04 23:02:19 +00:00
parent 8af0bea527
commit 4b4223b3c5
No known key found for this signature in database
GPG key ID: 34969C908CCA2804
125 changed files with 20059 additions and 142096 deletions

View file

@ -11,11 +11,11 @@ import (
"time" "time"
zlogsentry "github.com/archdx/zerolog-sentry" zlogsentry "github.com/archdx/zerolog-sentry"
"github.com/etkecc/go-crontab"
"github.com/etkecc/go-healthchecks/v2" "github.com/etkecc/go-healthchecks/v2"
"github.com/etkecc/go-linkpearl" "github.com/etkecc/go-linkpearl"
"github.com/getsentry/sentry-go" "github.com/getsentry/sentry-go"
_ "github.com/lib/pq" _ "github.com/lib/pq"
"github.com/mileusna/crontab"
"github.com/rs/zerolog" "github.com/rs/zerolog"
_ "modernc.org/sqlite" _ "modernc.org/sqlite"
@ -157,6 +157,9 @@ func initSMTP(cfg *config.Config) {
func initCron() { func initCron() {
cron = crontab.New() cron = crontab.New()
cron.SetPanicLogger(func(r any) {
log.Error().Any("recover", r).Msg("cronjob panicked")
})
err := cron.AddJob("* * * * *", q.Process) err := cron.AddJob("* * * * *", q.Process)
if err != nil { if err != nil {

23
go.mod
View file

@ -7,28 +7,28 @@ require (
github.com/emersion/go-msgauth v0.7.0 github.com/emersion/go-msgauth v0.7.0
github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6
github.com/emersion/go-smtp v0.24.0 github.com/emersion/go-smtp v0.24.0
github.com/etkecc/go-crontab v1.2.1
github.com/etkecc/go-env v1.2.1 github.com/etkecc/go-env v1.2.1
github.com/etkecc/go-fswatcher v1.0.1 github.com/etkecc/go-fswatcher v1.0.1
github.com/etkecc/go-healthchecks/v2 v2.2.2 github.com/etkecc/go-healthchecks/v2 v2.2.2
github.com/etkecc/go-kit v1.7.5 github.com/etkecc/go-kit v1.7.5
github.com/etkecc/go-linkpearl v0.0.0-20251009114554-5985fe44a189 github.com/etkecc/go-linkpearl v0.0.0-20251016112544-0d85fb3bccdc
github.com/etkecc/go-mxidwc v1.0.1 github.com/etkecc/go-mxidwc v1.0.1
github.com/etkecc/go-secgen v1.4.0 github.com/etkecc/go-secgen v1.4.0
github.com/etkecc/go-validator/v2 v2.2.6 github.com/etkecc/go-validator/v2 v2.2.6
github.com/fsnotify/fsnotify v1.9.0 github.com/fsnotify/fsnotify v1.9.0
github.com/gabriel-vasile/mimetype v1.4.10 github.com/gabriel-vasile/mimetype v1.4.11
github.com/getsentry/sentry-go v0.35.3 github.com/getsentry/sentry-go v0.36.2
github.com/jhillyerd/enmime/v2 v2.2.0 github.com/jhillyerd/enmime/v2 v2.2.0
github.com/kvannotten/mailstrip v0.0.0-20200711213611-0002f5c0467e github.com/kvannotten/mailstrip v0.0.0-20200711213611-0002f5c0467e
github.com/lib/pq v1.10.9 github.com/lib/pq v1.10.9
github.com/mcnijman/go-emailaddress v1.1.1 github.com/mcnijman/go-emailaddress v1.1.1
github.com/mileusna/crontab v1.2.0
github.com/raja/argon2pw v1.0.2-0.20210910183755-a391af63bd39 github.com/raja/argon2pw v1.0.2-0.20210910183755-a391af63bd39
github.com/rs/zerolog v1.34.0 github.com/rs/zerolog v1.34.0
github.com/swaggo/swag v1.16.3 github.com/swaggo/swag v1.16.3
golang.org/x/exp v0.0.0-20251002181428-27f1f14c8bb9 golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546
maunium.net/go/mautrix v0.25.1 maunium.net/go/mautrix v0.25.2
modernc.org/sqlite v1.39.0 modernc.org/sqlite v1.40.0
) )
require ( require (
@ -39,7 +39,8 @@ require (
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
github.com/buger/jsonparser v1.1.1 // indirect github.com/buger/jsonparser v1.1.1 // indirect
github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a // indirect github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a // indirect
github.com/clipperhouse/uax29/v2 v2.2.0 // indirect github.com/clipperhouse/stringish v0.1.1 // indirect
github.com/clipperhouse/uax29/v2 v2.3.0 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect
github.com/etkecc/go-trysmtp v1.1.5 // indirect github.com/etkecc/go-trysmtp v1.1.5 // indirect
github.com/fatih/color v1.18.0 // indirect github.com/fatih/color v1.18.0 // indirect
@ -61,7 +62,7 @@ require (
github.com/ncruces/go-strftime v1.0.0 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect
github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 // indirect github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 // indirect
github.com/olekukonko/errors v1.1.0 // indirect github.com/olekukonko/errors v1.1.0 // indirect
github.com/olekukonko/ll v0.1.1 // indirect github.com/olekukonko/ll v0.1.2 // indirect
github.com/olekukonko/tablewriter v1.1.0 // indirect github.com/olekukonko/tablewriter v1.1.0 // indirect
github.com/petermattis/goid v0.0.0-20250904145737-900bdf8bb490 // indirect github.com/petermattis/goid v0.0.0-20250904145737-900bdf8bb490 // indirect
github.com/pkg/errors v0.9.1 // indirect github.com/pkg/errors v0.9.1 // indirect
@ -73,12 +74,12 @@ require (
github.com/tidwall/pretty v1.2.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect
github.com/tidwall/sjson v1.2.5 // indirect github.com/tidwall/sjson v1.2.5 // indirect
github.com/yuin/goldmark v1.7.13 // indirect github.com/yuin/goldmark v1.7.13 // indirect
go.mau.fi/util v0.9.1 // indirect go.mau.fi/util v0.9.2 // indirect
golang.org/x/crypto v0.43.0 // indirect golang.org/x/crypto v0.43.0 // indirect
golang.org/x/net v0.46.0 // indirect golang.org/x/net v0.46.0 // indirect
golang.org/x/sys v0.37.0 // indirect golang.org/x/sys v0.37.0 // indirect
golang.org/x/text v0.30.0 // indirect golang.org/x/text v0.30.0 // indirect
golang.org/x/tools v0.37.0 // indirect golang.org/x/tools v0.38.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect
modernc.org/libc v1.66.10 // indirect modernc.org/libc v1.66.10 // indirect
modernc.org/mathutil v1.7.1 // indirect modernc.org/mathutil v1.7.1 // indirect

50
go.sum
View file

@ -16,8 +16,10 @@ github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMU
github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a h1:MISbI8sU/PSK/ztvmWKFcI7UGb5/HQT7B+i3a2myKgI= github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a h1:MISbI8sU/PSK/ztvmWKFcI7UGb5/HQT7B+i3a2myKgI=
github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a/go.mod h1:2GxOXOlEPAMFPfp014mK1SWq8G8BN8o7/dfYqJrVGn8= github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a/go.mod h1:2GxOXOlEPAMFPfp014mK1SWq8G8BN8o7/dfYqJrVGn8=
github.com/clipperhouse/uax29/v2 v2.2.0 h1:ChwIKnQN3kcZteTXMgb1wztSgaU+ZemkgWdohwgs8tY= github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs=
github.com/clipperhouse/uax29/v2 v2.2.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA=
github.com/clipperhouse/uax29/v2 v2.3.0 h1:SNdx9DVUqMoBuBoW3iLOj4FQv3dN5mDtuqwuhIGpJy4=
github.com/clipperhouse/uax29/v2 v2.3.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g=
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@ -31,6 +33,8 @@ github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 h1:oP4q0fw+fOSWn3
github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6/go.mod h1:iL2twTeMvZnrg54ZoPDNfJaJaqy0xIQFuBdrLsmspwQ= github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6/go.mod h1:iL2twTeMvZnrg54ZoPDNfJaJaqy0xIQFuBdrLsmspwQ=
github.com/emersion/go-smtp v0.24.0 h1:g6AfoF140mvW0vLNPD/LuCBLEAdlxOjIXqbIkJIS6Wk= github.com/emersion/go-smtp v0.24.0 h1:g6AfoF140mvW0vLNPD/LuCBLEAdlxOjIXqbIkJIS6Wk=
github.com/emersion/go-smtp v0.24.0/go.mod h1:ZtRRkbTyp2XTHCA+BmyTFTrj8xY4I+b4McvHxCU2gsQ= github.com/emersion/go-smtp v0.24.0/go.mod h1:ZtRRkbTyp2XTHCA+BmyTFTrj8xY4I+b4McvHxCU2gsQ=
github.com/etkecc/go-crontab v1.2.1 h1:tZM5K0dKt8drUY86+JdxfTQj+6iclgwT7PdeMxXZ4qI=
github.com/etkecc/go-crontab v1.2.1/go.mod h1:L2llhyxQDhbEU3uInVnnSsNZ+1P5NtAUuXkyUN0/BAg=
github.com/etkecc/go-env v1.2.1 h1:b/mIa8D1d9hc3rI8h5bEtBHsnKkBZ6UbmYPog3QIPTU= github.com/etkecc/go-env v1.2.1 h1:b/mIa8D1d9hc3rI8h5bEtBHsnKkBZ6UbmYPog3QIPTU=
github.com/etkecc/go-env v1.2.1/go.mod h1:yTs1DWEsllAZYA417r4V+OmMuYinGXtBzKkLGNvCv5c= github.com/etkecc/go-env v1.2.1/go.mod h1:yTs1DWEsllAZYA417r4V+OmMuYinGXtBzKkLGNvCv5c=
github.com/etkecc/go-fswatcher v1.0.1 h1:n9hqtjzTS3ETb9hcZe1EqYA1lkkhlzZlztu3hXwFDQc= github.com/etkecc/go-fswatcher v1.0.1 h1:n9hqtjzTS3ETb9hcZe1EqYA1lkkhlzZlztu3hXwFDQc=
@ -39,8 +43,8 @@ github.com/etkecc/go-healthchecks/v2 v2.2.2 h1:YV7e+Ga8JY3aZX4Qf6Q1Ca+DnqbT5Drjl
github.com/etkecc/go-healthchecks/v2 v2.2.2/go.mod h1:IowWGN4F6By6z0eh63+639OscylgtdvT9ITjfN6hnZA= github.com/etkecc/go-healthchecks/v2 v2.2.2/go.mod h1:IowWGN4F6By6z0eh63+639OscylgtdvT9ITjfN6hnZA=
github.com/etkecc/go-kit v1.7.5 h1:DbHTbJ69Jnt2d8mjOyKA9f2ARxvi8xgnYgVMlMZd3gI= github.com/etkecc/go-kit v1.7.5 h1:DbHTbJ69Jnt2d8mjOyKA9f2ARxvi8xgnYgVMlMZd3gI=
github.com/etkecc/go-kit v1.7.5/go.mod h1:ZFeQrvlMIV6OeZ4XJ090kkyNYRRN4+5wcg1vjnOKEGI= github.com/etkecc/go-kit v1.7.5/go.mod h1:ZFeQrvlMIV6OeZ4XJ090kkyNYRRN4+5wcg1vjnOKEGI=
github.com/etkecc/go-linkpearl v0.0.0-20251009114554-5985fe44a189 h1:isSNC9AsMdUMlnE9tVdTk/DRSNOQx2RvLhKxH3GnpXg= github.com/etkecc/go-linkpearl v0.0.0-20251016112544-0d85fb3bccdc h1:jUwnFg8OxXgp6JEmVyuR0sfUqV3pEbnWJqZnrroCM60=
github.com/etkecc/go-linkpearl v0.0.0-20251009114554-5985fe44a189/go.mod h1:ua01SU6PcE7NtTT5d1wFRUcrqcA5RWbvo9yRLu4LKCA= github.com/etkecc/go-linkpearl v0.0.0-20251016112544-0d85fb3bccdc/go.mod h1:bg9M+agdjBrD0nS3uGUh7iP59mwlVZMw2ah+z3JzNoY=
github.com/etkecc/go-mxidwc v1.0.1 h1:t4Kq3FxSlQjt1i7RpzE5q3cOWjJ0vrTzzGZRSpgh8mg= github.com/etkecc/go-mxidwc v1.0.1 h1:t4Kq3FxSlQjt1i7RpzE5q3cOWjJ0vrTzzGZRSpgh8mg=
github.com/etkecc/go-mxidwc v1.0.1/go.mod h1:WFlntcH4mdual/gNi6X7a6rSJERNuLjdrwM3K/tucQA= github.com/etkecc/go-mxidwc v1.0.1/go.mod h1:WFlntcH4mdual/gNi6X7a6rSJERNuLjdrwM3K/tucQA=
github.com/etkecc/go-secgen v1.4.0 h1:PapLpen3aIqG2LDu+U6KWDb4SMW2UGuvetpJjd86yUc= github.com/etkecc/go-secgen v1.4.0 h1:PapLpen3aIqG2LDu+U6KWDb4SMW2UGuvetpJjd86yUc=
@ -53,10 +57,10 @@ github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU=
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
github.com/gabriel-vasile/mimetype v1.4.10 h1:zyueNbySn/z8mJZHLt6IPw0KoZsiQNszIpU+bX4+ZK0= github.com/gabriel-vasile/mimetype v1.4.11 h1:AQvxbp830wPhHTqc1u7nzoLT+ZFxGY7emj5DR5DYFik=
github.com/gabriel-vasile/mimetype v1.4.10/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= github.com/gabriel-vasile/mimetype v1.4.11/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
github.com/getsentry/sentry-go v0.35.3 h1:u5IJaEqZyPdWqe/hKlBKBBnMTSxB/HenCqF3QLabeds= github.com/getsentry/sentry-go v0.36.2 h1:uhuxRPTrUy0dnSzTd0LrYXlBYygLkKY0hhlG5LXarzM=
github.com/getsentry/sentry-go v0.35.3/go.mod h1:mdL49ixwT2yi57k5eh7mpnDyPybixPzlzEJFu0Z76QA= github.com/getsentry/sentry-go v0.36.2/go.mod h1:p5Im24mJBeruET8Q4bbcMfCQ+F+Iadc4L48tB1apo2c=
github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
@ -119,8 +123,6 @@ github.com/mcnijman/go-emailaddress v1.1.1 h1:AGhgVDG3tCDaL0/Vc6erlPQjDuDN3dAT7r
github.com/mcnijman/go-emailaddress v1.1.1/go.mod h1:5whZrhS8Xp5LxO8zOD35BC+b76kROtsh+dPomeRt/II= github.com/mcnijman/go-emailaddress v1.1.1/go.mod h1:5whZrhS8Xp5LxO8zOD35BC+b76kROtsh+dPomeRt/II=
github.com/mikesmitty/edkey v0.0.0-20170222072505-3356ea4e686a h1:eU8j/ClY2Ty3qdHnn0TyW3ivFoPC/0F1gQZz8yTxbbE= github.com/mikesmitty/edkey v0.0.0-20170222072505-3356ea4e686a h1:eU8j/ClY2Ty3qdHnn0TyW3ivFoPC/0F1gQZz8yTxbbE=
github.com/mikesmitty/edkey v0.0.0-20170222072505-3356ea4e686a/go.mod h1:v8eSC2SMp9/7FTKUncp7fH9IwPfw+ysMObcEz5FWheQ= github.com/mikesmitty/edkey v0.0.0-20170222072505-3356ea4e686a/go.mod h1:v8eSC2SMp9/7FTKUncp7fH9IwPfw+ysMObcEz5FWheQ=
github.com/mileusna/crontab v1.2.0 h1:x9ZmE2A4p6CDqMEGQ+GbqsNtnmbdmWMQYShdQu8LvrU=
github.com/mileusna/crontab v1.2.0/go.mod h1:dbns64w/u3tUnGZGf8pAa76ZqOfeBX4olW4U1ZwExmc=
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
@ -128,8 +130,8 @@ github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 h1:zrbMGy9YXpIeTnGj
github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6/go.mod h1:rEKTHC9roVVicUIfZK7DYrdIoM0EOr8mK1Hj5s3JjH0= github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6/go.mod h1:rEKTHC9roVVicUIfZK7DYrdIoM0EOr8mK1Hj5s3JjH0=
github.com/olekukonko/errors v1.1.0 h1:RNuGIh15QdDenh+hNvKrJkmxxjV4hcS50Db478Ou5sM= github.com/olekukonko/errors v1.1.0 h1:RNuGIh15QdDenh+hNvKrJkmxxjV4hcS50Db478Ou5sM=
github.com/olekukonko/errors v1.1.0/go.mod h1:ppzxA5jBKcO1vIpCXQ9ZqgDh8iwODz6OXIGKU8r5m4Y= github.com/olekukonko/errors v1.1.0/go.mod h1:ppzxA5jBKcO1vIpCXQ9ZqgDh8iwODz6OXIGKU8r5m4Y=
github.com/olekukonko/ll v0.1.1 h1:9Dfeed5/Mgaxb9lHRAftLK9pVfYETvHn+If6lywVhJc= github.com/olekukonko/ll v0.1.2 h1:lkg/k/9mlsy0SxO5aC+WEpbdT5K83ddnNhAepz7TQc0=
github.com/olekukonko/ll v0.1.1/go.mod h1:2dJo+hYZcJMLMbKwHEWvxCUbAOLc/CXWS9noET22Mdo= github.com/olekukonko/ll v0.1.2/go.mod h1:b52bVQRRPObe+yyBl0TxNfhesL0nedD4Cht0/zx55Ew=
github.com/olekukonko/tablewriter v1.1.0 h1:N0LHrshF4T39KvI96fn6GT8HEjXRXYNDrDjKFDB7RIY= github.com/olekukonko/tablewriter v1.1.0 h1:N0LHrshF4T39KvI96fn6GT8HEjXRXYNDrDjKFDB7RIY=
github.com/olekukonko/tablewriter v1.1.0/go.mod h1:5c+EBPeSqvXnLLgkm9isDdzR3wjfBkHR9Nhfp3NWrzo= github.com/olekukonko/tablewriter v1.1.0/go.mod h1:5c+EBPeSqvXnLLgkm9isDdzR3wjfBkHR9Nhfp3NWrzo=
github.com/petermattis/goid v0.0.0-20250904145737-900bdf8bb490 h1:QTvNkZ5ylY0PGgA+Lih+GdboMLY/G9SEGLMEGVjTVA4= github.com/petermattis/goid v0.0.0-20250904145737-900bdf8bb490 h1:QTvNkZ5ylY0PGgA+Lih+GdboMLY/G9SEGLMEGVjTVA4=
@ -173,16 +175,16 @@ github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
github.com/yuin/goldmark v1.7.13 h1:GPddIs617DnBLFFVJFgpo1aBfe/4xcvMc3SB5t/D0pA= github.com/yuin/goldmark v1.7.13 h1:GPddIs617DnBLFFVJFgpo1aBfe/4xcvMc3SB5t/D0pA=
github.com/yuin/goldmark v1.7.13/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= github.com/yuin/goldmark v1.7.13/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg=
go.mau.fi/util v0.9.1 h1:A+XKHRsjKkFi2qOm4RriR1HqY2hoOXNS3WFHaC89r2Y= go.mau.fi/util v0.9.2 h1:+S4Z03iCsGqU2WY8X2gySFsFjaLlUHFRDVCYvVwynKM=
go.mau.fi/util v0.9.1/go.mod h1:M0bM9SyaOWJniaHs9hxEzz91r5ql6gYq6o1q5O1SsjQ= go.mau.fi/util v0.9.2/go.mod h1:055elBBCJSdhRsmub7ci9hXZPgGr1U6dYg44cSgRgoU=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04= golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04=
golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0= golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0=
golang.org/x/exp v0.0.0-20251002181428-27f1f14c8bb9 h1:TQwNpfvNkxAVlItJf6Cr5JTsVZoC/Sj7K3OZv2Pc14A= golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 h1:mgKeJMpvi0yx/sU5GsxQ7p6s2wtOnGAHZWCHUM4KGzY=
golang.org/x/exp v0.0.0-20251002181428-27f1f14c8bb9/go.mod h1:TwQYMMnGpvZyc+JpB/UAuTNIsVJifOlSkrZkhcvpVUk= golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546/go.mod h1:j/pmGrbnkbPtQfxEe5D0VQhZC6qKbfKifgD0oM7sR70=
golang.org/x/mod v0.28.0 h1:gQBtGhjxykdjY9YhZpSlZIsbnaE2+PgjfLWUQTnoZ1U= golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA=
golang.org/x/mod v0.28.0/go.mod h1:yfB/L0NOf/kmEbXjzCPOx1iK1fRutOydrCMsqRhEBxI= golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w=
golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM=
golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4= golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4=
@ -204,8 +206,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k= golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k=
golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM= golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.37.0 h1:DVSRzp7FwePZW356yEAChSdNcQo6Nsp+fex1SUW09lE= golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ=
golang.org/x/tools v0.37.0/go.mod h1:MBN5QPQtLMHVdvsbtarmTNukZDdgwdwlO5qGacAzF0w= golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@ -218,8 +220,8 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
maunium.net/go/mautrix v0.25.1 h1:+xe3eXtQNcDPU/HoWzvSOA5YX57iqlYI1TXf/fM0KWs= maunium.net/go/mautrix v0.25.2 h1:CUG23zp754yGOTMh9Q4mVSENS9FyweE/G+6ZsPDMCUU=
maunium.net/go/mautrix v0.25.1/go.mod h1:iSueLJ/2fBaNrsTObGqi1j0cl/loxrtAjmjay1scYD8= maunium.net/go/mautrix v0.25.2/go.mod h1:EWgYyp2iFZP7pnSm+rufHlO8YVnA2KnoNBDpwekiAwI=
modernc.org/cc/v4 v4.26.5 h1:xM3bX7Mve6G8K8b+T11ReenJOT+BmVqQj0FY5T4+5Y4= modernc.org/cc/v4 v4.26.5 h1:xM3bX7Mve6G8K8b+T11ReenJOT+BmVqQj0FY5T4+5Y4=
modernc.org/cc/v4 v4.26.5/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= modernc.org/cc/v4 v4.26.5/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0=
modernc.org/ccgo/v4 v4.28.1 h1:wPKYn5EC/mYTqBO373jKjvX2n+3+aK7+sICCv4Fjy1A= modernc.org/ccgo/v4 v4.28.1 h1:wPKYn5EC/mYTqBO373jKjvX2n+3+aK7+sICCv4Fjy1A=
@ -240,8 +242,8 @@ modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8=
modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
modernc.org/sqlite v1.39.0 h1:6bwu9Ooim0yVYA7IZn9demiQk/Ejp0BtTjBWFLymSeY= modernc.org/sqlite v1.40.0 h1:bNWEDlYhNPAUdUdBzjAvn8icAs/2gaKlj4vM+tQ6KdQ=
modernc.org/sqlite v1.39.0/go.mod h1:cPTJYSlgg3Sfg046yBShXENNtPrWrDX8bsbAQBzgQ5E= modernc.org/sqlite v1.40.0/go.mod h1:9fjQZ0mB1LLP0GYrp39oOJXx/I2sxEnZtzCmEQIKvGE=
modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=

2
vendor/github.com/clipperhouse/stringish/.gitignore generated vendored Normal file
View file

@ -0,0 +1,2 @@
.DS_Store
*.test

21
vendor/github.com/clipperhouse/stringish/LICENSE generated vendored Normal file
View file

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2025 Matt Sherman
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

64
vendor/github.com/clipperhouse/stringish/README.md generated vendored Normal file
View file

@ -0,0 +1,64 @@
# stringish
A small Go module that provides a generic type constraint for “string-like”
data, and a utf8 package that works with both strings and byte slices
without conversions.
```go
type Interface interface {
~[]byte | ~string
}
```
[![Go Reference](https://pkg.go.dev/badge/github.com/clipperhouse/stringish/utf8.svg)](https://pkg.go.dev/github.com/clipperhouse/stringish/utf8)
[![Test Status](https://github.com/clipperhouse/stringish/actions/workflows/gotest.yml/badge.svg)](https://github.com/clipperhouse/stringish/actions/workflows/gotest.yml)
## Install
```
go get github.com/clipperhouse/stringish
```
## Examples
```go
import (
"github.com/clipperhouse/stringish"
"github.com/clipperhouse/stringish/utf8"
)
s := "Hello, 世界"
r, size := utf8.DecodeRune(s) // not DecodeRuneInString 🎉
b := []byte("Hello, 世界")
r, size = utf8.DecodeRune(b) // same API!
func MyFoo[T stringish.Interface](s T) T {
// pass a string or a []byte
// iterate, slice, transform, whatever
}
```
## Motivation
Sometimes we want APIs to accept `string` or `[]byte` without having to convert
between those types. That conversion usually allocates!
By implementing with `stringish.Interface`, we can have a single API, and
single implementation for both types: one `Foo` instead of `Foo` and
`FooString`.
We have converted the
[`unicode/utf8` package](https://github.com/clipperhouse/stringish/blob/main/utf8/utf8.go)
as an example -- note the absence of`*InString` funcs. We might look at `x/text`
next.
## Used by
- clipperhouse/uax29: [stringish trie](https://github.com/clipperhouse/uax29/blob/master/graphemes/trie.go#L27), [stringish iterator](https://github.com/clipperhouse/uax29/blob/master/internal/iterators/iterator.go#L9), [stringish SplitFunc](https://github.com/clipperhouse/uax29/blob/master/graphemes/splitfunc.go#L21)
- [clipperhouse/displaywidth](https://github.com/clipperhouse/displaywidth)
## Prior discussion
- [Consideration of similar by the Go team](https://github.com/golang/go/issues/48643)

View file

@ -0,0 +1,5 @@
package stringish
type Interface interface {
~[]byte | ~string
}

View file

@ -1,5 +1,9 @@
An implementation of grapheme cluster boundaries from [Unicode text segmentation](https://unicode.org/reports/tr29/#Grapheme_Cluster_Boundaries) (UAX 29), for Unicode version 15.0.0. An implementation of grapheme cluster boundaries from [Unicode text segmentation](https://unicode.org/reports/tr29/#Grapheme_Cluster_Boundaries) (UAX 29), for Unicode version 15.0.0.
[![Documentation](https://pkg.go.dev/badge/github.com/clipperhouse/uax29/v2/graphemes.svg)](https://pkg.go.dev/github.com/clipperhouse/uax29/v2/graphemes)
![Tests](https://github.com/clipperhouse/uax29/actions/workflows/gotest.yml/badge.svg)
![Fuzz](https://github.com/clipperhouse/uax29/actions/workflows/gofuzz.yml/badge.svg)
## Quick start ## Quick start
``` ```
@ -18,15 +22,14 @@ for tokens.Next() { // Next() returns true until end of data
} }
``` ```
[![Documentation](https://pkg.go.dev/badge/github.com/clipperhouse/uax29/v2/graphemes.svg)](https://pkg.go.dev/github.com/clipperhouse/uax29/v2/graphemes)
_A grapheme is a “single visible character”, which might be a simple as a single letter, or a complex emoji that consists of several Unicode code points._ _A grapheme is a “single visible character”, which might be a simple as a single letter, or a complex emoji that consists of several Unicode code points._
## Conformance ## Conformance
We use the Unicode [test suite](https://unicode.org/reports/tr41/tr41-26.html#Tests29). Status: We use the Unicode [test suite](https://unicode.org/reports/tr41/tr41-26.html#Tests29).
![Go](https://github.com/clipperhouse/uax29/actions/workflows/gotest.yml/badge.svg) ![Tests](https://github.com/clipperhouse/uax29/actions/workflows/gotest.yml/badge.svg)
![Fuzz](https://github.com/clipperhouse/uax29/actions/workflows/gofuzz.yml/badge.svg)
## APIs ## APIs
@ -71,9 +74,18 @@ for tokens.Next() { // Next() returns true until end of data
} }
``` ```
### Performance ### Benchmarks
On a Mac M2 laptop, we see around 200MB/s, or around 100 million graphemes per second. You should see ~constant memory, and no allocations. On a Mac M2 laptop, we see around 200MB/s, or around 100 million graphemes per second, and no allocations.
```
goos: darwin
goarch: arm64
pkg: github.com/clipperhouse/uax29/graphemes/comparative
cpu: Apple M2
BenchmarkGraphemes/clipperhouse/uax29-8 173805 ns/op 201.16 MB/s 0 B/op 0 allocs/op
BenchmarkGraphemes/rivo/uniseg-8 2045128 ns/op 17.10 MB/s 0 B/op 0 allocs/op
```
### Invalid inputs ### Invalid inputs

View file

@ -1,8 +1,11 @@
package graphemes package graphemes
import "github.com/clipperhouse/uax29/v2/internal/iterators" import (
"github.com/clipperhouse/stringish"
"github.com/clipperhouse/uax29/v2/internal/iterators"
)
type Iterator[T iterators.Stringish] struct { type Iterator[T stringish.Interface] struct {
*iterators.Iterator[T] *iterators.Iterator[T]
} }

View file

@ -3,7 +3,7 @@ package graphemes
import ( import (
"bufio" "bufio"
"github.com/clipperhouse/uax29/v2/internal/iterators" "github.com/clipperhouse/stringish"
) )
// is determines if lookup intersects propert(ies) // is determines if lookup intersects propert(ies)
@ -18,7 +18,7 @@ const _Ignore = _Extend
// See https://unicode.org/reports/tr29/#Grapheme_Cluster_Boundaries. // See https://unicode.org/reports/tr29/#Grapheme_Cluster_Boundaries.
var SplitFunc bufio.SplitFunc = splitFunc[[]byte] var SplitFunc bufio.SplitFunc = splitFunc[[]byte]
func splitFunc[T iterators.Stringish](data T, atEOF bool) (advance int, token T, err error) { func splitFunc[T stringish.Interface](data T, atEOF bool) (advance int, token T, err error) {
var empty T var empty T
if len(data) == 0 { if len(data) == 0 {
return 0, empty, nil return 0, empty, nil

View file

@ -1,10 +1,10 @@
package graphemes package graphemes
import "github.com/clipperhouse/stringish"
// generated by github.com/clipperhouse/uax29/v2 // generated by github.com/clipperhouse/uax29/v2
// from https://www.unicode.org/Public/15.0.0/ucd/auxiliary/GraphemeBreakProperty.txt // from https://www.unicode.org/Public/15.0.0/ucd/auxiliary/GraphemeBreakProperty.txt
import "github.com/clipperhouse/uax29/v2/internal/iterators"
type property uint16 type property uint16
const ( const (
@ -27,7 +27,7 @@ const (
// lookup returns the trie value for the first UTF-8 encoding in s and // lookup returns the trie value for the first UTF-8 encoding in s and
// the width in bytes of this encoding. The size will be 0 if s does not // the width in bytes of this encoding. The size will be 0 if s does not
// hold enough bytes to complete the encoding. len(s) must be greater than 0. // hold enough bytes to complete the encoding. len(s) must be greater than 0.
func lookup[T iterators.Stringish](s T) (v property, sz int) { func lookup[T stringish.Interface](s T) (v property, sz int) {
c0 := s[0] c0 := s[0]
switch { switch {
case c0 < 0x80: // is ASCII case c0 < 0x80: // is ASCII

View file

@ -1,14 +1,12 @@
package iterators package iterators
type Stringish interface { import "github.com/clipperhouse/stringish"
[]byte | string
}
type SplitFunc[T Stringish] func(T, bool) (int, T, error) type SplitFunc[T stringish.Interface] func(T, bool) (int, T, error)
// Iterator is a generic iterator for words that are either []byte or string. // Iterator is a generic iterator for words that are either []byte or string.
// Iterate while Next() is true, and access the word via Value(). // Iterate while Next() is true, and access the word via Value().
type Iterator[T Stringish] struct { type Iterator[T stringish.Interface] struct {
split SplitFunc[T] split SplitFunc[T]
data T data T
start int start int
@ -16,7 +14,7 @@ type Iterator[T Stringish] struct {
} }
// New creates a new Iterator for the given data and SplitFunc. // New creates a new Iterator for the given data and SplitFunc.
func New[T Stringish](split SplitFunc[T], data T) *Iterator[T] { func New[T stringish.Interface](split SplitFunc[T], data T) *Iterator[T] {
return &Iterator[T]{ return &Iterator[T]{
split: split, split: split,
data: data, data: data,
@ -83,3 +81,20 @@ func (iter *Iterator[T]) Reset() {
iter.start = 0 iter.start = 0
iter.pos = 0 iter.pos = 0
} }
func (iter *Iterator[T]) First() T {
if len(iter.data) == 0 {
return iter.data
}
advance, _, err := iter.split(iter.data, true)
if err != nil {
panic(err)
}
if advance <= 0 {
panic("SplitFunc returned a zero or negative advance")
}
if advance > len(iter.data) {
panic("SplitFunc advanced beyond the end of the data")
}
return iter.data[:advance]
}

1
vendor/github.com/etkecc/go-crontab/.gitignore generated vendored Normal file
View file

@ -0,0 +1 @@
/cover.out

150
vendor/github.com/etkecc/go-crontab/.golangci.yml generated vendored Normal file
View file

@ -0,0 +1,150 @@
version: "2"
run:
concurrency: 4
modules-download-mode: readonly
issues-exit-code: 1
tests: true
output:
formats:
text:
path: stdout
print-linter-name: true
print-issued-lines: true
linters:
enable:
- asasalint
- asciicheck
- bidichk
- bodyclose
- containedctx
- copyloopvar
- decorder
- dogsled
- dupl
- dupword
- durationcheck
- errchkjson
- errname
- errorlint
- exhaustive
- forcetypeassert
- gocognit
- gocritic
- gocyclo
- gosec
- gosmopolitan
- makezero
- mirror
- misspell
- nestif
- nolintlint
- prealloc
- predeclared
- revive
- sqlclosecheck
- unconvert
- unparam
- usestdlibvars
- wastedassign
settings:
decorder:
dec-order:
- const
- var
- type
- func
dogsled:
max-blank-identifiers: 3
errcheck:
check-type-assertions: true
check-blank: true
errchkjson:
report-no-exported: true
exhaustive:
check:
- switch
- map
default-signifies-exhaustive: true
gocognit:
min-complexity: 15
gocritic:
enabled-tags:
- diagnostic
- style
- performance
grouper:
const-require-single-const: true
import-require-single-import: true
var-require-single-var: true
misspell:
locale: US
nestif:
min-complexity: 5
unparam:
check-exported: true
usestdlibvars:
time-month: true
time-layout: true
crypto-hash: true
default-rpc-path: true
sql-isolation-level: true
tls-signature-scheme: true
constant-kind: true
exclusions:
generated: lax
presets:
- comments
- common-false-positives
- legacy
- std-error-handling
rules:
- linters:
- dupl
- errcheck
- gocognit
- gocyclo
- gosec
path: _test\.go
- linters:
- staticcheck
text: 'SA9003:'
- linters:
- lll
source: '^//go:generate '
- linters:
- revive
text: returns unexported type
- linters:
- revive
text: 'var-naming: avoid meaningless package names'
paths:
- mocks
- third_party$
- builtin$
- examples$
issues:
max-issues-per-linter: 0
max-same-issues: 0
new: false
formatters:
enable:
- gofmt
- gofumpt
- goimports
settings:
gofmt:
simplify: true
rewrite-rules:
- pattern: interface{}
replacement: any
- pattern: a[b:len(a)]
replacement: a[b:]
gofumpt:
extra-rules: true
exclusions:
generated: lax
paths:
- mocks
- third_party$
- builtin$
- examples$

View file

@ -1,10 +1,16 @@
# Go/Golang package for Crontab tickers [![GoDoc](https://godoc.org/github.com/mileusna/crontab?status.svg)](https://godoc.org/github.com/mileusna/crontab) # Go/Golang package for Crontab tickers [![GoDoc](https://godoc.org/github.com/etkecc/go-crontab?status.svg)](https://godoc.org/github.com/etkecc/go-crontab)
This package provides crontab tickers to golang apps, supporting crontab-like syntax like `* * * * *` or `*/2 * * * *` etc. This package provides crontab tickers to golang apps, supporting crontab-like syntax like `* * * * *` or `*/2 * * * *` etc.
This is a fork of [github.com/mileusna/crontab](https://github.com/mileusna/crontab) with small modifications:
- added `SetPanicLogger` function to set custom panic logger function used in panic recovery
- added linter and refactored code to pass linter checks
- changed minimal go version to 1.22
## Installation <a id="installation"></a> ## Installation <a id="installation"></a>
``` ```
go get github.com/mileusna/crontab go get github.com/etkecc/go-crontab
``` ```
## Example<a id="example"></a> ## Example<a id="example"></a>
@ -16,7 +22,7 @@ import (
"fmt" "fmt"
"log" "log"
"github.com/mileusna/crontab" "github.com/etkecc/go-crontab"
) )
func main() { func main() {

View file

@ -14,9 +14,10 @@ import (
// Crontab struct representing cron table // Crontab struct representing cron table
type Crontab struct { type Crontab struct {
ticker *time.Ticker mu sync.RWMutex
jobs []*job ticker *time.Ticker
sync.RWMutex jobs []*job
paniclog func(any)
} }
// job in cron table // job in cron table
@ -27,8 +28,9 @@ type job struct {
month map[int]struct{} month map[int]struct{}
dayOfWeek map[int]struct{} dayOfWeek map[int]struct{}
fn interface{} fn any
args []interface{} args []any
paniclog func(any)
sync.RWMutex sync.RWMutex
} }
@ -43,14 +45,17 @@ type tick struct {
// New initializes and returns new cron table // New initializes and returns new cron table
func New() *Crontab { func New() *Crontab {
return new(time.Minute) return newCrontab(time.Minute)
} }
// new creates new crontab, arg provided for testing purpose // newCrontab creates newCrontab crontab, arg provided for testing purpose
func new(t time.Duration) *Crontab { func newCrontab(t time.Duration) *Crontab {
c := &Crontab{ c := &Crontab{
ticker: time.NewTicker(t), ticker: time.NewTicker(t),
jobs: []*job{}, jobs: []*job{},
paniclog: func(r any) {
log.Println("Crontab error", r)
},
} }
go func() { go func() {
@ -71,21 +76,21 @@ func new(t time.Duration) *Crontab {
// * fn is not function // * fn is not function
// //
// * Provided args don't match the number and/or the type of fn args // * Provided args don't match the number and/or the type of fn args
func (c *Crontab) AddJob(schedule string, fn interface{}, args ...interface{}) error { func (c *Crontab) AddJob(schedule string, fn any, args ...any) error {
j, err := parseSchedule(schedule) j, err := parseSchedule(schedule)
c.Lock() c.mu.Lock()
defer c.Unlock() defer c.mu.Unlock()
if err != nil { if err != nil {
return err return err
} }
if fn == nil || reflect.ValueOf(fn).Kind() != reflect.Func { if fn == nil || reflect.ValueOf(fn).Kind() != reflect.Func {
return fmt.Errorf("Cron job must be func()") return errors.New("cron job must be func()")
} }
fnType := reflect.TypeOf(fn) fnType := reflect.TypeOf(fn)
if len(args) != fnType.NumIn() { if len(args) != fnType.NumIn() {
return fmt.Errorf("Number of func() params and number of provided params doesn't match") return errors.New("number of func() params and number of provided params doesn't match")
} }
for i := 0; i < fnType.NumIn(); i++ { for i := 0; i < fnType.NumIn(); i++ {
@ -95,10 +100,10 @@ func (c *Crontab) AddJob(schedule string, fn interface{}, args ...interface{}) e
if t1 != t2 { if t1 != t2 {
if t1.Kind() != reflect.Interface { if t1.Kind() != reflect.Interface {
return fmt.Errorf("Param with index %d shold be `%s` not `%s`", i, t1, t2) return fmt.Errorf("param with index %d shold be `%s` not `%s`", i, t1, t2)
} }
if !t2.Implements(t1) { if !t2.Implements(t1) {
return fmt.Errorf("Param with index %d of type `%s` doesn't implement interface `%s`", i, t2, t1) return fmt.Errorf("param with index %d of type `%s` doesn't implement interface `%s`", i, t2, t1)
} }
} }
} }
@ -106,13 +111,14 @@ func (c *Crontab) AddJob(schedule string, fn interface{}, args ...interface{}) e
// all checked, add job to cron tab // all checked, add job to cron tab
j.fn = fn j.fn = fn
j.args = args j.args = args
j.paniclog = c.paniclog
c.jobs = append(c.jobs, j) c.jobs = append(c.jobs, j)
return nil return nil
} }
// MustAddJob is like AddJob but panics if there is an problem with job // MustAddJob is like AddJob but panics if there is an problem with job
// //
// It simplifies initialization, since we usually add jobs at the beggining so you won't have to check for errors (it will panic when program starts). // It simplifies initialization, since we usually add jobs at the beginning so you won't have to check for errors (it will panic when program starts).
// It is a similar aproach as go's std lib package `regexp` and `regexp.Compile()` `regexp.MustCompile()` // It is a similar aproach as go's std lib package `regexp` and `regexp.Compile()` `regexp.MustCompile()`
// MustAddJob will panic if: // MustAddJob will panic if:
// //
@ -121,31 +127,36 @@ func (c *Crontab) AddJob(schedule string, fn interface{}, args ...interface{}) e
// * fn is not function // * fn is not function
// //
// * Provided args don't match the number and/or the type of fn args // * Provided args don't match the number and/or the type of fn args
func (c *Crontab) MustAddJob(schedule string, fn interface{}, args ...interface{}) { func (c *Crontab) MustAddJob(schedule string, fn any, args ...any) {
if err := c.AddJob(schedule, fn, args...); err != nil { if err := c.AddJob(schedule, fn, args...); err != nil {
panic(err) panic(err)
} }
} }
// SetPanicLogger function to set custom panic logger
func (c *Crontab) SetPanicLogger(f func(any)) {
c.paniclog = f
}
// Shutdown the cron table schedule // Shutdown the cron table schedule
// //
// Once stopped, it can't be restarted. // Once stopped, it can't be restarted.
// This function is pre-shuttdown helper for your app, there is no Start/Stop functionallity with crontab package. // This function is pre-shuttdown helper for your app, there is no Start/Stop functionality with crontab package.
func (c *Crontab) Shutdown() { func (c *Crontab) Shutdown() {
c.ticker.Stop() c.ticker.Stop()
} }
// Clear all jobs from cron table // Clear all jobs from cron table
func (c *Crontab) Clear() { func (c *Crontab) Clear() {
c.Lock() c.mu.Lock()
c.jobs = []*job{} c.jobs = []*job{}
c.Unlock() c.mu.Unlock()
} }
// RunAll jobs in cron table, shcheduled or not // RunAll jobs in cron table, shcheduled or not
func (c *Crontab) RunAll() { func (c *Crontab) RunAll() {
c.RLock() c.mu.RLock()
defer c.RUnlock() defer c.mu.RUnlock()
for _, j := range c.jobs { for _, j := range c.jobs {
go j.run() go j.run()
} }
@ -154,8 +165,8 @@ func (c *Crontab) RunAll() {
// RunScheduled jobs // RunScheduled jobs
func (c *Crontab) runScheduled(t time.Time) { func (c *Crontab) runScheduled(t time.Time) {
tick := getTick(t) tick := getTick(t)
c.RLock() c.mu.RLock()
defer c.RUnlock() defer c.mu.RUnlock()
for _, j := range c.jobs { for _, j := range c.jobs {
if j.tick(tick) { if j.tick(tick) {
@ -170,7 +181,7 @@ func (j *job) run() {
j.RLock() j.RLock()
defer func() { defer func() {
if r := recover(); r != nil { if r := recover(); r != nil {
log.Println("Crontab error", r) j.paniclog(r)
} }
}() }()
v := reflect.ValueOf(j.fn) v := reflect.ValueOf(j.fn)
@ -224,7 +235,7 @@ func parseSchedule(s string) (*job, error) {
s = matchSpaces.ReplaceAllLiteralString(s, " ") s = matchSpaces.ReplaceAllLiteralString(s, " ")
parts := strings.Split(s, " ") parts := strings.Split(s, " ")
if len(parts) != 5 { if len(parts) != 5 {
return j, errors.New("Schedule string must have five components like * * * * *") return j, errors.New("schedule string must have five components like * * * * *")
} }
j.min, err = parsePart(parts[0], 0, 59) j.min, err = parsePart(parts[0], 0, 59)
@ -267,34 +278,35 @@ func parseSchedule(s string) (*job, error) {
} }
// parsePart parse individual schedule part from schedule string // parsePart parse individual schedule part from schedule string
func parsePart(s string, min, max int) (map[int]struct{}, error) { //
//nolint:gocognit // TODO: refactor
func parsePart(s string, minimum, maximum int) (map[int]struct{}, error) {
r := make(map[int]struct{}) r := make(map[int]struct{})
// wildcard pattern // wildcard pattern
if s == "*" { if s == "*" {
for i := min; i <= max; i++ { for i := minimum; i <= maximum; i++ {
r[i] = struct{}{} r[i] = struct{}{}
} }
return r, nil return r, nil
} }
// */2 1-59/5 pattern // */2 1-59/5 pattern
if matches := matchN.FindStringSubmatch(s); matches != nil { if matches := matchN.FindStringSubmatch(s); matches != nil { //nolint:nestif // TODO: refactor
localMin := min localMin := minimum
localMax := max localMax := maximum
if matches[1] != "" && matches[1] != "*" { if matches[1] != "" && matches[1] != "*" {
if rng := matchRange.FindStringSubmatch(matches[1]); rng != nil { if rng := matchRange.FindStringSubmatch(matches[1]); rng != nil {
localMin, _ = strconv.Atoi(rng[1]) localMin, _ = strconv.Atoi(rng[1]) //nolint:errcheck // checked below
localMax, _ = strconv.Atoi(rng[2]) localMax, _ = strconv.Atoi(rng[2]) //nolint:errcheck // checked below
if localMin < min || localMax > max { if localMin < minimum || localMax > maximum {
return nil, fmt.Errorf("Out of range for %s in %s. %s must be in range %d-%d", rng[1], s, rng[1], min, max) return nil, fmt.Errorf("out of range for %s in %s. %s must be in range %d-%d", rng[1], s, rng[1], minimum, maximum)
} }
} else { } else {
return nil, fmt.Errorf("Unable to parse %s part in %s", matches[1], s) return nil, fmt.Errorf("unable to parse %s part in %s", matches[1], s)
} }
} }
n, _ := strconv.Atoi(matches[2]) n, _ := strconv.Atoi(matches[2]) //nolint:errcheck // regexp ensures this is number
for i := localMin; i <= localMax; i += n { for i := localMin; i <= localMax; i += n {
r[i] = struct{}{} r[i] = struct{}{}
} }
@ -305,26 +317,26 @@ func parsePart(s string, min, max int) (map[int]struct{}, error) {
parts := strings.Split(s, ",") parts := strings.Split(s, ",")
for _, x := range parts { for _, x := range parts {
if rng := matchRange.FindStringSubmatch(x); rng != nil { if rng := matchRange.FindStringSubmatch(x); rng != nil {
localMin, _ := strconv.Atoi(rng[1]) localMin, _ := strconv.Atoi(rng[1]) //nolint:errcheck // checked below
localMax, _ := strconv.Atoi(rng[2]) localMax, _ := strconv.Atoi(rng[2]) //nolint:errcheck // checked below
if localMin < min || localMax > max { if localMin < minimum || localMax > maximum {
return nil, fmt.Errorf("Out of range for %s in %s. %s must be in range %d-%d", x, s, x, min, max) return nil, fmt.Errorf("out of range for %s in %s. %s must be in range %d-%d", x, s, x, minimum, maximum)
} }
for i := localMin; i <= localMax; i++ { for i := localMin; i <= localMax; i++ {
r[i] = struct{}{} r[i] = struct{}{}
} }
} else if i, err := strconv.Atoi(x); err == nil { } else if i, err := strconv.Atoi(x); err == nil {
if i < min || i > max { if i < minimum || i > maximum {
return nil, fmt.Errorf("Out of range for %d in %s. %d must be in range %d-%d", i, s, i, min, max) return nil, fmt.Errorf("out of range for %d in %s. %d must be in range %d-%d", i, s, i, minimum, maximum)
} }
r[i] = struct{}{} r[i] = struct{}{}
} else { } else {
return nil, fmt.Errorf("Unable to parse %s part in %s", x, s) return nil, fmt.Errorf("unable to parse %s part in %s", x, s)
} }
} }
if len(r) == 0 { if len(r) == 0 {
return nil, fmt.Errorf("Unable to parse %s", s) return nil, fmt.Errorf("unable to parse %s", s)
} }
return r, nil return r, nil
@ -340,3 +352,5 @@ func getTick(t time.Time) tick {
dayOfWeek: int(t.Weekday()), dayOfWeek: int(t.Weekday()),
} }
} }
// vim: ft=go

22
vendor/github.com/etkecc/go-crontab/justfile generated vendored Normal file
View file

@ -0,0 +1,22 @@
# show help by default
default:
@just --list --justfile {{ justfile() }}
# update go deps
update *flags:
go get {{ flags }} .
go mod tidy
# run linter
lint:
golangci-lint run ./...
# automatically fix liter issues
lintfix:
golangci-lint run --fix ./...
# run unit tests
test:
@go test -cover -coverprofile=cover.out -coverpkg=./... -covermode=set ./...
@go tool cover -func=cover.out
-@rm -f cover.out

View file

@ -0,0 +1,9 @@
# Github is obeying this ignore file by default.
# Run this command on local to ignore formatting commits in `git blame`
# git config blame.ignoreRevsFile .git-blame-ignore-revs
# Added a new column to supported_mimes.md
# The supported_mimes.md file was a nice way to find when a file format was
# introduced. However, when I changed to add a new column in the table, the
# whole git blame got poisoned for the file.
eb497f9bc5d31c6eab2929a112051218670137ba

View file

@ -97,7 +97,9 @@ or from a [file](https://pkg.go.dev/github.com/gabriel-vasile/mimetype#DetectFil
</div> </div>
## Contributing ## Contributing
Contributions are unexpected but welcome. When submitting a PR for detection of Contributions are never expected but very much welcome.
a new file format, please make sure to add a record to the list of testcases [mimetype_tests](https://github.com/gabriel-vasile/mimetype_tests/actions/workflows/test.yml)
from [mimetype_test.go](mimetype_test.go). For complex files a record can be added shows which file formats are most often misidentified and can help prioritise.
in the [testdata](testdata) directory. When submitting a PR for detection of a new file format, please make sure to
add a record to the list of testcases in [mimetype_test.go](mimetype_test.go).
For complex files a record can be added in the [testdata](testdata) directory.

View file

@ -2,6 +2,7 @@ package charset
import ( import (
"bytes" "bytes"
"strings"
"unicode/utf8" "unicode/utf8"
"github.com/gabriel-vasile/mimetype/internal/markup" "github.com/gabriel-vasile/mimetype/internal/markup"
@ -141,27 +142,25 @@ func FromXML(content []byte) string {
return FromPlain(content) return FromPlain(content)
} }
func fromXML(s scan.Bytes) string { func fromXML(s scan.Bytes) string {
xml := []byte("<?XML") xml := []byte("<?xml")
lxml := len(xml) lxml := len(xml)
for { for {
if len(s) == 0 { s.TrimLWS()
return ""
}
for scan.ByteIsWS(s.Peek()) {
s.Advance(1)
}
if len(s) <= lxml { if len(s) <= lxml {
return "" return ""
} }
if !s.Match(xml, scan.IgnoreCase) {
s = s[1:] // safe to slice instead of s.Advance(1) because bounds are checked i, k := s.Search(xml, 0)
continue if i == -1 {
return ""
} }
aName, aVal, hasMore := "", "", true s.Advance(i + k)
var aName, aVal []byte
hasMore := true
for hasMore { for hasMore {
aName, aVal, hasMore = markup.GetAnAttribute(&s) aName, aVal, hasMore = markup.GetAnAttribute(&s)
if aName == "encoding" && aVal != "" { if scan.Bytes(aName).Match([]byte("encoding"), 0) != -1 && len(aVal) != 0 {
return aVal return string(aVal)
} }
} }
} }
@ -198,10 +197,10 @@ func fromHTML(s scan.Bytes) string {
return "" return ""
} }
// Abort when <body is reached. // Abort when <body is reached.
if s.Match(body, scan.IgnoreCase) { if s.Match(body, scan.IgnoreCase) != -1 {
return "" return ""
} }
if !s.Match(meta, scan.IgnoreCase) { if s.Match(meta, scan.IgnoreCase) == -1 {
s = s[1:] // safe to slice instead of s.Advance(1) because bounds are checked s = s[1:] // safe to slice instead of s.Advance(1) because bounds are checked
continue continue
} }
@ -215,14 +214,16 @@ func fromHTML(s scan.Bytes) string {
needPragma := dontKnow needPragma := dontKnow
charset := "" charset := ""
aName, aVal, hasMore := "", "", true var aNameB, aValB []byte
hasMore := true
for hasMore { for hasMore {
aName, aVal, hasMore = markup.GetAnAttribute(&s) aNameB, aValB, hasMore = markup.GetAnAttribute(&s)
aName := strings.ToLower(string(aNameB))
if attrList[aName] { if attrList[aName] {
continue continue
} }
// processing step // processing step
if len(aName) == 0 && len(aVal) == 0 { if len(aName) == 0 && len(aValB) == 0 {
if needPragma == dontKnow { if needPragma == dontKnow {
continue continue
} }
@ -231,15 +232,18 @@ func fromHTML(s scan.Bytes) string {
} }
} }
attrList[aName] = true attrList[aName] = true
if aName == "http-equiv" && scan.Bytes(aVal).Match([]byte("CONTENT-TYPE"), scan.IgnoreCase) { switch aName {
gotPragma = true case "http-equiv":
} else if aName == "content" { if scan.Bytes(aValB).Match([]byte("CONTENT-TYPE"), scan.IgnoreCase) != -1 {
charset = string(extractCharsetFromMeta(scan.Bytes(aVal))) gotPragma = true
}
case "content":
charset = string(extractCharsetFromMeta(scan.Bytes(aValB)))
if len(charset) != 0 { if len(charset) != 0 {
needPragma = doNeedPragma needPragma = doNeedPragma
} }
} else if aName == "charset" { case "charset":
charset = aVal charset = string(aValB)
needPragma = doNotNeedPragma needPragma = doNotNeedPragma
} }
} }

View file

@ -257,8 +257,11 @@ out:
return 0 return 0
} }
// openArray is used instead of an inline []byte{'['} to avoid mem alllocs.
var openArray = []byte{'['}
func (p *parserState) consumeArray(b []byte, qs []query, lvl int) (n int) { func (p *parserState) consumeArray(b []byte, qs []query, lvl int) (n int) {
p.appendPath([]byte{'['}, qs) p.appendPath(openArray, qs)
if len(b) == 0 { if len(b) == 0 {
return 0 return 0
} }

View file

@ -5,43 +5,80 @@ import (
"encoding/binary" "encoding/binary"
) )
var ( // SevenZ matches a 7z archive.
// SevenZ matches a 7z archive. func SevenZ(raw []byte, _ uint32) bool {
SevenZ = prefix([]byte{0x37, 0x7A, 0xBC, 0xAF, 0x27, 0x1C}) return bytes.HasPrefix(raw, []byte{0x37, 0x7A, 0xBC, 0xAF, 0x27, 0x1C})
// Gzip matches gzip files based on http://www.zlib.org/rfc-gzip.html#header-trailer. }
Gzip = prefix([]byte{0x1f, 0x8b})
// Fits matches an Flexible Image Transport System file. // Gzip matches gzip files based on http://www.zlib.org/rfc-gzip.html#header-trailer.
Fits = prefix([]byte{ func Gzip(raw []byte, _ uint32) bool {
return bytes.HasPrefix(raw, []byte{0x1f, 0x8b})
}
// Fits matches an Flexible Image Transport System file.
func Fits(raw []byte, _ uint32) bool {
return bytes.HasPrefix(raw, []byte{
0x53, 0x49, 0x4D, 0x50, 0x4C, 0x45, 0x20, 0x20, 0x3D, 0x20, 0x53, 0x49, 0x4D, 0x50, 0x4C, 0x45, 0x20, 0x20, 0x3D, 0x20,
0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x54, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x54,
}) })
// Xar matches an eXtensible ARchive format file. }
Xar = prefix([]byte{0x78, 0x61, 0x72, 0x21})
// Bz2 matches a bzip2 file. // Xar matches an eXtensible ARchive format file.
Bz2 = prefix([]byte{0x42, 0x5A, 0x68}) func Xar(raw []byte, _ uint32) bool {
// Ar matches an ar (Unix) archive file. return bytes.HasPrefix(raw, []byte{0x78, 0x61, 0x72, 0x21})
Ar = prefix([]byte{0x21, 0x3C, 0x61, 0x72, 0x63, 0x68, 0x3E}) }
// Deb matches a Debian package file.
Deb = offset([]byte{ // Bz2 matches a bzip2 file.
func Bz2(raw []byte, _ uint32) bool {
return bytes.HasPrefix(raw, []byte{0x42, 0x5A, 0x68})
}
// Ar matches an ar (Unix) archive file.
func Ar(raw []byte, _ uint32) bool {
return bytes.HasPrefix(raw, []byte{0x21, 0x3C, 0x61, 0x72, 0x63, 0x68, 0x3E})
}
// Deb matches a Debian package file.
func Deb(raw []byte, _ uint32) bool {
return offset(raw, []byte{
0x64, 0x65, 0x62, 0x69, 0x61, 0x6E, 0x2D, 0x64, 0x65, 0x62, 0x69, 0x61, 0x6E, 0x2D,
0x62, 0x69, 0x6E, 0x61, 0x72, 0x79, 0x62, 0x69, 0x6E, 0x61, 0x72, 0x79,
}, 8) }, 8)
// Warc matches a Web ARChive file. }
Warc = prefix([]byte("WARC/1.0"), []byte("WARC/1.1"))
// Cab matches a Microsoft Cabinet archive file. // Warc matches a Web ARChive file.
Cab = prefix([]byte("MSCF\x00\x00\x00\x00")) func Warc(raw []byte, _ uint32) bool {
// Xz matches an xz compressed stream based on https://tukaani.org/xz/xz-file-format.txt. return bytes.HasPrefix(raw, []byte("WARC/1.0")) ||
Xz = prefix([]byte{0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00}) bytes.HasPrefix(raw, []byte("WARC/1.1"))
// Lzip matches an Lzip compressed file. }
Lzip = prefix([]byte{0x4c, 0x5a, 0x49, 0x50})
// RPM matches an RPM or Delta RPM package file. // Cab matches a Microsoft Cabinet archive file.
RPM = prefix([]byte{0xed, 0xab, 0xee, 0xdb}, []byte("drpm")) func Cab(raw []byte, _ uint32) bool {
// Cpio matches a cpio archive file. return bytes.HasPrefix(raw, []byte("MSCF\x00\x00\x00\x00"))
Cpio = prefix([]byte("070707"), []byte("070701"), []byte("070702")) }
// RAR matches a RAR archive file.
RAR = prefix([]byte("Rar!\x1A\x07\x00"), []byte("Rar!\x1A\x07\x01\x00")) // Xz matches an xz compressed stream based on https://tukaani.org/xz/xz-file-format.txt.
) func Xz(raw []byte, _ uint32) bool {
return bytes.HasPrefix(raw, []byte{0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00})
}
// Lzip matches an Lzip compressed file.
func Lzip(raw []byte, _ uint32) bool {
return bytes.HasPrefix(raw, []byte{0x4c, 0x5a, 0x49, 0x50})
}
// RPM matches an RPM or Delta RPM package file.
func RPM(raw []byte, _ uint32) bool {
return bytes.HasPrefix(raw, []byte{0xed, 0xab, 0xee, 0xdb}) ||
bytes.HasPrefix(raw, []byte("drpm"))
}
// RAR matches a RAR archive file.
func RAR(raw []byte, _ uint32) bool {
return bytes.HasPrefix(raw, []byte("Rar!\x1A\x07\x00")) ||
bytes.HasPrefix(raw, []byte("Rar!\x1A\x07\x01\x00"))
}
// InstallShieldCab matches an InstallShield Cabinet archive file. // InstallShieldCab matches an InstallShield Cabinet archive file.
func InstallShieldCab(raw []byte, _ uint32) bool { func InstallShieldCab(raw []byte, _ uint32) bool {
@ -78,6 +115,17 @@ func CRX(raw []byte, limit uint32) bool {
return Zip(raw[zipOffset:], limit) return Zip(raw[zipOffset:], limit)
} }
// Cpio matches a cpio archive file.
func Cpio(raw []byte, _ uint32) bool {
if len(raw) < 6 {
return false
}
return binary.LittleEndian.Uint16(raw) == 070707 || // binary cpio
bytes.HasPrefix(raw, []byte("070707")) || // portable ASCII cpios
bytes.HasPrefix(raw, []byte("070701")) ||
bytes.HasPrefix(raw, []byte("070702"))
}
// Tar matches a (t)ape (ar)chive file. // Tar matches a (t)ape (ar)chive file.
// Tar files are divided into 512 bytes records. First record contains a 257 // Tar files are divided into 512 bytes records. First record contains a 257
// bytes header padded with NUL. // bytes header padded with NUL.

View file

@ -5,26 +5,50 @@ import (
"encoding/binary" "encoding/binary"
) )
var ( // Flac matches a Free Lossless Audio Codec file.
// Flac matches a Free Lossless Audio Codec file. func Flac(raw []byte, _ uint32) bool {
Flac = prefix([]byte("\x66\x4C\x61\x43\x00\x00\x00\x22")) return bytes.HasPrefix(raw, []byte("\x66\x4C\x61\x43\x00\x00\x00\x22"))
// Midi matches a Musical Instrument Digital Interface file. }
Midi = prefix([]byte("\x4D\x54\x68\x64"))
// Ape matches a Monkey's Audio file. // Midi matches a Musical Instrument Digital Interface file.
Ape = prefix([]byte("\x4D\x41\x43\x20\x96\x0F\x00\x00\x34\x00\x00\x00\x18\x00\x00\x00\x90\xE3")) func Midi(raw []byte, _ uint32) bool {
// MusePack matches a Musepack file. return bytes.HasPrefix(raw, []byte("\x4D\x54\x68\x64"))
MusePack = prefix([]byte("MPCK")) }
// Au matches a Sun Microsystems au file.
Au = prefix([]byte("\x2E\x73\x6E\x64")) // Ape matches a Monkey's Audio file.
// Amr matches an Adaptive Multi-Rate file. func Ape(raw []byte, _ uint32) bool {
Amr = prefix([]byte("\x23\x21\x41\x4D\x52")) return bytes.HasPrefix(raw, []byte("\x4D\x41\x43\x20\x96\x0F\x00\x00\x34\x00\x00\x00\x18\x00\x00\x00\x90\xE3"))
// Voc matches a Creative Voice file. }
Voc = prefix([]byte("Creative Voice File"))
// M3u matches a Playlist file. // MusePack matches a Musepack file.
M3u = prefix([]byte("#EXTM3U")) func MusePack(raw []byte, _ uint32) bool {
// AAC matches an Advanced Audio Coding file. return bytes.HasPrefix(raw, []byte("MPCK"))
AAC = prefix([]byte{0xFF, 0xF1}, []byte{0xFF, 0xF9}) }
)
// Au matches a Sun Microsystems au file.
func Au(raw []byte, _ uint32) bool {
return bytes.HasPrefix(raw, []byte("\x2E\x73\x6E\x64"))
}
// Amr matches an Adaptive Multi-Rate file.
func Amr(raw []byte, _ uint32) bool {
return bytes.HasPrefix(raw, []byte("\x23\x21\x41\x4D\x52"))
}
// Voc matches a Creative Voice file.
func Voc(raw []byte, _ uint32) bool {
return bytes.HasPrefix(raw, []byte("Creative Voice File"))
}
// M3u matches a Playlist file.
func M3u(raw []byte, _ uint32) bool {
return bytes.HasPrefix(raw, []byte("#EXTM3U"))
}
// AAC matches an Advanced Audio Coding file.
func AAC(raw []byte, _ uint32) bool {
return len(raw) > 1 && ((raw[0] == 0xFF && raw[1] == 0xF1) || (raw[0] == 0xFF && raw[1] == 0xF9))
}
// Mp3 matches an mp3 file. // Mp3 matches an mp3 file.
func Mp3(raw []byte, limit uint32) bool { func Mp3(raw []byte, limit uint32) bool {

View file

@ -6,26 +6,52 @@ import (
"encoding/binary" "encoding/binary"
) )
var ( // Lnk matches Microsoft lnk binary format.
// Lnk matches Microsoft lnk binary format. func Lnk(raw []byte, _ uint32) bool {
Lnk = prefix([]byte{0x4C, 0x00, 0x00, 0x00, 0x01, 0x14, 0x02, 0x00}) return bytes.HasPrefix(raw, []byte{0x4C, 0x00, 0x00, 0x00, 0x01, 0x14, 0x02, 0x00})
// Wasm matches a web assembly File Format file. }
Wasm = prefix([]byte{0x00, 0x61, 0x73, 0x6D})
// Exe matches a Windows/DOS executable file. // Wasm matches a web assembly File Format file.
Exe = prefix([]byte{0x4D, 0x5A}) func Wasm(raw []byte, _ uint32) bool {
// Elf matches an Executable and Linkable Format file. return bytes.HasPrefix(raw, []byte{0x00, 0x61, 0x73, 0x6D})
Elf = prefix([]byte{0x7F, 0x45, 0x4C, 0x46}) }
// Nes matches a Nintendo Entertainment system ROM file.
Nes = prefix([]byte{0x4E, 0x45, 0x53, 0x1A}) // Exe matches a Windows/DOS executable file.
// SWF matches an Adobe Flash swf file. func Exe(raw []byte, _ uint32) bool {
SWF = prefix([]byte("CWS"), []byte("FWS"), []byte("ZWS")) return len(raw) > 1 && raw[0] == 0x4D && raw[1] == 0x5A
// Torrent has bencoded text in the beginning. }
Torrent = prefix([]byte("d8:announce"))
// PAR1 matches a parquet file. // Elf matches an Executable and Linkable Format file.
Par1 = prefix([]byte{0x50, 0x41, 0x52, 0x31}) func Elf(raw []byte, _ uint32) bool {
// CBOR matches a Concise Binary Object Representation https://cbor.io/ return bytes.HasPrefix(raw, []byte{0x7F, 0x45, 0x4C, 0x46})
CBOR = prefix([]byte{0xD9, 0xD9, 0xF7}) }
)
// Nes matches a Nintendo Entertainment system ROM file.
func Nes(raw []byte, _ uint32) bool {
return bytes.HasPrefix(raw, []byte{0x4E, 0x45, 0x53, 0x1A})
}
// SWF matches an Adobe Flash swf file.
func SWF(raw []byte, _ uint32) bool {
return bytes.HasPrefix(raw, []byte("CWS")) ||
bytes.HasPrefix(raw, []byte("FWS")) ||
bytes.HasPrefix(raw, []byte("ZWS"))
}
// Torrent has bencoded text in the beginning.
func Torrent(raw []byte, _ uint32) bool {
return bytes.HasPrefix(raw, []byte("d8:announce"))
}
// PAR1 matches a parquet file.
func Par1(raw []byte, _ uint32) bool {
return bytes.HasPrefix(raw, []byte{0x50, 0x41, 0x52, 0x31})
}
// CBOR matches a Concise Binary Object Representation https://cbor.io/
func CBOR(raw []byte, _ uint32) bool {
return bytes.HasPrefix(raw, []byte{0xD9, 0xD9, 0xF7})
}
// Java bytecode and Mach-O binaries share the same magic number. // Java bytecode and Mach-O binaries share the same magic number.
// More info here https://github.com/threatstack/libmagic/blob/master/magic/Magdir/cafebabe // More info here https://github.com/threatstack/libmagic/blob/master/magic/Magdir/cafebabe
@ -168,8 +194,10 @@ func Marc(raw []byte, limit uint32) bool {
// //
// [glTF specification]: https://registry.khronos.org/glTF/specs/2.0/glTF-2.0.html // [glTF specification]: https://registry.khronos.org/glTF/specs/2.0/glTF-2.0.html
// [IANA glTF entry]: https://www.iana.org/assignments/media-types/model/gltf-binary // [IANA glTF entry]: https://www.iana.org/assignments/media-types/model/gltf-binary
var GLB = prefix([]byte("\x67\x6C\x54\x46\x02\x00\x00\x00"), func GLB(raw []byte, _ uint32) bool {
[]byte("\x67\x6C\x54\x46\x01\x00\x00\x00")) return bytes.HasPrefix(raw, []byte("\x67\x6C\x54\x46\x02\x00\x00\x00")) ||
bytes.HasPrefix(raw, []byte("\x67\x6C\x54\x46\x01\x00\x00\x00"))
}
// TzIf matches a Time Zone Information Format (TZif) file. // TzIf matches a Time Zone Information Format (TZif) file.
// See more: https://tools.ietf.org/id/draft-murchison-tzdist-tzif-00.html#rfc.section.3 // See more: https://tools.ietf.org/id/draft-murchison-tzdist-tzif-00.html#rfc.section.3

View file

@ -1,13 +1,21 @@
package magic package magic
var ( import "bytes"
// Sqlite matches an SQLite database file.
Sqlite = prefix([]byte{ // Sqlite matches an SQLite database file.
func Sqlite(raw []byte, _ uint32) bool {
return bytes.HasPrefix(raw, []byte{
0x53, 0x51, 0x4c, 0x69, 0x74, 0x65, 0x20, 0x66, 0x53, 0x51, 0x4c, 0x69, 0x74, 0x65, 0x20, 0x66,
0x6f, 0x72, 0x6d, 0x61, 0x74, 0x20, 0x33, 0x00, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x20, 0x33, 0x00,
}) })
// MsAccessAce matches Microsoft Access dababase file. }
MsAccessAce = offset([]byte("Standard ACE DB"), 4)
// MsAccessMdb matches legacy Microsoft Access database file (JET, 2003 and earlier). // MsAccessAce matches Microsoft Access dababase file.
MsAccessMdb = offset([]byte("Standard Jet DB"), 4) func MsAccessAce(raw []byte, _ uint32) bool {
) return offset(raw, []byte("Standard ACE DB"), 4)
}
// MsAccessMdb matches legacy Microsoft Access database file (JET, 2003 and earlier).
func MsAccessMdb(raw []byte, _ uint32) bool {
return offset(raw, []byte("Standard Jet DB"), 4)
}

View file

@ -5,14 +5,31 @@ import (
"encoding/binary" "encoding/binary"
) )
var ( // Pdf matches a Portable Document Format file.
// Fdf matches a Forms Data Format file. // https://github.com/file/file/blob/11010cc805546a3e35597e67e1129a481aed40e8/magic/Magdir/pdf
Fdf = prefix([]byte("%FDF")) func Pdf(raw []byte, _ uint32) bool {
// Mobi matches a Mobi file. // usual pdf signature
Mobi = offset([]byte("BOOKMOBI"), 60) return bytes.HasPrefix(raw, []byte("%PDF-")) ||
// Lit matches a Microsoft Lit file. // new-line prefixed signature
Lit = prefix([]byte("ITOLITLS")) bytes.HasPrefix(raw, []byte("\012%PDF-")) ||
) // UTF-8 BOM prefixed signature
bytes.HasPrefix(raw, []byte("\xef\xbb\xbf%PDF-"))
}
// Fdf matches a Forms Data Format file.
func Fdf(raw []byte, _ uint32) bool {
return bytes.HasPrefix(raw, []byte("%FDF"))
}
// Mobi matches a Mobi file.
func Mobi(raw []byte, _ uint32) bool {
return offset(raw, []byte("BOOKMOBI"), 60)
}
// Lit matches a Microsoft Lit file.
func Lit(raw []byte, _ uint32) bool {
return bytes.HasPrefix(raw, []byte("ITOLITLS"))
}
// PDF matches a Portable Document Format file. // PDF matches a Portable Document Format file.
// The %PDF- header should be the first thing inside the file but many // The %PDF- header should be the first thing inside the file but many

View file

@ -4,14 +4,20 @@ import (
"bytes" "bytes"
) )
var ( // Woff matches a Web Open Font Format file.
// Woff matches a Web Open Font Format file. func Woff(raw []byte, _ uint32) bool {
Woff = prefix([]byte("wOFF")) return bytes.HasPrefix(raw, []byte("wOFF"))
// Woff2 matches a Web Open Font Format version 2 file. }
Woff2 = prefix([]byte("wOF2"))
// Otf matches an OpenType font file. // Woff2 matches a Web Open Font Format version 2 file.
Otf = prefix([]byte{0x4F, 0x54, 0x54, 0x4F, 0x00}) func Woff2(raw []byte, _ uint32) bool {
) return bytes.HasPrefix(raw, []byte("wOF2"))
}
// Otf matches an OpenType font file.
func Otf(raw []byte, _ uint32) bool {
return bytes.HasPrefix(raw, []byte{0x4F, 0x54, 0x54, 0x4F, 0x00})
}
// Ttf matches a TrueType font file. // Ttf matches a TrueType font file.
func Ttf(raw []byte, limit uint32) bool { func Ttf(raw []byte, limit uint32) bool {

View file

@ -4,24 +4,33 @@ import (
"bytes" "bytes"
) )
var ( // AVIF matches an AV1 Image File Format still or animated.
// AVIF matches an AV1 Image File Format still or animated. // Wikipedia page seems outdated listing image/avif-sequence for animations.
// Wikipedia page seems outdated listing image/avif-sequence for animations. // https://github.com/AOMediaCodec/av1-avif/issues/59
// https://github.com/AOMediaCodec/av1-avif/issues/59 func AVIF(raw []byte, _ uint32) bool {
AVIF = ftyp([]byte("avif"), []byte("avis")) return ftyp(raw, []byte("avif"), []byte("avis"))
// ThreeGP matches a 3GPP file. }
ThreeGP = ftyp(
// ThreeGP matches a 3GPP file.
func ThreeGP(raw []byte, _ uint32) bool {
return ftyp(raw,
[]byte("3gp1"), []byte("3gp2"), []byte("3gp3"), []byte("3gp4"), []byte("3gp1"), []byte("3gp2"), []byte("3gp3"), []byte("3gp4"),
[]byte("3gp5"), []byte("3gp6"), []byte("3gp7"), []byte("3gs7"), []byte("3gp5"), []byte("3gp6"), []byte("3gp7"), []byte("3gs7"),
[]byte("3ge6"), []byte("3ge7"), []byte("3gg6"), []byte("3ge6"), []byte("3ge7"), []byte("3gg6"),
) )
// ThreeG2 matches a 3GPP2 file. }
ThreeG2 = ftyp(
// ThreeG2 matches a 3GPP2 file.
func ThreeG2(raw []byte, _ uint32) bool {
return ftyp(raw,
[]byte("3g24"), []byte("3g25"), []byte("3g26"), []byte("3g2a"), []byte("3g24"), []byte("3g25"), []byte("3g26"), []byte("3g2a"),
[]byte("3g2b"), []byte("3g2c"), []byte("KDDI"), []byte("3g2b"), []byte("3g2c"), []byte("KDDI"),
) )
// AMp4 matches an audio MP4 file. }
AMp4 = ftyp(
// AMp4 matches an audio MP4 file.
func AMp4(raw []byte, _ uint32) bool {
return ftyp(raw,
// audio for Adobe Flash Player 9+ // audio for Adobe Flash Player 9+
[]byte("F4A "), []byte("F4B "), []byte("F4A "), []byte("F4B "),
// Apple iTunes AAC-LC (.M4A) Audio // Apple iTunes AAC-LC (.M4A) Audio
@ -31,33 +40,61 @@ var (
// Nero Digital AAC Audio // Nero Digital AAC Audio
[]byte("NDAS"), []byte("NDAS"),
) )
// Mqv matches a Sony / Mobile QuickTime file. }
Mqv = ftyp([]byte("mqt "))
// M4a matches an audio M4A file. // Mqv matches a Sony / Mobile QuickTime file.
M4a = ftyp([]byte("M4A ")) func Mqv(raw []byte, _ uint32) bool {
// M4v matches an Appl4 M4V video file. return ftyp(raw, []byte("mqt "))
M4v = ftyp([]byte("M4V "), []byte("M4VH"), []byte("M4VP")) }
// Heic matches a High Efficiency Image Coding (HEIC) file.
Heic = ftyp([]byte("heic"), []byte("heix")) // M4a matches an audio M4A file.
// HeicSequence matches a High Efficiency Image Coding (HEIC) file sequence. func M4a(raw []byte, _ uint32) bool {
HeicSequence = ftyp([]byte("hevc"), []byte("hevx")) return ftyp(raw, []byte("M4A "))
// Heif matches a High Efficiency Image File Format (HEIF) file. }
Heif = ftyp([]byte("mif1"), []byte("heim"), []byte("heis"), []byte("avic"))
// HeifSequence matches a High Efficiency Image File Format (HEIF) file sequence. // M4v matches an Appl4 M4V video file.
HeifSequence = ftyp([]byte("msf1"), []byte("hevm"), []byte("hevs"), []byte("avcs")) func M4v(raw []byte, _ uint32) bool {
// Mj2 matches a Motion JPEG 2000 file: https://en.wikipedia.org/wiki/Motion_JPEG_2000. return ftyp(raw, []byte("M4V "), []byte("M4VH"), []byte("M4VP"))
Mj2 = ftyp([]byte("mj2s"), []byte("mjp2"), []byte("MFSM"), []byte("MGSV")) }
// Dvb matches a Digital Video Broadcasting file: https://dvb.org.
// https://cconcolato.github.io/mp4ra/filetype.html // Heic matches a High Efficiency Image Coding (HEIC) file.
// https://github.com/file/file/blob/512840337ead1076519332d24fefcaa8fac36e06/magic/Magdir/animation#L135-L154 func Heic(raw []byte, _ uint32) bool {
Dvb = ftyp( return ftyp(raw, []byte("heic"), []byte("heix"))
}
// HeicSequence matches a High Efficiency Image Coding (HEIC) file sequence.
func HeicSequence(raw []byte, _ uint32) bool {
return ftyp(raw, []byte("hevc"), []byte("hevx"))
}
// Heif matches a High Efficiency Image File Format (HEIF) file.
func Heif(raw []byte, _ uint32) bool {
return ftyp(raw, []byte("mif1"), []byte("heim"), []byte("heis"), []byte("avic"))
}
// HeifSequence matches a High Efficiency Image File Format (HEIF) file sequence.
func HeifSequence(raw []byte, _ uint32) bool {
return ftyp(raw, []byte("msf1"), []byte("hevm"), []byte("hevs"), []byte("avcs"))
}
// Mj2 matches a Motion JPEG 2000 file: https://en.wikipedia.org/wiki/Motion_JPEG_2000.
func Mj2(raw []byte, _ uint32) bool {
return ftyp(raw, []byte("mj2s"), []byte("mjp2"), []byte("MFSM"), []byte("MGSV"))
}
// Dvb matches a Digital Video Broadcasting file: https://dvb.org.
// https://cconcolato.github.io/mp4ra/filetype.html
// https://github.com/file/file/blob/512840337ead1076519332d24fefcaa8fac36e06/magic/Magdir/animation#L135-L154
func Dvb(raw []byte, _ uint32) bool {
return ftyp(raw,
[]byte("dby1"), []byte("dsms"), []byte("dts1"), []byte("dts2"), []byte("dby1"), []byte("dsms"), []byte("dts1"), []byte("dts2"),
[]byte("dts3"), []byte("dxo "), []byte("dmb1"), []byte("dmpf"), []byte("dts3"), []byte("dxo "), []byte("dmb1"), []byte("dmpf"),
[]byte("drc1"), []byte("dv1a"), []byte("dv1b"), []byte("dv2a"), []byte("drc1"), []byte("dv1a"), []byte("dv1b"), []byte("dv2a"),
[]byte("dv2b"), []byte("dv3a"), []byte("dv3b"), []byte("dvr1"), []byte("dv2b"), []byte("dv3a"), []byte("dv3b"), []byte("dvr1"),
[]byte("dvt1"), []byte("emsg")) []byte("dvt1"), []byte("emsg"))
// TODO: add support for remaining video formats at ftyps.com. }
)
// TODO: add support for remaining video formats at ftyps.com.
// QuickTime matches a QuickTime File Format file. // QuickTime matches a QuickTime File Format file.
// https://www.loc.gov/preservation/digital/formats/fdd/fdd000052.shtml // https://www.loc.gov/preservation/digital/formats/fdd/fdd000052.shtml

View file

@ -2,66 +2,127 @@ package magic
import "bytes" import "bytes"
var ( // Png matches a Portable Network Graphics file.
// Png matches a Portable Network Graphics file. // https://www.w3.org/TR/PNG/
// https://www.w3.org/TR/PNG/ func Png(raw []byte, _ uint32) bool {
Png = prefix([]byte{0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A}) return bytes.HasPrefix(raw, []byte{0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A})
// Apng matches an Animated Portable Network Graphics file. }
// https://wiki.mozilla.org/APNG_Specification
Apng = offset([]byte("acTL"), 37)
// Jpg matches a Joint Photographic Experts Group file.
Jpg = prefix([]byte{0xFF, 0xD8, 0xFF})
// Jp2 matches a JPEG 2000 Image file (ISO 15444-1).
Jp2 = jpeg2k([]byte{0x6a, 0x70, 0x32, 0x20})
// Jpx matches a JPEG 2000 Image file (ISO 15444-2).
Jpx = jpeg2k([]byte{0x6a, 0x70, 0x78, 0x20})
// Jpm matches a JPEG 2000 Image file (ISO 15444-6).
Jpm = jpeg2k([]byte{0x6a, 0x70, 0x6D, 0x20})
// Gif matches a Graphics Interchange Format file.
Gif = prefix([]byte("GIF87a"), []byte("GIF89a"))
// Bmp matches a bitmap image file.
Bmp = prefix([]byte{0x42, 0x4D})
// Ps matches a PostScript file.
Ps = prefix([]byte("%!PS-Adobe-"))
// Psd matches a Photoshop Document file.
Psd = prefix([]byte("8BPS"))
// Ico matches an ICO file.
Ico = prefix([]byte{0x00, 0x00, 0x01, 0x00}, []byte{0x00, 0x00, 0x02, 0x00})
// Icns matches an ICNS (Apple Icon Image format) file.
Icns = prefix([]byte("icns"))
// Tiff matches a Tagged Image File Format file.
Tiff = prefix([]byte{0x49, 0x49, 0x2A, 0x00}, []byte{0x4D, 0x4D, 0x00, 0x2A})
// Bpg matches a Better Portable Graphics file.
Bpg = prefix([]byte{0x42, 0x50, 0x47, 0xFB})
// Xcf matches GIMP image data.
Xcf = prefix([]byte("gimp xcf"))
// Pat matches GIMP pattern data.
Pat = offset([]byte("GPAT"), 20)
// Gbr matches GIMP brush data.
Gbr = offset([]byte("GIMP"), 20)
// Hdr matches Radiance HDR image.
// https://web.archive.org/web/20060913152809/http://local.wasp.uwa.edu.au/~pbourke/dataformats/pic/
Hdr = prefix([]byte("#?RADIANCE\n"))
// Xpm matches X PixMap image data.
Xpm = prefix([]byte{0x2F, 0x2A, 0x20, 0x58, 0x50, 0x4D, 0x20, 0x2A, 0x2F})
// Jxs matches a JPEG XS coded image file (ISO/IEC 21122-3).
Jxs = prefix([]byte{0x00, 0x00, 0x00, 0x0C, 0x4A, 0x58, 0x53, 0x20, 0x0D, 0x0A, 0x87, 0x0A})
// Jxr matches Microsoft HD JXR photo file.
Jxr = prefix([]byte{0x49, 0x49, 0xBC, 0x01})
)
func jpeg2k(sig []byte) Detector { // Apng matches an Animated Portable Network Graphics file.
return func(raw []byte, _ uint32) bool { // https://wiki.mozilla.org/APNG_Specification
if len(raw) < 24 { func Apng(raw []byte, _ uint32) bool {
return false return offset(raw, []byte("acTL"), 37)
} }
if !bytes.Equal(raw[4:8], []byte{0x6A, 0x50, 0x20, 0x20}) && // Jpg matches a Joint Photographic Experts Group file.
!bytes.Equal(raw[4:8], []byte{0x6A, 0x50, 0x32, 0x20}) { func Jpg(raw []byte, _ uint32) bool {
return false return bytes.HasPrefix(raw, []byte{0xFF, 0xD8, 0xFF})
} }
return bytes.Equal(raw[20:24], sig)
// Jp2 matches a JPEG 2000 Image file (ISO 15444-1).
func Jp2(raw []byte, _ uint32) bool {
return jpeg2k(raw, []byte{0x6a, 0x70, 0x32, 0x20})
}
// Jpx matches a JPEG 2000 Image file (ISO 15444-2).
func Jpx(raw []byte, _ uint32) bool {
return jpeg2k(raw, []byte{0x6a, 0x70, 0x78, 0x20})
}
// Jpm matches a JPEG 2000 Image file (ISO 15444-6).
func Jpm(raw []byte, _ uint32) bool {
return jpeg2k(raw, []byte{0x6a, 0x70, 0x6D, 0x20})
}
// Gif matches a Graphics Interchange Format file.
func Gif(raw []byte, _ uint32) bool {
return bytes.HasPrefix(raw, []byte("GIF87a")) ||
bytes.HasPrefix(raw, []byte("GIF89a"))
}
// Bmp matches a bitmap image file.
func Bmp(raw []byte, _ uint32) bool {
return bytes.HasPrefix(raw, []byte{0x42, 0x4D})
}
// Ps matches a PostScript file.
func Ps(raw []byte, _ uint32) bool {
return bytes.HasPrefix(raw, []byte("%!PS-Adobe-"))
}
// Psd matches a Photoshop Document file.
func Psd(raw []byte, _ uint32) bool {
return bytes.HasPrefix(raw, []byte("8BPS"))
}
// Ico matches an ICO file.
func Ico(raw []byte, _ uint32) bool {
return bytes.HasPrefix(raw, []byte{0x00, 0x00, 0x01, 0x00}) ||
bytes.HasPrefix(raw, []byte{0x00, 0x00, 0x02, 0x00})
}
// Icns matches an ICNS (Apple Icon Image format) file.
func Icns(raw []byte, _ uint32) bool {
return bytes.HasPrefix(raw, []byte("icns"))
}
// Tiff matches a Tagged Image File Format file.
func Tiff(raw []byte, _ uint32) bool {
return bytes.HasPrefix(raw, []byte{0x49, 0x49, 0x2A, 0x00}) ||
bytes.HasPrefix(raw, []byte{0x4D, 0x4D, 0x00, 0x2A})
}
// Bpg matches a Better Portable Graphics file.
func Bpg(raw []byte, _ uint32) bool {
return bytes.HasPrefix(raw, []byte{0x42, 0x50, 0x47, 0xFB})
}
// Xcf matches GIMP image data.
func Xcf(raw []byte, _ uint32) bool {
return bytes.HasPrefix(raw, []byte("gimp xcf"))
}
// Pat matches GIMP pattern data.
func Pat(raw []byte, _ uint32) bool {
return offset(raw, []byte("GPAT"), 20)
}
// Gbr matches GIMP brush data.
func Gbr(raw []byte, _ uint32) bool {
return offset(raw, []byte("GIMP"), 20)
}
// Hdr matches Radiance HDR image.
// https://web.archive.org/web/20060913152809/http://local.wasp.uwa.edu.au/~pbourke/dataformats/pic/
func Hdr(raw []byte, _ uint32) bool {
return bytes.HasPrefix(raw, []byte("#?RADIANCE\n"))
}
// Xpm matches X PixMap image data.
func Xpm(raw []byte, _ uint32) bool {
return bytes.HasPrefix(raw, []byte{0x2F, 0x2A, 0x20, 0x58, 0x50, 0x4D, 0x20, 0x2A, 0x2F})
}
// Jxs matches a JPEG XS coded image file (ISO/IEC 21122-3).
func Jxs(raw []byte, _ uint32) bool {
return bytes.HasPrefix(raw, []byte{0x00, 0x00, 0x00, 0x0C, 0x4A, 0x58, 0x53, 0x20, 0x0D, 0x0A, 0x87, 0x0A})
}
// Jxr matches Microsoft HD JXR photo file.
func Jxr(raw []byte, _ uint32) bool {
return bytes.HasPrefix(raw, []byte{0x49, 0x49, 0xBC, 0x01})
}
func jpeg2k(raw []byte, sig []byte) bool {
if len(raw) < 24 {
return false
} }
if !bytes.Equal(raw[4:8], []byte{0x6A, 0x50, 0x20, 0x20}) &&
!bytes.Equal(raw[4:8], []byte{0x6A, 0x50, 0x32, 0x20}) {
return false
}
return bytes.Equal(raw[20:24], sig)
} }
// Webp matches a WebP file. // Webp matches a WebP file.
@ -108,3 +169,20 @@ func Jxl(raw []byte, _ uint32) bool {
return bytes.HasPrefix(raw, []byte{0xFF, 0x0A}) || return bytes.HasPrefix(raw, []byte{0xFF, 0x0A}) ||
bytes.HasPrefix(raw, []byte("\x00\x00\x00\x0cJXL\x20\x0d\x0a\x87\x0a")) bytes.HasPrefix(raw, []byte("\x00\x00\x00\x0cJXL\x20\x0d\x0a\x87\x0a"))
} }
// DXF matches Drawing Exchange Format AutoCAD file.
// There does not seem to be a clear specification and the files in the wild
// differ wildly.
// https://images.autodesk.com/adsk/files/autocad_2012_pdf_dxf-reference_enu.pdf
//
// I collected these signatures by downloading a few dozen files from
// http://cd.textfiles.com/amigaenv/DXF/OBJEKTE/ and
// https://sembiance.com/fileFormatSamples/poly/dxf/ and then
// xxd -l 16 {} | sort | uniq.
// These signatures are only for the ASCII version of DXF. There is a binary version too.
func DXF(raw []byte, _ uint32) bool {
return bytes.HasPrefix(raw, []byte(" 0\x0ASECTION\x0A")) ||
bytes.HasPrefix(raw, []byte(" 0\x0D\x0ASECTION\x0D\x0A")) ||
bytes.HasPrefix(raw, []byte("0\x0ASECTION\x0A")) ||
bytes.HasPrefix(raw, []byte("0\x0D\x0ASECTION\x0D\x0A"))
}

View file

@ -3,7 +3,6 @@ package magic
import ( import (
"bytes" "bytes"
"fmt"
"github.com/gabriel-vasile/mimetype/internal/scan" "github.com/gabriel-vasile/mimetype/internal/scan"
) )
@ -22,37 +21,20 @@ type (
} }
) )
// prefix creates a Detector which returns true if any of the provided signatures // offset returns true if the provided signature can be
// is the prefix of the raw input.
func prefix(sigs ...[]byte) Detector {
return func(raw []byte, limit uint32) bool {
for _, s := range sigs {
if bytes.HasPrefix(raw, s) {
return true
}
}
return false
}
}
// offset creates a Detector which returns true if the provided signature can be
// found at offset in the raw input. // found at offset in the raw input.
func offset(sig []byte, offset int) Detector { func offset(raw []byte, sig []byte, offset int) bool {
return func(raw []byte, limit uint32) bool { return len(raw) > offset && bytes.HasPrefix(raw[offset:], sig)
return len(raw) > offset && bytes.HasPrefix(raw[offset:], sig)
}
} }
// ciPrefix is like prefix but the check is case insensitive. // ciPrefix is like prefix but the check is case insensitive.
func ciPrefix(sigs ...[]byte) Detector { func ciPrefix(raw []byte, sigs ...[]byte) bool {
return func(raw []byte, limit uint32) bool { for _, s := range sigs {
for _, s := range sigs { if ciCheck(s, raw) {
if ciCheck(s, raw) { return true
return true
}
} }
return false
} }
return false
} }
func ciCheck(sig, raw []byte) bool { func ciCheck(sig, raw []byte) bool {
if len(raw) < len(sig)+1 { if len(raw) < len(sig)+1 {
@ -72,22 +54,18 @@ func ciCheck(sig, raw []byte) bool {
return true return true
} }
// xml creates a Detector which returns true if any of the provided XML signatures // xml returns true if any of the provided XML signatures matches the raw input.
// matches the raw input. func xml(b scan.Bytes, sigs ...xmlSig) bool {
func xml(sigs ...xmlSig) Detector { b.TrimLWS()
return func(raw []byte, limit uint32) bool { if len(b) == 0 {
b := scan.Bytes(raw)
b.TrimLWS()
if len(b) == 0 {
return false
}
for _, s := range sigs {
if xmlCheck(s, b) {
return true
}
}
return false return false
} }
for _, s := range sigs {
if xmlCheck(s, b) {
return true
}
}
return false
} }
func xmlCheck(sig xmlSig, raw []byte) bool { func xmlCheck(sig xmlSig, raw []byte) bool {
raw = raw[:min(len(raw), 512)] raw = raw[:min(len(raw), 512)]
@ -103,28 +81,24 @@ func xmlCheck(sig xmlSig, raw []byte) bool {
return localNameIndex != -1 && localNameIndex < bytes.Index(raw, sig.xmlns) return localNameIndex != -1 && localNameIndex < bytes.Index(raw, sig.xmlns)
} }
// markup creates a Detector which returns true is any of the HTML signatures // markup returns true is any of the HTML signatures matches the raw input.
// matches the raw input. func markup(b scan.Bytes, sigs ...[]byte) bool {
func markup(sigs ...[]byte) Detector { if bytes.HasPrefix(b, []byte{0xEF, 0xBB, 0xBF}) {
return func(raw []byte, limit uint32) bool { // We skip the UTF-8 BOM if present to ensure we correctly
b := scan.Bytes(raw) // process any leading whitespace. The presence of the BOM
if bytes.HasPrefix(b, []byte{0xEF, 0xBB, 0xBF}) { // is taken into account during charset detection in charset.go.
// We skip the UTF-8 BOM if present to ensure we correctly b.Advance(3)
// process any leading whitespace. The presence of the BOM }
// is taken into account during charset detection in charset.go. b.TrimLWS()
b.Advance(3) if len(b) == 0 {
}
b.TrimLWS()
if len(b) == 0 {
return false
}
for _, s := range sigs {
if markupCheck(s, b) {
return true
}
}
return false return false
} }
for _, s := range sigs {
if markupCheck(s, b) {
return true
}
}
return false
} }
func markupCheck(sig, raw []byte) bool { func markupCheck(sig, raw []byte) bool {
if len(raw) < len(sig)+1 { if len(raw) < len(sig)+1 {
@ -149,29 +123,17 @@ func markupCheck(sig, raw []byte) bool {
return true return true
} }
// ftyp creates a Detector which returns true if any of the FTYP signatures // ftyp returns true if any of the FTYP signatures matches the raw input.
// matches the raw input. func ftyp(raw []byte, sigs ...[]byte) bool {
func ftyp(sigs ...[]byte) Detector { if len(raw) < 12 {
return func(raw []byte, limit uint32) bool {
if len(raw) < 12 {
return false
}
for _, s := range sigs {
if bytes.Equal(raw[8:12], s) {
return true
}
}
return false return false
} }
} for _, s := range sigs {
if bytes.Equal(raw[8:12], s) {
func newXMLSig(localName, xmlns string) xmlSig { return true
ret := xmlSig{xmlns: []byte(xmlns)} }
if localName != "" {
ret.localName = []byte(fmt.Sprintf("<%s", localName))
} }
return false
return ret
} }
// A valid shebang starts with the "#!" characters, // A valid shebang starts with the "#!" characters,
@ -184,29 +146,17 @@ func newXMLSig(localName, xmlns string) xmlSig {
// #! /usr/bin/env php // #! /usr/bin/env php
// //
// /usr/bin/env is the interpreter, php is the first and only argument. // /usr/bin/env is the interpreter, php is the first and only argument.
func shebang(sigs ...[]byte) Detector { func shebang(b scan.Bytes, matchFlags scan.Flags, sigs ...[]byte) bool {
return func(raw []byte, limit uint32) bool { line := b.Line()
b := scan.Bytes(raw) if len(line) < 2 || line[0] != '#' || line[1] != '!' {
line := b.Line() return false
for _, s := range sigs { }
if shebangCheck(s, line) { line = line[2:]
return true line.TrimLWS()
} for _, s := range sigs {
if line.Match(s, matchFlags) != -1 {
return true
} }
return false
} }
} return false
func shebangCheck(sig []byte, raw scan.Bytes) bool {
if len(raw) < len(sig)+2 {
return false
}
if raw[0] != '#' || raw[1] != '!' {
return false
}
raw.Advance(2) // skip #! we checked above
raw.TrimLWS()
raw.TrimRWS()
return bytes.Equal(raw, sig)
} }

View file

@ -44,17 +44,6 @@ func Ole(raw []byte, limit uint32) bool {
return bytes.HasPrefix(raw, []byte{0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1}) return bytes.HasPrefix(raw, []byte{0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1})
} }
// Aaf matches an Advanced Authoring Format file.
// See: https://pyaaf.readthedocs.io/en/latest/about.html
// See: https://en.wikipedia.org/wiki/Advanced_Authoring_Format
func Aaf(raw []byte, limit uint32) bool {
if len(raw) < 31 {
return false
}
return bytes.HasPrefix(raw[8:], []byte{0x41, 0x41, 0x46, 0x42, 0x0D, 0x00, 0x4F, 0x4D}) &&
(raw[30] == 0x09 || raw[30] == 0x0C)
}
// Doc matches a Microsoft Word 97-2003 file. // Doc matches a Microsoft Word 97-2003 file.
// See: https://github.com/decalage2/oletools/blob/412ee36ae45e70f42123e835871bac956d958461/oletools/common/clsid.py // See: https://github.com/decalage2/oletools/blob/412ee36ae45e70f42123e835871bac956d958461/oletools/common/clsid.py
func Doc(raw []byte, _ uint32) bool { func Doc(raw []byte, _ uint32) bool {
@ -209,3 +198,14 @@ func matchOleClsid(in []byte, clsid []byte) bool {
return bytes.HasPrefix(in[clsidOffset:], clsid) return bytes.HasPrefix(in[clsidOffset:], clsid)
} }
// WPD matches a WordPerfect document.
func WPD(raw []byte, _ uint32) bool {
if len(raw) < 10 {
return false
}
if !bytes.HasPrefix(raw, []byte("\xffWPC")) {
return false
}
return raw[8] == 1 && raw[9] == 10
}

View file

@ -10,9 +10,9 @@ import (
"github.com/gabriel-vasile/mimetype/internal/scan" "github.com/gabriel-vasile/mimetype/internal/scan"
) )
var ( // HTML matches a Hypertext Markup Language file.
// HTML matches a Hypertext Markup Language file. func HTML(raw []byte, _ uint32) bool {
HTML = markup( return markup(raw,
[]byte("<!DOCTYPE HTML"), []byte("<!DOCTYPE HTML"),
[]byte("<HTML"), []byte("<HTML"),
[]byte("<HEAD"), []byte("<HEAD"),
@ -31,133 +31,254 @@ var (
[]byte("<P"), []byte("<P"),
[]byte("<!--"), []byte("<!--"),
) )
// XML matches an Extensible Markup Language file. }
XML = markup([]byte("<?XML"))
// Owl2 matches an Owl ontology file. // XML matches an Extensible Markup Language file.
Owl2 = xml(newXMLSig("Ontology", `xmlns="http://www.w3.org/2002/07/owl#"`)) func XML(raw []byte, _ uint32) bool {
// Rss matches a Rich Site Summary file. return markup(raw, []byte("<?XML"))
Rss = xml(newXMLSig("rss", "")) }
// Atom matches an Atom Syndication Format file.
Atom = xml(newXMLSig("feed", `xmlns="http://www.w3.org/2005/Atom"`)) // Owl2 matches an Owl ontology file.
// Kml matches a Keyhole Markup Language file. func Owl2(raw []byte, _ uint32) bool {
Kml = xml( return xml(raw,
newXMLSig("kml", `xmlns="http://www.opengis.net/kml/2.2"`), xmlSig{[]byte("<Ontology"), []byte(`xmlns="http://www.w3.org/2002/07/owl#"`)},
newXMLSig("kml", `xmlns="http://earth.google.com/kml/2.0"`),
newXMLSig("kml", `xmlns="http://earth.google.com/kml/2.1"`),
newXMLSig("kml", `xmlns="http://earth.google.com/kml/2.2"`),
) )
// Xliff matches a XML Localization Interchange File Format file. }
Xliff = xml(newXMLSig("xliff", `xmlns="urn:oasis:names:tc:xliff:document:1.2"`))
// Collada matches a COLLAborative Design Activity file. // Rss matches a Rich Site Summary file.
Collada = xml(newXMLSig("COLLADA", `xmlns="http://www.collada.org/2005/11/COLLADASchema"`)) func Rss(raw []byte, _ uint32) bool {
// Gml matches a Geography Markup Language file. return xml(raw,
Gml = xml( xmlSig{[]byte("<rss"), []byte{}},
newXMLSig("", `xmlns:gml="http://www.opengis.net/gml"`),
newXMLSig("", `xmlns:gml="http://www.opengis.net/gml/3.2"`),
newXMLSig("", `xmlns:gml="http://www.opengis.net/gml/3.3/exr"`),
) )
// Gpx matches a GPS Exchange Format file. }
Gpx = xml(newXMLSig("gpx", `xmlns="http://www.topografix.com/GPX/1/1"`))
// Tcx matches a Training Center XML file. // Atom matches an Atom Syndication Format file.
Tcx = xml(newXMLSig("TrainingCenterDatabase", `xmlns="http://www.garmin.com/xmlschemas/TrainingCenterDatabase/v2"`)) func Atom(raw []byte, _ uint32) bool {
// X3d matches an Extensible 3D Graphics file. return xml(raw,
X3d = xml(newXMLSig("X3D", `xmlns:xsd="http://www.w3.org/2001/XMLSchema-instance"`)) xmlSig{[]byte("<feed"), []byte(`xmlns="http://www.w3.org/2005/Atom"`)},
// Amf matches an Additive Manufacturing XML file. )
Amf = xml(newXMLSig("amf", "")) }
// Threemf matches a 3D Manufacturing Format file.
Threemf = xml(newXMLSig("model", `xmlns="http://schemas.microsoft.com/3dmanufacturing/core/2015/02"`)) // Kml matches a Keyhole Markup Language file.
// Xfdf matches a XML Forms Data Format file. func Kml(raw []byte, _ uint32) bool {
Xfdf = xml(newXMLSig("xfdf", `xmlns="http://ns.adobe.com/xfdf/"`)) return xml(raw,
// VCard matches a Virtual Contact File. xmlSig{[]byte("<kml"), []byte(`xmlns="http://www.opengis.net/kml/2.2"`)},
VCard = ciPrefix([]byte("BEGIN:VCARD\n"), []byte("BEGIN:VCARD\r\n")) xmlSig{[]byte("<kml"), []byte(`xmlns="http://earth.google.com/kml/2.0"`)},
// ICalendar matches a iCalendar file. xmlSig{[]byte("<kml"), []byte(`xmlns="http://earth.google.com/kml/2.1"`)},
ICalendar = ciPrefix([]byte("BEGIN:VCALENDAR\n"), []byte("BEGIN:VCALENDAR\r\n")) xmlSig{[]byte("<kml"), []byte(`xmlns="http://earth.google.com/kml/2.2"`)},
phpPageF = ciPrefix( )
}
// Xliff matches a XML Localization Interchange File Format file.
func Xliff(raw []byte, _ uint32) bool {
return xml(raw,
xmlSig{[]byte("<xliff"), []byte(`xmlns="urn:oasis:names:tc:xliff:document:1.2"`)},
)
}
// Collada matches a COLLAborative Design Activity file.
func Collada(raw []byte, _ uint32) bool {
return xml(raw,
xmlSig{[]byte("<COLLADA"), []byte(`xmlns="http://www.collada.org/2005/11/COLLADASchema"`)},
)
}
// Gml matches a Geography Markup Language file.
func Gml(raw []byte, _ uint32) bool {
return xml(raw,
xmlSig{[]byte{}, []byte(`xmlns:gml="http://www.opengis.net/gml"`)},
xmlSig{[]byte{}, []byte(`xmlns:gml="http://www.opengis.net/gml/3.2"`)},
xmlSig{[]byte{}, []byte(`xmlns:gml="http://www.opengis.net/gml/3.3/exr"`)},
)
}
// Gpx matches a GPS Exchange Format file.
func Gpx(raw []byte, _ uint32) bool {
return xml(raw,
xmlSig{[]byte("<gpx"), []byte(`xmlns="http://www.topografix.com/GPX/1/1"`)},
)
}
// Tcx matches a Training Center XML file.
func Tcx(raw []byte, _ uint32) bool {
return xml(raw,
xmlSig{[]byte("<TrainingCenterDatabase"), []byte(`xmlns="http://www.garmin.com/xmlschemas/TrainingCenterDatabase/v2"`)},
)
}
// X3d matches an Extensible 3D Graphics file.
func X3d(raw []byte, _ uint32) bool {
return xml(raw,
xmlSig{[]byte("<X3D"), []byte(`xmlns:xsd="http://www.w3.org/2001/XMLSchema-instance"`)},
)
}
// Amf matches an Additive Manufacturing XML file.
func Amf(raw []byte, _ uint32) bool {
return xml(raw, xmlSig{[]byte("<amf"), []byte{}})
}
// Threemf matches a 3D Manufacturing Format file.
func Threemf(raw []byte, _ uint32) bool {
return xml(raw,
xmlSig{[]byte("<model"), []byte(`xmlns="http://schemas.microsoft.com/3dmanufacturing/core/2015/02"`)},
)
}
// Xfdf matches a XML Forms Data Format file.
func Xfdf(raw []byte, _ uint32) bool {
return xml(raw, xmlSig{[]byte("<xfdf"), []byte(`xmlns="http://ns.adobe.com/xfdf/"`)})
}
// VCard matches a Virtual Contact File.
func VCard(raw []byte, _ uint32) bool {
return ciPrefix(raw, []byte("BEGIN:VCARD\n"), []byte("BEGIN:VCARD\r\n"))
}
// ICalendar matches a iCalendar file.
func ICalendar(raw []byte, _ uint32) bool {
return ciPrefix(raw, []byte("BEGIN:VCALENDAR\n"), []byte("BEGIN:VCALENDAR\r\n"))
}
func phpPageF(raw []byte, _ uint32) bool {
return ciPrefix(raw,
[]byte("<?PHP"), []byte("<?PHP"),
[]byte("<?\n"), []byte("<?\n"),
[]byte("<?\r"), []byte("<?\r"),
[]byte("<? "), []byte("<? "),
) )
phpScriptF = shebang( }
func phpScriptF(raw []byte, _ uint32) bool {
return shebang(raw,
scan.CompactWS,
[]byte("/usr/local/bin/php"), []byte("/usr/local/bin/php"),
[]byte("/usr/bin/php"), []byte("/usr/bin/php"),
[]byte("/usr/bin/env php"), []byte("/usr/bin/env php"),
[]byte("/usr/bin/env -S php"),
) )
// Js matches a Javascript file. }
Js = shebang(
// Js matches a Javascript file.
func Js(raw []byte, _ uint32) bool {
return shebang(raw,
scan.CompactWS,
[]byte("/bin/node"), []byte("/bin/node"),
[]byte("/usr/bin/node"), []byte("/usr/bin/node"),
[]byte("/bin/nodejs"), []byte("/bin/nodejs"),
[]byte("/usr/bin/nodejs"), []byte("/usr/bin/nodejs"),
[]byte("/usr/bin/env node"), []byte("/usr/bin/env node"),
[]byte("/usr/bin/env -S node"),
[]byte("/usr/bin/env nodejs"), []byte("/usr/bin/env nodejs"),
[]byte("/usr/bin/env -S nodejs"),
) )
// Lua matches a Lua programming language file. }
Lua = shebang(
// Lua matches a Lua programming language file.
func Lua(raw []byte, _ uint32) bool {
return shebang(raw,
scan.CompactWS|scan.FullWord,
[]byte("/usr/bin/lua"), []byte("/usr/bin/lua"),
[]byte("/usr/local/bin/lua"), []byte("/usr/local/bin/lua"),
[]byte("/usr/bin/env lua"), []byte("/usr/bin/env lua"),
[]byte("/usr/bin/env -S lua"),
) )
// Perl matches a Perl programming language file. }
Perl = shebang(
// Perl matches a Perl programming language file.
func Perl(raw []byte, _ uint32) bool {
return shebang(raw,
scan.CompactWS|scan.FullWord,
[]byte("/usr/bin/perl"), []byte("/usr/bin/perl"),
[]byte("/usr/bin/env perl"), []byte("/usr/bin/env perl"),
[]byte("/usr/bin/env -S perl"),
) )
// Python matches a Python programming language file. }
Python = shebang(
// Python matches a Python programming language file.
func Python(raw []byte, _ uint32) bool {
return shebang(raw,
scan.CompactWS,
[]byte("/usr/bin/python"), []byte("/usr/bin/python"),
[]byte("/usr/local/bin/python"), []byte("/usr/local/bin/python"),
[]byte("/usr/bin/env python"), []byte("/usr/bin/env python"),
[]byte("/usr/bin/env -S python"),
[]byte("/usr/bin/python2"), []byte("/usr/bin/python2"),
[]byte("/usr/local/bin/python2"), []byte("/usr/local/bin/python2"),
[]byte("/usr/bin/env python2"), []byte("/usr/bin/env python2"),
[]byte("/usr/bin/env -S python2"),
[]byte("/usr/bin/python3"), []byte("/usr/bin/python3"),
[]byte("/usr/local/bin/python3"), []byte("/usr/local/bin/python3"),
[]byte("/usr/bin/env python3"), []byte("/usr/bin/env python3"),
[]byte("/usr/bin/env -S python3"),
) )
// Ruby matches a Ruby programming language file.
Ruby = shebang( }
// Ruby matches a Ruby programming language file.
func Ruby(raw []byte, _ uint32) bool {
return shebang(raw,
scan.CompactWS,
[]byte("/usr/bin/ruby"), []byte("/usr/bin/ruby"),
[]byte("/usr/local/bin/ruby"), []byte("/usr/local/bin/ruby"),
[]byte("/usr/bin/env ruby"), []byte("/usr/bin/env ruby"),
[]byte("/usr/bin/env -S ruby"),
) )
// Tcl matches a Tcl programming language file. }
Tcl = shebang(
// Tcl matches a Tcl programming language file.
func Tcl(raw []byte, _ uint32) bool {
return shebang(raw,
scan.CompactWS,
[]byte("/usr/bin/tcl"), []byte("/usr/bin/tcl"),
[]byte("/usr/local/bin/tcl"), []byte("/usr/local/bin/tcl"),
[]byte("/usr/bin/env tcl"), []byte("/usr/bin/env tcl"),
[]byte("/usr/bin/env -S tcl"),
[]byte("/usr/bin/tclsh"), []byte("/usr/bin/tclsh"),
[]byte("/usr/local/bin/tclsh"), []byte("/usr/local/bin/tclsh"),
[]byte("/usr/bin/env tclsh"), []byte("/usr/bin/env tclsh"),
[]byte("/usr/bin/env -S tclsh"),
[]byte("/usr/bin/wish"), []byte("/usr/bin/wish"),
[]byte("/usr/local/bin/wish"), []byte("/usr/local/bin/wish"),
[]byte("/usr/bin/env wish"), []byte("/usr/bin/env wish"),
[]byte("/usr/bin/env -S wish"),
) )
// Rtf matches a Rich Text Format file. }
Rtf = prefix([]byte("{\\rtf"))
// Shell matches a shell script file. // Rtf matches a Rich Text Format file.
Shell = shebang( func Rtf(raw []byte, _ uint32) bool {
return bytes.HasPrefix(raw, []byte("{\\rtf"))
}
// Shell matches a shell script file.
func Shell(raw []byte, _ uint32) bool {
return shebang(raw,
scan.CompactWS|scan.FullWord,
[]byte("/bin/sh"), []byte("/bin/sh"),
[]byte("/bin/bash"), []byte("/bin/bash"),
[]byte("/usr/local/bin/bash"), []byte("/usr/local/bin/bash"),
[]byte("/usr/bin/env bash"), []byte("/usr/bin/env bash"),
[]byte("/usr/bin/env -S bash"),
[]byte("/bin/csh"), []byte("/bin/csh"),
[]byte("/usr/local/bin/csh"), []byte("/usr/local/bin/csh"),
[]byte("/usr/bin/env csh"), []byte("/usr/bin/env csh"),
[]byte("/usr/bin/env -S csh"),
[]byte("/bin/dash"), []byte("/bin/dash"),
[]byte("/usr/local/bin/dash"), []byte("/usr/local/bin/dash"),
[]byte("/usr/bin/env dash"), []byte("/usr/bin/env dash"),
[]byte("/usr/bin/env -S dash"),
[]byte("/bin/ksh"), []byte("/bin/ksh"),
[]byte("/usr/local/bin/ksh"), []byte("/usr/local/bin/ksh"),
[]byte("/usr/bin/env ksh"), []byte("/usr/bin/env ksh"),
[]byte("/usr/bin/env -S ksh"),
[]byte("/bin/tcsh"), []byte("/bin/tcsh"),
[]byte("/usr/local/bin/tcsh"), []byte("/usr/local/bin/tcsh"),
[]byte("/usr/bin/env tcsh"), []byte("/usr/bin/env tcsh"),
[]byte("/usr/bin/env -S tcsh"),
[]byte("/bin/zsh"), []byte("/bin/zsh"),
[]byte("/usr/local/bin/zsh"), []byte("/usr/local/bin/zsh"),
[]byte("/usr/bin/env zsh"), []byte("/usr/bin/env zsh"),
[]byte("/usr/bin/env -S zsh"),
) )
) }
// Text matches a plain text file. // Text matches a plain text file.
// //
@ -183,10 +304,14 @@ func Text(raw []byte, _ uint32) bool {
// XHTML matches an XHTML file. This check depends on the XML check to have passed. // XHTML matches an XHTML file. This check depends on the XML check to have passed.
func XHTML(raw []byte, limit uint32) bool { func XHTML(raw []byte, limit uint32) bool {
raw = raw[:min(len(raw), 4096)] raw = raw[:min(len(raw), 1024)]
b := scan.Bytes(raw) b := scan.Bytes(raw)
return b.Search([]byte("<!DOCTYPE HTML"), scan.CompactWS|scan.IgnoreCase) != -1 || i, _ := b.Search([]byte("<!DOCTYPE HTML"), scan.CompactWS|scan.IgnoreCase)
b.Search([]byte("<HTML XMLNS="), scan.CompactWS|scan.IgnoreCase) != -1 if i != -1 {
return true
}
i, _ = b.Search([]byte("<HTML XMLNS="), scan.CompactWS|scan.IgnoreCase)
return i != -1
} }
// Php matches a PHP: Hypertext Preprocessor file. // Php matches a PHP: Hypertext Preprocessor file.
@ -294,11 +419,12 @@ func svgWithoutXMLDeclaration(s scan.Bytes) bool {
return false return false
} }
targetName, targetVal := "xmlns", "http://www.w3.org/2000/svg" targetName, targetVal := []byte("xmlns"), []byte("http://www.w3.org/2000/svg")
aName, aVal, hasMore := "", "", true var aName, aVal []byte
hasMore := true
for hasMore { for hasMore {
aName, aVal, hasMore = mkup.GetAnAttribute(&s) aName, aVal, hasMore = mkup.GetAnAttribute(&s)
if aName == targetName && aVal == targetVal { if bytes.Equal(aName, targetName) && bytes.Equal(aVal, targetVal) {
return true return true
} }
if !hasMore { if !hasMore {
@ -325,10 +451,11 @@ func svgWithXMLDeclaration(s scan.Bytes) bool {
// version is a required attribute for XML. // version is a required attribute for XML.
hasVersion := false hasVersion := false
aName, hasMore := "", true var aName []byte
hasMore := true
for hasMore { for hasMore {
aName, _, hasMore = mkup.GetAnAttribute(&s) aName, _, hasMore = mkup.GetAnAttribute(&s)
if aName == "version" { if bytes.Equal(aName, []byte("version")) {
hasVersion = true hasVersion = true
break break
} }

View file

@ -4,17 +4,23 @@ import (
"bytes" "bytes"
) )
var ( // Flv matches a Flash video file.
// Flv matches a Flash video file. func Flv(raw []byte, _ uint32) bool {
Flv = prefix([]byte("\x46\x4C\x56\x01")) return bytes.HasPrefix(raw, []byte("\x46\x4C\x56\x01"))
// Asf matches an Advanced Systems Format file. }
Asf = prefix([]byte{
// Asf matches an Advanced Systems Format file.
func Asf(raw []byte, _ uint32) bool {
return bytes.HasPrefix(raw, []byte{
0x30, 0x26, 0xB2, 0x75, 0x8E, 0x66, 0xCF, 0x11, 0x30, 0x26, 0xB2, 0x75, 0x8E, 0x66, 0xCF, 0x11,
0xA6, 0xD9, 0x00, 0xAA, 0x00, 0x62, 0xCE, 0x6C, 0xA6, 0xD9, 0x00, 0xAA, 0x00, 0x62, 0xCE, 0x6C,
}) })
// Rmvb matches a RealMedia Variable Bitrate file. }
Rmvb = prefix([]byte{0x2E, 0x52, 0x4D, 0x46})
) // Rmvb matches a RealMedia Variable Bitrate file.
func Rmvb(raw []byte, _ uint32) bool {
return bytes.HasPrefix(raw, []byte{0x2E, 0x52, 0x4D, 0x46})
}
// WebM matches a WebM file. // WebM matches a WebM file.
func WebM(raw []byte, limit uint32) bool { func WebM(raw []byte, limit uint32) bool {

View file

@ -6,32 +6,65 @@ import (
"github.com/gabriel-vasile/mimetype/internal/scan" "github.com/gabriel-vasile/mimetype/internal/scan"
) )
var ( // Odt matches an OpenDocument Text file.
// Odt matches an OpenDocument Text file. func Odt(raw []byte, _ uint32) bool {
Odt = offset([]byte("mimetypeapplication/vnd.oasis.opendocument.text"), 30) return offset(raw, []byte("mimetypeapplication/vnd.oasis.opendocument.text"), 30)
// Ott matches an OpenDocument Text Template file. }
Ott = offset([]byte("mimetypeapplication/vnd.oasis.opendocument.text-template"), 30)
// Ods matches an OpenDocument Spreadsheet file. // Ott matches an OpenDocument Text Template file.
Ods = offset([]byte("mimetypeapplication/vnd.oasis.opendocument.spreadsheet"), 30) func Ott(raw []byte, _ uint32) bool {
// Ots matches an OpenDocument Spreadsheet Template file. return offset(raw, []byte("mimetypeapplication/vnd.oasis.opendocument.text-template"), 30)
Ots = offset([]byte("mimetypeapplication/vnd.oasis.opendocument.spreadsheet-template"), 30) }
// Odp matches an OpenDocument Presentation file.
Odp = offset([]byte("mimetypeapplication/vnd.oasis.opendocument.presentation"), 30) // Ods matches an OpenDocument Spreadsheet file.
// Otp matches an OpenDocument Presentation Template file. func Ods(raw []byte, _ uint32) bool {
Otp = offset([]byte("mimetypeapplication/vnd.oasis.opendocument.presentation-template"), 30) return offset(raw, []byte("mimetypeapplication/vnd.oasis.opendocument.spreadsheet"), 30)
// Odg matches an OpenDocument Drawing file. }
Odg = offset([]byte("mimetypeapplication/vnd.oasis.opendocument.graphics"), 30)
// Otg matches an OpenDocument Drawing Template file. // Ots matches an OpenDocument Spreadsheet Template file.
Otg = offset([]byte("mimetypeapplication/vnd.oasis.opendocument.graphics-template"), 30) func Ots(raw []byte, _ uint32) bool {
// Odf matches an OpenDocument Formula file. return offset(raw, []byte("mimetypeapplication/vnd.oasis.opendocument.spreadsheet-template"), 30)
Odf = offset([]byte("mimetypeapplication/vnd.oasis.opendocument.formula"), 30) }
// Odc matches an OpenDocument Chart file.
Odc = offset([]byte("mimetypeapplication/vnd.oasis.opendocument.chart"), 30) // Odp matches an OpenDocument Presentation file.
// Epub matches an EPUB file. func Odp(raw []byte, _ uint32) bool {
Epub = offset([]byte("mimetypeapplication/epub+zip"), 30) return offset(raw, []byte("mimetypeapplication/vnd.oasis.opendocument.presentation"), 30)
// Sxc matches an OpenOffice Spreadsheet file. }
Sxc = offset([]byte("mimetypeapplication/vnd.sun.xml.calc"), 30)
) // Otp matches an OpenDocument Presentation Template file.
func Otp(raw []byte, _ uint32) bool {
return offset(raw, []byte("mimetypeapplication/vnd.oasis.opendocument.presentation-template"), 30)
}
// Odg matches an OpenDocument Drawing file.
func Odg(raw []byte, _ uint32) bool {
return offset(raw, []byte("mimetypeapplication/vnd.oasis.opendocument.graphics"), 30)
}
// Otg matches an OpenDocument Drawing Template file.
func Otg(raw []byte, _ uint32) bool {
return offset(raw, []byte("mimetypeapplication/vnd.oasis.opendocument.graphics-template"), 30)
}
// Odf matches an OpenDocument Formula file.
func Odf(raw []byte, _ uint32) bool {
return offset(raw, []byte("mimetypeapplication/vnd.oasis.opendocument.formula"), 30)
}
// Odc matches an OpenDocument Chart file.
func Odc(raw []byte, _ uint32) bool {
return offset(raw, []byte("mimetypeapplication/vnd.oasis.opendocument.chart"), 30)
}
// Epub matches an EPUB file.
func Epub(raw []byte, _ uint32) bool {
return offset(raw, []byte("mimetypeapplication/epub+zip"), 30)
}
// Sxc matches an OpenOffice Spreadsheet file.
func Sxc(raw []byte, _ uint32) bool {
return offset(raw, []byte("mimetypeapplication/vnd.sun.xml.calc"), 30)
}
// Zip matches a zip archive. // Zip matches a zip archive.
func Zip(raw []byte, limit uint32) bool { func Zip(raw []byte, limit uint32) bool {
@ -134,11 +167,11 @@ func msoxml(raw scan.Bytes, searchFor zipEntries, stopAfter int) bool {
// If the first is not one of the next usually expected entries, // If the first is not one of the next usually expected entries,
// then abort this check. // then abort this check.
if i == 0 { if i == 0 {
if !bytes.Equal(f, []byte("[Content_Types].xml")) && if !bytes.Equal(f, []byte("[Content_Types].xml")) && // this is a file
!bytes.Equal(f, []byte("_rels/.rels")) && !bytes.HasPrefix(f, []byte("_rels/")) && // these are directories
!bytes.Equal(f, []byte("docProps")) && !bytes.HasPrefix(f, []byte("docProps/")) &&
!bytes.Equal(f, []byte("customXml")) && !bytes.HasPrefix(f, []byte("customXml/")) &&
!bytes.Equal(f, []byte("[trash]")) { !bytes.HasPrefix(f, []byte("[trash]/")) {
return false return false
} }
} }

View file

@ -8,46 +8,48 @@ import (
"github.com/gabriel-vasile/mimetype/internal/scan" "github.com/gabriel-vasile/mimetype/internal/scan"
) )
func GetAnAttribute(s *scan.Bytes) (name, val string, hasMore bool) { // GetAnAttribute assumes we passed over an SGML tag and extracts first
// attribute and its value.
//
// Initially, this code existed inside charset/charset.go, because it was part of
// implementing the https://html.spec.whatwg.org/multipage/parsing.html#prescan-a-byte-stream-to-determine-its-encoding
// algorithm. But because extracting an attribute from a tag is the same for
// both HTML and XML, then the code was moved here.
func GetAnAttribute(s *scan.Bytes) (name, val []byte, hasMore bool) {
for scan.ByteIsWS(s.Peek()) || s.Peek() == '/' { for scan.ByteIsWS(s.Peek()) || s.Peek() == '/' {
s.Advance(1) s.Advance(1)
} }
if s.Peek() == '>' { if s.Peek() == '>' {
return "", "", false return nil, nil, false
} }
// Allocate 10 to avoid resizes. origS, end := *s, 0
// Attribute names and values are continuous slices of bytes in input,
// so we could do without allocating and returning slices of input.
nameB := make([]byte, 0, 10)
// step 4 and 5 // step 4 and 5
for { for {
// bap means byte at position in the specification. // bap means byte at position in the specification.
bap := s.Pop() bap := s.Pop()
if bap == 0 { if bap == 0 {
return "", "", false return nil, nil, false
} }
if bap == '=' && len(nameB) > 0 { if bap == '=' && end > 0 {
val, hasMore := getAValue(s) val, hasMore := getAValue(s)
return string(nameB), string(val), hasMore return origS[:end], val, hasMore
} else if scan.ByteIsWS(bap) { } else if scan.ByteIsWS(bap) {
for scan.ByteIsWS(s.Peek()) { for scan.ByteIsWS(s.Peek()) {
s.Advance(1) s.Advance(1)
} }
if s.Peek() != '=' { if s.Peek() != '=' {
return string(nameB), "", true return origS[:end], nil, true
} }
s.Advance(1) s.Advance(1)
for scan.ByteIsWS(s.Peek()) { for scan.ByteIsWS(s.Peek()) {
s.Advance(1) s.Advance(1)
} }
val, hasMore := getAValue(s) val, hasMore := getAValue(s)
return string(nameB), string(val), hasMore return origS[:end], val, hasMore
} else if bap == '/' || bap == '>' { } else if bap == '/' || bap == '>' {
return string(nameB), "", false return origS[:end], nil, false
} else if bap >= 'A' && bap <= 'Z' { } else { // for any ASCII, non-ASCII, just advance
nameB = append(nameB, bap+0x20) end++
} else {
nameB = append(nameB, bap)
} }
} }
} }

View file

@ -138,46 +138,85 @@ func (b *Bytes) Uint16() (uint16, bool) {
return v, true return v, true
} }
type Flags int
const ( const (
CompactWS = 1 << iota // CompactWS will make one whitespace from pattern to match one or more spaces from input.
CompactWS Flags = 1 << iota
// IgnoreCase will match lower case from pattern with lower case from input.
// IgnoreCase will match upper case from pattern with both lower and upper case from input.
// This flag is not really well named,
IgnoreCase IgnoreCase
// FullWord ensures the input ends with a full word (it's followed by spaces.)
FullWord
) )
// Search for occurences of pattern p inside b at any index. // Search for occurences of pattern p inside b at any index.
func (b Bytes) Search(p []byte, flags int) int { // It returns the index where p was found in b and how many bytes were needed
// for matching the pattern.
func (b Bytes) Search(p []byte, flags Flags) (i int, l int) {
lb, lp := len(b), len(p)
if lp == 0 {
return 0, 0
}
if lb == 0 {
return -1, 0
}
if flags == 0 { if flags == 0 {
return bytes.Index(b, p) if i = bytes.Index(b, p); i == -1 {
return -1, 0
} else {
return i, lp
}
} }
lb, lp := len(b), len(p)
for i := range b { for i := range b {
if lb-i < lp { if lb-i < lp {
return -1 return -1, 0
} }
if b[i:].Match(p, flags) { if l = b[i:].Match(p, flags); l != -1 {
return i return i, l
} }
} }
return 0 return -1, 0
} }
// Match pattern p at index 0 of b. // Match returns how many bytes were needed to match pattern p.
func (b Bytes) Match(p []byte, flags int) bool { // It returns -1 if p does not match b.
func (b Bytes) Match(p []byte, flags Flags) int {
l := len(b)
if len(p) == 0 {
return 0
}
if l == 0 {
return -1
}
// If no flags, or scanning for full word at the end of pattern then
// do a fast HasPrefix check.
// For other flags it's not possible to use HasPrefix.
if flags == 0 || flags&FullWord > 0 {
if bytes.HasPrefix(b, p) {
b = b[len(p):]
p = p[len(p):]
goto out
}
return -1
}
for len(b) > 0 { for len(b) > 0 {
// If we finished all we we're looking for from p. // If we finished all we were looking for from p.
if len(p) == 0 { if len(p) == 0 {
return true goto out
} }
if flags&IgnoreCase > 0 && isUpper(p[0]) { if flags&IgnoreCase > 0 && isUpper(p[0]) {
if upper(b[0]) != p[0] { if upper(b[0]) != p[0] {
return false return -1
} }
b, p = b[1:], p[1:] b, p = b[1:], p[1:]
} else if flags&CompactWS > 0 && ByteIsWS(p[0]) { } else if flags&CompactWS > 0 && ByteIsWS(p[0]) {
p = p[1:] p = p[1:]
if !ByteIsWS(b[0]) { if !ByteIsWS(b[0]) {
return false return -1
} }
b = b[1:] b = b[1:]
if !ByteIsWS(p[0]) { if !ByteIsWS(p[0]) {
@ -185,12 +224,22 @@ func (b Bytes) Match(p []byte, flags int) bool {
} }
} else { } else {
if b[0] != p[0] { if b[0] != p[0] {
return false return -1
} }
b, p = b[1:], p[1:] b, p = b[1:], p[1:]
} }
} }
return true out:
// If p still has leftover characters, it means it didn't fully match b.
if len(p) > 0 {
return -1
}
if flags&FullWord > 0 {
if len(b) > 0 && !ByteIsWS(b[0]) {
return -1
}
}
return l - len(b)
} }
func isUpper(c byte) bool { func isUpper(c byte) bool {

View file

@ -2,6 +2,7 @@ package mimetype
import ( import (
"mime" "mime"
"strings"
"github.com/gabriel-vasile/mimetype/internal/charset" "github.com/gabriel-vasile/mimetype/internal/charset"
"github.com/gabriel-vasile/mimetype/internal/magic" "github.com/gabriel-vasile/mimetype/internal/magic"
@ -109,7 +110,7 @@ func (m *MIME) match(in []byte, readLimit uint32) *MIME {
// Limit the number of bytes searched for to 1024. // Limit the number of bytes searched for to 1024.
charset = f(in[:min(len(in), 1024)]) charset = f(in[:min(len(in), 1024)])
} }
if m == root { if m == root || charset == "" {
return m return m
} }
@ -126,6 +127,27 @@ func (m *MIME) flatten() []*MIME {
return out return out
} }
// hierarchy returns an easy to read list of ancestors for m.
// For example, application/json would return json>txt>root.
func (m *MIME) hierarchy() string {
h := ""
for m := m; m != nil; m = m.Parent() {
e := strings.TrimPrefix(m.Extension(), ".")
if e == "" {
// There are some MIME without extensions. When generating the hierarchy,
// it would be confusing to use empty string as extension.
// Use the subtype instead; ex: application/x-executable -> x-executable.
e = strings.Split(m.String(), "/")[1]
if m.Is("application/octet-stream") {
// for octet-stream use root, because it's short and used in many places
e = "root"
}
}
h += ">" + e
}
return strings.TrimPrefix(h, ">")
}
// clone creates a new MIME with the provided optional MIME parameters. // clone creates a new MIME with the provided optional MIME parameters.
func (m *MIME) clone(charset string) *MIME { func (m *MIME) clone(charset string) *MIME {
clonedMIME := m.mime clonedMIME := m.mime
@ -155,7 +177,10 @@ func (m *MIME) cloneHierarchy(charset string) *MIME {
} }
func (m *MIME) lookup(mime string) *MIME { func (m *MIME) lookup(mime string) *MIME {
for _, n := range append(m.aliases, m.mime) { if mime == m.mime {
return m
}
for _, n := range m.aliases {
if n == mime { if n == mime {
return m return m
} }

View file

@ -12,7 +12,7 @@ import (
"sync/atomic" "sync/atomic"
) )
var defaultLimit uint32 = 3072 const defaultLimit uint32 = 3072
// readLimit is the maximum number of bytes from the input used when detecting. // readLimit is the maximum number of bytes from the input used when detecting.
var readLimit uint32 = defaultLimit var readLimit uint32 = defaultLimit
@ -112,15 +112,18 @@ func SetLimit(limit uint32) {
} }
// Extend adds detection for other file formats. // Extend adds detection for other file formats.
// It is equivalent to calling Extend() on the root mime type "application/octet-stream". // It is equivalent to calling Extend() on the root MIME type "application/octet-stream".
func Extend(detector func(raw []byte, limit uint32) bool, mime, extension string, aliases ...string) { func Extend(detector func(raw []byte, limit uint32) bool, mime, extension string, aliases ...string) {
root.Extend(detector, mime, extension, aliases...) root.Extend(detector, mime, extension, aliases...)
} }
// Lookup finds a MIME object by its string representation. // Lookup finds a MIME object by its string representation.
// The representation can be the main mime type, or any of its aliases. // The representation can be the main MIME type, or any of its aliases.
func Lookup(mime string) *MIME { func Lookup(m string) *MIME {
// We store the MIME types without optional params, so
// perform parsing to extract the target MIME type without optional params.
m, _, _ = mime.ParseMediaType(m)
mu.RLock() mu.RLock()
defer mu.RUnlock() defer mu.RUnlock()
return root.lookup(mime) return root.lookup(m)
} }

View file

@ -1,196 +1,197 @@
## 191 Supported MIME types ## 192 Supported MIME types
This file is automatically generated when running tests. Do not edit manually. This file is automatically generated when running tests. Do not edit manually.
Extension | MIME type | Aliases Extension | MIME type <br> Aliases | Hierarchy
--------- | --------- | ------- --------- | ---------------------- | ---------
**n/a** | application/octet-stream | - **n/a** | **application/octet-stream** | root
**.xpm** | image/x-xpixmap | - **.xpm** | **image/x-xpixmap** | xpm>root
**.7z** | application/x-7z-compressed | - **.7z** | **application/x-7z-compressed** | 7z>root
**.zip** | application/zip | application/x-zip, application/x-zip-compressed **.zip** | **application/zip** <br> application/x-zip, application/x-zip-compressed | zip>root
**.docx** | application/vnd.openxmlformats-officedocument.wordprocessingml.document | - **.docx** | **application/vnd.openxmlformats-officedocument.wordprocessingml.document** | docx>zip>root
**.pptx** | application/vnd.openxmlformats-officedocument.presentationml.presentation | - **.pptx** | **application/vnd.openxmlformats-officedocument.presentationml.presentation** | pptx>zip>root
**.xlsx** | application/vnd.openxmlformats-officedocument.spreadsheetml.sheet | - **.xlsx** | **application/vnd.openxmlformats-officedocument.spreadsheetml.sheet** | xlsx>zip>root
**.epub** | application/epub+zip | - **.epub** | **application/epub+zip** | epub>zip>root
**.apk** | application/vnd.android.package-archive | - **.apk** | **application/vnd.android.package-archive** | apk>zip>root
**.jar** | application/java-archive | application/jar, application/jar-archive, application/x-java-archive **.jar** | **application/java-archive** <br> application/jar, application/jar-archive, application/x-java-archive | jar>zip>root
**.odt** | application/vnd.oasis.opendocument.text | application/x-vnd.oasis.opendocument.text **.odt** | **application/vnd.oasis.opendocument.text** <br> application/x-vnd.oasis.opendocument.text | odt>zip>root
**.ott** | application/vnd.oasis.opendocument.text-template | application/x-vnd.oasis.opendocument.text-template **.ott** | **application/vnd.oasis.opendocument.text-template** <br> application/x-vnd.oasis.opendocument.text-template | ott>odt>zip>root
**.ods** | application/vnd.oasis.opendocument.spreadsheet | application/x-vnd.oasis.opendocument.spreadsheet **.ods** | **application/vnd.oasis.opendocument.spreadsheet** <br> application/x-vnd.oasis.opendocument.spreadsheet | ods>zip>root
**.ots** | application/vnd.oasis.opendocument.spreadsheet-template | application/x-vnd.oasis.opendocument.spreadsheet-template **.ots** | **application/vnd.oasis.opendocument.spreadsheet-template** <br> application/x-vnd.oasis.opendocument.spreadsheet-template | ots>ods>zip>root
**.odp** | application/vnd.oasis.opendocument.presentation | application/x-vnd.oasis.opendocument.presentation **.odp** | **application/vnd.oasis.opendocument.presentation** <br> application/x-vnd.oasis.opendocument.presentation | odp>zip>root
**.otp** | application/vnd.oasis.opendocument.presentation-template | application/x-vnd.oasis.opendocument.presentation-template **.otp** | **application/vnd.oasis.opendocument.presentation-template** <br> application/x-vnd.oasis.opendocument.presentation-template | otp>odp>zip>root
**.odg** | application/vnd.oasis.opendocument.graphics | application/x-vnd.oasis.opendocument.graphics **.odg** | **application/vnd.oasis.opendocument.graphics** <br> application/x-vnd.oasis.opendocument.graphics | odg>zip>root
**.otg** | application/vnd.oasis.opendocument.graphics-template | application/x-vnd.oasis.opendocument.graphics-template **.otg** | **application/vnd.oasis.opendocument.graphics-template** <br> application/x-vnd.oasis.opendocument.graphics-template | otg>odg>zip>root
**.odf** | application/vnd.oasis.opendocument.formula | application/x-vnd.oasis.opendocument.formula **.odf** | **application/vnd.oasis.opendocument.formula** <br> application/x-vnd.oasis.opendocument.formula | odf>zip>root
**.odc** | application/vnd.oasis.opendocument.chart | application/x-vnd.oasis.opendocument.chart **.odc** | **application/vnd.oasis.opendocument.chart** <br> application/x-vnd.oasis.opendocument.chart | odc>zip>root
**.sxc** | application/vnd.sun.xml.calc | - **.sxc** | **application/vnd.sun.xml.calc** | sxc>zip>root
**.kmz** | application/vnd.google-earth.kmz | - **.kmz** | **application/vnd.google-earth.kmz** | kmz>zip>root
**.vsdx** | application/vnd.ms-visio.drawing.main+xml | - **.vsdx** | **application/vnd.ms-visio.drawing.main+xml** | vsdx>zip>root
**.pdf** | application/pdf | application/x-pdf **.pdf** | **application/pdf** <br> application/x-pdf | pdf>root
**.fdf** | application/vnd.fdf | - **.fdf** | **application/vnd.fdf** | fdf>root
**n/a** | application/x-ole-storage | - **n/a** | **application/x-ole-storage** | x-ole-storage>root
**.msi** | application/x-ms-installer | application/x-windows-installer, application/x-msi **.msi** | **application/x-ms-installer** <br> application/x-windows-installer, application/x-msi | msi>x-ole-storage>root
**.aaf** | application/octet-stream | - **.msg** | **application/vnd.ms-outlook** | msg>x-ole-storage>root
**.msg** | application/vnd.ms-outlook | - **.xls** | **application/vnd.ms-excel** <br> application/msexcel | xls>x-ole-storage>root
**.xls** | application/vnd.ms-excel | application/msexcel **.pub** | **application/vnd.ms-publisher** | pub>x-ole-storage>root
**.pub** | application/vnd.ms-publisher | - **.ppt** | **application/vnd.ms-powerpoint** <br> application/mspowerpoint | ppt>x-ole-storage>root
**.ppt** | application/vnd.ms-powerpoint | application/mspowerpoint **.doc** | **application/msword** <br> application/vnd.ms-word | doc>x-ole-storage>root
**.doc** | application/msword | application/vnd.ms-word **.ps** | **application/postscript** | ps>root
**.ps** | application/postscript | - **.psd** | **image/vnd.adobe.photoshop** <br> image/x-psd, application/photoshop | psd>root
**.psd** | image/vnd.adobe.photoshop | image/x-psd, application/photoshop **.p7s** | **application/pkcs7-signature** | p7s>root
**.p7s** | application/pkcs7-signature | - **.ogg** | **application/ogg** <br> application/x-ogg | ogg>root
**.ogg** | application/ogg | application/x-ogg **.oga** | **audio/ogg** | oga>ogg>root
**.oga** | audio/ogg | - **.ogv** | **video/ogg** | ogv>ogg>root
**.ogv** | video/ogg | - **.png** | **image/png** | png>root
**.png** | image/png | - **.png** | **image/vnd.mozilla.apng** | png>png>root
**.png** | image/vnd.mozilla.apng | - **.jpg** | **image/jpeg** | jpg>root
**.jpg** | image/jpeg | - **.jxl** | **image/jxl** | jxl>root
**.jxl** | image/jxl | - **.jp2** | **image/jp2** | jp2>root
**.jp2** | image/jp2 | - **.jpf** | **image/jpx** | jpf>root
**.jpf** | image/jpx | - **.jpm** | **image/jpm** <br> video/jpm | jpm>root
**.jpm** | image/jpm | video/jpm **.jxs** | **image/jxs** | jxs>root
**.jxs** | image/jxs | - **.gif** | **image/gif** | gif>root
**.gif** | image/gif | - **.webp** | **image/webp** | webp>root
**.webp** | image/webp | - **.exe** | **application/vnd.microsoft.portable-executable** | exe>root
**.exe** | application/vnd.microsoft.portable-executable | - **n/a** | **application/x-elf** | x-elf>root
**n/a** | application/x-elf | - **n/a** | **application/x-object** | x-object>x-elf>root
**n/a** | application/x-object | - **n/a** | **application/x-executable** | x-executable>x-elf>root
**n/a** | application/x-executable | - **.so** | **application/x-sharedlib** | so>x-elf>root
**.so** | application/x-sharedlib | - **n/a** | **application/x-coredump** | x-coredump>x-elf>root
**n/a** | application/x-coredump | - **.a** | **application/x-archive** <br> application/x-unix-archive | a>root
**.a** | application/x-archive | application/x-unix-archive **.deb** | **application/vnd.debian.binary-package** | deb>a>root
**.deb** | application/vnd.debian.binary-package | - **.tar** | **application/x-tar** | tar>root
**.tar** | application/x-tar | - **.xar** | **application/x-xar** | xar>root
**.xar** | application/x-xar | - **.bz2** | **application/x-bzip2** | bz2>root
**.bz2** | application/x-bzip2 | - **.fits** | **application/fits** <br> image/fits | fits>root
**.fits** | application/fits | image/fits **.tiff** | **image/tiff** | tiff>root
**.tiff** | image/tiff | - **.bmp** | **image/bmp** <br> image/x-bmp, image/x-ms-bmp | bmp>root
**.bmp** | image/bmp | image/x-bmp, image/x-ms-bmp **.123** | **application/vnd.lotus-1-2-3** | 123>root
**.123** | application/vnd.lotus-1-2-3 | - **.ico** | **image/x-icon** | ico>root
**.ico** | image/x-icon | - **.mp3** | **audio/mpeg** <br> audio/x-mpeg, audio/mp3 | mp3>root
**.mp3** | audio/mpeg | audio/x-mpeg, audio/mp3 **.flac** | **audio/flac** | flac>root
**.flac** | audio/flac | - **.midi** | **audio/midi** <br> audio/mid, audio/sp-midi, audio/x-mid, audio/x-midi | midi>root
**.midi** | audio/midi | audio/mid, audio/sp-midi, audio/x-mid, audio/x-midi **.ape** | **audio/ape** | ape>root
**.ape** | audio/ape | - **.mpc** | **audio/musepack** | mpc>root
**.mpc** | audio/musepack | - **.amr** | **audio/amr** <br> audio/amr-nb | amr>root
**.amr** | audio/amr | audio/amr-nb **.wav** | **audio/wav** <br> audio/x-wav, audio/vnd.wave, audio/wave | wav>root
**.wav** | audio/wav | audio/x-wav, audio/vnd.wave, audio/wave **.aiff** | **audio/aiff** <br> audio/x-aiff | aiff>root
**.aiff** | audio/aiff | audio/x-aiff **.au** | **audio/basic** | au>root
**.au** | audio/basic | - **.mpeg** | **video/mpeg** | mpeg>root
**.mpeg** | video/mpeg | - **.mov** | **video/quicktime** | mov>root
**.mov** | video/quicktime | - **.mp4** | **video/mp4** | mp4>root
**.mp4** | video/mp4 | - **.avif** | **image/avif** | avif>mp4>root
**.avif** | image/avif | - **.3gp** | **video/3gpp** <br> video/3gp, audio/3gpp | 3gp>mp4>root
**.3gp** | video/3gpp | video/3gp, audio/3gpp **.3g2** | **video/3gpp2** <br> video/3g2, audio/3gpp2 | 3g2>mp4>root
**.3g2** | video/3gpp2 | video/3g2, audio/3gpp2 **.mp4** | **audio/mp4** <br> audio/x-mp4a | mp4>mp4>root
**.mp4** | audio/mp4 | audio/x-mp4a **.mqv** | **video/quicktime** | mqv>mp4>root
**.mqv** | video/quicktime | - **.m4a** | **audio/x-m4a** | m4a>mp4>root
**.m4a** | audio/x-m4a | - **.m4v** | **video/x-m4v** | m4v>mp4>root
**.m4v** | video/x-m4v | - **.heic** | **image/heic** | heic>mp4>root
**.heic** | image/heic | - **.heic** | **image/heic-sequence** | heic>mp4>root
**.heic** | image/heic-sequence | - **.heif** | **image/heif** | heif>mp4>root
**.heif** | image/heif | - **.heif** | **image/heif-sequence** | heif>mp4>root
**.heif** | image/heif-sequence | - **.mj2** | **video/mj2** | mj2>mp4>root
**.mj2** | video/mj2 | - **.dvb** | **video/vnd.dvb.file** | dvb>mp4>root
**.dvb** | video/vnd.dvb.file | - **.webm** | **video/webm** <br> audio/webm | webm>root
**.webm** | video/webm | audio/webm **.avi** | **video/x-msvideo** <br> video/avi, video/msvideo | avi>root
**.avi** | video/x-msvideo | video/avi, video/msvideo **.flv** | **video/x-flv** | flv>root
**.flv** | video/x-flv | - **.mkv** | **video/x-matroska** | mkv>root
**.mkv** | video/x-matroska | - **.asf** | **video/x-ms-asf** <br> video/asf, video/x-ms-wmv | asf>root
**.asf** | video/x-ms-asf | video/asf, video/x-ms-wmv **.aac** | **audio/aac** | aac>root
**.aac** | audio/aac | - **.voc** | **audio/x-unknown** | voc>root
**.voc** | audio/x-unknown | - **.m3u** | **application/vnd.apple.mpegurl** <br> audio/mpegurl | m3u>root
**.m3u** | application/vnd.apple.mpegurl | audio/mpegurl **.rmvb** | **application/vnd.rn-realmedia-vbr** | rmvb>root
**.rmvb** | application/vnd.rn-realmedia-vbr | - **.gz** | **application/gzip** <br> application/x-gzip, application/x-gunzip, application/gzipped, application/gzip-compressed, application/x-gzip-compressed, gzip/document | gz>root
**.gz** | application/gzip | application/x-gzip, application/x-gunzip, application/gzipped, application/gzip-compressed, application/x-gzip-compressed, gzip/document **.class** | **application/x-java-applet** | class>root
**.class** | application/x-java-applet | - **.swf** | **application/x-shockwave-flash** | swf>root
**.swf** | application/x-shockwave-flash | - **.crx** | **application/x-chrome-extension** | crx>root
**.crx** | application/x-chrome-extension | - **.ttf** | **font/ttf** <br> font/sfnt, application/x-font-ttf, application/font-sfnt | ttf>root
**.ttf** | font/ttf | font/sfnt, application/x-font-ttf, application/font-sfnt **.woff** | **font/woff** | woff>root
**.woff** | font/woff | - **.woff2** | **font/woff2** | woff2>root
**.woff2** | font/woff2 | - **.otf** | **font/otf** | otf>root
**.otf** | font/otf | - **.ttc** | **font/collection** | ttc>root
**.ttc** | font/collection | - **.eot** | **application/vnd.ms-fontobject** | eot>root
**.eot** | application/vnd.ms-fontobject | - **.wasm** | **application/wasm** | wasm>root
**.wasm** | application/wasm | - **.shx** | **application/vnd.shx** | shx>root
**.shx** | application/vnd.shx | - **.shp** | **application/vnd.shp** | shp>shx>root
**.shp** | application/vnd.shp | - **.dbf** | **application/x-dbf** | dbf>root
**.dbf** | application/x-dbf | - **.dcm** | **application/dicom** | dcm>root
**.dcm** | application/dicom | - **.rar** | **application/x-rar-compressed** <br> application/x-rar | rar>root
**.rar** | application/x-rar-compressed | application/x-rar **.djvu** | **image/vnd.djvu** | djvu>root
**.djvu** | image/vnd.djvu | - **.mobi** | **application/x-mobipocket-ebook** | mobi>root
**.mobi** | application/x-mobipocket-ebook | - **.lit** | **application/x-ms-reader** | lit>root
**.lit** | application/x-ms-reader | - **.bpg** | **image/bpg** | bpg>root
**.bpg** | image/bpg | - **.cbor** | **application/cbor** | cbor>root
**.cbor** | application/cbor | - **.sqlite** | **application/vnd.sqlite3** <br> application/x-sqlite3 | sqlite>root
**.sqlite** | application/vnd.sqlite3 | application/x-sqlite3 **.dwg** | **image/vnd.dwg** <br> image/x-dwg, application/acad, application/x-acad, application/autocad_dwg, application/dwg, application/x-dwg, application/x-autocad, drawing/dwg | dwg>root
**.dwg** | image/vnd.dwg | image/x-dwg, application/acad, application/x-acad, application/autocad_dwg, application/dwg, application/x-dwg, application/x-autocad, drawing/dwg **.nes** | **application/vnd.nintendo.snes.rom** | nes>root
**.nes** | application/vnd.nintendo.snes.rom | - **.lnk** | **application/x-ms-shortcut** | lnk>root
**.lnk** | application/x-ms-shortcut | - **.macho** | **application/x-mach-binary** | macho>root
**.macho** | application/x-mach-binary | - **.qcp** | **audio/qcelp** | qcp>root
**.qcp** | audio/qcelp | - **.icns** | **image/x-icns** | icns>root
**.icns** | image/x-icns | - **.hdr** | **image/vnd.radiance** | hdr>root
**.hdr** | image/vnd.radiance | - **.mrc** | **application/marc** | mrc>root
**.mrc** | application/marc | - **.mdb** | **application/x-msaccess** | mdb>root
**.mdb** | application/x-msaccess | - **.accdb** | **application/x-msaccess** | accdb>root
**.accdb** | application/x-msaccess | - **.zst** | **application/zstd** | zst>root
**.zst** | application/zstd | - **.cab** | **application/vnd.ms-cab-compressed** | cab>root
**.cab** | application/vnd.ms-cab-compressed | - **.rpm** | **application/x-rpm** | rpm>root
**.rpm** | application/x-rpm | - **.xz** | **application/x-xz** | xz>root
**.xz** | application/x-xz | - **.lz** | **application/lzip** <br> application/x-lzip | lz>root
**.lz** | application/lzip | application/x-lzip **.torrent** | **application/x-bittorrent** | torrent>root
**.torrent** | application/x-bittorrent | - **.cpio** | **application/x-cpio** | cpio>root
**.cpio** | application/x-cpio | - **n/a** | **application/tzif** | tzif>root
**n/a** | application/tzif | - **.xcf** | **image/x-xcf** | xcf>root
**.xcf** | image/x-xcf | - **.pat** | **image/x-gimp-pat** | pat>root
**.pat** | image/x-gimp-pat | - **.gbr** | **image/x-gimp-gbr** | gbr>root
**.gbr** | image/x-gimp-gbr | - **.glb** | **model/gltf-binary** | glb>root
**.glb** | model/gltf-binary | - **.cab** | **application/x-installshield** | cab>root
**.cab** | application/x-installshield | - **.jxr** | **image/jxr** <br> image/vnd.ms-photo | jxr>root
**.jxr** | image/jxr | image/vnd.ms-photo **.parquet** | **application/vnd.apache.parquet** <br> application/x-parquet | parquet>root
**.parquet** | application/vnd.apache.parquet | application/x-parquet **.one** | **application/onenote** | one>root
**.one** | application/onenote | - **.chm** | **application/vnd.ms-htmlhelp** | chm>root
**.chm** | application/vnd.ms-htmlhelp | - **.wpd** | **application/vnd.wordperfect** | wpd>root
**.txt** | text/plain | - **.dxf** | **image/vnd.dxf** | dxf>root
**.svg** | image/svg+xml | - **.txt** | **text/plain** | txt>root
**.html** | text/html | - **.svg** | **image/svg+xml** | svg>txt>root
**.xml** | text/xml | application/xml **.html** | **text/html** | html>txt>root
**.rss** | application/rss+xml | text/rss **.xml** | **text/xml** <br> application/xml | xml>txt>root
**.atom** | application/atom+xml | - **.rss** | **application/rss+xml** <br> text/rss | rss>xml>txt>root
**.x3d** | model/x3d+xml | - **.atom** | **application/atom+xml** | atom>xml>txt>root
**.kml** | application/vnd.google-earth.kml+xml | - **.x3d** | **model/x3d+xml** | x3d>xml>txt>root
**.xlf** | application/x-xliff+xml | - **.kml** | **application/vnd.google-earth.kml+xml** | kml>xml>txt>root
**.dae** | model/vnd.collada+xml | - **.xlf** | **application/x-xliff+xml** | xlf>xml>txt>root
**.gml** | application/gml+xml | - **.dae** | **model/vnd.collada+xml** | dae>xml>txt>root
**.gpx** | application/gpx+xml | - **.gml** | **application/gml+xml** | gml>xml>txt>root
**.tcx** | application/vnd.garmin.tcx+xml | - **.gpx** | **application/gpx+xml** | gpx>xml>txt>root
**.amf** | application/x-amf | - **.tcx** | **application/vnd.garmin.tcx+xml** | tcx>xml>txt>root
**.3mf** | application/vnd.ms-package.3dmanufacturing-3dmodel+xml | - **.amf** | **application/x-amf** | amf>xml>txt>root
**.xfdf** | application/vnd.adobe.xfdf | - **.3mf** | **application/vnd.ms-package.3dmanufacturing-3dmodel+xml** | 3mf>xml>txt>root
**.owl** | application/owl+xml | - **.xfdf** | **application/vnd.adobe.xfdf** | xfdf>xml>txt>root
**.html** | application/xhtml+xml | - **.owl** | **application/owl+xml** | owl>xml>txt>root
**.php** | text/x-php | - **.html** | **application/xhtml+xml** | html>xml>txt>root
**.js** | text/javascript | application/x-javascript, application/javascript **.php** | **text/x-php** | php>txt>root
**.lua** | text/x-lua | - **.js** | **text/javascript** <br> application/x-javascript, application/javascript | js>txt>root
**.pl** | text/x-perl | - **.lua** | **text/x-lua** | lua>txt>root
**.py** | text/x-python | text/x-script.python, application/x-python **.pl** | **text/x-perl** | pl>txt>root
**.rb** | text/x-ruby | application/x-ruby **.py** | **text/x-python** <br> text/x-script.python, application/x-python | py>txt>root
**.json** | application/json | - **.rb** | **text/x-ruby** <br> application/x-ruby | rb>txt>root
**.geojson** | application/geo+json | - **.json** | **application/json** | json>txt>root
**.har** | application/json | - **.geojson** | **application/geo+json** | geojson>json>txt>root
**.gltf** | model/gltf+json | - **.har** | **application/json** | har>json>txt>root
**.ndjson** | application/x-ndjson | - **.gltf** | **model/gltf+json** | gltf>json>txt>root
**.rtf** | text/rtf | application/rtf **.ndjson** | **application/x-ndjson** | ndjson>txt>root
**.srt** | application/x-subrip | application/x-srt, text/x-srt **.rtf** | **text/rtf** <br> application/rtf | rtf>txt>root
**.tcl** | text/x-tcl | application/x-tcl **.srt** | **application/x-subrip** <br> application/x-srt, text/x-srt | srt>txt>root
**.csv** | text/csv | - **.tcl** | **text/x-tcl** <br> application/x-tcl | tcl>txt>root
**.tsv** | text/tab-separated-values | - **.csv** | **text/csv** | csv>txt>root
**.vcf** | text/vcard | - **.tsv** | **text/tab-separated-values** | tsv>txt>root
**.ics** | text/calendar | - **.vcf** | **text/vcard** | vcf>txt>root
**.warc** | application/warc | - **.ics** | **text/calendar** | ics>txt>root
**.vtt** | text/vtt | - **.warc** | **application/warc** | warc>txt>root
**.sh** | text/x-shellscript | text/x-sh, application/x-shellscript, application/x-sh **.vtt** | **text/vtt** | vtt>txt>root
**.pbm** | image/x-portable-bitmap | - **.sh** | **text/x-shellscript** <br> text/x-sh, application/x-shellscript, application/x-sh | sh>txt>root
**.pgm** | image/x-portable-graymap | - **.pbm** | **image/x-portable-bitmap** | pbm>txt>root
**.ppm** | image/x-portable-pixmap | - **.pgm** | **image/x-portable-graymap** | pgm>txt>root
**.pam** | image/x-portable-arbitrarymap | - **.ppm** | **image/x-portable-pixmap** | ppm>txt>root
**.pam** | **image/x-portable-arbitrarymap** | pam>txt>root

View file

@ -24,7 +24,7 @@ var root = newMIME("application/octet-stream", "",
woff2, otf, ttc, eot, wasm, shx, dbf, dcm, rar, djvu, mobi, lit, bpg, cbor, woff2, otf, ttc, eot, wasm, shx, dbf, dcm, rar, djvu, mobi, lit, bpg, cbor,
sqlite3, dwg, nes, lnk, macho, qcp, icns, hdr, mrc, mdb, accdb, zstd, cab, sqlite3, dwg, nes, lnk, macho, qcp, icns, hdr, mrc, mdb, accdb, zstd, cab,
rpm, xz, lzip, torrent, cpio, tzif, xcf, pat, gbr, glb, cabIS, jxr, parquet, rpm, xz, lzip, torrent, cpio, tzif, xcf, pat, gbr, glb, cabIS, jxr, parquet,
oneNote, chm, oneNote, chm, wpd, dxf,
// Keep text last because it is the slowest check. // Keep text last because it is the slowest check.
text, text,
) )
@ -65,10 +65,9 @@ var (
jar = newMIME("application/java-archive", ".jar", magic.Jar). jar = newMIME("application/java-archive", ".jar", magic.Jar).
alias("application/jar", "application/jar-archive", "application/x-java-archive") alias("application/jar", "application/jar-archive", "application/x-java-archive")
apk = newMIME("application/vnd.android.package-archive", ".apk", magic.APK) apk = newMIME("application/vnd.android.package-archive", ".apk", magic.APK)
ole = newMIME("application/x-ole-storage", "", magic.Ole, msi, aaf, msg, xls, pub, ppt, doc) ole = newMIME("application/x-ole-storage", "", magic.Ole, msi, msg, xls, pub, ppt, doc)
msi = newMIME("application/x-ms-installer", ".msi", magic.Msi). msi = newMIME("application/x-ms-installer", ".msi", magic.Msi).
alias("application/x-windows-installer", "application/x-msi") alias("application/x-windows-installer", "application/x-msi")
aaf = newMIME("application/octet-stream", ".aaf", magic.Aaf)
doc = newMIME("application/msword", ".doc", magic.Doc). doc = newMIME("application/msword", ".doc", magic.Doc).
alias("application/vnd.ms-word") alias("application/vnd.ms-word")
ppt = newMIME("application/vnd.ms-powerpoint", ".ppt", magic.Ppt). ppt = newMIME("application/vnd.ms-powerpoint", ".ppt", magic.Ppt).
@ -286,4 +285,6 @@ var (
cbor = newMIME("application/cbor", ".cbor", magic.CBOR) cbor = newMIME("application/cbor", ".cbor", magic.CBOR)
oneNote = newMIME("application/onenote", ".one", magic.One) oneNote = newMIME("application/onenote", ".one", magic.One)
chm = newMIME("application/vnd.ms-htmlhelp", ".chm", magic.CHM) chm = newMIME("application/vnd.ms-htmlhelp", ".chm", magic.CHM)
wpd = newMIME("application/vnd.wordperfect", ".wpd", magic.WPD)
dxf = newMIME("image/vnd.dxf", ".dxf", magic.DXF)
) )

View file

@ -1,5 +1,63 @@
# Changelog # Changelog
## 0.36.2
The Sentry SDK team is happy to announce the immediate availability of Sentry Go SDK v0.36.2.
### Bug Fixes
- Fix context propagation for logs to ensure logger instances correctly inherit span and hub information from their creation context ([#1118](https://github.com/getsentry/sentry-go/pull/1118))
- Logs now properly propagate trace context from the logger's original context, even when emitted in a different context
- The logger will first check the emission context, then fall back to its creation context, and finally to the current hub
## 0.36.1
The Sentry SDK team is happy to announce the immediate availability of Sentry Go SDK v0.36.1.
### Bug Fixes
- Prevent panic when converting error chains containing non-comparable error types by using a safe fallback for visited detection in exception conversion ([#1113](https://github.com/getsentry/sentry-go/pull/1113))
## 0.36.0
The Sentry SDK team is happy to announce the immediate availability of Sentry Go SDK v0.36.0.
### Breaking Changes
- Behavioral change for the `MaxBreadcrumbs` client option. Removed the hard limit of 100 breadcrumbs, allowing users to set a larger limit and also changed the default limit from 30 to 100 ([#1106](https://github.com/getsentry/sentry-go/pull/1106)))
- The changes to error handling ([#1075](https://github.com/getsentry/sentry-go/pull/1075)) will affect issue grouping. It is expected that any wrapped and complex errors will be grouped under a new issue group.
### Features
- Add support for improved issue grouping with enhanced error chain handling ([#1075](https://github.com/getsentry/sentry-go/pull/1075))
The SDK now provides better handling of complex error scenarios, particularly when dealing with multiple related errors or error chains. This feature automatically detects and properly structures errors created with Go's `errors.Join()` function and other multi-error patterns.
```go
// Multiple errors are now properly grouped and displayed in Sentry
err1 := errors.New("err1")
err2 := errors.New("err2")
combinedErr := errors.Join(err1, err2)
// When captured, these will be shown as related exceptions in Sentry
sentry.CaptureException(combinedErr)
```
- Add `TraceIgnoreStatusCodes` option to allow filtering of HTTP transactions based on status codes ([#1089](https://github.com/getsentry/sentry-go/pull/1089))
- Configure which HTTP status codes should not be traced by providing single codes or ranges
- Example: `TraceIgnoreStatusCodes: [][]int{{404}, {500, 599}}` ignores 404 and server errors 500-599
### Bug Fixes
- Fix logs being incorrectly filtered by `BeforeSend` callback ([#1109](https://github.com/getsentry/sentry-go/pull/1109))
- Logs now bypass the `processEvent` method and are sent directly to the transport
- This ensures logs are only filtered by `BeforeSendLog`, not by the error/message `BeforeSend` callback
### Misc
- Add support for Go 1.25 and drop support for Go 1.22 ([#1103](https://github.com/getsentry/sentry-go/pull/1103))
## 0.35.3 ## 0.35.3
The Sentry SDK team is happy to announce the immediate availability of Sentry Go SDK v0.35.3. The Sentry SDK team is happy to announce the immediate availability of Sentry Go SDK v0.35.3.

View file

@ -70,7 +70,7 @@ mod-tidy: ## Check go.mod tidiness
set -e ; \ set -e ; \
for dir in $(ALL_GO_MOD_DIRS); do \ for dir in $(ALL_GO_MOD_DIRS); do \
echo ">>> Running 'go mod tidy' for module: $${dir}"; \ echo ">>> Running 'go mod tidy' for module: $${dir}"; \
(cd "$${dir}" && go mod tidy -go=1.22 -compat=1.22); \ (cd "$${dir}" && go mod tidy -go=1.23 -compat=1.23); \
done; \ done; \
git diff --exit-code; git diff --exit-code;
.PHONY: mod-tidy .PHONY: mod-tidy

View file

@ -125,7 +125,8 @@ func (l *BatchLogger) run(ctx context.Context) {
func (l *BatchLogger) processEvent(logs []Log) { func (l *BatchLogger) processEvent(logs []Log) {
event := NewEvent() event := NewEvent()
event.Timestamp = time.Now() event.Timestamp = time.Now()
event.EventID = EventID(uuid())
event.Type = logEvent.Type event.Type = logEvent.Type
event.Logs = logs event.Logs = logs
l.client.CaptureEvent(event, nil, nil) l.client.Transport.SendEvent(event)
} }

View file

@ -5,7 +5,6 @@ import (
"crypto/x509" "crypto/x509"
"fmt" "fmt"
"io" "io"
"log"
"math/rand" "math/rand"
"net/http" "net/http"
"os" "os"
@ -15,24 +14,31 @@ import (
"time" "time"
"github.com/getsentry/sentry-go/internal/debug" "github.com/getsentry/sentry-go/internal/debug"
"github.com/getsentry/sentry-go/internal/debuglog"
) )
// The identifier of the SDK. // The identifier of the SDK.
const sdkIdentifier = "sentry.go" const sdkIdentifier = "sentry.go"
// maxErrorDepth is the maximum number of errors reported in a chain of errors. const (
// This protects the SDK from an arbitrarily long chain of wrapped errors. // maxErrorDepth is the maximum number of errors reported in a chain of errors.
// // This protects the SDK from an arbitrarily long chain of wrapped errors.
// An additional consideration is that arguably reporting a long chain of errors //
// is of little use when debugging production errors with Sentry. The Sentry UI // An additional consideration is that arguably reporting a long chain of errors
// is not optimized for long chains either. The top-level error together with a // is of little use when debugging production errors with Sentry. The Sentry UI
// stack trace is often the most useful information. // is not optimized for long chains either. The top-level error together with a
const maxErrorDepth = 10 // stack trace is often the most useful information.
maxErrorDepth = 100
// defaultMaxSpans limits the default number of recorded spans per transaction. The limit is // defaultMaxSpans limits the default number of recorded spans per transaction. The limit is
// meant to bound memory usage and prevent too large transaction events that // meant to bound memory usage and prevent too large transaction events that
// would be rejected by Sentry. // would be rejected by Sentry.
const defaultMaxSpans = 1000 defaultMaxSpans = 1000
// defaultMaxBreadcrumbs is the default maximum number of breadcrumbs added to
// an event. Can be overwritten with the MaxBreadcrumbs option.
defaultMaxBreadcrumbs = 100
)
// hostname is the host name reported by the kernel. It is precomputed once to // hostname is the host name reported by the kernel. It is precomputed once to
// avoid syscalls when capturing events. // avoid syscalls when capturing events.
@ -78,8 +84,8 @@ type usageError struct {
} }
// DebugLogger is an instance of log.Logger that is used to provide debug information about running Sentry Client // DebugLogger is an instance of log.Logger that is used to provide debug information about running Sentry Client
// can be enabled by either using DebugLogger.SetOutput directly or with Debug client option. // can be enabled by either using debuglog.SetOutput directly or with Debug client option.
var DebugLogger = log.New(io.Discard, "[Sentry] ", log.LstdFlags) var DebugLogger = debuglog.GetLogger()
// EventProcessor is a function that processes an event. // EventProcessor is a function that processes an event.
// Event processors are used to change an event before it is sent to Sentry. // Event processors are used to change an event before it is sent to Sentry.
@ -228,6 +234,21 @@ type ClientOptions struct {
Tags map[string]string Tags map[string]string
// EnableLogs controls when logs should be emitted. // EnableLogs controls when logs should be emitted.
EnableLogs bool EnableLogs bool
// TraceIgnoreStatusCodes is a list of HTTP status codes that should not be traced.
// Each element can be either:
// - A single-element slice [code] for a specific status code
// - A two-element slice [min, max] for a range of status codes (inclusive)
// When an HTTP request results in a status code that matches any of these codes or ranges,
// the transaction will not be sent to Sentry.
//
// Examples:
// [][]int{{404}} // ignore only status code 404
// [][]int{{400, 405}} // ignore status codes 400-405
// [][]int{{404}, {500}} // ignore status codes 404 and 500
// [][]int{{404}, {400, 405}, {500, 599}} // ignore 404, range 400-405, and range 500-599
//
// By default, this is empty and all status codes are traced.
TraceIgnoreStatusCodes [][]int
} }
// Client is the underlying processor that is used by the main API and Hub // Client is the underlying processor that is used by the main API and Hub
@ -281,7 +302,7 @@ func NewClient(options ClientOptions) (*Client, error) {
if debugWriter == nil { if debugWriter == nil {
debugWriter = os.Stderr debugWriter = os.Stderr
} }
DebugLogger.SetOutput(debugWriter) debuglog.SetOutput(debugWriter)
} }
if options.Dsn == "" { if options.Dsn == "" {
@ -386,12 +407,12 @@ func (client *Client) setupIntegrations() {
for _, integration := range integrations { for _, integration := range integrations {
if client.integrationAlreadyInstalled(integration.Name()) { if client.integrationAlreadyInstalled(integration.Name()) {
DebugLogger.Printf("Integration %s is already installed\n", integration.Name()) debuglog.Printf("Integration %s is already installed\n", integration.Name())
continue continue
} }
client.integrations = append(client.integrations, integration) client.integrations = append(client.integrations, integration)
integration.SetupOnce(client) integration.SetupOnce(client)
DebugLogger.Printf("Integration installed: %s\n", integration.Name()) debuglog.Printf("Integration installed: %s\n", integration.Name())
} }
sort.Slice(client.integrations, func(i, j int) bool { sort.Slice(client.integrations, func(i, j int) bool {
@ -632,7 +653,7 @@ func (client *Client) processEvent(event *Event, hint *EventHint, scope EventMod
// options.TracesSampler when they are started. Other events // options.TracesSampler when they are started. Other events
// (errors, messages) are sampled here. Does not apply to check-ins. // (errors, messages) are sampled here. Does not apply to check-ins.
if event.Type != transactionType && event.Type != checkInType && !sample(client.options.SampleRate) { if event.Type != transactionType && event.Type != checkInType && !sample(client.options.SampleRate) {
DebugLogger.Println("Event dropped due to SampleRate hit.") debuglog.Println("Event dropped due to SampleRate hit.")
return nil return nil
} }
@ -648,7 +669,7 @@ func (client *Client) processEvent(event *Event, hint *EventHint, scope EventMod
case transactionType: case transactionType:
if client.options.BeforeSendTransaction != nil { if client.options.BeforeSendTransaction != nil {
if event = client.options.BeforeSendTransaction(event, hint); event == nil { if event = client.options.BeforeSendTransaction(event, hint); event == nil {
DebugLogger.Println("Transaction dropped due to BeforeSendTransaction callback.") debuglog.Println("Transaction dropped due to BeforeSendTransaction callback.")
return nil return nil
} }
} }
@ -656,7 +677,7 @@ func (client *Client) processEvent(event *Event, hint *EventHint, scope EventMod
default: default:
if client.options.BeforeSend != nil { if client.options.BeforeSend != nil {
if event = client.options.BeforeSend(event, hint); event == nil { if event = client.options.BeforeSend(event, hint); event == nil {
DebugLogger.Println("Event dropped due to BeforeSend callback.") debuglog.Println("Event dropped due to BeforeSend callback.")
return nil return nil
} }
} }
@ -723,7 +744,7 @@ func (client *Client) prepareEvent(event *Event, hint *EventHint, scope EventMod
id := event.EventID id := event.EventID
event = processor(event, hint) event = processor(event, hint)
if event == nil { if event == nil {
DebugLogger.Printf("Event dropped by one of the Client EventProcessors: %s\n", id) debuglog.Printf("Event dropped by one of the Client EventProcessors: %s\n", id)
return nil return nil
} }
} }
@ -732,7 +753,7 @@ func (client *Client) prepareEvent(event *Event, hint *EventHint, scope EventMod
id := event.EventID id := event.EventID
event = processor(event, hint) event = processor(event, hint)
if event == nil { if event == nil {
DebugLogger.Printf("Event dropped by one of the Global EventProcessors: %s\n", id) debuglog.Printf("Event dropped by one of the Global EventProcessors: %s\n", id)
return nil return nil
} }
} }

View file

@ -1,233 +1,37 @@
package sentry package sentry
import ( import (
"encoding/json" "github.com/getsentry/sentry-go/internal/protocol"
"fmt"
"net/url"
"strconv"
"strings"
"time"
) )
type scheme string // Re-export protocol types to maintain public API compatibility
const ( // Dsn is used as the remote address source to client transport.
schemeHTTP scheme = "http" type Dsn struct {
schemeHTTPS scheme = "https" protocol.Dsn
)
func (scheme scheme) defaultPort() int {
switch scheme {
case schemeHTTPS:
return 443
case schemeHTTP:
return 80
default:
return 80
}
} }
// DsnParseError represents an error that occurs if a Sentry // DsnParseError represents an error that occurs if a Sentry
// DSN cannot be parsed. // DSN cannot be parsed.
type DsnParseError struct { type DsnParseError = protocol.DsnParseError
Message string
}
func (e DsnParseError) Error() string {
return "[Sentry] DsnParseError: " + e.Message
}
// Dsn is used as the remote address source to client transport.
type Dsn struct {
scheme scheme
publicKey string
secretKey string
host string
port int
path string
projectID string
}
// NewDsn creates a Dsn by parsing rawURL. Most users will never call this // NewDsn creates a Dsn by parsing rawURL. Most users will never call this
// function directly. It is provided for use in custom Transport // function directly. It is provided for use in custom Transport
// implementations. // implementations.
func NewDsn(rawURL string) (*Dsn, error) { func NewDsn(rawURL string) (*Dsn, error) {
// Parse protocolDsn, err := protocol.NewDsn(rawURL)
parsedURL, err := url.Parse(rawURL)
if err != nil { if err != nil {
return nil, &DsnParseError{fmt.Sprintf("invalid url: %v", err)} return nil, err
} }
return &Dsn{Dsn: *protocolDsn}, nil
// Scheme
var scheme scheme
switch parsedURL.Scheme {
case "http":
scheme = schemeHTTP
case "https":
scheme = schemeHTTPS
default:
return nil, &DsnParseError{"invalid scheme"}
}
// PublicKey
publicKey := parsedURL.User.Username()
if publicKey == "" {
return nil, &DsnParseError{"empty username"}
}
// SecretKey
var secretKey string
if parsedSecretKey, ok := parsedURL.User.Password(); ok {
secretKey = parsedSecretKey
}
// Host
host := parsedURL.Hostname()
if host == "" {
return nil, &DsnParseError{"empty host"}
}
// Port
var port int
if p := parsedURL.Port(); p != "" {
port, err = strconv.Atoi(p)
if err != nil {
return nil, &DsnParseError{"invalid port"}
}
} else {
port = scheme.defaultPort()
}
// ProjectID
if parsedURL.Path == "" || parsedURL.Path == "/" {
return nil, &DsnParseError{"empty project id"}
}
pathSegments := strings.Split(parsedURL.Path[1:], "/")
projectID := pathSegments[len(pathSegments)-1]
if projectID == "" {
return nil, &DsnParseError{"empty project id"}
}
// Path
var path string
if len(pathSegments) > 1 {
path = "/" + strings.Join(pathSegments[0:len(pathSegments)-1], "/")
}
return &Dsn{
scheme: scheme,
publicKey: publicKey,
secretKey: secretKey,
host: host,
port: port,
path: path,
projectID: projectID,
}, nil
} }
// String formats Dsn struct into a valid string url. // RequestHeaders returns all the necessary headers that have to be used in the transport when sending events
func (dsn Dsn) String() string {
var url string
url += fmt.Sprintf("%s://%s", dsn.scheme, dsn.publicKey)
if dsn.secretKey != "" {
url += fmt.Sprintf(":%s", dsn.secretKey)
}
url += fmt.Sprintf("@%s", dsn.host)
if dsn.port != dsn.scheme.defaultPort() {
url += fmt.Sprintf(":%d", dsn.port)
}
if dsn.path != "" {
url += dsn.path
}
url += fmt.Sprintf("/%s", dsn.projectID)
return url
}
// Get the scheme of the DSN.
func (dsn Dsn) GetScheme() string {
return string(dsn.scheme)
}
// Get the public key of the DSN.
func (dsn Dsn) GetPublicKey() string {
return dsn.publicKey
}
// Get the secret key of the DSN.
func (dsn Dsn) GetSecretKey() string {
return dsn.secretKey
}
// Get the host of the DSN.
func (dsn Dsn) GetHost() string {
return dsn.host
}
// Get the port of the DSN.
func (dsn Dsn) GetPort() int {
return dsn.port
}
// Get the path of the DSN.
func (dsn Dsn) GetPath() string {
return dsn.path
}
// Get the project ID of the DSN.
func (dsn Dsn) GetProjectID() string {
return dsn.projectID
}
// GetAPIURL returns the URL of the envelope endpoint of the project
// associated with the DSN.
func (dsn Dsn) GetAPIURL() *url.URL {
var rawURL string
rawURL += fmt.Sprintf("%s://%s", dsn.scheme, dsn.host)
if dsn.port != dsn.scheme.defaultPort() {
rawURL += fmt.Sprintf(":%d", dsn.port)
}
if dsn.path != "" {
rawURL += dsn.path
}
rawURL += fmt.Sprintf("/api/%s/%s/", dsn.projectID, "envelope")
parsedURL, _ := url.Parse(rawURL)
return parsedURL
}
// RequestHeaders returns all the necessary headers that have to be used in the transport when seinding events
// to the /store endpoint. // to the /store endpoint.
// //
// Deprecated: This method shall only be used if you want to implement your own transport that sends events to // Deprecated: This method shall only be used if you want to implement your own transport that sends events to
// the /store endpoint. If you're using the transport provided by the SDK, all necessary headers to authenticate // the /store endpoint. If you're using the transport provided by the SDK, all necessary headers to authenticate
// against the /envelope endpoint are added automatically. // against the /envelope endpoint are added automatically.
func (dsn Dsn) RequestHeaders() map[string]string { func (dsn Dsn) RequestHeaders() map[string]string {
auth := fmt.Sprintf("Sentry sentry_version=%s, sentry_timestamp=%d, "+ return dsn.Dsn.RequestHeaders(SDKVersion)
"sentry_client=sentry.go/%s, sentry_key=%s", apiVersion, time.Now().Unix(), SDKVersion, dsn.publicKey)
if dsn.secretKey != "" {
auth = fmt.Sprintf("%s, sentry_secret=%s", auth, dsn.secretKey)
}
return map[string]string{
"Content-Type": "application/json",
"X-Sentry-Auth": auth,
}
}
// MarshalJSON converts the Dsn struct to JSON.
func (dsn Dsn) MarshalJSON() ([]byte, error) {
return json.Marshal(dsn.String())
}
// UnmarshalJSON converts JSON data to the Dsn struct.
func (dsn *Dsn) UnmarshalJSON(data []byte) error {
var str string
_ = json.Unmarshal(data, &str)
newDsn, err := NewDsn(str)
if err != nil {
return err
}
*dsn = *newDsn
return nil
} }

View file

@ -60,7 +60,7 @@ func DynamicSamplingContextFromTransaction(span *Span) DynamicSamplingContext {
} }
if dsn := client.dsn; dsn != nil { if dsn := client.dsn; dsn != nil {
if publicKey := dsn.publicKey; publicKey != "" { if publicKey := dsn.GetPublicKey(); publicKey != "" {
entries["public_key"] = publicKey entries["public_key"] = publicKey
} }
} }
@ -136,7 +136,7 @@ func DynamicSamplingContextFromScope(scope *Scope, client *Client) DynamicSampli
} }
if dsn := client.dsn; dsn != nil { if dsn := client.dsn; dsn != nil {
if publicKey := dsn.publicKey; publicKey != "" { if publicKey := dsn.GetPublicKey(); publicKey != "" {
entries["public_key"] = publicKey entries["public_key"] = publicKey
} }
} }

132
vendor/github.com/getsentry/sentry-go/exception.go generated vendored Normal file
View file

@ -0,0 +1,132 @@
package sentry
import (
"fmt"
"reflect"
"slices"
)
const (
MechanismTypeGeneric string = "generic"
MechanismTypeChained string = "chained"
MechanismTypeUnwrap string = "unwrap"
MechanismSourceCause string = "cause"
)
type visited struct {
comparable map[error]struct{}
msgs map[string]struct{}
}
func (v *visited) seenError(err error) bool {
t := reflect.TypeOf(err)
if t == nil {
return false
}
if t.Comparable() {
if _, ok := v.comparable[err]; ok {
return true
}
v.comparable[err] = struct{}{}
return false
}
key := t.String() + err.Error()
if _, ok := v.msgs[key]; ok {
return true
}
v.msgs[key] = struct{}{}
return false
}
func convertErrorToExceptions(err error, maxErrorDepth int) []Exception {
var exceptions []Exception
vis := &visited{
make(map[error]struct{}),
make(map[string]struct{}),
}
convertErrorDFS(err, &exceptions, nil, "", vis, maxErrorDepth, 0)
// mechanism type is used for debugging purposes, but since we can't really distinguish the origin of who invoked
// captureException, we set it to nil if the error is not chained.
if len(exceptions) == 1 {
exceptions[0].Mechanism = nil
}
slices.Reverse(exceptions)
// Add a trace of the current stack to the top level(outermost) error in a chain if
// it doesn't have a stack trace yet.
// We only add to the most recent error to avoid duplication and because the
// current stack is most likely unrelated to errors deeper in the chain.
if len(exceptions) > 0 && exceptions[len(exceptions)-1].Stacktrace == nil {
exceptions[len(exceptions)-1].Stacktrace = NewStacktrace()
}
return exceptions
}
func convertErrorDFS(err error, exceptions *[]Exception, parentID *int, source string, visited *visited, maxErrorDepth int, currentDepth int) {
if err == nil {
return
}
if visited.seenError(err) {
return
}
_, isExceptionGroup := err.(interface{ Unwrap() []error })
exception := Exception{
Value: err.Error(),
Type: reflect.TypeOf(err).String(),
Stacktrace: ExtractStacktrace(err),
}
currentID := len(*exceptions)
var mechanismType string
if parentID == nil {
mechanismType = MechanismTypeGeneric
source = ""
} else {
mechanismType = MechanismTypeChained
}
exception.Mechanism = &Mechanism{
Type: mechanismType,
ExceptionID: currentID,
ParentID: parentID,
Source: source,
IsExceptionGroup: isExceptionGroup,
}
*exceptions = append(*exceptions, exception)
if maxErrorDepth >= 0 && currentDepth >= maxErrorDepth {
return
}
switch v := err.(type) {
case interface{ Unwrap() []error }:
unwrapped := v.Unwrap()
for i := range unwrapped {
if unwrapped[i] != nil {
childSource := fmt.Sprintf("errors[%d]", i)
convertErrorDFS(unwrapped[i], exceptions, &currentID, childSource, visited, maxErrorDepth, currentDepth+1)
}
}
case interface{ Unwrap() error }:
unwrapped := v.Unwrap()
if unwrapped != nil {
convertErrorDFS(unwrapped, exceptions, &currentID, MechanismTypeUnwrap, visited, maxErrorDepth, currentDepth+1)
}
case interface{ Cause() error }:
cause := v.Cause()
if cause != nil {
convertErrorDFS(cause, exceptions, &currentID, MechanismSourceCause, visited, maxErrorDepth, currentDepth+1)
}
}
}

View file

@ -5,6 +5,8 @@ import (
"fmt" "fmt"
"sync" "sync"
"time" "time"
"github.com/getsentry/sentry-go/internal/debuglog"
) )
type contextKey int type contextKey int
@ -18,14 +20,6 @@ const (
RequestContextKey = contextKey(2) RequestContextKey = contextKey(2)
) )
// defaultMaxBreadcrumbs is the default maximum number of breadcrumbs added to
// an event. Can be overwritten with the maxBreadcrumbs option.
const defaultMaxBreadcrumbs = 30
// maxBreadcrumbs is the absolute maximum number of breadcrumbs added to an
// event. The maxBreadcrumbs option cannot be set higher than this value.
const maxBreadcrumbs = 100
// currentHub is the initial Hub with no Client bound and an empty Scope. // currentHub is the initial Hub with no Client bound and an empty Scope.
var currentHub = NewHub(nil, NewScope()) var currentHub = NewHub(nil, NewScope())
@ -289,7 +283,7 @@ func (hub *Hub) AddBreadcrumb(breadcrumb *Breadcrumb, hint *BreadcrumbHint) {
// If there's no client, just store it on the scope straight away // If there's no client, just store it on the scope straight away
if client == nil { if client == nil {
hub.Scope().AddBreadcrumb(breadcrumb, maxBreadcrumbs) hub.Scope().AddBreadcrumb(breadcrumb, defaultMaxBreadcrumbs)
return return
} }
@ -299,8 +293,6 @@ func (hub *Hub) AddBreadcrumb(breadcrumb *Breadcrumb, hint *BreadcrumbHint) {
return return
case limit == 0: case limit == 0:
limit = defaultMaxBreadcrumbs limit = defaultMaxBreadcrumbs
case limit > maxBreadcrumbs:
limit = maxBreadcrumbs
} }
if client.options.BeforeBreadcrumb != nil { if client.options.BeforeBreadcrumb != nil {
@ -308,7 +300,7 @@ func (hub *Hub) AddBreadcrumb(breadcrumb *Breadcrumb, hint *BreadcrumbHint) {
hint = &BreadcrumbHint{} hint = &BreadcrumbHint{}
} }
if breadcrumb = client.options.BeforeBreadcrumb(breadcrumb, hint); breadcrumb == nil { if breadcrumb = client.options.BeforeBreadcrumb(breadcrumb, hint); breadcrumb == nil {
DebugLogger.Println("breadcrumb dropped due to BeforeBreadcrumb callback.") debuglog.Println("breadcrumb dropped due to BeforeBreadcrumb callback.")
return return
} }
} }

View file

@ -8,6 +8,8 @@ import (
"runtime/debug" "runtime/debug"
"strings" "strings"
"sync" "sync"
"github.com/getsentry/sentry-go/internal/debuglog"
) )
// ================================ // ================================
@ -32,7 +34,7 @@ func (mi *modulesIntegration) processor(event *Event, _ *EventHint) *Event {
mi.once.Do(func() { mi.once.Do(func() {
info, ok := debug.ReadBuildInfo() info, ok := debug.ReadBuildInfo()
if !ok { if !ok {
DebugLogger.Print("The Modules integration is not available in binaries built without module support.") debuglog.Print("The Modules integration is not available in binaries built without module support.")
return return
} }
mi.modules = extractModules(info) mi.modules = extractModules(info)
@ -141,7 +143,7 @@ func (iei *ignoreErrorsIntegration) processor(event *Event, _ *EventHint) *Event
for _, suspect := range suspects { for _, suspect := range suspects {
for _, pattern := range iei.ignoreErrors { for _, pattern := range iei.ignoreErrors {
if pattern.Match([]byte(suspect)) || strings.Contains(suspect, pattern.String()) { if pattern.Match([]byte(suspect)) || strings.Contains(suspect, pattern.String()) {
DebugLogger.Printf("Event dropped due to being matched by `IgnoreErrors` option."+ debuglog.Printf("Event dropped due to being matched by `IgnoreErrors` option."+
"| Value matched: %s | Filter used: %s", suspect, pattern) "| Value matched: %s | Filter used: %s", suspect, pattern)
return nil return nil
} }
@ -203,7 +205,7 @@ func (iei *ignoreTransactionsIntegration) processor(event *Event, _ *EventHint)
for _, pattern := range iei.ignoreTransactions { for _, pattern := range iei.ignoreTransactions {
if pattern.Match([]byte(suspect)) || strings.Contains(suspect, pattern.String()) { if pattern.Match([]byte(suspect)) || strings.Contains(suspect, pattern.String()) {
DebugLogger.Printf("Transaction dropped due to being matched by `IgnoreTransactions` option."+ debuglog.Printf("Transaction dropped due to being matched by `IgnoreTransactions` option."+
"| Value matched: %s | Filter used: %s", suspect, pattern) "| Value matched: %s | Filter used: %s", suspect, pattern)
return nil return nil
} }

View file

@ -3,16 +3,14 @@ package sentry
import ( import (
"context" "context"
"encoding/json" "encoding/json"
"errors"
"fmt" "fmt"
"net" "net"
"net/http" "net/http"
"reflect"
"slices"
"strings" "strings"
"time" "time"
"github.com/getsentry/sentry-go/attribute" "github.com/getsentry/sentry-go/attribute"
"github.com/getsentry/sentry-go/internal/protocol"
"github.com/getsentry/sentry-go/internal/ratelimit" "github.com/getsentry/sentry-go/internal/ratelimit"
) )
@ -41,18 +39,8 @@ const (
) )
// SdkInfo contains all metadata about the SDK. // SdkInfo contains all metadata about the SDK.
type SdkInfo struct { type SdkInfo = protocol.SdkInfo
Name string `json:"name,omitempty"` type SdkPackage = protocol.SdkPackage
Version string `json:"version,omitempty"`
Integrations []string `json:"integrations,omitempty"`
Packages []SdkPackage `json:"packages,omitempty"`
}
// SdkPackage describes a package that was installed.
type SdkPackage struct {
Name string `json:"name,omitempty"`
Version string `json:"version,omitempty"`
}
// TODO: This type could be more useful, as map of interface{} is too generic // TODO: This type could be more useful, as map of interface{} is too generic
// and requires a lot of type assertions in beforeBreadcrumb calls // and requires a lot of type assertions in beforeBreadcrumb calls
@ -249,11 +237,11 @@ var sensitiveHeaders = map[string]struct{}{
// NewRequest avoids operations that depend on network access. In particular, it // NewRequest avoids operations that depend on network access. In particular, it
// does not read r.Body. // does not read r.Body.
func NewRequest(r *http.Request) *Request { func NewRequest(r *http.Request) *Request {
protocol := schemeHTTP prot := protocol.SchemeHTTP
if r.TLS != nil || r.Header.Get("X-Forwarded-Proto") == "https" { if r.TLS != nil || r.Header.Get("X-Forwarded-Proto") == "https" {
protocol = schemeHTTPS prot = protocol.SchemeHTTPS
} }
url := fmt.Sprintf("%s://%s%s", protocol, r.Host, r.URL.Path) url := fmt.Sprintf("%s://%s%s", prot, r.Host, r.URL.Path)
var cookies string var cookies string
var env map[string]string var env map[string]string
@ -296,7 +284,7 @@ func NewRequest(r *http.Request) *Request {
// Mechanism is the mechanism by which an exception was generated and handled. // Mechanism is the mechanism by which an exception was generated and handled.
type Mechanism struct { type Mechanism struct {
Type string `json:"type,omitempty"` Type string `json:"type"`
Description string `json:"description,omitempty"` Description string `json:"description,omitempty"`
HelpLink string `json:"help_link,omitempty"` HelpLink string `json:"help_link,omitempty"`
Source string `json:"source,omitempty"` Source string `json:"source,omitempty"`
@ -425,64 +413,84 @@ func (e *Event) SetException(exception error, maxErrorDepth int) {
return return
} }
err := exception exceptions := convertErrorToExceptions(exception, maxErrorDepth)
if len(exceptions) == 0 {
for i := 0; err != nil && (i < maxErrorDepth || maxErrorDepth == -1); i++ {
// Add the current error to the exception slice with its details
e.Exception = append(e.Exception, Exception{
Value: err.Error(),
Type: reflect.TypeOf(err).String(),
Stacktrace: ExtractStacktrace(err),
})
// Attempt to unwrap the error using the standard library's Unwrap method.
// If errors.Unwrap returns nil, it means either there is no error to unwrap,
// or the error does not implement the Unwrap method.
unwrappedErr := errors.Unwrap(err)
if unwrappedErr != nil {
// The error was successfully unwrapped using the standard library's Unwrap method.
err = unwrappedErr
continue
}
cause, ok := err.(interface{ Cause() error })
if !ok {
// We cannot unwrap the error further.
break
}
// The error implements the Cause method, indicating it may have been wrapped
// using the github.com/pkg/errors package.
err = cause.Cause()
}
// Add a trace of the current stack to the most recent error in a chain if
// it doesn't have a stack trace yet.
// We only add to the most recent error to avoid duplication and because the
// current stack is most likely unrelated to errors deeper in the chain.
if e.Exception[0].Stacktrace == nil {
e.Exception[0].Stacktrace = NewStacktrace()
}
if len(e.Exception) <= 1 {
return return
} }
// event.Exception should be sorted such that the most recent error is last. e.Exception = exceptions
slices.Reverse(e.Exception) }
for i := range e.Exception { // ToEnvelope converts the Event to a Sentry envelope.
e.Exception[i].Mechanism = &Mechanism{ // This includes the event data and any attachments as separate envelope items.
IsExceptionGroup: true, func (e *Event) ToEnvelope(dsn *protocol.Dsn) (*protocol.Envelope, error) {
ExceptionID: i, return e.ToEnvelopeWithTime(dsn, time.Now())
Type: "generic", }
// ToEnvelopeWithTime converts the Event to a Sentry envelope with a specific sentAt time.
// This is primarily useful for testing with predictable timestamps.
func (e *Event) ToEnvelopeWithTime(dsn *protocol.Dsn, sentAt time.Time) (*protocol.Envelope, error) {
// Create envelope header with trace context
trace := make(map[string]string)
if dsc := e.sdkMetaData.dsc; dsc.HasEntries() {
for k, v := range dsc.Entries {
trace[k] = v
} }
if i == 0 {
continue
}
e.Exception[i].Mechanism.ParentID = Pointer(i - 1)
} }
header := &protocol.EnvelopeHeader{
EventID: string(e.EventID),
SentAt: sentAt,
Trace: trace,
}
if dsn != nil {
header.Dsn = dsn.String()
}
header.Sdk = &e.Sdk
envelope := protocol.NewEnvelope(header)
eventBody, err := json.Marshal(e)
if err != nil {
// Try fallback: remove problematic fields and retry
e.Breadcrumbs = nil
e.Contexts = nil
e.Extra = map[string]interface{}{
"info": fmt.Sprintf("Could not encode original event as JSON. "+
"Succeeded by removing Breadcrumbs, Contexts and Extra. "+
"Please verify the data you attach to the scope. "+
"Error: %s", err),
}
eventBody, err = json.Marshal(e)
if err != nil {
return nil, fmt.Errorf("event could not be marshaled even with fallback: %w", err)
}
DebugLogger.Printf("Event marshaling succeeded with fallback after removing problematic fields")
}
var mainItem *protocol.EnvelopeItem
switch e.Type {
case transactionType:
mainItem = protocol.NewEnvelopeItem(protocol.EnvelopeItemTypeTransaction, eventBody)
case checkInType:
mainItem = protocol.NewEnvelopeItem(protocol.EnvelopeItemTypeCheckIn, eventBody)
case logEvent.Type:
mainItem = protocol.NewLogItem(len(e.Logs), eventBody)
default:
mainItem = protocol.NewEnvelopeItem(protocol.EnvelopeItemTypeEvent, eventBody)
}
envelope.AddItem(mainItem)
for _, attachment := range e.Attachments {
attachmentItem := protocol.NewAttachmentItem(attachment.Filename, attachment.ContentType, attachment.Payload)
envelope.AddItem(attachmentItem)
}
return envelope, nil
} }
// TODO: Event.Contexts map[string]interface{} => map[string]EventContext, // TODO: Event.Contexts map[string]interface{} => map[string]EventContext,

View file

@ -0,0 +1,35 @@
package debuglog
import (
"io"
"log"
)
// logger is the global debug logger instance.
var logger = log.New(io.Discard, "[Sentry] ", log.LstdFlags)
// SetOutput changes the output destination of the logger.
func SetOutput(w io.Writer) {
logger.SetOutput(w)
}
// GetLogger returns the current logger instance.
// This function is thread-safe and can be called concurrently.
func GetLogger() *log.Logger {
return logger
}
// Printf calls Printf on the underlying logger.
func Printf(format string, args ...interface{}) {
logger.Printf(format, args...)
}
// Println calls Println on the underlying logger.
func Println(args ...interface{}) {
logger.Println(args...)
}
// Print calls Print on the underlying logger.
func Print(args ...interface{}) {
logger.Print(args...)
}

View file

@ -0,0 +1,571 @@
package http
import (
"bytes"
"context"
"crypto/tls"
"crypto/x509"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"sync"
"sync/atomic"
"time"
"github.com/getsentry/sentry-go/internal/debuglog"
"github.com/getsentry/sentry-go/internal/protocol"
"github.com/getsentry/sentry-go/internal/ratelimit"
)
const (
apiVersion = 7
defaultTimeout = time.Second * 30
defaultQueueSize = 1000
// maxDrainResponseBytes is the maximum number of bytes that transport
// implementations will read from response bodies when draining them.
maxDrainResponseBytes = 16 << 10
)
var (
ErrTransportQueueFull = errors.New("transport queue full")
ErrTransportClosed = errors.New("transport is closed")
)
type TransportOptions struct {
Dsn string
HTTPClient *http.Client
HTTPTransport http.RoundTripper
HTTPProxy string
HTTPSProxy string
CaCerts *x509.CertPool
}
func getProxyConfig(options TransportOptions) func(*http.Request) (*url.URL, error) {
if len(options.HTTPSProxy) > 0 {
return func(*http.Request) (*url.URL, error) {
return url.Parse(options.HTTPSProxy)
}
}
if len(options.HTTPProxy) > 0 {
return func(*http.Request) (*url.URL, error) {
return url.Parse(options.HTTPProxy)
}
}
return http.ProxyFromEnvironment
}
func getTLSConfig(options TransportOptions) *tls.Config {
if options.CaCerts != nil {
return &tls.Config{
RootCAs: options.CaCerts,
MinVersion: tls.VersionTLS12,
}
}
return nil
}
func getSentryRequestFromEnvelope(ctx context.Context, dsn *protocol.Dsn, envelope *protocol.Envelope) (r *http.Request, err error) {
defer func() {
if r != nil {
sdkName := envelope.Header.Sdk.Name
sdkVersion := envelope.Header.Sdk.Version
r.Header.Set("User-Agent", fmt.Sprintf("%s/%s", sdkName, sdkVersion))
r.Header.Set("Content-Type", "application/x-sentry-envelope")
auth := fmt.Sprintf("Sentry sentry_version=%d, "+
"sentry_client=%s/%s, sentry_key=%s", apiVersion, sdkName, sdkVersion, dsn.GetPublicKey())
if dsn.GetSecretKey() != "" {
auth = fmt.Sprintf("%s, sentry_secret=%s", auth, dsn.GetSecretKey())
}
r.Header.Set("X-Sentry-Auth", auth)
}
}()
var buf bytes.Buffer
_, err = envelope.WriteTo(&buf)
if err != nil {
return nil, err
}
return http.NewRequestWithContext(
ctx,
http.MethodPost,
dsn.GetAPIURL().String(),
&buf,
)
}
func categoryFromEnvelope(envelope *protocol.Envelope) ratelimit.Category {
if envelope == nil || len(envelope.Items) == 0 {
return ratelimit.CategoryAll
}
for _, item := range envelope.Items {
if item == nil || item.Header == nil {
continue
}
switch item.Header.Type {
case protocol.EnvelopeItemTypeEvent:
return ratelimit.CategoryError
case protocol.EnvelopeItemTypeTransaction:
return ratelimit.CategoryTransaction
case protocol.EnvelopeItemTypeCheckIn:
return ratelimit.CategoryMonitor
case protocol.EnvelopeItemTypeLog:
return ratelimit.CategoryLog
case protocol.EnvelopeItemTypeAttachment:
continue
default:
return ratelimit.CategoryAll
}
}
return ratelimit.CategoryAll
}
// SyncTransport is a blocking implementation of Transport.
//
// Clients using this transport will send requests to Sentry sequentially and
// block until a response is returned.
//
// The blocking behavior is useful in a limited set of use cases. For example,
// use it when deploying code to a Function as a Service ("Serverless")
// platform, where any work happening in a background goroutine is not
// guaranteed to execute.
//
// For most cases, prefer AsyncTransport.
type SyncTransport struct {
dsn *protocol.Dsn
client *http.Client
transport http.RoundTripper
mu sync.Mutex
limits ratelimit.Map
Timeout time.Duration
}
func NewSyncTransport(options TransportOptions) protocol.TelemetryTransport {
dsn, err := protocol.NewDsn(options.Dsn)
if err != nil || dsn == nil {
debuglog.Printf("Transport is disabled: invalid dsn: %v\n", err)
return NewNoopTransport()
}
transport := &SyncTransport{
Timeout: defaultTimeout,
limits: make(ratelimit.Map),
dsn: dsn,
}
if options.HTTPTransport != nil {
transport.transport = options.HTTPTransport
} else {
transport.transport = &http.Transport{
Proxy: getProxyConfig(options),
TLSClientConfig: getTLSConfig(options),
}
}
if options.HTTPClient != nil {
transport.client = options.HTTPClient
} else {
transport.client = &http.Client{
Transport: transport.transport,
Timeout: transport.Timeout,
}
}
return transport
}
func (t *SyncTransport) SendEnvelope(envelope *protocol.Envelope) error {
return t.SendEnvelopeWithContext(context.Background(), envelope)
}
func (t *SyncTransport) Close() {}
func (t *SyncTransport) SendEvent(event protocol.EnvelopeConvertible) {
envelope, err := event.ToEnvelope(t.dsn)
if err != nil {
debuglog.Printf("Failed to convert to envelope: %v", err)
return
}
if envelope == nil {
debuglog.Printf("Error: event with empty envelope")
return
}
if err := t.SendEnvelope(envelope); err != nil {
debuglog.Printf("Error sending the envelope: %v", err)
}
}
func (t *SyncTransport) IsRateLimited(category ratelimit.Category) bool {
return t.disabled(category)
}
func (t *SyncTransport) SendEnvelopeWithContext(ctx context.Context, envelope *protocol.Envelope) error {
if envelope == nil {
debuglog.Printf("Error: provided empty envelope")
return nil
}
category := categoryFromEnvelope(envelope)
if t.disabled(category) {
return nil
}
request, err := getSentryRequestFromEnvelope(ctx, t.dsn, envelope)
if err != nil {
debuglog.Printf("There was an issue creating the request: %v", err)
return err
}
response, err := t.client.Do(request)
if err != nil {
debuglog.Printf("There was an issue with sending an event: %v", err)
return err
}
if response.StatusCode >= 400 && response.StatusCode <= 599 {
b, err := io.ReadAll(response.Body)
if err != nil {
debuglog.Printf("Error while reading response body: %v", err)
}
debuglog.Printf("Sending %s failed with the following error: %s", envelope.Header.EventID, string(b))
}
t.mu.Lock()
if t.limits == nil {
t.limits = make(ratelimit.Map)
}
t.limits.Merge(ratelimit.FromResponse(response))
t.mu.Unlock()
_, _ = io.CopyN(io.Discard, response.Body, maxDrainResponseBytes)
return response.Body.Close()
}
func (t *SyncTransport) Flush(_ time.Duration) bool {
return true
}
func (t *SyncTransport) FlushWithContext(_ context.Context) bool {
return true
}
func (t *SyncTransport) disabled(c ratelimit.Category) bool {
t.mu.Lock()
defer t.mu.Unlock()
disabled := t.limits.IsRateLimited(c)
if disabled {
debuglog.Printf("Too many requests for %q, backing off till: %v", c, t.limits.Deadline(c))
}
return disabled
}
// AsyncTransport is the default, non-blocking, implementation of Transport.
//
// Clients using this transport will enqueue requests in a queue and return to
// the caller before any network communication has happened. Requests are sent
// to Sentry sequentially from a background goroutine.
type AsyncTransport struct {
dsn *protocol.Dsn
client *http.Client
transport http.RoundTripper
queue chan *protocol.Envelope
mu sync.RWMutex
limits ratelimit.Map
done chan struct{}
wg sync.WaitGroup
flushRequest chan chan struct{}
sentCount int64
droppedCount int64
errorCount int64
QueueSize int
Timeout time.Duration
startOnce sync.Once
closeOnce sync.Once
}
func NewAsyncTransport(options TransportOptions) protocol.TelemetryTransport {
dsn, err := protocol.NewDsn(options.Dsn)
if err != nil || dsn == nil {
debuglog.Printf("Transport is disabled: invalid dsn: %v", err)
return NewNoopTransport()
}
transport := &AsyncTransport{
QueueSize: defaultQueueSize,
Timeout: defaultTimeout,
done: make(chan struct{}),
limits: make(ratelimit.Map),
dsn: dsn,
}
transport.queue = make(chan *protocol.Envelope, transport.QueueSize)
transport.flushRequest = make(chan chan struct{})
if options.HTTPTransport != nil {
transport.transport = options.HTTPTransport
} else {
transport.transport = &http.Transport{
Proxy: getProxyConfig(options),
TLSClientConfig: getTLSConfig(options),
}
}
if options.HTTPClient != nil {
transport.client = options.HTTPClient
} else {
transport.client = &http.Client{
Transport: transport.transport,
Timeout: transport.Timeout,
}
}
transport.start()
return transport
}
func (t *AsyncTransport) start() {
t.startOnce.Do(func() {
t.wg.Add(1)
go t.worker()
})
}
func (t *AsyncTransport) SendEnvelope(envelope *protocol.Envelope) error {
select {
case <-t.done:
return ErrTransportClosed
default:
}
category := categoryFromEnvelope(envelope)
if t.isRateLimited(category) {
return nil
}
select {
case t.queue <- envelope:
return nil
default:
atomic.AddInt64(&t.droppedCount, 1)
return ErrTransportQueueFull
}
}
func (t *AsyncTransport) SendEvent(event protocol.EnvelopeConvertible) {
envelope, err := event.ToEnvelope(t.dsn)
if err != nil {
debuglog.Printf("Failed to convert to envelope: %v", err)
return
}
if envelope == nil {
debuglog.Printf("Error: event with empty envelope")
return
}
if err := t.SendEnvelope(envelope); err != nil {
debuglog.Printf("Error sending the envelope: %v", err)
}
}
func (t *AsyncTransport) Flush(timeout time.Duration) bool {
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
return t.FlushWithContext(ctx)
}
func (t *AsyncTransport) FlushWithContext(ctx context.Context) bool {
flushResponse := make(chan struct{})
select {
case t.flushRequest <- flushResponse:
select {
case <-flushResponse:
return true
case <-ctx.Done():
return false
}
case <-ctx.Done():
return false
}
}
func (t *AsyncTransport) Close() {
t.closeOnce.Do(func() {
close(t.done)
close(t.queue)
close(t.flushRequest)
t.wg.Wait()
})
}
func (t *AsyncTransport) IsRateLimited(category ratelimit.Category) bool {
return t.isRateLimited(category)
}
func (t *AsyncTransport) worker() {
defer t.wg.Done()
for {
select {
case <-t.done:
return
case envelope, open := <-t.queue:
if !open {
return
}
t.processEnvelope(envelope)
case flushResponse, open := <-t.flushRequest:
if !open {
return
}
t.drainQueue()
close(flushResponse)
}
}
}
func (t *AsyncTransport) drainQueue() {
for {
select {
case envelope, open := <-t.queue:
if !open {
return
}
t.processEnvelope(envelope)
default:
return
}
}
}
func (t *AsyncTransport) processEnvelope(envelope *protocol.Envelope) {
if t.sendEnvelopeHTTP(envelope) {
atomic.AddInt64(&t.sentCount, 1)
} else {
atomic.AddInt64(&t.errorCount, 1)
}
}
func (t *AsyncTransport) sendEnvelopeHTTP(envelope *protocol.Envelope) bool {
category := categoryFromEnvelope(envelope)
if t.isRateLimited(category) {
return false
}
ctx, cancel := context.WithTimeout(context.Background(), defaultTimeout)
defer cancel()
request, err := getSentryRequestFromEnvelope(ctx, t.dsn, envelope)
if err != nil {
debuglog.Printf("Failed to create request from envelope: %v", err)
return false
}
response, err := t.client.Do(request)
if err != nil {
debuglog.Printf("HTTP request failed: %v", err)
return false
}
defer response.Body.Close()
success := t.handleResponse(response)
t.mu.Lock()
if t.limits == nil {
t.limits = make(ratelimit.Map)
}
t.limits.Merge(ratelimit.FromResponse(response))
t.mu.Unlock()
_, _ = io.CopyN(io.Discard, response.Body, maxDrainResponseBytes)
return success
}
func (t *AsyncTransport) handleResponse(response *http.Response) bool {
if response.StatusCode >= 200 && response.StatusCode < 300 {
return true
}
if response.StatusCode >= 400 && response.StatusCode < 500 {
if body, err := io.ReadAll(io.LimitReader(response.Body, maxDrainResponseBytes)); err == nil {
debuglog.Printf("Client error %d: %s", response.StatusCode, string(body))
}
return false
}
if response.StatusCode >= 500 {
debuglog.Printf("Server error %d", response.StatusCode)
return false
}
debuglog.Printf("Unexpected status code %d", response.StatusCode)
return false
}
func (t *AsyncTransport) isRateLimited(category ratelimit.Category) bool {
t.mu.RLock()
defer t.mu.RUnlock()
limited := t.limits.IsRateLimited(category)
if limited {
debuglog.Printf("Rate limited for category %q until %v", category, t.limits.Deadline(category))
}
return limited
}
// NoopTransport is a transport implementation that drops all events.
// Used internally when an empty or invalid DSN is provided.
type NoopTransport struct{}
func NewNoopTransport() *NoopTransport {
debuglog.Println("Transport initialized with invalid DSN. Using NoopTransport. No events will be delivered.")
return &NoopTransport{}
}
func (t *NoopTransport) SendEnvelope(_ *protocol.Envelope) error {
debuglog.Println("Envelope dropped due to NoopTransport usage.")
return nil
}
func (t *NoopTransport) SendEvent(_ protocol.EnvelopeConvertible) {
debuglog.Println("Event dropped due to NoopTransport usage.")
}
func (t *NoopTransport) IsRateLimited(_ ratelimit.Category) bool {
return false
}
func (t *NoopTransport) Flush(_ time.Duration) bool {
return true
}
func (t *NoopTransport) FlushWithContext(_ context.Context) bool {
return true
}
func (t *NoopTransport) Close() {
// Nothing to close
}

View file

@ -0,0 +1,236 @@
package protocol
import (
"encoding/json"
"fmt"
"net/url"
"strconv"
"strings"
"time"
)
// apiVersion is the version of the Sentry API.
const apiVersion = "7"
type scheme string
const (
SchemeHTTP scheme = "http"
SchemeHTTPS scheme = "https"
)
func (scheme scheme) defaultPort() int {
switch scheme {
case SchemeHTTPS:
return 443
case SchemeHTTP:
return 80
default:
return 80
}
}
// DsnParseError represents an error that occurs if a Sentry
// DSN cannot be parsed.
type DsnParseError struct {
Message string
}
func (e DsnParseError) Error() string {
return "[Sentry] DsnParseError: " + e.Message
}
// Dsn is used as the remote address source to client transport.
type Dsn struct {
scheme scheme
publicKey string
secretKey string
host string
port int
path string
projectID string
}
// NewDsn creates a Dsn by parsing rawURL. Most users will never call this
// function directly. It is provided for use in custom Transport
// implementations.
func NewDsn(rawURL string) (*Dsn, error) {
// Parse
parsedURL, err := url.Parse(rawURL)
if err != nil {
return nil, &DsnParseError{fmt.Sprintf("invalid url: %v", err)}
}
// Scheme
var scheme scheme
switch parsedURL.Scheme {
case "http":
scheme = SchemeHTTP
case "https":
scheme = SchemeHTTPS
default:
return nil, &DsnParseError{"invalid scheme"}
}
// PublicKey
publicKey := parsedURL.User.Username()
if publicKey == "" {
return nil, &DsnParseError{"empty username"}
}
// SecretKey
var secretKey string
if parsedSecretKey, ok := parsedURL.User.Password(); ok {
secretKey = parsedSecretKey
}
// Host
host := parsedURL.Hostname()
if host == "" {
return nil, &DsnParseError{"empty host"}
}
// Port
var port int
if p := parsedURL.Port(); p != "" {
port, err = strconv.Atoi(p)
if err != nil {
return nil, &DsnParseError{"invalid port"}
}
} else {
port = scheme.defaultPort()
}
// ProjectID
if parsedURL.Path == "" || parsedURL.Path == "/" {
return nil, &DsnParseError{"empty project id"}
}
pathSegments := strings.Split(parsedURL.Path[1:], "/")
projectID := pathSegments[len(pathSegments)-1]
if projectID == "" {
return nil, &DsnParseError{"empty project id"}
}
// Path
var path string
if len(pathSegments) > 1 {
path = "/" + strings.Join(pathSegments[0:len(pathSegments)-1], "/")
}
return &Dsn{
scheme: scheme,
publicKey: publicKey,
secretKey: secretKey,
host: host,
port: port,
path: path,
projectID: projectID,
}, nil
}
// String formats Dsn struct into a valid string url.
func (dsn Dsn) String() string {
var url string
url += fmt.Sprintf("%s://%s", dsn.scheme, dsn.publicKey)
if dsn.secretKey != "" {
url += fmt.Sprintf(":%s", dsn.secretKey)
}
url += fmt.Sprintf("@%s", dsn.host)
if dsn.port != dsn.scheme.defaultPort() {
url += fmt.Sprintf(":%d", dsn.port)
}
if dsn.path != "" {
url += dsn.path
}
url += fmt.Sprintf("/%s", dsn.projectID)
return url
}
// Get the scheme of the DSN.
func (dsn Dsn) GetScheme() string {
return string(dsn.scheme)
}
// Get the public key of the DSN.
func (dsn Dsn) GetPublicKey() string {
return dsn.publicKey
}
// Get the secret key of the DSN.
func (dsn Dsn) GetSecretKey() string {
return dsn.secretKey
}
// Get the host of the DSN.
func (dsn Dsn) GetHost() string {
return dsn.host
}
// Get the port of the DSN.
func (dsn Dsn) GetPort() int {
return dsn.port
}
// Get the path of the DSN.
func (dsn Dsn) GetPath() string {
return dsn.path
}
// Get the project ID of the DSN.
func (dsn Dsn) GetProjectID() string {
return dsn.projectID
}
// GetAPIURL returns the URL of the envelope endpoint of the project
// associated with the DSN.
func (dsn Dsn) GetAPIURL() *url.URL {
var rawURL string
rawURL += fmt.Sprintf("%s://%s", dsn.scheme, dsn.host)
if dsn.port != dsn.scheme.defaultPort() {
rawURL += fmt.Sprintf(":%d", dsn.port)
}
if dsn.path != "" {
rawURL += dsn.path
}
rawURL += fmt.Sprintf("/api/%s/%s/", dsn.projectID, "envelope")
parsedURL, _ := url.Parse(rawURL)
return parsedURL
}
// RequestHeaders returns all the necessary headers that have to be used in the transport when sending events
// to the /store endpoint.
//
// Deprecated: This method shall only be used if you want to implement your own transport that sends events to
// the /store endpoint. If you're using the transport provided by the SDK, all necessary headers to authenticate
// against the /envelope endpoint are added automatically.
func (dsn Dsn) RequestHeaders(sdkVersion string) map[string]string {
auth := fmt.Sprintf("Sentry sentry_version=%s, sentry_timestamp=%d, "+
"sentry_client=sentry.go/%s, sentry_key=%s", apiVersion, time.Now().Unix(), sdkVersion, dsn.publicKey)
if dsn.secretKey != "" {
auth = fmt.Sprintf("%s, sentry_secret=%s", auth, dsn.secretKey)
}
return map[string]string{
"Content-Type": "application/json",
"X-Sentry-Auth": auth,
}
}
// MarshalJSON converts the Dsn struct to JSON.
func (dsn Dsn) MarshalJSON() ([]byte, error) {
return json.Marshal(dsn.String())
}
// UnmarshalJSON converts JSON data to the Dsn struct.
func (dsn *Dsn) UnmarshalJSON(data []byte) error {
var str string
_ = json.Unmarshal(data, &str)
newDsn, err := NewDsn(str)
if err != nil {
return err
}
*dsn = *newDsn
return nil
}

View file

@ -0,0 +1,213 @@
package protocol
import (
"bytes"
"encoding/json"
"fmt"
"io"
"time"
)
// Envelope represents a Sentry envelope containing headers and items.
type Envelope struct {
Header *EnvelopeHeader `json:"-"`
Items []*EnvelopeItem `json:"-"`
}
// EnvelopeHeader represents the header of a Sentry envelope.
type EnvelopeHeader struct {
// EventID is the unique identifier for this event
EventID string `json:"event_id"`
// SentAt is the timestamp when the event was sent from the SDK as string in RFC 3339 format.
// Used for clock drift correction of the event timestamp. The time zone must be UTC.
SentAt time.Time `json:"sent_at,omitempty"`
// Dsn can be used for self-authenticated envelopes.
// This means that the envelope has all the information necessary to be sent to sentry.
// In this case the full DSN must be stored in this key.
Dsn string `json:"dsn,omitempty"`
// Sdk carries the same payload as the sdk interface in the event payload but can be carried for all events.
// This means that SDK information can be carried for minidumps, session data and other submissions.
Sdk *SdkInfo `json:"sdk,omitempty"`
// Trace contains the [Dynamic Sampling Context](https://develop.sentry.dev/sdk/telemetry/traces/dynamic-sampling-context/)
Trace map[string]string `json:"trace,omitempty"`
}
// EnvelopeItemType represents the type of envelope item.
type EnvelopeItemType string
// Constants for envelope item types as defined in the Sentry documentation.
const (
EnvelopeItemTypeEvent EnvelopeItemType = "event"
EnvelopeItemTypeTransaction EnvelopeItemType = "transaction"
EnvelopeItemTypeCheckIn EnvelopeItemType = "check_in"
EnvelopeItemTypeAttachment EnvelopeItemType = "attachment"
EnvelopeItemTypeLog EnvelopeItemType = "log"
)
// EnvelopeItemHeader represents the header of an envelope item.
type EnvelopeItemHeader struct {
// Type specifies the type of this Item and its contents.
// Based on the Item type, more headers may be required.
Type EnvelopeItemType `json:"type"`
// Length is the length of the payload in bytes.
// If no length is specified, the payload implicitly goes to the next newline.
// For payloads containing newline characters, the length must be specified.
Length *int `json:"length,omitempty"`
// Filename is the name of the attachment file (used for attachments)
Filename string `json:"filename,omitempty"`
// ContentType is the MIME type of the item payload (used for attachments and some other item types)
ContentType string `json:"content_type,omitempty"`
// ItemCount is the number of items in a batch (used for logs)
ItemCount *int `json:"item_count,omitempty"`
}
// EnvelopeItem represents a single item within an envelope.
type EnvelopeItem struct {
Header *EnvelopeItemHeader `json:"-"`
Payload []byte `json:"-"`
}
// NewEnvelope creates a new envelope with the given header.
func NewEnvelope(header *EnvelopeHeader) *Envelope {
return &Envelope{
Header: header,
Items: make([]*EnvelopeItem, 0),
}
}
// AddItem adds an item to the envelope.
func (e *Envelope) AddItem(item *EnvelopeItem) {
e.Items = append(e.Items, item)
}
// Serialize serializes the envelope to the Sentry envelope format.
//
// Format: Headers "\n" { Item } [ "\n" ]
// Item: Headers "\n" Payload "\n".
func (e *Envelope) Serialize() ([]byte, error) {
var buf bytes.Buffer
headerBytes, err := json.Marshal(e.Header)
if err != nil {
return nil, fmt.Errorf("failed to marshal envelope header: %w", err)
}
if _, err := buf.Write(headerBytes); err != nil {
return nil, fmt.Errorf("failed to write envelope header: %w", err)
}
if _, err := buf.WriteString("\n"); err != nil {
return nil, fmt.Errorf("failed to write newline after envelope header: %w", err)
}
for _, item := range e.Items {
if err := e.writeItem(&buf, item); err != nil {
return nil, fmt.Errorf("failed to write envelope item: %w", err)
}
}
return buf.Bytes(), nil
}
// WriteTo writes the envelope to the given writer in the Sentry envelope format.
func (e *Envelope) WriteTo(w io.Writer) (int64, error) {
data, err := e.Serialize()
if err != nil {
return 0, err
}
n, err := w.Write(data)
return int64(n), err
}
// writeItem writes a single envelope item to the buffer.
func (e *Envelope) writeItem(buf *bytes.Buffer, item *EnvelopeItem) error {
headerBytes, err := json.Marshal(item.Header)
if err != nil {
return fmt.Errorf("failed to marshal item header: %w", err)
}
if _, err := buf.Write(headerBytes); err != nil {
return fmt.Errorf("failed to write item header: %w", err)
}
if _, err := buf.WriteString("\n"); err != nil {
return fmt.Errorf("failed to write newline after item header: %w", err)
}
if len(item.Payload) > 0 {
if _, err := buf.Write(item.Payload); err != nil {
return fmt.Errorf("failed to write item payload: %w", err)
}
}
if _, err := buf.WriteString("\n"); err != nil {
return fmt.Errorf("failed to write newline after item payload: %w", err)
}
return nil
}
// Size returns the total size of the envelope when serialized.
func (e *Envelope) Size() (int, error) {
data, err := e.Serialize()
if err != nil {
return 0, err
}
return len(data), nil
}
// MarshalJSON converts the EnvelopeHeader to JSON.
func (h *EnvelopeHeader) MarshalJSON() ([]byte, error) {
type header EnvelopeHeader
return json.Marshal((*header)(h))
}
// NewEnvelopeItem creates a new envelope item with the specified type and payload.
func NewEnvelopeItem(itemType EnvelopeItemType, payload []byte) *EnvelopeItem {
length := len(payload)
return &EnvelopeItem{
Header: &EnvelopeItemHeader{
Type: itemType,
Length: &length,
},
Payload: payload,
}
}
// NewAttachmentItem creates a new envelope item for an attachment.
// Parameters: filename, contentType, payload.
func NewAttachmentItem(filename, contentType string, payload []byte) *EnvelopeItem {
length := len(payload)
return &EnvelopeItem{
Header: &EnvelopeItemHeader{
Type: EnvelopeItemTypeAttachment,
Length: &length,
ContentType: contentType,
Filename: filename,
},
Payload: payload,
}
}
// NewLogItem creates a new envelope item for logs.
func NewLogItem(itemCount int, payload []byte) *EnvelopeItem {
length := len(payload)
return &EnvelopeItem{
Header: &EnvelopeItemHeader{
Type: EnvelopeItemTypeLog,
Length: &length,
ItemCount: &itemCount,
ContentType: "application/vnd.sentry.items.log+json",
},
Payload: payload,
}
}

View file

@ -0,0 +1,40 @@
package protocol
import (
"context"
"time"
"github.com/getsentry/sentry-go/internal/ratelimit"
)
// EnvelopeConvertible represents any type that can be converted to a Sentry envelope.
// This interface allows the telemetry buffers to be generic while still working with
// concrete types like Event.
type EnvelopeConvertible interface {
// ToEnvelope converts the item to a Sentry envelope.
ToEnvelope(dsn *Dsn) (*Envelope, error)
}
// TelemetryTransport represents the envelope-first transport interface.
// This interface is designed for the telemetry buffer system and provides
// non-blocking sends with backpressure signals.
type TelemetryTransport interface {
// SendEnvelope sends an envelope to Sentry. Returns immediately with
// backpressure error if the queue is full.
SendEnvelope(envelope *Envelope) error
// SendEvent sends an event to Sentry.
SendEvent(event EnvelopeConvertible)
// IsRateLimited checks if a specific category is currently rate limited
IsRateLimited(category ratelimit.Category) bool
// Flush waits for all pending envelopes to be sent, with timeout
Flush(timeout time.Duration) bool
// FlushWithContext waits for all pending envelopes to be sent
FlushWithContext(ctx context.Context) bool
// Close shuts down the transport gracefully
Close()
}

View file

@ -0,0 +1,15 @@
package protocol
// SdkInfo contains SDK metadata.
type SdkInfo struct {
Name string `json:"name,omitempty"`
Version string `json:"version,omitempty"`
Integrations []string `json:"integrations,omitempty"`
Packages []SdkPackage `json:"packages,omitempty"`
}
// SdkPackage describes a package that was installed.
type SdkPackage struct {
Name string `json:"name,omitempty"`
Version string `json:"version,omitempty"`
}

View file

@ -57,3 +57,47 @@ func (c Category) String() string {
return rv return rv
} }
} }
// Priority represents the importance level of a category for buffer management.
type Priority int
const (
PriorityCritical Priority = iota + 1
PriorityHigh
PriorityMedium
PriorityLow
PriorityLowest
)
func (p Priority) String() string {
switch p {
case PriorityCritical:
return "critical"
case PriorityHigh:
return "high"
case PriorityMedium:
return "medium"
case PriorityLow:
return "low"
case PriorityLowest:
return "lowest"
default:
return "unknown"
}
}
// GetPriority returns the priority level for this category.
func (c Category) GetPriority() Priority {
switch c {
case CategoryError:
return PriorityCritical
case CategoryMonitor:
return PriorityHigh
case CategoryLog:
return PriorityMedium
case CategoryTransaction:
return PriorityLow
default:
return PriorityMedium
}
}

View file

@ -10,6 +10,7 @@ import (
"time" "time"
"github.com/getsentry/sentry-go/attribute" "github.com/getsentry/sentry-go/attribute"
"github.com/getsentry/sentry-go/internal/debuglog"
) )
type LogLevel string type LogLevel string
@ -74,7 +75,7 @@ func NewLogger(ctx context.Context) Logger {
} }
} }
DebugLogger.Println("fallback to noopLogger: enableLogs disabled") debuglog.Println("fallback to noopLogger: enableLogs disabled")
return &noopLogger{} // fallback: does nothing return &noopLogger{} // fallback: does nothing
} }
@ -89,20 +90,30 @@ func (l *sentryLogger) log(ctx context.Context, level LogLevel, severity int, me
if message == "" { if message == "" {
return return
} }
hub := GetHubFromContext(ctx)
if hub == nil {
hub = CurrentHub()
}
var traceID TraceID var traceID TraceID
var spanID SpanID var spanID SpanID
var span *Span var span *Span
var user User var user User
span = SpanFromContext(ctx)
if span == nil {
span = SpanFromContext(l.ctx)
}
hub := GetHubFromContext(ctx)
if hub == nil {
hub = GetHubFromContext(l.ctx)
}
if hub == nil {
hub = CurrentHub()
}
scope := hub.Scope() scope := hub.Scope()
if scope != nil { if scope != nil {
scope.mu.Lock() scope.mu.Lock()
span = scope.span // Use span from hub only as last resort
if span == nil {
span = scope.span
}
if span != nil { if span != nil {
traceID = span.TraceID traceID = span.TraceID
spanID = span.SpanID spanID = span.SpanID
@ -187,7 +198,7 @@ func (l *sentryLogger) log(ctx context.Context, level LogLevel, severity int, me
} }
if l.client.options.Debug { if l.client.options.Debug {
DebugLogger.Printf(message, args...) debuglog.Printf(message, args...)
} }
} }
@ -198,7 +209,7 @@ func (l *sentryLogger) SetAttributes(attrs ...attribute.Builder) {
for _, v := range attrs { for _, v := range attrs {
t, ok := mapTypesToStr[v.Value.Type()] t, ok := mapTypesToStr[v.Value.Type()]
if !ok || t == "" { if !ok || t == "" {
DebugLogger.Printf("invalid attribute type set: %v", t) debuglog.Printf("invalid attribute type set: %v", t)
continue continue
} }

View file

@ -6,6 +6,7 @@ import (
"os" "os"
"github.com/getsentry/sentry-go/attribute" "github.com/getsentry/sentry-go/attribute"
"github.com/getsentry/sentry-go/internal/debuglog"
) )
// Fallback, no-op logger if logging is disabled. // Fallback, no-op logger if logging is disabled.
@ -46,7 +47,7 @@ func (n *noopLogEntry) Attributes(_ ...attribute.Builder) LogEntry {
} }
func (n *noopLogEntry) Emit(args ...interface{}) { func (n *noopLogEntry) Emit(args ...interface{}) {
DebugLogger.Printf("Log with level=[%v] is being dropped. Turn on logging via EnableLogs", n.level) debuglog.Printf("Log with level=[%v] is being dropped. Turn on logging via EnableLogs", n.level)
if n.level == LogLevelFatal { if n.level == LogLevelFatal {
if n.shouldPanic { if n.shouldPanic {
panic(args) panic(args)
@ -56,7 +57,7 @@ func (n *noopLogEntry) Emit(args ...interface{}) {
} }
func (n *noopLogEntry) Emitf(message string, args ...interface{}) { func (n *noopLogEntry) Emitf(message string, args ...interface{}) {
DebugLogger.Printf("Log with level=[%v] is being dropped. Turn on logging via EnableLogs", n.level) debuglog.Printf("Log with level=[%v] is being dropped. Turn on logging via EnableLogs", n.level)
if n.level == LogLevelFatal { if n.level == LogLevelFatal {
if n.shouldPanic { if n.shouldPanic {
panic(fmt.Sprintf(message, args...)) panic(fmt.Sprintf(message, args...))
@ -96,7 +97,7 @@ func (*noopLogger) Panic() LogEntry {
} }
func (*noopLogger) SetAttributes(...attribute.Builder) { func (*noopLogger) SetAttributes(...attribute.Builder) {
DebugLogger.Printf("No attributes attached. Turn on logging via EnableLogs") debuglog.Printf("No attributes attached. Turn on logging via EnableLogs")
} }
func (*noopLogger) Write(_ []byte) (n int, err error) { func (*noopLogger) Write(_ []byte) (n int, err error) {

View file

@ -6,6 +6,8 @@ import (
"net/http" "net/http"
"sync" "sync"
"time" "time"
"github.com/getsentry/sentry-go/internal/debuglog"
) )
// Scope holds contextual data for the current scope. // Scope holds contextual data for the current scope.
@ -472,7 +474,7 @@ func (scope *Scope) ApplyToEvent(event *Event, hint *EventHint, client *Client)
id := event.EventID id := event.EventID
event = processor(event, hint) event = processor(event, hint)
if event == nil { if event == nil {
DebugLogger.Printf("Event dropped by one of the Scope EventProcessors: %s\n", id) debuglog.Printf("Event dropped by one of the Scope EventProcessors: %s\n", id)
return nil return nil
} }
} }

View file

@ -6,7 +6,7 @@ import (
) )
// The version of the SDK. // The version of the SDK.
const SDKVersion = "0.35.3" const SDKVersion = "0.36.2"
// apiVersion is the minimum version of the Sentry API compatible with the // apiVersion is the minimum version of the Sentry API compatible with the
// sentry-go SDK. // sentry-go SDK.

View file

@ -2,6 +2,8 @@ package sentry
import ( import (
"sync" "sync"
"github.com/getsentry/sentry-go/internal/debuglog"
) )
// A spanRecorder stores a span tree that makes up a transaction. Safe for // A spanRecorder stores a span tree that makes up a transaction. Safe for
@ -24,7 +26,7 @@ func (r *spanRecorder) record(s *Span) {
if len(r.spans) >= maxSpans { if len(r.spans) >= maxSpans {
r.overflowOnce.Do(func() { r.overflowOnce.Do(func() {
root := r.spans[0] root := r.spans[0]
DebugLogger.Printf("Too many spans: dropping spans from transaction with TraceID=%s SpanID=%s limit=%d", debuglog.Printf("Too many spans: dropping spans from transaction with TraceID=%s SpanID=%s limit=%d",
root.TraceID, root.SpanID, maxSpans) root.TraceID, root.SpanID, maxSpans)
}) })
// TODO(tracing): mark the transaction event in some way to // TODO(tracing): mark the transaction event in some way to

View file

@ -12,6 +12,8 @@ import (
"strings" "strings"
"sync" "sync"
"time" "time"
"github.com/getsentry/sentry-go/internal/debuglog"
) )
const ( const (
@ -347,6 +349,58 @@ func (s *Span) SetDynamicSamplingContext(dsc DynamicSamplingContext) {
} }
} }
// shouldIgnoreStatusCode checks if the transaction should be ignored based on HTTP status code.
func (s *Span) shouldIgnoreStatusCode() bool {
if !s.IsTransaction() {
return false
}
ignoreStatusCodes := s.clientOptions().TraceIgnoreStatusCodes
if len(ignoreStatusCodes) == 0 {
return false
}
s.mu.Lock()
statusCodeData, exists := s.Data["http.response.status_code"]
s.mu.Unlock()
if !exists {
return false
}
statusCode, ok := statusCodeData.(int)
if !ok {
return false
}
for _, ignoredRange := range ignoreStatusCodes {
switch len(ignoredRange) {
case 1:
// Single status code
if statusCode == ignoredRange[0] {
s.mu.Lock()
s.Sampled = SampledFalse
s.mu.Unlock()
debuglog.Printf("dropping transaction with status code %v: found in TraceIgnoreStatusCodes", statusCode)
return true
}
case 2:
// Range of status codes [min, max]
if ignoredRange[0] <= statusCode && statusCode <= ignoredRange[1] {
s.mu.Lock()
s.Sampled = SampledFalse
s.mu.Unlock()
debuglog.Printf("dropping transaction with status code %v: found in TraceIgnoreStatusCodes range [%d, %d]", statusCode, ignoredRange[0], ignoredRange[1])
return true
}
default:
debuglog.Printf("incorrect TraceIgnoreStatusCodes format: %v", ignoredRange)
}
}
return false
}
// doFinish runs the actual Span.Finish() logic. // doFinish runs the actual Span.Finish() logic.
func (s *Span) doFinish() { func (s *Span) doFinish() {
if s.EndTime.IsZero() { if s.EndTime.IsZero() {
@ -360,6 +414,10 @@ func (s *Span) doFinish() {
} }
} }
if s.shouldIgnoreStatusCode() {
return
}
if !s.Sampled.Bool() { if !s.Sampled.Bool() {
return return
} }
@ -449,14 +507,14 @@ func (s *Span) sample() Sampled {
// https://develop.sentry.dev/sdk/performance/#sampling // https://develop.sentry.dev/sdk/performance/#sampling
// #1 tracing is not enabled. // #1 tracing is not enabled.
if !clientOptions.EnableTracing { if !clientOptions.EnableTracing {
DebugLogger.Printf("Dropping transaction: EnableTracing is set to %t", clientOptions.EnableTracing) debuglog.Printf("Dropping transaction: EnableTracing is set to %t", clientOptions.EnableTracing)
s.sampleRate = 0.0 s.sampleRate = 0.0
return SampledFalse return SampledFalse
} }
// #2 explicit sampling decision via StartSpan/StartTransaction options. // #2 explicit sampling decision via StartSpan/StartTransaction options.
if s.explicitSampled != SampledUndefined { if s.explicitSampled != SampledUndefined {
DebugLogger.Printf("Using explicit sampling decision from StartSpan/StartTransaction: %v", s.explicitSampled) debuglog.Printf("Using explicit sampling decision from StartSpan/StartTransaction: %v", s.explicitSampled)
switch s.explicitSampled { switch s.explicitSampled {
case SampledTrue: case SampledTrue:
s.sampleRate = 1.0 s.sampleRate = 1.0
@ -489,25 +547,25 @@ func (s *Span) sample() Sampled {
s.dynamicSamplingContext.Entries["sample_rate"] = strconv.FormatFloat(tracesSamplerSampleRate, 'f', -1, 64) s.dynamicSamplingContext.Entries["sample_rate"] = strconv.FormatFloat(tracesSamplerSampleRate, 'f', -1, 64)
} }
if tracesSamplerSampleRate < 0.0 || tracesSamplerSampleRate > 1.0 { if tracesSamplerSampleRate < 0.0 || tracesSamplerSampleRate > 1.0 {
DebugLogger.Printf("Dropping transaction: Returned TracesSampler rate is out of range [0.0, 1.0]: %f", tracesSamplerSampleRate) debuglog.Printf("Dropping transaction: Returned TracesSampler rate is out of range [0.0, 1.0]: %f", tracesSamplerSampleRate)
return SampledFalse return SampledFalse
} }
if tracesSamplerSampleRate == 0.0 { if tracesSamplerSampleRate == 0.0 {
DebugLogger.Printf("Dropping transaction: Returned TracesSampler rate is: %f", tracesSamplerSampleRate) debuglog.Printf("Dropping transaction: Returned TracesSampler rate is: %f", tracesSamplerSampleRate)
return SampledFalse return SampledFalse
} }
if rng.Float64() < tracesSamplerSampleRate { if rng.Float64() < tracesSamplerSampleRate {
return SampledTrue return SampledTrue
} }
DebugLogger.Printf("Dropping transaction: TracesSampler returned rate: %f", tracesSamplerSampleRate) debuglog.Printf("Dropping transaction: TracesSampler returned rate: %f", tracesSamplerSampleRate)
return SampledFalse return SampledFalse
} }
// #4 inherit parent decision. // #4 inherit parent decision.
if s.Sampled != SampledUndefined { if s.Sampled != SampledUndefined {
DebugLogger.Printf("Using sampling decision from parent: %v", s.Sampled) debuglog.Printf("Using sampling decision from parent: %v", s.Sampled)
switch s.Sampled { switch s.Sampled {
case SampledTrue: case SampledTrue:
s.sampleRate = 1.0 s.sampleRate = 1.0
@ -525,11 +583,11 @@ func (s *Span) sample() Sampled {
s.dynamicSamplingContext.Entries["sample_rate"] = strconv.FormatFloat(sampleRate, 'f', -1, 64) s.dynamicSamplingContext.Entries["sample_rate"] = strconv.FormatFloat(sampleRate, 'f', -1, 64)
} }
if sampleRate < 0.0 || sampleRate > 1.0 { if sampleRate < 0.0 || sampleRate > 1.0 {
DebugLogger.Printf("Dropping transaction: TracesSampleRate out of range [0.0, 1.0]: %f", sampleRate) debuglog.Printf("Dropping transaction: TracesSampleRate out of range [0.0, 1.0]: %f", sampleRate)
return SampledFalse return SampledFalse
} }
if sampleRate == 0.0 { if sampleRate == 0.0 {
DebugLogger.Printf("Dropping transaction: TracesSampleRate rate is: %f", sampleRate) debuglog.Printf("Dropping transaction: TracesSampleRate rate is: %f", sampleRate)
return SampledFalse return SampledFalse
} }
@ -552,7 +610,7 @@ func (s *Span) toEvent() *Event {
finished := make([]*Span, 0, len(children)) finished := make([]*Span, 0, len(children))
for _, child := range children { for _, child := range children {
if child.EndTime.IsZero() { if child.EndTime.IsZero() {
DebugLogger.Printf("Dropped unfinished span: Op=%q TraceID=%s SpanID=%s", child.Op, child.TraceID, child.SpanID) debuglog.Printf("Dropped unfinished span: Op=%q TraceID=%s SpanID=%s", child.Op, child.TraceID, child.SpanID)
continue continue
} }
finished = append(finished, child) finished = append(finished, child)

View file

@ -13,6 +13,9 @@ import (
"sync" "sync"
"time" "time"
"github.com/getsentry/sentry-go/internal/debuglog"
httpinternal "github.com/getsentry/sentry-go/internal/http"
"github.com/getsentry/sentry-go/internal/protocol"
"github.com/getsentry/sentry-go/internal/ratelimit" "github.com/getsentry/sentry-go/internal/ratelimit"
) )
@ -87,14 +90,14 @@ func getRequestBodyFromEvent(event *Event) []byte {
} }
body, err = json.Marshal(event) body, err = json.Marshal(event)
if err == nil { if err == nil {
DebugLogger.Println(msg) debuglog.Println(msg)
return body return body
} }
// This should _only_ happen when Event.Exception[0].Stacktrace.Frames[0].Vars is unserializable // This should _only_ happen when Event.Exception[0].Stacktrace.Frames[0].Vars is unserializable
// Which won't ever happen, as we don't use it now (although it's the part of public interface accepted by Sentry) // Which won't ever happen, as we don't use it now (although it's the part of public interface accepted by Sentry)
// Juuust in case something, somehow goes utterly wrong. // Juuust in case something, somehow goes utterly wrong.
DebugLogger.Println("Event couldn't be marshaled, even with stripped contextual data. Skipping delivery. " + debuglog.Println("Event couldn't be marshaled, even with stripped contextual data. Skipping delivery. " +
"Please notify the SDK owners with possibly broken payload.") "Please notify the SDK owners with possibly broken payload.")
return nil return nil
} }
@ -227,13 +230,13 @@ func getRequestFromEvent(ctx context.Context, event *Event, dsn *Dsn) (r *http.R
r.Header.Set("Content-Type", "application/x-sentry-envelope") r.Header.Set("Content-Type", "application/x-sentry-envelope")
auth := fmt.Sprintf("Sentry sentry_version=%s, "+ auth := fmt.Sprintf("Sentry sentry_version=%s, "+
"sentry_client=%s/%s, sentry_key=%s", apiVersion, event.Sdk.Name, event.Sdk.Version, dsn.publicKey) "sentry_client=%s/%s, sentry_key=%s", apiVersion, event.Sdk.Name, event.Sdk.Version, dsn.GetPublicKey())
// The key sentry_secret is effectively deprecated and no longer needs to be set. // The key sentry_secret is effectively deprecated and no longer needs to be set.
// However, since it was required in older self-hosted versions, // However, since it was required in older self-hosted versions,
// it should still passed through to Sentry if set. // it should still passed through to Sentry if set.
if dsn.secretKey != "" { if dsn.GetSecretKey() != "" {
auth = fmt.Sprintf("%s, sentry_secret=%s", auth, dsn.secretKey) auth = fmt.Sprintf("%s, sentry_secret=%s", auth, dsn.GetSecretKey())
} }
r.Header.Set("X-Sentry-Auth", auth) r.Header.Set("X-Sentry-Auth", auth)
@ -321,7 +324,7 @@ func NewHTTPTransport() *HTTPTransport {
func (t *HTTPTransport) Configure(options ClientOptions) { func (t *HTTPTransport) Configure(options ClientOptions) {
dsn, err := NewDsn(options.Dsn) dsn, err := NewDsn(options.Dsn)
if err != nil { if err != nil {
DebugLogger.Printf("%v\n", err) debuglog.Printf("%v\n", err)
return return
} }
t.dsn = dsn t.dsn = dsn
@ -403,17 +406,17 @@ func (t *HTTPTransport) SendEventWithContext(ctx context.Context, event *Event)
if event.Type == transactionType { if event.Type == transactionType {
eventType = "transaction" eventType = "transaction"
} else { } else {
eventType = fmt.Sprintf("%s event", event.Level) eventType = fmt.Sprintf("%s event", event.Type)
} }
DebugLogger.Printf( debuglog.Printf(
"Sending %s [%s] to %s project: %s", "Sending %s [%s] to %s project: %s",
eventType, eventType,
event.EventID, event.EventID,
t.dsn.host, t.dsn.GetHost(),
t.dsn.projectID, t.dsn.GetProjectID(),
) )
default: default:
DebugLogger.Println("Event dropped due to transport buffer being full.") debuglog.Println("Event dropped due to transport buffer being full.")
} }
t.buffer <- b t.buffer <- b
@ -477,14 +480,14 @@ started:
// Wait until the current batch is done or the timeout. // Wait until the current batch is done or the timeout.
select { select {
case <-b.done: case <-b.done:
DebugLogger.Println("Buffer flushed successfully.") debuglog.Println("Buffer flushed successfully.")
return true return true
case <-timeout: case <-timeout:
goto fail goto fail
} }
fail: fail:
DebugLogger.Println("Buffer flushing was canceled or timed out.") debuglog.Println("Buffer flushing was canceled or timed out.")
return false return false
} }
@ -524,15 +527,15 @@ func (t *HTTPTransport) worker() {
response, err := t.client.Do(item.request) response, err := t.client.Do(item.request)
if err != nil { if err != nil {
DebugLogger.Printf("There was an issue with sending an event: %v", err) debuglog.Printf("There was an issue with sending an event: %v", err)
continue continue
} }
if response.StatusCode >= 400 && response.StatusCode <= 599 { if response.StatusCode >= 400 && response.StatusCode <= 599 {
b, err := io.ReadAll(response.Body) b, err := io.ReadAll(response.Body)
if err != nil { if err != nil {
DebugLogger.Printf("Error while reading response code: %v", err) debuglog.Printf("Error while reading response code: %v", err)
} }
DebugLogger.Printf("Sending %s failed with the following error: %s", eventType, string(b)) debuglog.Printf("Sending %s failed with the following error: %s", eventType, string(b))
} }
t.mu.Lock() t.mu.Lock()
@ -559,7 +562,7 @@ func (t *HTTPTransport) disabled(c ratelimit.Category) bool {
defer t.mu.RUnlock() defer t.mu.RUnlock()
disabled := t.limits.IsRateLimited(c) disabled := t.limits.IsRateLimited(c)
if disabled { if disabled {
DebugLogger.Printf("Too many requests for %q, backing off till: %v", c, t.limits.Deadline(c)) debuglog.Printf("Too many requests for %q, backing off till: %v", c, t.limits.Deadline(c))
} }
return disabled return disabled
} }
@ -605,7 +608,7 @@ func NewHTTPSyncTransport() *HTTPSyncTransport {
func (t *HTTPSyncTransport) Configure(options ClientOptions) { func (t *HTTPSyncTransport) Configure(options ClientOptions) {
dsn, err := NewDsn(options.Dsn) dsn, err := NewDsn(options.Dsn)
if err != nil { if err != nil {
DebugLogger.Printf("%v\n", err) debuglog.Printf("%v\n", err)
return return
} }
t.dsn = dsn t.dsn = dsn
@ -660,25 +663,25 @@ func (t *HTTPSyncTransport) SendEventWithContext(ctx context.Context, event *Eve
default: default:
eventIdentifier = fmt.Sprintf("%s event", event.Level) eventIdentifier = fmt.Sprintf("%s event", event.Level)
} }
DebugLogger.Printf( debuglog.Printf(
"Sending %s [%s] to %s project: %s", "Sending %s [%s] to %s project: %s",
eventIdentifier, eventIdentifier,
event.EventID, event.EventID,
t.dsn.host, t.dsn.GetHost(),
t.dsn.projectID, t.dsn.GetProjectID(),
) )
response, err := t.client.Do(request) response, err := t.client.Do(request)
if err != nil { if err != nil {
DebugLogger.Printf("There was an issue with sending an event: %v", err) debuglog.Printf("There was an issue with sending an event: %v", err)
return return
} }
if response.StatusCode >= 400 && response.StatusCode <= 599 { if response.StatusCode >= 400 && response.StatusCode <= 599 {
b, err := io.ReadAll(response.Body) b, err := io.ReadAll(response.Body)
if err != nil { if err != nil {
DebugLogger.Printf("Error while reading response code: %v", err) debuglog.Printf("Error while reading response code: %v", err)
} }
DebugLogger.Printf("Sending %s failed with the following error: %s", eventIdentifier, string(b)) debuglog.Printf("Sending %s failed with the following error: %s", eventIdentifier, string(b))
} }
t.mu.Lock() t.mu.Lock()
@ -710,7 +713,7 @@ func (t *HTTPSyncTransport) disabled(c ratelimit.Category) bool {
defer t.mu.Unlock() defer t.mu.Unlock()
disabled := t.limits.IsRateLimited(c) disabled := t.limits.IsRateLimited(c)
if disabled { if disabled {
DebugLogger.Printf("Too many requests for %q, backing off till: %v", c, t.limits.Deadline(c)) debuglog.Printf("Too many requests for %q, backing off till: %v", c, t.limits.Deadline(c))
} }
return disabled return disabled
} }
@ -726,11 +729,11 @@ type noopTransport struct{}
var _ Transport = noopTransport{} var _ Transport = noopTransport{}
func (noopTransport) Configure(ClientOptions) { func (noopTransport) Configure(ClientOptions) {
DebugLogger.Println("Sentry client initialized with an empty DSN. Using noopTransport. No events will be delivered.") debuglog.Println("Sentry client initialized with an empty DSN. Using noopTransport. No events will be delivered.")
} }
func (noopTransport) SendEvent(*Event) { func (noopTransport) SendEvent(*Event) {
DebugLogger.Println("Event dropped due to noopTransport usage.") debuglog.Println("Event dropped due to noopTransport usage.")
} }
func (noopTransport) Flush(time.Duration) bool { func (noopTransport) Flush(time.Duration) bool {
@ -742,3 +745,60 @@ func (noopTransport) FlushWithContext(context.Context) bool {
} }
func (noopTransport) Close() {} func (noopTransport) Close() {}
// ================================
// Internal Transport Adapters
// ================================
// newInternalAsyncTransport creates a new AsyncTransport from internal/http
// wrapped to satisfy the Transport interface.
//
// This is not yet exposed in the public API and is for internal experimentation.
func newInternalAsyncTransport() Transport {
return &internalAsyncTransportAdapter{}
}
// internalAsyncTransportAdapter wraps the internal AsyncTransport to implement
// the root-level Transport interface.
type internalAsyncTransportAdapter struct {
transport protocol.TelemetryTransport
dsn *protocol.Dsn
}
func (a *internalAsyncTransportAdapter) Configure(options ClientOptions) {
transportOptions := httpinternal.TransportOptions{
Dsn: options.Dsn,
HTTPClient: options.HTTPClient,
HTTPTransport: options.HTTPTransport,
HTTPProxy: options.HTTPProxy,
HTTPSProxy: options.HTTPSProxy,
CaCerts: options.CaCerts,
}
a.transport = httpinternal.NewAsyncTransport(transportOptions)
if options.Dsn != "" {
dsn, err := protocol.NewDsn(options.Dsn)
if err != nil {
debuglog.Printf("Failed to parse DSN in adapter: %v\n", err)
} else {
a.dsn = dsn
}
}
}
func (a *internalAsyncTransportAdapter) SendEvent(event *Event) {
a.transport.SendEvent(event)
}
func (a *internalAsyncTransportAdapter) Flush(timeout time.Duration) bool {
return a.transport.Flush(timeout)
}
func (a *internalAsyncTransportAdapter) FlushWithContext(ctx context.Context) bool {
return a.transport.FlushWithContext(ctx)
}
func (a *internalAsyncTransportAdapter) Close() {
a.transport.Close()
}

View file

@ -10,6 +10,7 @@ import (
"strings" "strings"
"time" "time"
"github.com/getsentry/sentry-go/internal/debuglog"
exec "golang.org/x/sys/execabs" exec "golang.org/x/sys/execabs"
) )
@ -62,7 +63,7 @@ func defaultRelease() (release string) {
} }
for _, e := range envs { for _, e := range envs {
if release = os.Getenv(e); release != "" { if release = os.Getenv(e); release != "" {
DebugLogger.Printf("Using release from environment variable %s: %s", e, release) debuglog.Printf("Using release from environment variable %s: %s", e, release)
return release return release
} }
} }
@ -89,23 +90,23 @@ func defaultRelease() (release string) {
if err, ok := err.(*exec.ExitError); ok && len(err.Stderr) > 0 { if err, ok := err.(*exec.ExitError); ok && len(err.Stderr) > 0 {
fmt.Fprintf(&s, ": %s", err.Stderr) fmt.Fprintf(&s, ": %s", err.Stderr)
} }
DebugLogger.Print(s.String()) debuglog.Print(s.String())
} else { } else {
release = strings.TrimSpace(string(b)) release = strings.TrimSpace(string(b))
DebugLogger.Printf("Using release from Git: %s", release) debuglog.Printf("Using release from Git: %s", release)
return release return release
} }
} }
DebugLogger.Print("Some Sentry features will not be available. See https://docs.sentry.io/product/releases/.") debuglog.Print("Some Sentry features will not be available. See https://docs.sentry.io/product/releases/.")
DebugLogger.Print("To stop seeing this message, pass a Release to sentry.Init or set the SENTRY_RELEASE environment variable.") debuglog.Print("To stop seeing this message, pass a Release to sentry.Init or set the SENTRY_RELEASE environment variable.")
return "" return ""
} }
func revisionFromBuildInfo(info *debug.BuildInfo) string { func revisionFromBuildInfo(info *debug.BuildInfo) string {
for _, setting := range info.Settings { for _, setting := range info.Settings {
if setting.Key == "vcs.revision" && setting.Value != "" { if setting.Key == "vcs.revision" && setting.Value != "" {
DebugLogger.Printf("Using release from debug info: %s", setting.Value) debuglog.Printf("Using release from debug info: %s", setting.Value)
return setting.Value return setting.Value
} }
} }

View file

@ -163,7 +163,7 @@ func (o *Inspector) structToMap(val reflect.Value) map[string]interface{} {
case reflect.Bool: case reflect.Bool:
result[fieldName] = *(*bool)(ptr) result[fieldName] = *(*bool)(ptr)
default: default:
result[fieldName] = fmt.Sprintf("<unexported %s>", field.Type().String()) result[fieldName] = fmt.Sprintf("*unexported %s*", field.Type().String())
} }
} }
} }

View file

@ -495,7 +495,7 @@ func (l *Logger) Fatal(args ...any) {
os.Exit(1) os.Exit(1)
} }
l.log(lx.LevelError, lx.ClassText, cat.Space(args...), nil, true) l.log(lx.LevelError, lx.ClassText, cat.Space(args...), nil, false)
os.Exit(1) os.Exit(1)
} }

View file

@ -11,65 +11,75 @@ import (
var Functions = make(map[string]any) var Functions = make(map[string]any)
func setPragmas(conn *sqlite3.SQLiteConn, pragmas ...string) (err error) {
for _, pragma := range pragmas {
if _, err = conn.Exec(pragma, []driver.Value{}); err != nil {
return
}
}
return
}
func registerFuncsAndSetTrace(conn *sqlite3.SQLiteConn) (err error) {
for name, fn := range Functions {
err = conn.RegisterFunc(name, fn, true)
if err != nil {
return
}
}
err = DoSetTrace(conn)
return
}
func init() { func init() {
sql.Register("litestream", &sqlite3.SQLiteDriver{ sql.Register("litestream", &sqlite3.SQLiteDriver{
ConnectHook: func(conn *sqlite3.SQLiteConn) (err error) { ConnectHook: func(conn *sqlite3.SQLiteConn) (err error) {
for name, fn := range Functions { if err = registerFuncsAndSetTrace(conn); err != nil {
err = conn.RegisterFunc(name, fn, true)
if err != nil {
return
}
}
err = DoSetTrace(conn)
if err != nil {
return return
} }
if err = conn.SetFileControlInt("main", sqlite3.SQLITE_FCNTL_PERSIST_WAL, 1); err != nil { if err = conn.SetFileControlInt("main", sqlite3.SQLITE_FCNTL_PERSIST_WAL, 1); err != nil {
return return
} }
if _, err = conn.Exec("PRAGMA foreign_keys = ON", []driver.Value{}); err != nil { err = setPragmas(
return conn,
} "PRAGMA foreign_keys = ON",
if _, err = conn.Exec("PRAGMA journal_mode = WAL", []driver.Value{}); err != nil { "PRAGMA journal_mode = WAL",
return "PRAGMA wal_autocheckpoint = 0",
} "PRAGMA synchronous = NORMAL",
if _, err = conn.Exec("PRAGMA wal_autocheckpoint = 0", []driver.Value{}); err != nil { "PRAGMA busy_timeout = 5000",
return )
}
if _, err = conn.Exec("PRAGMA synchronous = NORMAL", []driver.Value{}); err != nil {
return
}
if _, err = conn.Exec("PRAGMA busy_timeout = 5000", []driver.Value{}); err != nil {
return
}
return return
}, },
}) })
sql.Register("sqlite3-fk-wal", &sqlite3.SQLiteDriver{ sql.Register("sqlite3-fk-wal", &sqlite3.SQLiteDriver{
ConnectHook: func(conn *sqlite3.SQLiteConn) (err error) { ConnectHook: func(conn *sqlite3.SQLiteConn) (err error) {
for name, fn := range Functions { if err = registerFuncsAndSetTrace(conn); err != nil {
err = conn.RegisterFunc(name, fn, true)
if err != nil {
return
}
}
err = DoSetTrace(conn)
if err != nil {
return return
} }
if _, err = conn.Exec("PRAGMA foreign_keys = ON", []driver.Value{}); err != nil { err = setPragmas(
return conn,
} "PRAGMA foreign_keys = ON",
if _, err = conn.Exec("PRAGMA journal_mode = WAL", []driver.Value{}); err != nil { "PRAGMA journal_mode = WAL",
return "PRAGMA synchronous = NORMAL",
} "PRAGMA busy_timeout = 5000",
if _, err = conn.Exec("PRAGMA synchronous = NORMAL", []driver.Value{}); err != nil { )
return return
} },
if _, err = conn.Exec("PRAGMA busy_timeout = 5000", []driver.Value{}); err != nil { })
sql.Register("sqlite3-fk-wal-fullsync", &sqlite3.SQLiteDriver{
ConnectHook: func(conn *sqlite3.SQLiteConn) (err error) {
if err = registerFuncsAndSetTrace(conn); err != nil {
return return
} }
err = setPragmas(
conn,
"PRAGMA foreign_keys = ON",
"PRAGMA journal_mode = WAL",
"PRAGMA synchronous = FULL",
"PRAGMA busy_timeout = 5000",
)
return return
}, },
}) })

View file

@ -114,5 +114,6 @@
// whole file is limited to that dialect. // whole file is limited to that dialect.
// //
// If the filter ends with `(lines commented)`, then ALL lines chosen // If the filter ends with `(lines commented)`, then ALL lines chosen
// by the filter will be uncommented. // by the filter will be uncommented. The `--` comment prefix must be
// at the beginning of the line with no whitespace ahead of it.
package dbutil package dbutil

View file

@ -173,6 +173,9 @@ func (db *Database) filterSQLUpgrade(lines [][]byte) (string, error) {
} }
if dialect == db.Dialect { if dialect == db.Dialect {
if uncomment { if uncomment {
if !bytes.HasPrefix(lines[i], []byte("--")) {
return "", fmt.Errorf("line %d isn't commented even though the dialect filter claimed it is (-- must be at beginning of line)", i+1)
}
output = append(output, bytes.TrimPrefix(lines[i], []byte("--"))) output = append(output, bytes.TrimPrefix(lines[i], []byte("--")))
} else { } else {
output = append(output, lines[i]) output = append(output, lines[i])

19
vendor/go.mau.fi/util/exmaps/clone.go vendored Normal file
View file

@ -0,0 +1,19 @@
// Copyright (c) 2025 Tulir Asokan
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
package exmaps
import (
"maps"
)
// NonNilClone returns a copy of the given map, or an empty map if the input is nil.
func NonNilClone[Key comparable, Value any](m map[Key]Value) map[Key]Value {
if m == nil {
return make(map[Key]Value)
}
return maps.Clone(m)
}

View file

@ -6,7 +6,11 @@
package exsync package exsync
import "sync" import (
"iter"
"maps"
"sync"
)
// Map is a simple map with a built-in mutex. // Map is a simple map with a built-in mutex.
type Map[Key comparable, Value any] struct { type Map[Key comparable, Value any] struct {
@ -15,8 +19,13 @@ type Map[Key comparable, Value any] struct {
} }
func NewMap[Key comparable, Value any]() *Map[Key, Value] { func NewMap[Key comparable, Value any]() *Map[Key, Value] {
return NewMapWithData(make(map[Key]Value))
}
// NewMapWithData constructs a Map with the given map as data. Accessing the map directly after passing it here is not safe.
func NewMapWithData[Key comparable, Value any](data map[Key]Value) *Map[Key, Value] {
return &Map[Key, Value]{ return &Map[Key, Value]{
data: make(map[Key]Value), data: data,
} }
} }
@ -77,18 +86,50 @@ func (sm *Map[Key, Value]) GetOrSet(key Key, value Value) (actual Value, wasGet
return return
} }
// Clear removes all items from the map.
func (sm *Map[Key, Value]) Clear() {
sm.lock.Lock()
clear(sm.data)
sm.lock.Unlock()
}
// Len returns the number of items in the map.
func (sm *Map[Key, Value]) Len() int {
sm.lock.RLock()
l := len(sm.data)
sm.lock.RUnlock()
return l
}
// CopyFrom copies all key/value pairs from the given map into this map, overriding any existing keys.
// Keys present in this map but not in the given map are not removed.
func (sm *Map[Key, Value]) CopyFrom(other map[Key]Value) {
sm.lock.Lock()
maps.Copy(sm.data, other)
sm.lock.Unlock()
}
// Clone returns a copy of the map. // Clone returns a copy of the map.
func (sm *Map[Key, Value]) Clone() *Map[Key, Value] { func (sm *Map[Key, Value]) Clone() *Map[Key, Value] {
return &Map[Key, Value]{data: sm.CopyData()} return NewMapWithData(sm.CopyData())
} }
// CopyData returns a copy of the data in the map as a normal (non-atomic) map. // CopyData returns a copy of the data in the map as a normal (non-atomic) map.
func (sm *Map[Key, Value]) CopyData() map[Key]Value { func (sm *Map[Key, Value]) CopyData() map[Key]Value {
sm.lock.RLock() sm.lock.RLock()
copied := make(map[Key]Value, len(sm.data)) copied := maps.Clone(sm.data)
for key, value := range sm.data {
copied[key] = value
}
sm.lock.RUnlock() sm.lock.RUnlock()
return copied return copied
} }
func (sm *Map[Key, Value]) Iter() iter.Seq2[Key, Value] {
return func(yield func(Key, Value) bool) {
sm.lock.RLock()
defer sm.lock.RUnlock()
for k, v := range sm.data {
if !yield(k, v) {
return
}
}
}
}

View file

@ -7,6 +7,7 @@
package exsync package exsync
import ( import (
"iter"
"sync" "sync"
) )
@ -134,3 +135,15 @@ func (s *Set[T]) AsList() []T {
s.l.RUnlock() s.l.RUnlock()
return list return list
} }
func (s *Set[T]) Iter() iter.Seq[T] {
return func(yield func(T) bool) {
s.l.RLock()
defer s.l.RUnlock()
for item := range s.m {
if !yield(item) {
return
}
}
}
}

View file

@ -10,12 +10,19 @@ import (
"time" "time"
) )
func zeroSafeUnixToTime(val int64, fn func(int64) time.Time) time.Time {
if val == 0 {
return time.Time{}
}
return fn(val)
}
func UM(time time.Time) UnixMilli { func UM(time time.Time) UnixMilli {
return UnixMilli{Time: time} return UnixMilli{Time: time}
} }
func UMInt(ts int64) UnixMilli { func UMInt(ts int64) UnixMilli {
return UM(time.UnixMilli(ts)) return UM(zeroSafeUnixToTime(ts, time.UnixMilli))
} }
func UnixMilliNow() UnixMilli { func UnixMilliNow() UnixMilli {
@ -26,8 +33,8 @@ func UMicro(time time.Time) UnixMicro {
return UnixMicro{Time: time} return UnixMicro{Time: time}
} }
func UMicroInto(ts int64) UnixMicro { func UMicroInt(ts int64) UnixMicro {
return UMicro(time.UnixMicro(ts)) return UMicro(zeroSafeUnixToTime(ts, time.UnixMicro))
} }
func UnixMicroNow() UnixMicro { func UnixMicroNow() UnixMicro {
@ -39,7 +46,9 @@ func UN(time time.Time) UnixNano {
} }
func UNInt(ts int64) UnixNano { func UNInt(ts int64) UnixNano {
return UN(time.Unix(0, ts)) return UN(zeroSafeUnixToTime(ts, func(i int64) time.Time {
return time.Unix(0, i)
}))
} }
func UnixNanoNow() UnixNano { func UnixNanoNow() UnixNano {
@ -51,7 +60,9 @@ func U(time time.Time) Unix {
} }
func UInt(ts int64) Unix { func UInt(ts int64) Unix {
return U(time.Unix(ts, 0)) return U(zeroSafeUnixToTime(ts, func(i int64) time.Time {
return time.Unix(i, 0)
}))
} }
func UnixNow() Unix { func UnixNow() Unix {

View file

@ -57,6 +57,13 @@ func anyIntegerToTime(src any, unixConv func(int64) time.Time, into *time.Time)
return nil return nil
} }
func zeroSafeUnix(t time.Time, method func(time.Time) int64) int64 {
if t.IsZero() {
return 0
}
return method(t)
}
var _ sql.Scanner = &UnixMilli{} var _ sql.Scanner = &UnixMilli{}
var _ driver.Valuer = UnixMilli{} var _ driver.Valuer = UnixMilli{}
@ -65,9 +72,6 @@ type UnixMilli struct {
} }
func (um UnixMilli) MarshalJSON() ([]byte, error) { func (um UnixMilli) MarshalJSON() ([]byte, error) {
if um.IsZero() {
return []byte{'0'}, nil
}
return json.Marshal(um.UnixMilli()) return json.Marshal(um.UnixMilli())
} }
@ -83,6 +87,11 @@ func (um *UnixMilli) Scan(src any) error {
return anyIntegerToTime(src, time.UnixMilli, &um.Time) return anyIntegerToTime(src, time.UnixMilli, &um.Time)
} }
func (um UnixMilli) Unix() int64 { return zeroSafeUnix(um.Time, time.Time.Unix) }
func (um UnixMilli) UnixMilli() int64 { return zeroSafeUnix(um.Time, time.Time.UnixMilli) }
func (um UnixMilli) UnixMicro() int64 { return zeroSafeUnix(um.Time, time.Time.UnixMicro) }
func (um UnixMilli) UnixNano() int64 { return zeroSafeUnix(um.Time, time.Time.UnixNano) }
var _ sql.Scanner = &UnixMicro{} var _ sql.Scanner = &UnixMicro{}
var _ driver.Valuer = UnixMicro{} var _ driver.Valuer = UnixMicro{}
@ -91,9 +100,6 @@ type UnixMicro struct {
} }
func (um UnixMicro) MarshalJSON() ([]byte, error) { func (um UnixMicro) MarshalJSON() ([]byte, error) {
if um.IsZero() {
return []byte{'0'}, nil
}
return json.Marshal(um.UnixMicro()) return json.Marshal(um.UnixMicro())
} }
@ -109,6 +115,11 @@ func (um *UnixMicro) Scan(src any) error {
return anyIntegerToTime(src, time.UnixMicro, &um.Time) return anyIntegerToTime(src, time.UnixMicro, &um.Time)
} }
func (um UnixMicro) Unix() int64 { return zeroSafeUnix(um.Time, time.Time.Unix) }
func (um UnixMicro) UnixMilli() int64 { return zeroSafeUnix(um.Time, time.Time.UnixMilli) }
func (um UnixMicro) UnixMicro() int64 { return zeroSafeUnix(um.Time, time.Time.UnixMicro) }
func (um UnixMicro) UnixNano() int64 { return zeroSafeUnix(um.Time, time.Time.UnixNano) }
var _ sql.Scanner = &UnixNano{} var _ sql.Scanner = &UnixNano{}
var _ driver.Valuer = UnixNano{} var _ driver.Valuer = UnixNano{}
@ -117,9 +128,6 @@ type UnixNano struct {
} }
func (un UnixNano) MarshalJSON() ([]byte, error) { func (un UnixNano) MarshalJSON() ([]byte, error) {
if un.IsZero() {
return []byte{'0'}, nil
}
return json.Marshal(un.UnixNano()) return json.Marshal(un.UnixNano())
} }
@ -139,14 +147,16 @@ func (un *UnixNano) Scan(src any) error {
}, &un.Time) }, &un.Time)
} }
func (un UnixNano) Unix() int64 { return zeroSafeUnix(un.Time, time.Time.Unix) }
func (un UnixNano) UnixMilli() int64 { return zeroSafeUnix(un.Time, time.Time.UnixMilli) }
func (un UnixNano) UnixMicro() int64 { return zeroSafeUnix(un.Time, time.Time.UnixMicro) }
func (un UnixNano) UnixNano() int64 { return zeroSafeUnix(un.Time, time.Time.UnixNano) }
type Unix struct { type Unix struct {
time.Time time.Time
} }
func (u Unix) MarshalJSON() ([]byte, error) { func (u Unix) MarshalJSON() ([]byte, error) {
if u.IsZero() {
return []byte{'0'}, nil
}
return json.Marshal(u.Unix()) return json.Marshal(u.Unix())
} }
@ -168,3 +178,8 @@ func (u *Unix) Scan(src any) error {
return time.Unix(i, 0) return time.Unix(i, 0)
}, &u.Time) }, &u.Time)
} }
func (u Unix) Unix() int64 { return zeroSafeUnix(u.Time, time.Time.Unix) }
func (u Unix) UnixMilli() int64 { return zeroSafeUnix(u.Time, time.Time.UnixMilli) }
func (u Unix) UnixMicro() int64 { return zeroSafeUnix(u.Time, time.Time.UnixMicro) }
func (u Unix) UnixNano() int64 { return zeroSafeUnix(u.Time, time.Time.UnixNano) }

View file

@ -209,48 +209,46 @@ func DeleteImport(fset *token.FileSet, f *ast.File, path string) (deleted bool)
// DeleteNamedImport deletes the import with the given name and path from the file f, if present. // DeleteNamedImport deletes the import with the given name and path from the file f, if present.
// If there are duplicate import declarations, all matching ones are deleted. // If there are duplicate import declarations, all matching ones are deleted.
func DeleteNamedImport(fset *token.FileSet, f *ast.File, name, path string) (deleted bool) { func DeleteNamedImport(fset *token.FileSet, f *ast.File, name, path string) (deleted bool) {
var delspecs []*ast.ImportSpec var (
var delcomments []*ast.CommentGroup delspecs = make(map[*ast.ImportSpec]bool)
delcomments = make(map[*ast.CommentGroup]bool)
)
// Find the import nodes that import path, if any. // Find the import nodes that import path, if any.
for i := 0; i < len(f.Decls); i++ { for i := 0; i < len(f.Decls); i++ {
decl := f.Decls[i] gen, ok := f.Decls[i].(*ast.GenDecl)
gen, ok := decl.(*ast.GenDecl)
if !ok || gen.Tok != token.IMPORT { if !ok || gen.Tok != token.IMPORT {
continue continue
} }
for j := 0; j < len(gen.Specs); j++ { for j := 0; j < len(gen.Specs); j++ {
spec := gen.Specs[j] impspec := gen.Specs[j].(*ast.ImportSpec)
impspec := spec.(*ast.ImportSpec)
if importName(impspec) != name || importPath(impspec) != path { if importName(impspec) != name || importPath(impspec) != path {
continue continue
} }
// We found an import spec that imports path. // We found an import spec that imports path.
// Delete it. // Delete it.
delspecs = append(delspecs, impspec) delspecs[impspec] = true
deleted = true deleted = true
copy(gen.Specs[j:], gen.Specs[j+1:]) gen.Specs = slices.Delete(gen.Specs, j, j+1)
gen.Specs = gen.Specs[:len(gen.Specs)-1]
// If this was the last import spec in this decl, // If this was the last import spec in this decl,
// delete the decl, too. // delete the decl, too.
if len(gen.Specs) == 0 { if len(gen.Specs) == 0 {
copy(f.Decls[i:], f.Decls[i+1:]) f.Decls = slices.Delete(f.Decls, i, i+1)
f.Decls = f.Decls[:len(f.Decls)-1]
i-- i--
break break
} else if len(gen.Specs) == 1 { } else if len(gen.Specs) == 1 {
if impspec.Doc != nil { if impspec.Doc != nil {
delcomments = append(delcomments, impspec.Doc) delcomments[impspec.Doc] = true
} }
if impspec.Comment != nil { if impspec.Comment != nil {
delcomments = append(delcomments, impspec.Comment) delcomments[impspec.Comment] = true
} }
for _, cg := range f.Comments { for _, cg := range f.Comments {
// Found comment on the same line as the import spec. // Found comment on the same line as the import spec.
if cg.End() < impspec.Pos() && fset.Position(cg.End()).Line == fset.Position(impspec.Pos()).Line { if cg.End() < impspec.Pos() && fset.Position(cg.End()).Line == fset.Position(impspec.Pos()).Line {
delcomments = append(delcomments, cg) delcomments[cg] = true
break break
} }
} }
@ -294,38 +292,21 @@ func DeleteNamedImport(fset *token.FileSet, f *ast.File, name, path string) (del
} }
// Delete imports from f.Imports. // Delete imports from f.Imports.
for i := 0; i < len(f.Imports); i++ { before := len(f.Imports)
imp := f.Imports[i] f.Imports = slices.DeleteFunc(f.Imports, func(imp *ast.ImportSpec) bool {
for j, del := range delspecs { _, ok := delspecs[imp]
if imp == del { return ok
copy(f.Imports[i:], f.Imports[i+1:]) })
f.Imports = f.Imports[:len(f.Imports)-1] if len(f.Imports)+len(delspecs) != before {
copy(delspecs[j:], delspecs[j+1:]) // This can happen when the AST is invalid (i.e. imports differ between f.Decls and f.Imports).
delspecs = delspecs[:len(delspecs)-1] panic(fmt.Sprintf("deleted specs from Decls but not Imports: %v", delspecs))
i--
break
}
}
} }
// Delete comments from f.Comments. // Delete comments from f.Comments.
for i := 0; i < len(f.Comments); i++ { f.Comments = slices.DeleteFunc(f.Comments, func(cg *ast.CommentGroup) bool {
cg := f.Comments[i] _, ok := delcomments[cg]
for j, del := range delcomments { return ok
if cg == del { })
copy(f.Comments[i:], f.Comments[i+1:])
f.Comments = f.Comments[:len(f.Comments)-1]
copy(delcomments[j:], delcomments[j+1:])
delcomments = delcomments[:len(delcomments)-1]
i--
break
}
}
}
if len(delspecs) > 0 {
panic(fmt.Sprintf("deleted specs from Decls but not Imports: %v", delspecs))
}
return return
} }

View file

@ -175,7 +175,7 @@ func ExpandPatterns(ctxt *build.Context, patterns []string) map[string]bool {
for _, pkg := range all { for _, pkg := range all {
doPkg(pkg, neg) doPkg(pkg, neg)
} }
} else if dir := strings.TrimSuffix(arg, "/..."); dir != arg { } else if dir, ok := strings.CutSuffix(arg, "/..."); ok {
// dir/... matches all packages beneath dir // dir/... matches all packages beneath dir
for _, pkg := range all { for _, pkg := range all {
if strings.HasPrefix(pkg, dir) && if strings.HasPrefix(pkg, dir) &&

View file

@ -43,7 +43,7 @@ func (v *TagsFlag) Set(s string) error {
// Starting in Go 1.13, the -tags flag is a comma-separated list of build tags. // Starting in Go 1.13, the -tags flag is a comma-separated list of build tags.
*v = []string{} *v = []string{}
for _, s := range strings.Split(s, ",") { for s := range strings.SplitSeq(s, ",") {
if s != "" { if s != "" {
*v = append(*v, s) *v = append(*v, s)
} }

View file

@ -1,3 +1,31 @@
## v0.25.2 (2025-10-16)
* **Breaking change *(id)*** Split `UserID.ParseAndValidate` into
`ParseAndValidateRelaxed` and `ParseAndValidateStrict`. Strict is the old
behavior, but most users likely want the relaxed version, as there are real
users whose user IDs aren't valid under the strict rules.
* *(crypto)* Added helper methods for generating and verifying with recovery
keys.
* *(bridgev2/matrix)* Added config option to automatically generate a recovery
key for the bridge bot and self-sign the bridge's device.
* *(bridgev2/matrix)* Added initial support for using appservice/MSC3202 mode
for encryption with standard servers like Synapse.
* *(bridgev2)* Added optional support for implicit read receipts.
* *(bridgev2)* Added interface for deleting chats on remote network.
* *(bridgev2)* Added local enforcement of media duration and size limits.
* *(bridgev2)* Extended event duration logging to log any event taking too long.
* *(bridgev2)* Improved validation in group creation provisioning API.
* *(event)* Added event type constant for poll end events.
* *(client)* Added wrapper for searching user directory.
* *(client)* Improved support for managing [MSC4140] delayed events.
* *(crypto/helper)* Changed default sync handling to not block on waiting for
decryption keys. On initial sync, keys won't be requested at all by default.
* *(crypto)* Fixed olm unwedging not working (regressed in v0.25.1).
* *(bridgev2)* Fixed various bugs with migrating to split portals.
* *(event)* Fixed poll start events having incorrect null `m.relates_to`.
* *(client)* Fixed `RespUserProfile` losing standard fields when re-marshaling.
* *(federation)* Fixed various bugs in event auth.
## v0.25.1 (2025-09-16) ## v0.25.1 (2025-09-16)
* *(client)* Fixed HTTP method of delete devices API call * *(client)* Fixed HTTP method of delete devices API call

View file

@ -323,6 +323,7 @@ const (
LogBodyContextKey contextKey = iota LogBodyContextKey contextKey = iota
LogRequestIDContextKey LogRequestIDContextKey
MaxAttemptsContextKey MaxAttemptsContextKey
SyncTokenContextKey
) )
func (cli *Client) RequestStart(req *http.Request) { func (cli *Client) RequestStart(req *http.Request) {
@ -418,8 +419,18 @@ var requestID int32
var logSensitiveContent = os.Getenv("MAUTRIX_LOG_SENSITIVE_CONTENT") == "yes" var logSensitiveContent = os.Getenv("MAUTRIX_LOG_SENSITIVE_CONTENT") == "yes"
func (params *FullRequest) compileRequest(ctx context.Context) (*http.Request, error) { func (params *FullRequest) compileRequest(ctx context.Context) (*http.Request, error) {
reqID := atomic.AddInt32(&requestID, 1)
logger := zerolog.Ctx(ctx)
if logger.GetLevel() == zerolog.Disabled || logger == zerolog.DefaultContextLogger {
logger = params.Logger
}
ctx = logger.With().
Int32("req_id", reqID).
Logger().WithContext(ctx)
var logBody any var logBody any
reqBody := params.RequestBody var reqBody io.Reader
var reqLen int64
if params.RequestJSON != nil { if params.RequestJSON != nil {
jsonStr, err := json.Marshal(params.RequestJSON) jsonStr, err := json.Marshal(params.RequestJSON)
if err != nil { if err != nil {
@ -434,12 +445,22 @@ func (params *FullRequest) compileRequest(ctx context.Context) (*http.Request, e
logBody = params.RequestJSON logBody = params.RequestJSON
} }
reqBody = bytes.NewReader(jsonStr) reqBody = bytes.NewReader(jsonStr)
reqLen = int64(len(jsonStr))
} else if params.RequestBytes != nil { } else if params.RequestBytes != nil {
logBody = fmt.Sprintf("<%d bytes>", len(params.RequestBytes)) logBody = fmt.Sprintf("<%d bytes>", len(params.RequestBytes))
reqBody = bytes.NewReader(params.RequestBytes) reqBody = bytes.NewReader(params.RequestBytes)
params.RequestLength = int64(len(params.RequestBytes)) reqLen = int64(len(params.RequestBytes))
} else if params.RequestLength > 0 && params.RequestBody != nil { } else if params.RequestBody != nil {
logBody = fmt.Sprintf("<%d bytes>", params.RequestLength) logBody = "<unknown stream of bytes>"
reqLen = -1
if params.RequestLength > 0 {
logBody = fmt.Sprintf("<%d bytes>", params.RequestLength)
reqLen = params.RequestLength
} else if params.RequestLength == 0 {
zerolog.Ctx(ctx).Warn().
Msg("RequestBody passed without specifying request length")
}
reqBody = params.RequestBody
if rsc, ok := params.RequestBody.(io.ReadSeekCloser); ok { if rsc, ok := params.RequestBody.(io.ReadSeekCloser); ok {
// Prevent HTTP from closing the request body, it might be needed for retries // Prevent HTTP from closing the request body, it might be needed for retries
reqBody = nopCloseSeeker{rsc} reqBody = nopCloseSeeker{rsc}
@ -448,15 +469,8 @@ func (params *FullRequest) compileRequest(ctx context.Context) (*http.Request, e
params.RequestJSON = struct{}{} params.RequestJSON = struct{}{}
logBody = params.RequestJSON logBody = params.RequestJSON
reqBody = bytes.NewReader([]byte("{}")) reqBody = bytes.NewReader([]byte("{}"))
reqLen = 2
} }
reqID := atomic.AddInt32(&requestID, 1)
logger := zerolog.Ctx(ctx)
if logger.GetLevel() == zerolog.Disabled || logger == zerolog.DefaultContextLogger {
logger = params.Logger
}
ctx = logger.With().
Int32("req_id", reqID).
Logger().WithContext(ctx)
ctx = context.WithValue(ctx, LogBodyContextKey, logBody) ctx = context.WithValue(ctx, LogBodyContextKey, logBody)
ctx = context.WithValue(ctx, LogRequestIDContextKey, int(reqID)) ctx = context.WithValue(ctx, LogRequestIDContextKey, int(reqID))
req, err := http.NewRequestWithContext(ctx, params.Method, params.URL, reqBody) req, err := http.NewRequestWithContext(ctx, params.Method, params.URL, reqBody)
@ -472,9 +486,7 @@ func (params *FullRequest) compileRequest(ctx context.Context) (*http.Request, e
if params.RequestJSON != nil { if params.RequestJSON != nil {
req.Header.Set("Content-Type", "application/json") req.Header.Set("Content-Type", "application/json")
} }
if params.RequestLength > 0 && params.RequestBody != nil { req.ContentLength = reqLen
req.ContentLength = params.RequestLength
}
return req, nil return req, nil
} }
@ -1055,6 +1067,15 @@ func (cli *Client) GetProfile(ctx context.Context, mxid id.UserID) (resp *RespUs
return return
} }
func (cli *Client) SearchUserDirectory(ctx context.Context, query string, limit int) (resp *RespSearchUserDirectory, err error) {
urlPath := cli.BuildClientURL("v3", "user_directory", "search")
_, err = cli.MakeRequest(ctx, http.MethodPost, urlPath, &ReqSearchUserDirectory{
SearchTerm: query,
Limit: limit,
}, &resp)
return
}
func (cli *Client) GetMutualRooms(ctx context.Context, otherUserID id.UserID, extras ...ReqMutualRooms) (resp *RespMutualRooms, err error) { func (cli *Client) GetMutualRooms(ctx context.Context, otherUserID id.UserID, extras ...ReqMutualRooms) (resp *RespMutualRooms, err error) {
if cli.SpecVersions != nil && !cli.SpecVersions.Supports(FeatureMutualRooms) { if cli.SpecVersions != nil && !cli.SpecVersions.Supports(FeatureMutualRooms) {
err = fmt.Errorf("server does not support fetching mutual rooms") err = fmt.Errorf("server does not support fetching mutual rooms")
@ -1105,6 +1126,9 @@ func (cli *Client) SetDisplayName(ctx context.Context, displayName string) (err
// SetProfileField sets an arbitrary profile field. See https://spec.matrix.org/v1.16/client-server-api/#put_matrixclientv3profileuseridkeyname // SetProfileField sets an arbitrary profile field. See https://spec.matrix.org/v1.16/client-server-api/#put_matrixclientv3profileuseridkeyname
func (cli *Client) SetProfileField(ctx context.Context, key string, value any) (err error) { func (cli *Client) SetProfileField(ctx context.Context, key string, value any) (err error) {
urlPath := cli.BuildClientURL("v3", "profile", cli.UserID, key) urlPath := cli.BuildClientURL("v3", "profile", cli.UserID, key)
if key != "displayname" && key != "avatar_url" && !cli.SpecVersions.Supports(FeatureArbitraryProfileFields) && cli.SpecVersions.Supports(FeatureUnstableProfileFields) {
urlPath = cli.BuildClientURL("unstable", "uk.tcpip.msc4133", "profile", cli.UserID, key)
}
_, err = cli.MakeRequest(ctx, http.MethodPut, urlPath, map[string]any{ _, err = cli.MakeRequest(ctx, http.MethodPut, urlPath, map[string]any{
key: value, key: value,
}, nil) }, nil)
@ -1114,6 +1138,9 @@ func (cli *Client) SetProfileField(ctx context.Context, key string, value any) (
// DeleteProfileField deletes an arbitrary profile field. See https://spec.matrix.org/v1.16/client-server-api/#put_matrixclientv3profileuseridkeyname // DeleteProfileField deletes an arbitrary profile field. See https://spec.matrix.org/v1.16/client-server-api/#put_matrixclientv3profileuseridkeyname
func (cli *Client) DeleteProfileField(ctx context.Context, key string) (err error) { func (cli *Client) DeleteProfileField(ctx context.Context, key string) (err error) {
urlPath := cli.BuildClientURL("v3", "profile", cli.UserID, key) urlPath := cli.BuildClientURL("v3", "profile", cli.UserID, key)
if key != "displayname" && key != "avatar_url" && !cli.SpecVersions.Supports(FeatureArbitraryProfileFields) && cli.SpecVersions.Supports(FeatureUnstableProfileFields) {
urlPath = cli.BuildClientURL("unstable", "uk.tcpip.msc4133", "profile", cli.UserID, key)
}
_, err = cli.MakeRequest(ctx, http.MethodDelete, urlPath, nil, nil) _, err = cli.MakeRequest(ctx, http.MethodDelete, urlPath, nil, nil)
return return
} }
@ -1121,6 +1148,9 @@ func (cli *Client) DeleteProfileField(ctx context.Context, key string) (err erro
// GetProfileField gets an arbitrary profile field and parses the response into the given struct. See https://spec.matrix.org/unstable/client-server-api/#get_matrixclientv3profileuseridkeyname // GetProfileField gets an arbitrary profile field and parses the response into the given struct. See https://spec.matrix.org/unstable/client-server-api/#get_matrixclientv3profileuseridkeyname
func (cli *Client) GetProfileField(ctx context.Context, userID id.UserID, key string, into any) (err error) { func (cli *Client) GetProfileField(ctx context.Context, userID id.UserID, key string, into any) (err error) {
urlPath := cli.BuildClientURL("v3", "profile", userID, key) urlPath := cli.BuildClientURL("v3", "profile", userID, key)
if key != "displayname" && key != "avatar_url" && !cli.SpecVersions.Supports(FeatureArbitraryProfileFields) && cli.SpecVersions.Supports(FeatureUnstableProfileFields) {
urlPath = cli.BuildClientURL("unstable", "uk.tcpip.msc4133", "profile", cli.UserID, key)
}
_, err = cli.MakeRequest(ctx, http.MethodGet, urlPath, nil, into) _, err = cli.MakeRequest(ctx, http.MethodGet, urlPath, nil, into)
return return
} }
@ -1283,6 +1313,32 @@ func (cli *Client) SendMassagedStateEvent(ctx context.Context, roomID id.RoomID,
return return
} }
func (cli *Client) DelayedEvents(ctx context.Context, req *ReqDelayedEvents) (resp *RespDelayedEvents, err error) {
query := map[string]string{}
if req.DelayID != "" {
query["delay_id"] = string(req.DelayID)
}
if req.Status != "" {
query["status"] = string(req.Status)
}
if req.NextBatch != "" {
query["next_batch"] = req.NextBatch
}
urlPath := cli.BuildURLWithQuery(ClientURLPath{"unstable", "org.matrix.msc4140", "delayed_events"}, query)
_, err = cli.MakeRequest(ctx, http.MethodGet, urlPath, req, &resp)
// Migration: merge old keys with new ones
if resp != nil {
resp.Scheduled = append(resp.Scheduled, resp.DelayedEvents...)
resp.DelayedEvents = nil
resp.Finalised = append(resp.Finalised, resp.FinalisedEvents...)
resp.FinalisedEvents = nil
}
return
}
func (cli *Client) UpdateDelayedEvent(ctx context.Context, req *ReqUpdateDelayedEvent) (resp *RespUpdateDelayedEvent, err error) { func (cli *Client) UpdateDelayedEvent(ctx context.Context, req *ReqUpdateDelayedEvent) (resp *RespUpdateDelayedEvent, err error) {
urlPath := cli.BuildClientURL("unstable", "org.matrix.msc4140", "delayed_events", req.DelayID) urlPath := cli.BuildClientURL("unstable", "org.matrix.msc4140", "delayed_events", req.DelayID)
_, err = cli.MakeRequest(ctx, http.MethodPost, urlPath, req, &resp) _, err = cli.MakeRequest(ctx, http.MethodPost, urlPath, req, &resp)

View file

@ -20,6 +20,20 @@ type CrossSigningPublicKeysCache struct {
UserSigningKey id.Ed25519 UserSigningKey id.Ed25519
} }
func (mach *OlmMachine) GetOwnVerificationStatus(ctx context.Context) (hasKeys, isVerified bool, err error) {
pubkeys := mach.GetOwnCrossSigningPublicKeys(ctx)
if pubkeys != nil {
hasKeys = true
isVerified, err = mach.CryptoStore.IsKeySignedBy(
ctx, mach.Client.UserID, mach.GetAccount().SigningKey(), mach.Client.UserID, pubkeys.SelfSigningKey,
)
if err != nil {
err = fmt.Errorf("failed to check if current device is signed by own self-signing key: %w", err)
}
}
return
}
func (mach *OlmMachine) GetOwnCrossSigningPublicKeys(ctx context.Context) *CrossSigningPublicKeysCache { func (mach *OlmMachine) GetOwnCrossSigningPublicKeys(ctx context.Context) *CrossSigningPublicKeysCache {
if mach.crossSigningPubkeys != nil { if mach.crossSigningPubkeys != nil {
return mach.crossSigningPubkeys return mach.crossSigningPubkeys

View file

@ -71,6 +71,42 @@ func (mach *OlmMachine) GenerateAndUploadCrossSigningKeysWithPassword(ctx contex
}, passphrase) }, passphrase)
} }
func (mach *OlmMachine) VerifyWithRecoveryKey(ctx context.Context, recoveryKey string) error {
keyID, keyData, err := mach.SSSS.GetDefaultKeyData(ctx)
if err != nil {
return fmt.Errorf("failed to get default SSSS key data: %w", err)
}
key, err := keyData.VerifyRecoveryKey(keyID, recoveryKey)
if err != nil {
return err
}
err = mach.FetchCrossSigningKeysFromSSSS(ctx, key)
if err != nil {
return fmt.Errorf("failed to fetch cross-signing keys from SSSS: %w", err)
}
err = mach.SignOwnDevice(ctx, mach.OwnIdentity())
if err != nil {
return fmt.Errorf("failed to sign own device: %w", err)
}
err = mach.SignOwnMasterKey(ctx)
if err != nil {
return fmt.Errorf("failed to sign own master key: %w", err)
}
return nil
}
func (mach *OlmMachine) GenerateAndVerifyWithRecoveryKey(ctx context.Context) (recoveryKey string, err error) {
recoveryKey, _, err = mach.GenerateAndUploadCrossSigningKeys(ctx, nil, "")
if err != nil {
err = fmt.Errorf("failed to generate and upload cross-signing keys: %w", err)
} else if err = mach.SignOwnDevice(ctx, mach.OwnIdentity()); err != nil {
err = fmt.Errorf("failed to sign own device: %w", err)
} else if err = mach.SignOwnMasterKey(ctx); err != nil {
err = fmt.Errorf("failed to sign own master key: %w", err)
}
return
}
// GenerateAndUploadCrossSigningKeys generates a new key with all corresponding cross-signing keys. // GenerateAndUploadCrossSigningKeys generates a new key with all corresponding cross-signing keys.
// //
// A passphrase can be provided to generate the SSSS key. If the passphrase is empty, a random key // A passphrase can be provided to generate the SSSS key. If the passphrase is empty, a random key
@ -97,12 +133,12 @@ func (mach *OlmMachine) GenerateAndUploadCrossSigningKeys(ctx context.Context, u
// Publish cross-signing keys // Publish cross-signing keys
err = mach.PublishCrossSigningKeys(ctx, keysCache, uiaCallback) err = mach.PublishCrossSigningKeys(ctx, keysCache, uiaCallback)
if err != nil { if err != nil {
return "", nil, fmt.Errorf("failed to publish cross-signing keys: %w", err) return key.RecoveryKey(), keysCache, fmt.Errorf("failed to publish cross-signing keys: %w", err)
} }
err = mach.SSSS.SetDefaultKeyID(ctx, key.ID) err = mach.SSSS.SetDefaultKeyID(ctx, key.ID)
if err != nil { if err != nil {
return "", nil, fmt.Errorf("failed to mark %s as the default key: %w", key.ID, err) return key.RecoveryKey(), keysCache, fmt.Errorf("failed to mark %s as the default key: %w", key.ID, err)
} }
return key.RecoveryKey(), keysCache, nil return key.RecoveryKey(), keysCache, nil

View file

@ -225,13 +225,6 @@ func (helper *CryptoHelper) Init(ctx context.Context) error {
helper.ASEventProcessor.On(event.EventEncrypted, helper.HandleEncrypted) helper.ASEventProcessor.On(event.EventEncrypted, helper.HandleEncrypted)
} }
if helper.client.SetAppServiceDeviceID {
err = helper.mach.ShareKeys(ctx, -1)
if err != nil {
return fmt.Errorf("failed to share keys: %w", err)
}
}
return nil return nil
} }
@ -268,21 +261,21 @@ func (helper *CryptoHelper) verifyDeviceKeysOnServer(ctx context.Context) error
if !ok || len(device.Keys) == 0 { if !ok || len(device.Keys) == 0 {
if isShared { if isShared {
return fmt.Errorf("olm account is marked as shared, keys seem to have disappeared from the server") return fmt.Errorf("olm account is marked as shared, keys seem to have disappeared from the server")
} else {
helper.log.Debug().Msg("Olm account not shared and keys not on server, so device is probably fine")
return nil
} }
helper.log.Debug().Msg("Olm account not shared and keys not on server, sharing initial keys")
err = helper.mach.ShareKeys(ctx, -1)
if err != nil {
return fmt.Errorf("failed to share keys: %w", err)
}
return nil
} else if !isShared { } else if !isShared {
return fmt.Errorf("olm account is not marked as shared, but there are keys on the server") return fmt.Errorf("olm account is not marked as shared, but there are keys on the server")
} else if ed := device.Keys.GetEd25519(helper.client.DeviceID); ownID.SigningKey != ed { } else if ed := device.Keys.GetEd25519(helper.client.DeviceID); ownID.SigningKey != ed {
return fmt.Errorf("mismatching identity key on server (%q != %q)", ownID.SigningKey, ed) return fmt.Errorf("mismatching identity key on server (%q != %q)", ownID.SigningKey, ed)
}
if !isShared {
helper.log.Debug().Msg("Olm account not marked as shared, but keys on server match?")
} else { } else {
helper.log.Debug().Msg("Olm account marked as shared and keys on server match, device is fine") helper.log.Debug().Msg("Olm account marked as shared and keys on server match, device is fine")
return nil
} }
return nil
} }
var NoSessionFound = crypto.NoSessionFound var NoSessionFound = crypto.NoSessionFound
@ -304,24 +297,14 @@ func (helper *CryptoHelper) HandleEncrypted(ctx context.Context, evt *event.Even
ctx = log.WithContext(ctx) ctx = log.WithContext(ctx)
decrypted, err := helper.Decrypt(ctx, evt) decrypted, err := helper.Decrypt(ctx, evt)
if errors.Is(err, NoSessionFound) { if errors.Is(err, NoSessionFound) && ctx.Value(mautrix.SyncTokenContextKey) != "" {
log.Debug(). go helper.waitForSession(ctx, evt)
Int("wait_seconds", int(initialSessionWaitTimeout.Seconds())). } else if err != nil {
Msg("Couldn't find session, waiting for keys to arrive...")
if helper.mach.WaitForSession(ctx, evt.RoomID, content.SenderKey, content.SessionID, initialSessionWaitTimeout) {
log.Debug().Msg("Got keys after waiting, trying to decrypt event again")
decrypted, err = helper.Decrypt(ctx, evt)
} else {
go helper.waitLongerForSession(ctx, log, evt)
return
}
}
if err != nil {
log.Warn().Err(err).Msg("Failed to decrypt event") log.Warn().Err(err).Msg("Failed to decrypt event")
helper.DecryptErrorCallback(evt, err) helper.DecryptErrorCallback(evt, err)
return } else {
helper.postDecrypt(ctx, decrypted)
} }
helper.postDecrypt(ctx, decrypted)
} }
func (helper *CryptoHelper) postDecrypt(ctx context.Context, decrypted *event.Event) { func (helper *CryptoHelper) postDecrypt(ctx context.Context, decrypted *event.Event) {
@ -362,7 +345,29 @@ func (helper *CryptoHelper) RequestSession(ctx context.Context, roomID id.RoomID
} }
} }
func (helper *CryptoHelper) waitLongerForSession(ctx context.Context, log zerolog.Logger, evt *event.Event) { func (helper *CryptoHelper) waitForSession(ctx context.Context, evt *event.Event) {
log := zerolog.Ctx(ctx)
content := evt.Content.AsEncrypted()
log.Debug().
Int("wait_seconds", int(initialSessionWaitTimeout.Seconds())).
Msg("Couldn't find session, waiting for keys to arrive...")
if helper.mach.WaitForSession(ctx, evt.RoomID, content.SenderKey, content.SessionID, initialSessionWaitTimeout) {
log.Debug().Msg("Got keys after waiting, trying to decrypt event again")
decrypted, err := helper.Decrypt(ctx, evt)
if err != nil {
log.Warn().Err(err).Msg("Failed to decrypt event")
helper.DecryptErrorCallback(evt, err)
} else {
helper.postDecrypt(ctx, decrypted)
}
} else {
go helper.waitLongerForSession(ctx, evt)
}
}
func (helper *CryptoHelper) waitLongerForSession(ctx context.Context, evt *event.Event) {
log := zerolog.Ctx(ctx)
content := evt.Content.AsEncrypted() content := evt.Content.AsEncrypted()
log.Debug().Int("wait_seconds", int(extendedSessionWaitTimeout.Seconds())).Msg("Couldn't find session, requesting keys and waiting longer...") log.Debug().Int("wait_seconds", int(extendedSessionWaitTimeout.Seconds())).Msg("Couldn't find session, requesting keys and waiting longer...")

View file

@ -729,7 +729,7 @@ func (mach *OlmMachine) ShareKeys(ctx context.Context, currentOTKCount int) erro
start := time.Now() start := time.Now()
mach.otkUploadLock.Lock() mach.otkUploadLock.Lock()
defer mach.otkUploadLock.Unlock() defer mach.otkUploadLock.Unlock()
if mach.lastOTKUpload.Add(1*time.Minute).After(start) || currentOTKCount < 0 { if mach.lastOTKUpload.Add(1*time.Minute).After(start) || (currentOTKCount < 0 && mach.account.Shared) {
log.Debug().Msg("Checking OTK count from server due to suspiciously close share keys requests or negative OTK count") log.Debug().Msg("Checking OTK count from server due to suspiciously close share keys requests or negative OTK count")
resp, err := mach.Client.UploadKeys(ctx, &mautrix.ReqUploadKeys{}) resp, err := mach.Client.UploadKeys(ctx, &mautrix.ReqUploadKeys{})
if err != nil { if err != nil {

View file

@ -253,7 +253,7 @@ func (store *SQLCryptoStore) GetLatestSession(ctx context.Context, key id.Sender
// GetNewestSessionCreationTS gets the creation timestamp of the most recently created session with the given sender key. // GetNewestSessionCreationTS gets the creation timestamp of the most recently created session with the given sender key.
// This will exclude sessions that have never been used to encrypt or decrypt a message. // This will exclude sessions that have never been used to encrypt or decrypt a message.
func (store *SQLCryptoStore) GetNewestSessionCreationTS(ctx context.Context, key id.SenderKey) (createdAt time.Time, err error) { func (store *SQLCryptoStore) GetNewestSessionCreationTS(ctx context.Context, key id.SenderKey) (createdAt time.Time, err error) {
err = store.DB.QueryRow(ctx, "SELECT created_at FROM crypto_olm_session WHERE sender_key=$1 AND account_id=$2 AND (encrypted_at <> created_at OR decrypted_at <> created_at) ORDER BY created_at DESC LIMIT 1", err = store.DB.QueryRow(ctx, "SELECT created_at FROM crypto_olm_session WHERE sender_key=$1 AND account_id=$2 AND (last_encrypted <> created_at OR last_decrypted <> created_at) ORDER BY created_at DESC LIMIT 1",
key, store.AccountID).Scan(&createdAt) key, store.AccountID).Scan(&createdAt)
if errors.Is(err, sql.ErrNoRows) { if errors.Is(err, sql.ErrNoRows) {
err = nil err = nil
@ -664,6 +664,20 @@ func (store *SQLCryptoStore) IsOutboundGroupSessionShared(ctx context.Context, u
// ValidateMessageIndex returns whether the given event information match the ones stored in the database // ValidateMessageIndex returns whether the given event information match the ones stored in the database
// for the given sender key, session ID and index. If the index hasn't been stored, this will store it. // for the given sender key, session ID and index. If the index hasn't been stored, this will store it.
func (store *SQLCryptoStore) ValidateMessageIndex(ctx context.Context, senderKey id.SenderKey, sessionID id.SessionID, eventID id.EventID, index uint, timestamp int64) (bool, error) { func (store *SQLCryptoStore) ValidateMessageIndex(ctx context.Context, senderKey id.SenderKey, sessionID id.SessionID, eventID id.EventID, index uint, timestamp int64) (bool, error) {
if eventID == "" && timestamp == 0 {
var notOK bool
const validateEmptyQuery = `
SELECT EXISTS(SELECT 1 FROM crypto_message_index WHERE sender_key=$1 AND session_id=$2 AND "index"=$3)
`
err := store.DB.QueryRow(ctx, validateEmptyQuery, senderKey, sessionID, index).Scan(&notOK)
if notOK {
zerolog.Ctx(ctx).Debug().
Uint("message_index", index).
Msg("Rejecting event without event ID and timestamp due to already knowing them")
}
return !notOK, err
}
const validateQuery = ` const validateQuery = `
INSERT INTO crypto_message_index (sender_key, session_id, "index", event_id, timestamp) INSERT INTO crypto_message_index (sender_key, session_id, "index", event_id, timestamp)
VALUES ($1, $2, $3, $4, $5) VALUES ($1, $2, $3, $4, $5)

View file

@ -7,6 +7,8 @@
package ssss package ssss
import ( import (
"crypto/hmac"
"crypto/sha256"
"encoding/base64" "encoding/base64"
"fmt" "fmt"
"strings" "strings"
@ -108,12 +110,18 @@ func (key *Key) Decrypt(eventType string, data EncryptedKeyData) ([]byte, error)
return nil, err return nil, err
} }
mac, err := base64.RawStdEncoding.DecodeString(strings.TrimRight(data.MAC, "="))
if err != nil {
return nil, err
}
// derive the AES and HMAC keys for the requested event type using the SSSS key // derive the AES and HMAC keys for the requested event type using the SSSS key
aesKey, hmacKey := utils.DeriveKeysSHA256(key.Key, eventType) aesKey, hmacKey := utils.DeriveKeysSHA256(key.Key, eventType)
// compare the stored MAC with the one we calculated from the ciphertext // compare the stored MAC with the one we calculated from the ciphertext
calcMac := utils.HMACSHA256B64(payload, hmacKey) h := hmac.New(sha256.New, hmacKey[:])
if strings.TrimRight(data.MAC, "=") != calcMac { h.Write(payload)
if !hmac.Equal(h.Sum(nil), mac) {
return nil, ErrKeyDataMACMismatch return nil, ErrKeyDataMACMismatch
} }

View file

@ -525,6 +525,9 @@ func (gs *MemoryStore) ValidateMessageIndex(_ context.Context, senderKey id.Send
} }
val, ok := gs.MessageIndices[key] val, ok := gs.MessageIndices[key]
if !ok { if !ok {
if eventID == "" && timestamp == 0 {
return true, nil
}
gs.MessageIndices[key] = messageIndexValue{ gs.MessageIndices[key] = messageIndexValue{
EventID: eventID, EventID: eventID,
Timestamp: timestamp, Timestamp: timestamp,

View file

@ -13,6 +13,7 @@ import (
"net/http" "net/http"
"go.mau.fi/util/exhttp" "go.mau.fi/util/exhttp"
"go.mau.fi/util/exmaps"
"golang.org/x/exp/maps" "golang.org/x/exp/maps"
) )
@ -144,10 +145,7 @@ func (e *RespError) UnmarshalJSON(data []byte) error {
} }
func (e *RespError) MarshalJSON() ([]byte, error) { func (e *RespError) MarshalJSON() ([]byte, error) {
data := maps.Clone(e.ExtraData) data := exmaps.NonNilClone(e.ExtraData)
if data == nil {
data = make(map[string]any)
}
data["errcode"] = e.ErrCode data["errcode"] = e.ErrCode
data["error"] = e.Err data["error"] = e.Err
return json.Marshal(data) return json.Marshal(data)
@ -177,6 +175,12 @@ func (e RespError) WithStatus(status int) RespError {
return e return e
} }
func (e RespError) WithExtraData(extraData map[string]any) RespError {
e.ExtraData = exmaps.NonNilClone(e.ExtraData)
maps.Copy(e.ExtraData, extraData)
return e
}
// Error returns the errcode and error message. // Error returns the errcode and error message.
func (e RespError) Error() string { func (e RespError) Error() string {
return e.ErrCode + ": " + e.Err return e.ErrCode + ": " + e.Err

View file

@ -105,3 +105,15 @@ func (bmec *BeeperMuteEventContent) GetMutedUntilTime() time.Time {
} }
return time.Time{} return time.Time{}
} }
func (bmec *BeeperMuteEventContent) GetMuteDuration() time.Duration {
ts := bmec.GetMutedUntilTime()
now := time.Now()
if ts.Before(now) {
return 0
} else if ts == MutedForever {
return -1
} else {
return ts.Sub(now)
}
}

View file

@ -86,6 +86,10 @@ type BeeperRoomKeyAckEventContent struct {
FirstMessageIndex int `json:"first_message_index"` FirstMessageIndex int `json:"first_message_index"`
} }
type BeeperChatDeleteEventContent struct {
DeleteForEveryone bool `json:"delete_for_everyone,omitempty"`
}
type IntOrString int type IntOrString int
func (ios *IntOrString) UnmarshalJSON(data []byte) error { func (ios *IntOrString) UnmarshalJSON(data []byte) error {

View file

@ -55,6 +55,11 @@ export interface RoomFeatures {
allowed_reactions?: string[] allowed_reactions?: string[]
/** Whether custom emoji reactions are allowed. */ /** Whether custom emoji reactions are allowed. */
custom_emoji_reactions?: boolean custom_emoji_reactions?: boolean
/** Whether deleting the chat for yourself is supported. */
delete_chat?: boolean
/** Whether deleting the chat for all participants is supported. */
delete_chat_for_everyone?: boolean
} }
declare type integer = number declare type integer = number

View file

@ -18,6 +18,7 @@ import (
"go.mau.fi/util/exerrors" "go.mau.fi/util/exerrors"
"go.mau.fi/util/jsontime" "go.mau.fi/util/jsontime"
"go.mau.fi/util/ptr"
"golang.org/x/exp/constraints" "golang.org/x/exp/constraints"
"golang.org/x/exp/maps" "golang.org/x/exp/maps"
) )
@ -51,11 +52,12 @@ type RoomFeatures struct {
AllowedReactions []string `json:"allowed_reactions,omitempty"` AllowedReactions []string `json:"allowed_reactions,omitempty"`
CustomEmojiReactions bool `json:"custom_emoji_reactions,omitempty"` CustomEmojiReactions bool `json:"custom_emoji_reactions,omitempty"`
ReadReceipts bool `json:"read_receipts,omitempty"` ReadReceipts bool `json:"read_receipts,omitempty"`
TypingNotifications bool `json:"typing_notifications,omitempty"` TypingNotifications bool `json:"typing_notifications,omitempty"`
Archive bool `json:"archive,omitempty"` Archive bool `json:"archive,omitempty"`
MarkAsUnread bool `json:"mark_as_unread,omitempty"` MarkAsUnread bool `json:"mark_as_unread,omitempty"`
DeleteChat bool `json:"delete_chat,omitempty"` DeleteChat bool `json:"delete_chat,omitempty"`
DeleteChatForEveryone bool `json:"delete_chat_for_everyone,omitempty"`
} }
func (rf *RoomFeatures) GetID() string { func (rf *RoomFeatures) GetID() string {
@ -65,10 +67,32 @@ func (rf *RoomFeatures) GetID() string {
return base64.RawURLEncoding.EncodeToString(rf.Hash()) return base64.RawURLEncoding.EncodeToString(rf.Hash())
} }
func (rf *RoomFeatures) Clone() *RoomFeatures {
if rf == nil {
return nil
}
clone := *rf
clone.File = clone.File.Clone()
clone.Formatting = maps.Clone(clone.Formatting)
clone.EditMaxAge = ptr.Clone(clone.EditMaxAge)
clone.DeleteMaxAge = ptr.Clone(clone.DeleteMaxAge)
clone.DisappearingTimer = clone.DisappearingTimer.Clone()
clone.AllowedReactions = slices.Clone(clone.AllowedReactions)
return &clone
}
type FormattingFeatureMap map[FormattingFeature]CapabilitySupportLevel type FormattingFeatureMap map[FormattingFeature]CapabilitySupportLevel
type FileFeatureMap map[CapabilityMsgType]*FileFeatures type FileFeatureMap map[CapabilityMsgType]*FileFeatures
func (ffm FileFeatureMap) Clone() FileFeatureMap {
dup := maps.Clone(ffm)
for key, value := range dup {
dup[key] = value.Clone()
}
return dup
}
type DisappearingTimerCapability struct { type DisappearingTimerCapability struct {
Types []DisappearingType `json:"types"` Types []DisappearingType `json:"types"`
Timers []jsontime.Milliseconds `json:"timers,omitempty"` Timers []jsontime.Milliseconds `json:"timers,omitempty"`
@ -76,6 +100,16 @@ type DisappearingTimerCapability struct {
OmitEmptyTimer bool `json:"omit_empty_timer,omitempty"` OmitEmptyTimer bool `json:"omit_empty_timer,omitempty"`
} }
func (dtc *DisappearingTimerCapability) Clone() *DisappearingTimerCapability {
if dtc == nil {
return nil
}
clone := *dtc
clone.Types = slices.Clone(clone.Types)
clone.Timers = slices.Clone(clone.Timers)
return &clone
}
func (dtc *DisappearingTimerCapability) Supports(content *BeeperDisappearingTimer) bool { func (dtc *DisappearingTimerCapability) Supports(content *BeeperDisappearingTimer) bool {
if dtc == nil || content == nil || content.Type == DisappearingTypeNone { if dtc == nil || content == nil || content.Type == DisappearingTypeNone {
return true return true
@ -262,6 +296,7 @@ func (rf *RoomFeatures) Hash() []byte {
hashBool(hasher, "archive", rf.Archive) hashBool(hasher, "archive", rf.Archive)
hashBool(hasher, "mark_as_unread", rf.MarkAsUnread) hashBool(hasher, "mark_as_unread", rf.MarkAsUnread)
hashBool(hasher, "delete_chat", rf.DeleteChat) hashBool(hasher, "delete_chat", rf.DeleteChat)
hashBool(hasher, "delete_chat_for_everyone", rf.DeleteChatForEveryone)
return hasher.Sum(nil) return hasher.Sum(nil)
} }
@ -294,3 +329,13 @@ func (ff *FileFeatures) Hash() []byte {
hashBool(hasher, "view_once", ff.ViewOnce) hashBool(hasher, "view_once", ff.ViewOnce)
return hasher.Sum(nil) return hasher.Sum(nil)
} }
func (ff *FileFeatures) Clone() *FileFeatures {
if ff == nil {
return nil
}
clone := *ff
clone.MimeTypes = maps.Clone(clone.MimeTypes)
clone.MaxDuration = ptr.Clone(clone.MaxDuration)
return &clone
}

View file

@ -63,6 +63,7 @@ var TypeMap = map[Type]reflect.Type{
BeeperMessageStatus: reflect.TypeOf(BeeperMessageStatusEventContent{}), BeeperMessageStatus: reflect.TypeOf(BeeperMessageStatusEventContent{}),
BeeperTranscription: reflect.TypeOf(BeeperTranscriptionEventContent{}), BeeperTranscription: reflect.TypeOf(BeeperTranscriptionEventContent{}),
BeeperDeleteChat: reflect.TypeOf(BeeperChatDeleteEventContent{}),
AccountDataRoomTags: reflect.TypeOf(TagEventContent{}), AccountDataRoomTags: reflect.TypeOf(TagEventContent{}),
AccountDataDirectChats: reflect.TypeOf(DirectChatsEventContent{}), AccountDataDirectChats: reflect.TypeOf(DirectChatsEventContent{}),

70
vendor/maunium.net/go/mautrix/event/delayed.go generated vendored Normal file
View file

@ -0,0 +1,70 @@
package event
import (
"encoding/json"
"go.mau.fi/util/jsontime"
"maunium.net/go/mautrix/id"
)
type ScheduledDelayedEvent struct {
DelayID id.DelayID `json:"delay_id"`
RoomID id.RoomID `json:"room_id"`
Type Type `json:"type"`
StateKey *string `json:"state_key,omitempty"`
Delay int64 `json:"delay"`
RunningSince jsontime.UnixMilli `json:"running_since"`
Content Content `json:"content"`
}
func (e ScheduledDelayedEvent) AsEvent(eventID id.EventID, ts jsontime.UnixMilli) (*Event, error) {
evt := &Event{
ID: eventID,
RoomID: e.RoomID,
Type: e.Type,
StateKey: e.StateKey,
Content: e.Content,
Timestamp: ts.UnixMilli(),
}
return evt, evt.Content.ParseRaw(evt.Type)
}
type FinalisedDelayedEvent struct {
DelayedEvent *ScheduledDelayedEvent `json:"scheduled_event"`
Outcome DelayOutcome `json:"outcome"`
Reason DelayReason `json:"reason"`
Error json.RawMessage `json:"error,omitempty"`
EventID id.EventID `json:"event_id,omitempty"`
Timestamp jsontime.UnixMilli `json:"origin_server_ts"`
}
type DelayStatus string
var (
DelayStatusScheduled DelayStatus = "scheduled"
DelayStatusFinalised DelayStatus = "finalised"
)
type DelayAction string
var (
DelayActionSend DelayAction = "send"
DelayActionCancel DelayAction = "cancel"
DelayActionRestart DelayAction = "restart"
)
type DelayOutcome string
var (
DelayOutcomeSend DelayOutcome = "send"
DelayOutcomeCancel DelayOutcome = "cancel"
)
type DelayReason string
var (
DelayReasonAction DelayReason = "action"
DelayReasonError DelayReason = "error"
DelayReasonDelay DelayReason = "delay"
)

View file

@ -35,7 +35,7 @@ type MSC1767Message struct {
} }
type PollStartEventContent struct { type PollStartEventContent struct {
RelatesTo *RelatesTo `json:"m.relates_to"` RelatesTo *RelatesTo `json:"m.relates_to,omitempty"`
Mentions *Mentions `json:"m.mentions,omitempty"` Mentions *Mentions `json:"m.mentions,omitempty"`
PollStart struct { PollStart struct {
Kind string `json:"kind"` Kind string `json:"kind"`

View file

@ -231,6 +231,8 @@ type BridgeEventContent struct {
BeeperRoomType string `json:"com.beeper.room_type,omitempty"` BeeperRoomType string `json:"com.beeper.room_type,omitempty"`
BeeperRoomTypeV2 string `json:"com.beeper.room_type.v2,omitempty"` BeeperRoomTypeV2 string `json:"com.beeper.room_type.v2,omitempty"`
TempSlackRemoteIDMigratedFlag bool `json:"com.beeper.slack_remote_id_migrated,omitempty"`
} }
// DisappearingType represents the type of a disappearing message timer. // DisappearingType represents the type of a disappearing message timer.

View file

@ -128,7 +128,7 @@ func (et *Type) GuessClass() TypeClass {
InRoomVerificationKey.Type, InRoomVerificationMAC.Type, InRoomVerificationCancel.Type, InRoomVerificationKey.Type, InRoomVerificationMAC.Type, InRoomVerificationCancel.Type,
CallInvite.Type, CallCandidates.Type, CallAnswer.Type, CallReject.Type, CallSelectAnswer.Type, CallInvite.Type, CallCandidates.Type, CallAnswer.Type, CallReject.Type, CallSelectAnswer.Type,
CallNegotiate.Type, CallHangup.Type, BeeperMessageStatus.Type, EventUnstablePollStart.Type, EventUnstablePollResponse.Type, CallNegotiate.Type, CallHangup.Type, BeeperMessageStatus.Type, EventUnstablePollStart.Type, EventUnstablePollResponse.Type,
BeeperTranscription.Type: EventUnstablePollEnd.Type, BeeperTranscription.Type, BeeperDeleteChat.Type:
return MessageEventType return MessageEventType
case ToDeviceRoomKey.Type, ToDeviceRoomKeyRequest.Type, ToDeviceForwardedRoomKey.Type, ToDeviceRoomKeyWithheld.Type, case ToDeviceRoomKey.Type, ToDeviceRoomKeyRequest.Type, ToDeviceForwardedRoomKey.Type, ToDeviceRoomKeyWithheld.Type,
ToDeviceBeeperRoomKeyAck.Type: ToDeviceBeeperRoomKeyAck.Type:
@ -236,6 +236,7 @@ var (
BeeperMessageStatus = Type{"com.beeper.message_send_status", MessageEventType} BeeperMessageStatus = Type{"com.beeper.message_send_status", MessageEventType}
BeeperTranscription = Type{"com.beeper.transcription", MessageEventType} BeeperTranscription = Type{"com.beeper.transcription", MessageEventType}
BeeperDeleteChat = Type{"com.beeper.delete_chat", MessageEventType}
EventUnstablePollStart = Type{Type: "org.matrix.msc3381.poll.start", Class: MessageEventType} EventUnstablePollStart = Type{Type: "org.matrix.msc3381.poll.start", Class: MessageEventType}
EventUnstablePollResponse = Type{Type: "org.matrix.msc3381.poll.response", Class: MessageEventType} EventUnstablePollResponse = Type{Type: "org.matrix.msc3381.poll.response", Class: MessageEventType}

View file

@ -32,6 +32,9 @@ type EventID string
// https://github.com/matrix-org/matrix-doc/pull/2716 // https://github.com/matrix-org/matrix-doc/pull/2716
type BatchID string type BatchID string
// A DelayID is a string identifying a delayed event.
type DelayID string
func (roomID RoomID) String() string { func (roomID RoomID) String() string {
return string(roomID) return string(roomID)
} }

View file

@ -104,16 +104,24 @@ func ValidateUserLocalpart(localpart string) error {
return nil return nil
} }
// ParseAndValidate parses the user ID into the localpart and server name like Parse, // ParseAndValidateStrict is a stricter version of ParseAndValidateRelaxed that checks the localpart to only allow non-historical localparts.
// and also validates that the localpart is allowed according to the user identifiers spec. // This should be used with care: there are real users still using historical localparts.
func (userID UserID) ParseAndValidate() (localpart, homeserver string, err error) { func (userID UserID) ParseAndValidateStrict() (localpart, homeserver string, err error) {
localpart, homeserver, err = userID.Parse() localpart, homeserver, err = userID.ParseAndValidateRelaxed()
if err == nil { if err == nil {
err = ValidateUserLocalpart(localpart) err = ValidateUserLocalpart(localpart)
} }
if err == nil && len(userID) > UserIDMaxLength { return
}
// ParseAndValidateRelaxed parses the user ID into the localpart and server name like Parse,
// and also validates that the user ID is not too long and that the server name is valid.
func (userID UserID) ParseAndValidateRelaxed() (localpart, homeserver string, err error) {
if len(userID) > UserIDMaxLength {
err = ErrUserIDTooLong err = ErrUserIDTooLong
return
} }
localpart, homeserver, err = userID.Parse()
if err == nil && !ValidateServerName(homeserver) { if err == nil && !ValidateServerName(homeserver) {
err = fmt.Errorf("%q %q", homeserver, ErrNoncompliantServerPart) err = fmt.Errorf("%q %q", homeserver, ErrNoncompliantServerPart)
} }
@ -121,7 +129,7 @@ func (userID UserID) ParseAndValidate() (localpart, homeserver string, err error
} }
func (userID UserID) ParseAndDecode() (localpart, homeserver string, err error) { func (userID UserID) ParseAndDecode() (localpart, homeserver string, err error) {
localpart, homeserver, err = userID.ParseAndValidate() localpart, homeserver, err = userID.ParseAndValidateStrict()
if err == nil { if err == nil {
localpart, err = DecodeUserLocalpart(localpart) localpart, err = DecodeUserLocalpart(localpart)
} }

Some files were not shown because too many files have changed in this diff Show more