implement imap quota extension (rfc 9208)

we only have a "storage" limit. for total disk usage. we don't have a limit on
messages (count) or mailboxes (count). also not on total annotation size, but
we don't have support annotations at all at the moment.

we don't implement setquota. with rfc 9208 that's allowed. with the previous
quota rfc 2087 it wasn't.

the status command can now return "DELETED-STORAGE". which should be the disk
space that can be reclaimed by removing messages with the \Deleted flags.
however, it's not very likely clients set the \Deleted flag without expunging
the message immediately. we don't want to go through all messages to calculate
the sum of message sizes with the deleted flag. we also don't currently track
that in MailboxCount. so we just respond with "0". not compliant, but let's
wait until someone complains.

when returning quota information, it is not possible to give the current usage
when no limit is configured. clients implementing rfc 9208 should probably
conclude from the presence of QUOTA=RES-* capabilities (only in rfc 9208, not
in 2087) and the absence of those limits in quota responses (or the absence of
an untagged quota response at all) that a resource type doesn't have a limit.
thunderbird will claim there is no quota information when no limit was
configured, so we can probably conclude that it implements rfc 2087, but not
rfc 9208.

we now also show the usage & limit on the account page.

for issue #115 by pmarini

imapclient/parse.go

@@ -116,7 +116,8 @@ func (c *Conn) xrespText() RespText {
 
 var knownCodes = stringMap(
 	// Without parameters.
-	"ALERT", "PARSE", "READ-ONLY", "READ-WRITE", "TRYCREATE", "UIDNOTSTICKY", "UNAVAILABLE", "AUTHENTICATIONFAILED", "AUTHORIZATIONFAILED", "EXPIRED", "PRIVACYREQUIRED", "CONTACTADMIN", "NOPERM", "INUSE", "EXPUNGEISSUED", "CORRUPTION", "SERVERBUG", "CLIENTBUG", "CANNOT", "LIMIT", "OVERQUOTA", "ALREADYEXISTS", "NONEXISTENT", "NOTSAVED", "HASCHILDREN", "CLOSED", "UNKNOWN-CTE", "OVERQUOTA",
+	"ALERT", "PARSE", "READ-ONLY", "READ-WRITE", "TRYCREATE", "UIDNOTSTICKY", "UNAVAILABLE", "AUTHENTICATIONFAILED", "AUTHORIZATIONFAILED", "EXPIRED", "PRIVACYREQUIRED", "CONTACTADMIN", "NOPERM", "INUSE", "EXPUNGEISSUED", "CORRUPTION", "SERVERBUG", "CLIENTBUG", "CANNOT", "LIMIT", "OVERQUOTA", "ALREADYEXISTS", "NONEXISTENT", "NOTSAVED", "HASCHILDREN", "CLOSED", "UNKNOWN-CTE",
+	"OVERQUOTA", // ../rfc/9208:472
 	// With parameters.
 	"BADCHARSET", "CAPABILITY", "PERMANENTFLAGS", "UIDNEXT", "UIDVALIDITY", "UNSEEN", "APPENDUID", "COPYUID",
 	"HIGHESTMODSEQ", "MODIFIED",
@@ -367,7 +368,7 @@ func (c *Conn) xuntagged() Untagged {
 			if len(attrs) > 0 {
 				c.xspace()
 			}
-			s := c.xword()
+			s := c.xatom()
 			c.xspace()
 			S := strings.ToUpper(s)
 			var num int64
@@ -396,6 +397,8 @@ func (c *Conn) xuntagged() Untagged {
 				}
 			case "HIGHESTMODSEQ":
 				num = c.xint64()
+			case "DELETED-STORAGE":
+				num = c.xint64()
 			default:
 				c.xerrorf("status: unknown attribute %q", s)
 			}
@@ -489,6 +492,49 @@ func (c *Conn) xuntagged() Untagged {
 		c.xcrlf()
 		return UntaggedVanished{earlier, NumSet{Ranges: uids}}
 
+	// ../rfc/9208:668 ../2087:242
+	case "QUOTAROOT":
+		c.xspace()
+		c.xastring()
+		var roots []string
+		for c.take(' ') {
+			root := c.xastring()
+			roots = append(roots, root)
+		}
+		c.xcrlf()
+		return UntaggedQuotaroot(roots)
+
+	// ../rfc/9208:666 ../rfc/2087:239
+	case "QUOTA":
+		c.xspace()
+		root := c.xastring()
+		c.xspace()
+		c.xtake("(")
+
+		xresource := func() QuotaResource {
+			name := c.xatom()
+			c.xspace()
+			usage := c.xint64()
+			c.xspace()
+			limit := c.xint64()
+			return QuotaResource{QuotaResourceName(strings.ToUpper(name)), usage, limit}
+		}
+
+		seen := map[QuotaResourceName]bool{}
+		l := []QuotaResource{xresource()}
+		seen[l[0].Name] = true
+		for c.take(' ') {
+			res := xresource()
+			if seen[res.Name] {
+				c.xerrorf("duplicate resource name %q", res.Name)
+			}
+			seen[res.Name] = true
+			l = append(l, res)
+		}
+		c.xtake(")")
+		c.xcrlf()
+		return UntaggedQuota{root, l}
+
 	default:
 		v, err := strconv.ParseUint(w, 10, 32)
 		if err == nil {
@@ -682,7 +728,7 @@ func (c *Conn) xatom() string {
 	var s string
 	for {
 		b, err := c.readbyte()
-		c.xcheckf(err, "read byte for flag")
+		c.xcheckf(err, "read byte for atom")
 		if b <= ' ' || strings.IndexByte("(){%*\"\\]", b) >= 0 {
 			c.r.UnreadByte()
 			if s == "" {

imapclient/protocol.go

@@ -255,6 +255,37 @@ type UntaggedVanished struct {
 	UIDs    NumSet
 }
 
+// UntaggedQuotaroot lists the roots for which quota can be present.
+type UntaggedQuotaroot []string
+
+// UntaggedQuota holds the quota for a quota root.
+type UntaggedQuota struct {
+	Root string
+
+	// Always has at least one. Any QUOTA=RES-* capability not mentioned has no limit
+	// or this quota root.
+	Resources []QuotaResource
+}
+
+// Resource types ../rfc/9208:533
+
+// QuotaResourceName is the name of a resource type. More can be defined in the
+// future and encountered in the wild. Always in upper case.
+type QuotaResourceName string
+
+const (
+	QuotaResourceStorage           = "STORAGE"
+	QuotaResourceMesssage          = "MESSAGE"
+	QuotaResourceMailbox           = "MAILBOX"
+	QuotaResourceAnnotationStorage = "ANNOTATION-STORAGE"
+)
+
+type QuotaResource struct {
+	Name  QuotaResourceName
+	Usage int64 // Currently in use. Count or disk size in 1024 byte blocks.
+	Limit int64 // Maximum allowed usage.
+}
+
 // ../rfc/2971:184
 
 type UntaggedID map[string]string

imapserver/parse.go

@@ -436,9 +436,9 @@ func (p *parser) xmboxOrPat() ([]string, bool) {
 	return l, true
 }
 
-// ../rfc/9051:7056, RECENT ../rfc/3501:5047, APPENDLIMIT ../rfc/7889:252, HIGHESTMODSEQ ../rfc/7162:2452
+// ../rfc/9051:7056, RECENT ../rfc/3501:5047, APPENDLIMIT ../rfc/7889:252, HIGHESTMODSEQ ../rfc/7162:2452, DELETED-STORAGE ../rfc/9208:696
 func (p *parser) xstatusAtt() string {
-	w := p.xtakelist("MESSAGES", "UIDNEXT", "UIDVALIDITY", "UNSEEN", "DELETED", "SIZE", "RECENT", "APPENDLIMIT", "HIGHESTMODSEQ")
+	w := p.xtakelist("MESSAGES", "UIDNEXT", "UIDVALIDITY", "UNSEEN", "DELETED-STORAGE", "DELETED", "SIZE", "RECENT", "APPENDLIMIT", "HIGHESTMODSEQ")
 	if w == "HIGHESTMODSEQ" {
 		// HIGHESTMODSEQ is a CONDSTORE-enabling parameter. ../rfc/7162:375
 		p.conn.enabled[capCondstore] = true

imapserver/quota_test.go

@@ -0,0 +1,54 @@
+package imapserver
+
+import (
+	"testing"
+
+	"github.com/mjl-/mox/imapclient"
+)
+
+func TestQuota1(t *testing.T) {
+	tc := start(t)
+	defer tc.close()
+
+	tc.client.Login("[email protected]", password0)
+
+	// We don't implement setquota.
+	tc.transactf("bad", `setquota "" (STORAGE 123)`)
+
+	tc.transactf("bad", "getquotaroot")             // Missing param.
+	tc.transactf("bad", "getquotaroot inbox bogus") // Too many params.
+
+	tc.transactf("bad", "getquota")     // Missing param.
+	tc.transactf("bad", "getquota a b") // Too many params.
+
+	// tc does not have a limit.
+	tc.transactf("ok", "getquotaroot inbox")
+	tc.xuntagged(imapclient.UntaggedQuotaroot([]string{""}))
+
+	tc.transactf("no", "getquota bogusroot")
+	tc.transactf("ok", `getquota ""`)
+	tc.xuntagged()
+
+	// Check that we get a DELETED-STORAGE status attribute with value 0, also if
+	// messages are marked deleted. We don't go through the trouble.
+	tc.transactf("ok", "status inbox (DELETED-STORAGE)")
+	tc.xuntagged(imapclient.UntaggedStatus{Mailbox: "Inbox", Attrs: map[string]int64{"DELETED-STORAGE": 0}})
+
+	// tclimit does have a limit.
+	tclimit := startArgs(t, false, false, true, true, "limit")
+	defer tclimit.close()
+
+	tclimit.client.Login("[email protected]", password0)
+
+	tclimit.transactf("ok", "getquotaroot inbox")
+	tclimit.xuntagged(
+		imapclient.UntaggedQuotaroot([]string{""}),
+		imapclient.UntaggedQuota{Root: "", Resources: []imapclient.QuotaResource{{Name: imapclient.QuotaResourceStorage, Usage: 0, Limit: 1}}},
+	)
+
+	tclimit.transactf("ok", `getquota ""`)
+	tclimit.xuntagged(imapclient.UntaggedQuota{Root: "", Resources: []imapclient.QuotaResource{{Name: imapclient.QuotaResourceStorage, Usage: 0, Limit: 1}}})
+
+	tclimit.transactf("ok", "status inbox (DELETED-STORAGE)")
+	tclimit.xuntagged(imapclient.UntaggedStatus{Mailbox: "Inbox", Attrs: map[string]int64{"DELETED-STORAGE": 0}})
+}

imapserver/server.go

@@ -154,12 +154,13 @@ var authFailDelay = time.Second  // After authentication failure.
 // CONDSTORE: ../rfc/7162:411
 // QRESYNC: ../rfc/7162:1323
 // STATUS=SIZE: ../rfc/8438 ../rfc/9051:8024
+// QUOTA QUOTA=RES-STORAGE: ../rfc/9208:111
 //
 // We always announce support for SCRAM PLUS-variants, also on connections without
 // TLS. The client should not be selecting PLUS variants on non-TLS connections,
 // instead opting to do the bare SCRAM variant without indicating the server claims
 // to support the PLUS variant (skipping the server downgrade detection check).
-const serverCapabilities = "IMAP4rev2 IMAP4rev1 ENABLE LITERAL+ IDLE SASL-IR BINARY UNSELECT UIDPLUS ESEARCH SEARCHRES MOVE UTF8=ACCEPT LIST-EXTENDED SPECIAL-USE LIST-STATUS AUTH=SCRAM-SHA-256-PLUS AUTH=SCRAM-SHA-256 AUTH=SCRAM-SHA-1-PLUS AUTH=SCRAM-SHA-1 AUTH=CRAM-MD5 ID APPENDLIMIT=9223372036854775807 CONDSTORE QRESYNC STATUS=SIZE"
+const serverCapabilities = "IMAP4rev2 IMAP4rev1 ENABLE LITERAL+ IDLE SASL-IR BINARY UNSELECT UIDPLUS ESEARCH SEARCHRES MOVE UTF8=ACCEPT LIST-EXTENDED SPECIAL-USE LIST-STATUS AUTH=SCRAM-SHA-256-PLUS AUTH=SCRAM-SHA-256 AUTH=SCRAM-SHA-1-PLUS AUTH=SCRAM-SHA-1 AUTH=CRAM-MD5 ID APPENDLIMIT=9223372036854775807 CONDSTORE QRESYNC STATUS=SIZE QUOTA QUOTA=RES-STORAGE"
 
 type conn struct {
 	cid               int64
@@ -239,7 +240,7 @@ func stateCommands(cmds ...string) map[string]struct{} {
 var (
 	commandsStateAny              = stateCommands("capability", "noop", "logout", "id")
 	commandsStateNotAuthenticated = stateCommands("starttls", "authenticate", "login")
-	commandsStateAuthenticated    = stateCommands("enable", "select", "examine", "create", "delete", "rename", "subscribe", "unsubscribe", "list", "namespace", "status", "append", "idle", "lsub")
+	commandsStateAuthenticated    = stateCommands("enable", "select", "examine", "create", "delete", "rename", "subscribe", "unsubscribe", "list", "namespace", "status", "append", "idle", "lsub", "getquotaroot", "getquota")
 	commandsStateSelected         = stateCommands("close", "unselect", "expunge", "search", "fetch", "store", "copy", "move", "uid expunge", "uid search", "uid fetch", "uid store", "uid copy", "uid move")
 )
 
@@ -256,20 +257,22 @@ var commands = map[string]func(c *conn, tag, cmd string, p *parser){
 	"login":        (*conn).cmdLogin,
 
 	// Authenticated and selected.
-	"enable":      (*conn).cmdEnable,
-	"select":      (*conn).cmdSelect,
-	"examine":     (*conn).cmdExamine,
-	"create":      (*conn).cmdCreate,
-	"delete":      (*conn).cmdDelete,
-	"rename":      (*conn).cmdRename,
-	"subscribe":   (*conn).cmdSubscribe,
-	"unsubscribe": (*conn).cmdUnsubscribe,
-	"list":        (*conn).cmdList,
-	"lsub":        (*conn).cmdLsub,
-	"namespace":   (*conn).cmdNamespace,
-	"status":      (*conn).cmdStatus,
-	"append":      (*conn).cmdAppend,
-	"idle":        (*conn).cmdIdle,
+	"enable":       (*conn).cmdEnable,
+	"select":       (*conn).cmdSelect,
+	"examine":      (*conn).cmdExamine,
+	"create":       (*conn).cmdCreate,
+	"delete":       (*conn).cmdDelete,
+	"rename":       (*conn).cmdRename,
+	"subscribe":    (*conn).cmdSubscribe,
+	"unsubscribe":  (*conn).cmdUnsubscribe,
+	"list":         (*conn).cmdList,
+	"lsub":         (*conn).cmdLsub,
+	"namespace":    (*conn).cmdNamespace,
+	"status":       (*conn).cmdStatus,
+	"append":       (*conn).cmdAppend,
+	"idle":         (*conn).cmdIdle,
+	"getquotaroot": (*conn).cmdGetquotaroot,
+	"getquota":     (*conn).cmdGetquota,
 
 	// Selected.
 	"check":       (*conn).cmdCheck,
@@ -2628,7 +2631,7 @@ func (c *conn) cmdStatus(tag, cmd string, p *parser) {
 	c.ok(tag, cmd)
 }
 
-// Response syntax: ../rfc/9051:6681 ../rfc/9051:7070 ../rfc/9051:7059 ../rfc/3501:4834
+// Response syntax: ../rfc/9051:6681 ../rfc/9051:7070 ../rfc/9051:7059 ../rfc/3501:4834 ../rfc/9208:712
 func (c *conn) xstatusLine(tx *bstore.Tx, mb store.Mailbox, attrs []string) string {
 	status := []string{}
 	for _, a := range attrs {
@@ -2654,6 +2657,15 @@ func (c *conn) xstatusLine(tx *bstore.Tx, mb store.Mailbox, attrs []string) stri
 		case "HIGHESTMODSEQ":
 			// ../rfc/7162:366
 			status = append(status, A, fmt.Sprintf("%d", c.xhighestModSeq(tx, mb.ID).Client()))
+		case "DELETED-STORAGE":
+			// ../rfc/9208:394
+			// How much storage space could be reclaimed by expunging messages with the
+			// \Deleted flag. We could keep track of this number and return it efficiently.
+			// Calculating it each time can be slow, and we don't know if clients request it.
+			// Clients are not likely to set the deleted flag without immediately expunging
+			// nowadays. Let's wait for something to need it to go through the trouble, and
+			// always return 0 for now.
+			status = append(status, A, "0")
 		default:
 			xsyntaxErrorf("unknown attribute %q", a)
 		}
@@ -2792,7 +2804,7 @@ func (c *conn) cmdAppend(tag, cmd string, p *parser) {
 			ok, maxSize, err := c.account.CanAddMessageSize(tx, m.Size)
 			xcheckf(err, "checking quota")
 			if !ok {
-				// ../rfc/9051:5155
+				// ../rfc/9051:5155 ../rfc/9208:472
 				xusercodeErrorf("OVERQUOTA", "account over maximum total message size %d", maxSize)
 			}
 
@@ -2872,6 +2884,87 @@ wait:
 	c.ok(tag, cmd)
 }
 
+// Return the quota root for a mailbox name and any current quota's.
+//
+// State: Authenticated and selected.
+func (c *conn) cmdGetquotaroot(tag, cmd string, p *parser) {
+	// Command: ../rfc/9208:278 ../rfc/2087:141
+
+	// Request syntax: ../rfc/9208:660 ../rfc/2087:233
+	p.xspace()
+	name := p.xmailbox()
+	p.xempty()
+
+	// This mailbox does not have to exist. Caller just wants to know which limits
+	// would apply. We only have one limit, so we don't use the name otherwise.
+	// ../rfc/9208:295
+	name = xcheckmailboxname(name, true)
+
+	// Get current usage for account.
+	var quota, size int64 // Account only has a quota if > 0.
+	c.account.WithRLock(func() {
+		quota = c.account.QuotaMessageSize()
+		if quota >= 0 {
+			c.xdbread(func(tx *bstore.Tx) {
+				du := store.DiskUsage{ID: 1}
+				err := tx.Get(&du)
+				xcheckf(err, "gather used quota")
+				size = du.MessageSize
+			})
+		}
+	})
+
+	// We only have one per account quota, we name it "" like the examples in the RFC.
+	// Response syntax: ../rfc/9208:668 ../rfc/2087:242
+	c.bwritelinef(`* QUOTAROOT %s ""`, astring(name).pack(c))
+
+	// We only write the quota response if there is a limit. The syntax doesn't allow
+	// an empty list, so we cannot send the current disk usage if there is no limit.
+	if quota > 0 {
+		// Response syntax: ../rfc/9208:666 ../rfc/2087:239
+		c.bwritelinef(`* QUOTA "" (STORAGE %d %d)`, (size+1024-1)/1024, (quota+1024-1)/1024)
+	}
+	c.ok(tag, cmd)
+}
+
+// Return the quota for a quota root.
+//
+// State: Authenticated and selected.
+func (c *conn) cmdGetquota(tag, cmd string, p *parser) {
+	// Command: ../rfc/9208:245 ../rfc/2087:123
+
+	// Request syntax: ../rfc/9208:658 ../rfc/2087:231
+	p.xspace()
+	root := p.xastring()
+	p.xempty()
+
+	// We only have a per-account root called "".
+	if root != "" {
+		xuserErrorf("unknown quota root")
+	}
+
+	var quota, size int64
+	c.account.WithRLock(func() {
+		quota = c.account.QuotaMessageSize()
+		if quota > 0 {
+			c.xdbread(func(tx *bstore.Tx) {
+				du := store.DiskUsage{ID: 1}
+				err := tx.Get(&du)
+				xcheckf(err, "gather used quota")
+				size = du.MessageSize
+			})
+		}
+	})
+
+	// We only write the quota response if there is a limit. The syntax doesn't allow
+	// an empty list, so we cannot send the current disk usage if there is no limit.
+	if quota > 0 {
+		// Response syntax: ../rfc/9208:666 ../rfc/2087:239
+		c.bwritelinef(`* QUOTA "" (STORAGE %d %d)`, (size+1024-1)/1024, (quota+1024-1)/1024)
+	}
+	c.ok(tag, cmd)
+}
+
 // Check is an old deprecated command that is supposed to execute some mailbox consistency checks.
 //
 // State: Selected
@@ -3267,7 +3360,7 @@ func (c *conn) cmdxCopy(isUID bool, tag, cmd string, p *parser) {
 			if ok, maxSize, err := c.account.CanAddMessageSize(tx, totalSize); err != nil {
 				xcheckf(err, "checking quota")
 			} else if !ok {
-				// ../rfc/9051:5155
+				// ../rfc/9051:5155 ../rfc/9208:472
 				xusercodeErrorf("OVERQUOTA", "account over maximum total message size %d", maxSize)
 			}
 			err = c.account.AddMessageSize(c.log, tx, totalSize)

rfc/index.txt

@@ -229,7 +229,7 @@ https://www.iana.org/assignments/message-headers/message-headers.xhtml
 8508	Roadmap	-	IMAP REPLACE Extension
 8514	Roadmap	-	Internet Message Access Protocol (IMAP) - SAVEDATE Extension
 8970	Roadmap	-	IMAP4 Extension: Message Preview Generation
-9208	Roadmap	-	IMAP QUOTA Extension
+9208	Partial	-	IMAP QUOTA Extension
 9394	Roadmap	-	IMAP PARTIAL Extension for Paged SEARCH and FETCH
 
 5198	-?	-	Unicode Format for Network Interchange