Author: Manos Pitsidianakis [el13635@mail.ntua.gr]
Hash: 92c12d3526d5fdf4a4eaaf49f91ecd718074cb06
Timestamp: Tue, 24 Nov 2020 10:04:04 +0000 (3 years ago)

+452 -9 +/-4 browse
melib/imap: implement OAUTH2 authentication
1diff --git a/contrib/oauth2.py b/contrib/oauth2.py
2new file mode 100755
3index 0000000..a1ab65f
4--- /dev/null
5+++ b/contrib/oauth2.py
6 @@ -0,0 +1,348 @@
7+ #!/usr/bin/env python3
8+ #
9+ # Copyright 2012 Google Inc.
10+ # Copyright 2020 Manos Pitsidianakis
11+ #
12+ # Licensed under the Apache License, Version 2.0 (the "License");
13+ # you may not use this file except in compliance with the License.
14+ # You may obtain a copy of the License at
15+ #
16+ # http://www.apache.org/licenses/LICENSE-2.0
17+ #
18+ # Unless required by applicable law or agreed to in writing, software
19+ # distributed under the License is distributed on an "AS IS" BASIS,
20+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21+ # See the License for the specific language governing permissions and
22+ # limitations under the License.
23+
24+ """Performs client tasks for testing IMAP OAuth2 authentication.
25+
26+ To use this script, you'll need to have registered with Google as an OAuth
27+ application and obtained an OAuth client ID and client secret.
28+ See https://developers.google.com/identity/protocols/OAuth2 for instructions on
29+ registering and for documentation of the APIs invoked by this code.
30+
31+ This script has 3 modes of operation.
32+
33+ 1. The first mode is used to generate and authorize an OAuth2 token, the
34+ first step in logging in via OAuth2.
35+
36+ oauth2 --user=xxx@gmail.com \
37+ --client_id=1038[...].apps.googleusercontent.com \
38+ --client_secret=VWFn8LIKAMC-MsjBMhJeOplZ \
39+ --generate_oauth2_token
40+
41+ The script will converse with Google and generate an oauth request
42+ token, then present you with a URL you should visit in your browser to
43+ authorize the token. Once you get the verification code from the Google
44+ website, enter it into the script to get your OAuth access token. The output
45+ from this command will contain the access token, a refresh token, and some
46+ metadata about the tokens. The access token can be used until it expires, and
47+ the refresh token lasts indefinitely, so you should record these values for
48+ reuse.
49+
50+ 2. The script will generate new access tokens using a refresh token.
51+
52+ oauth2 --user=xxx@gmail.com \
53+ --client_id=1038[...].apps.googleusercontent.com \
54+ --client_secret=VWFn8LIKAMC-MsjBMhJeOplZ \
55+ --refresh_token=1/Yzm6MRy4q1xi7Dx2DuWXNgT6s37OrP_DW_IoyTum4YA
56+
57+ 3. The script will generate an OAuth2 string that can be fed
58+ directly to IMAP or SMTP. This is triggered with the --generate_oauth2_string
59+ option.
60+
61+ oauth2 --generate_oauth2_string --user=xxx@gmail.com \
62+ --access_token=ya29.AGy[...]ezLg
63+
64+ The output of this mode will be a base64-encoded string. To use it, connect to a
65+ IMAPFE and pass it as the second argument to the AUTHENTICATE command.
66+
67+ a AUTHENTICATE XOAUTH2 a9sha9sfs[...]9dfja929dk==
68+ """
69+
70+ import base64
71+ import imaplib
72+ import json
73+ from optparse import OptionParser
74+ import smtplib
75+ import sys
76+ import urllib.request, urllib.parse, urllib.error
77+
78+
79+ def SetupOptionParser():
80+ # Usage message is the module's docstring.
81+ parser = OptionParser(usage=__doc__)
82+ parser.add_option('--generate_oauth2_token',
83+ action='store_true',
84+ dest='generate_oauth2_token',
85+ help='generates an OAuth2 token for testing')
86+ parser.add_option('--generate_oauth2_string',
87+ action='store_true',
88+ dest='generate_oauth2_string',
89+ help='generates an initial client response string for '
90+ 'OAuth2')
91+ parser.add_option('--client_id',
92+ default=None,
93+ help='Client ID of the application that is authenticating. '
94+ 'See OAuth2 documentation for details.')
95+ parser.add_option('--client_secret',
96+ default=None,
97+ help='Client secret of the application that is '
98+ 'authenticating. See OAuth2 documentation for '
99+ 'details.')
100+ parser.add_option('--access_token',
101+ default=None,
102+ help='OAuth2 access token')
103+ parser.add_option('--refresh_token',
104+ default=None,
105+ help='OAuth2 refresh token')
106+ parser.add_option('--scope',
107+ default='https://mail.google.com/',
108+ help='scope for the access token. Multiple scopes can be '
109+ 'listed separated by spaces with the whole argument '
110+ 'quoted.')
111+ parser.add_option('--test_imap_authentication',
112+ action='store_true',
113+ dest='test_imap_authentication',
114+ help='attempts to authenticate to IMAP')
115+ parser.add_option('--test_smtp_authentication',
116+ action='store_true',
117+ dest='test_smtp_authentication',
118+ help='attempts to authenticate to SMTP')
119+ parser.add_option('--user',
120+ default=None,
121+ help='email address of user whose account is being '
122+ 'accessed')
123+ parser.add_option('--quiet',
124+ action='store_true',
125+ default=False,
126+ dest='quiet',
127+ help='Omit verbose descriptions and only print '
128+ 'machine-readable outputs.')
129+ return parser
130+
131+
132+ # The URL root for accessing Google Accounts.
133+ GOOGLE_ACCOUNTS_BASE_URL = 'https://accounts.google.com'
134+
135+
136+ # Hardcoded dummy redirect URI for non-web apps.
137+ REDIRECT_URI = 'urn:ietf:wg:oauth:2.0:oob'
138+
139+
140+ def AccountsUrl(command):
141+ """Generates the Google Accounts URL.
142+
143+ Args:
144+ command: The command to execute.
145+
146+ Returns:
147+ A URL for the given command.
148+ """
149+ return '%s/%s' % (GOOGLE_ACCOUNTS_BASE_URL, command)
150+
151+
152+ def UrlEscape(text):
153+ # See OAUTH 5.1 for a definition of which characters need to be escaped.
154+ return urllib.parse.quote(text, safe='~-._')
155+
156+
157+ def UrlUnescape(text):
158+ # See OAUTH 5.1 for a definition of which characters need to be escaped.
159+ return urllib.parse.unquote(text)
160+
161+
162+ def FormatUrlParams(params):
163+ """Formats parameters into a URL query string.
164+
165+ Args:
166+ params: A key-value map.
167+
168+ Returns:
169+ A URL query string version of the given parameters.
170+ """
171+ param_fragments = []
172+ for param in sorted(iter(params.items()), key=lambda x: x[0]):
173+ param_fragments.append('%s=%s' % (param[0], UrlEscape(param[1])))
174+ return '&'.join(param_fragments)
175+
176+
177+ def GeneratePermissionUrl(client_id, scope='https://mail.google.com/'):
178+ """Generates the URL for authorizing access.
179+
180+ This uses the "OAuth2 for Installed Applications" flow described at
181+ https://developers.google.com/accounts/docs/OAuth2InstalledApp
182+
183+ Args:
184+ client_id: Client ID obtained by registering your app.
185+ scope: scope for access token, e.g. 'https://mail.google.com'
186+ Returns:
187+ A URL that the user should visit in their browser.
188+ """
189+ params = {}
190+ params['client_id'] = client_id
191+ params['redirect_uri'] = REDIRECT_URI
192+ params['scope'] = scope
193+ params['response_type'] = 'code'
194+ return '%s?%s' % (AccountsUrl('o/oauth2/auth'),
195+ FormatUrlParams(params))
196+
197+
198+ def AuthorizeTokens(client_id, client_secret, authorization_code):
199+ """Obtains OAuth access token and refresh token.
200+
201+ This uses the application portion of the "OAuth2 for Installed Applications"
202+ flow at https://developers.google.com/accounts/docs/OAuth2InstalledApp#handlingtheresponse
203+
204+ Args:
205+ client_id: Client ID obtained by registering your app.
206+ client_secret: Client secret obtained by registering your app.
207+ authorization_code: code generated by Google Accounts after user grants
208+ permission.
209+ Returns:
210+ The decoded response from the Google Accounts server, as a dict. Expected
211+ fields include 'access_token', 'expires_in', and 'refresh_token'.
212+ """
213+ params = {}
214+ params['client_id'] = client_id
215+ params['client_secret'] = client_secret
216+ params['code'] = authorization_code
217+ params['redirect_uri'] = REDIRECT_URI
218+ params['grant_type'] = 'authorization_code'
219+ request_url = AccountsUrl('o/oauth2/token')
220+
221+ response = urllib.request.urlopen(request_url, urllib.parse.urlencode(params).encode()).read()
222+ return json.loads(response)
223+
224+
225+ def RefreshToken(client_id, client_secret, refresh_token):
226+ """Obtains a new token given a refresh token.
227+
228+ See https://developers.google.com/accounts/docs/OAuth2InstalledApp#refresh
229+
230+ Args:
231+ client_id: Client ID obtained by registering your app.
232+ client_secret: Client secret obtained by registering your app.
233+ refresh_token: A previously-obtained refresh token.
234+ Returns:
235+ The decoded response from the Google Accounts server, as a dict. Expected
236+ fields include 'access_token', 'expires_in', and 'refresh_token'.
237+ """
238+ params = {}
239+ params['client_id'] = client_id
240+ params['client_secret'] = client_secret
241+ params['refresh_token'] = refresh_token
242+ params['grant_type'] = 'refresh_token'
243+ request_url = AccountsUrl('o/oauth2/token')
244+
245+ response = urllib.request.urlopen(request_url, urllib.parse.urlencode(params).encode()).read()
246+ return json.loads(response)
247+
248+
249+ def GenerateOAuth2String(username, access_token, base64_encode=True):
250+ """Generates an IMAP OAuth2 authentication string.
251+
252+ See https://developers.google.com/google-apps/gmail/oauth2_overview
253+
254+ Args:
255+ username: the username (email address) of the account to authenticate
256+ access_token: An OAuth2 access token.
257+ base64_encode: Whether to base64-encode the output.
258+
259+ Returns:
260+ The SASL argument for the OAuth2 mechanism.
261+ """
262+ auth_string = 'user=%s\1auth=Bearer %s\1\1' % (username, access_token)
263+ if base64_encode:
264+ auth_string = base64.b64encode(bytes(auth_string, 'utf-8'))
265+ return auth_string
266+
267+
268+ def TestImapAuthentication(user, auth_string):
269+ """Authenticates to IMAP with the given auth_string.
270+
271+ Prints a debug trace of the attempted IMAP connection.
272+
273+ Args:
274+ user: The Gmail username (full email address)
275+ auth_string: A valid OAuth2 string, as returned by GenerateOAuth2String.
276+ Must not be base64-encoded, since imaplib does its own base64-encoding.
277+ """
278+ print()
279+ imap_conn = imaplib.IMAP4_SSL('imap.gmail.com')
280+ imap_conn.debug = 4
281+ imap_conn.authenticate('XOAUTH2', lambda x: auth_string)
282+ imap_conn.select('INBOX')
283+
284+
285+ def TestSmtpAuthentication(user, auth_string):
286+ """Authenticates to SMTP with the given auth_string.
287+
288+ Args:
289+ user: The Gmail username (full email address)
290+ auth_string: A valid OAuth2 string, not base64-encoded, as returned by
291+ GenerateOAuth2String.
292+ """
293+ print()
294+ smtp_conn = smtplib.SMTP('smtp.gmail.com', 587)
295+ smtp_conn.set_debuglevel(True)
296+ smtp_conn.ehlo('test')
297+ smtp_conn.starttls()
298+ smtp_conn.docmd('AUTH', 'XOAUTH2 ' + base64.b64encode(auth_string))
299+
300+
301+ def RequireOptions(options, *args):
302+ missing = [arg for arg in args if getattr(options, arg) is None]
303+ if missing:
304+ print('Missing options: %s' % ' '.join(missing), file=sys.stderr)
305+ sys.exit(-1)
306+
307+
308+ def main(argv):
309+ options_parser = SetupOptionParser()
310+ (options, args) = options_parser.parse_args()
311+ if options.refresh_token:
312+ RequireOptions(options, 'client_id', 'client_secret')
313+ response = RefreshToken(options.client_id, options.client_secret,
314+ options.refresh_token)
315+ if options.quiet:
316+ print(response['access_token'])
317+ else:
318+ print('Access Token: %s' % response['access_token'])
319+ print('Access Token Expiration Seconds: %s' % response['expires_in'])
320+ elif options.generate_oauth2_string:
321+ RequireOptions(options, 'user', 'access_token')
322+ oauth2_string = GenerateOAuth2String(options.user, options.access_token)
323+ if options.quiet:
324+ print(oauth2_string.decode('utf-8'))
325+ else:
326+ print('OAuth2 argument:\n' + oauth2_string.decode('utf-8'))
327+ elif options.generate_oauth2_token:
328+ RequireOptions(options, 'client_id', 'client_secret')
329+ print('To authorize token, visit this url and follow the directions:')
330+ print(' %s' % GeneratePermissionUrl(options.client_id, options.scope))
331+ authorization_code = input('Enter verification code: ')
332+ response = AuthorizeTokens(options.client_id, options.client_secret,
333+ authorization_code)
334+ print('Refresh Token: %s' % response['refresh_token'])
335+ print('Access Token: %s' % response['access_token'])
336+ print('Access Token Expiration Seconds: %s' % response['expires_in'])
337+ elif options.test_imap_authentication:
338+ RequireOptions(options, 'user', 'access_token')
339+ TestImapAuthentication(options.user,
340+ GenerateOAuth2String(options.user, options.access_token,
341+ base64_encode=False))
342+ elif options.test_smtp_authentication:
343+ RequireOptions(options, 'user', 'access_token')
344+ TestSmtpAuthentication(options.user,
345+ GenerateOAuth2String(options.user, options.access_token,
346+ base64_encode=False))
347+ else:
348+ options_parser.print_help()
349+ print('Nothing to do, exiting.')
350+ return
351+
352+
353+ if __name__ == '__main__':
354+ main(sys.argv)
355 diff --git a/docs/meli.conf.5 b/docs/meli.conf.5
356index cc6e3c1..1d30628 100644
357--- a/docs/meli.conf.5
358+++ b/docs/meli.conf.5
359 @@ -235,11 +235,25 @@ Do not validate TLS certificates.
360 Use IDLE extension.
361 .\" default value
362 .Pq Em true
363+ .It Ic use_condstore Ar boolean
364+ .Pq Em optional
365+ Use CONDSTORE extension.
366+ .\" default value
367+ .Pq Em true
368 .It Ic use_deflate Ar boolean
369 .Pq Em optional
370 Use COMPRESS=DEFLATE extension (if built with DEFLATE support).
371 .\" default value
372 .Pq Em true
373+ .It Ic use_oauth2 Ar boolean
374+ .Pq Em optional
375+ Use OAUTH2 authentication.
376+ Can only be used with
377+ .Ic server_password_command
378+ which should return a base64-encoded OAUTH2 token ready to be passed to IMAP.
379+ For help on setup with Gmail, see Gmail section below.
380+ .\" default value
381+ .Pq Em false
382 .It Ic timeout Ar integer
383 .Pq Em optional
384 Timeout to use for server connections in seconds.
385 @@ -247,6 +261,35 @@ A timeout of 0 seconds means there's no timeout.
386 .\" default value
387 .Pq Em 16
388 .El
389+ .Ss Gmail
390+ Gmail has non-standard IMAP behaviors that need to be worked around.
391+ .Ss Gmail - sending mail
392+ Option
393+ .Ic store_sent_mail
394+ should be disabled since Gmail auto-saves sent mail by its own.
395+ .Ss Gmail OAUTH2
396+ To use OAUTH2, you must go through a process to register your own private "application" with Google that can use OAUTH2 tokens.
397+ For convenience in the meli repository under the
398+ .Pa contrib/
399+ directory you can find a python3 file named oauth2.py to generate and request the appropriate data to perform OAUTH2 authentication.
400+ Steps:
401+ .Bl -bullet -compact
402+ .It
403+ In Google APIs, create a custom OAuth client ID and note down the Client ID and Client Secret.
404+ You may need to create a consent screen; follow the steps described in the website.
405+ .It
406+ Run the oauth2.py script as follows (after adjusting binary paths and credentials):
407+ .Cm python3 oauth2.py --user=xxx@gmail.com --client_id=1038[...].apps.googleusercontent.com --client_secret=VWFn8LIKAMC-MsjBMhJeOplZ --generate_oauth2_token
408+ and follow the instructions.
409+ Note down the refresh token.
410+ .It
411+ In
412+ .Ic server_password_command
413+ enter a command like this (after adjusting binary paths and credentials):
414+ .Cm TOKEN=$(python3 oauth2.py --user=xxx@gmail.com --quiet --client_id=1038[...].apps.googleusercontent.com --client_secret=VWFn8LIKAMC-MsjBMhJeOplZ --refresh_token=1/Yzm6MRy4q1xi7Dx2DuWXNgT6s37OrP_DW_IoyTum4YA) && python3 oauth2.py --user=xxx@gmail.com --generate_oauth2_string --quiet --access_token=$TOKEN
415+ .It
416+ On startup, meli should evaluate this command which if successful must only return a base64-encoded token ready to be passed to IMAP.
417+ .El
418 .Ss JMAP only
419 JMAP specific options
420 .Bl -tag -width 36n
421 diff --git a/melib/src/backends/imap.rs b/melib/src/backends/imap.rs
422index a060ca9..e3186a6 100644
423--- a/melib/src/backends/imap.rs
424+++ b/melib/src/backends/imap.rs
425 @@ -64,6 +64,7 @@ pub type UIDVALIDITY = UID;
426 pub type MessageSequenceNumber = ImapNum;
427
428 pub static SUPPORTED_CAPABILITIES: &[&str] = &[
429+ "AUTH=OAUTH2",
430 #[cfg(feature = "deflate_compression")]
431 "COMPRESS=DEFLATE",
432 "CONDSTORE",
433 @@ -232,6 +233,7 @@ impl MailBackend for ImapType {
434 #[cfg(feature = "deflate_compression")]
435 deflate,
436 condstore,
437+ oauth2,
438 },
439 } = self.server_conf.protocol
440 {
441 @@ -273,6 +275,15 @@ impl MailBackend for ImapType {
442 };
443 }
444 }
445+ "AUTH=OAUTH2" => {
446+ if oauth2 {
447+ *status = MailBackendExtensionStatus::Enabled { comment: None };
448+ } else {
449+ *status = MailBackendExtensionStatus::Supported {
450+ comment: Some("Disabled by user configuration"),
451+ };
452+ }
453+ }
454 _ => {
455 if SUPPORTED_CAPABILITIES
456 .iter()
457 @@ -1218,7 +1229,14 @@ impl ImapType {
458 ) -> Result<Box<dyn MailBackend>> {
459 let server_hostname = get_conf_val!(s["server_hostname"])?;
460 let server_username = get_conf_val!(s["server_username"])?;
461+ let use_oauth2: bool = get_conf_val!(s["use_oauth2"], false)?;
462 let server_password = if !s.extra.contains_key("server_password_command") {
463+ if use_oauth2 {
464+ return Err(MeliError::new(format!(
465+ "({}) `use_oauth2` use requires `server_password_command` set with a command that returns an OAUTH2 token. Consult documentation for guidance.",
466+ s.name,
467+ )));
468+ }
469 get_conf_val!(s["server_password"])?.to_string()
470 } else {
471 let invocation = get_conf_val!(s["server_password_command"])?;
472 @@ -1275,6 +1293,7 @@ impl ImapType {
473 condstore: get_conf_val!(s["use_condstore"], true)?,
474 #[cfg(feature = "deflate_compression")]
475 deflate: get_conf_val!(s["use_deflate"], true)?,
476+ oauth2: use_oauth2,
477 },
478 },
479 timeout,
480 @@ -1463,7 +1482,14 @@ impl ImapType {
481 pub fn validate_config(s: &AccountSettings) -> Result<()> {
482 get_conf_val!(s["server_hostname"])?;
483 get_conf_val!(s["server_username"])?;
484+ let use_oauth2: bool = get_conf_val!(s["use_oauth2"], false)?;
485 if !s.extra.contains_key("server_password_command") {
486+ if use_oauth2 {
487+ return Err(MeliError::new(format!(
488+ "({}) `use_oauth2` use requires `server_password_command` set with a command that returns an OAUTH2 token. Consult documentation for guidance.",
489+ s.name,
490+ )));
491+ }
492 get_conf_val!(s["server_password"])?;
493 } else if s.extra.contains_key("server_password") {
494 return Err(MeliError::new(format!(
495 diff --git a/melib/src/backends/imap/connection.rs b/melib/src/backends/imap/connection.rs
496index 69aa622..1d2353c 100644
497--- a/melib/src/backends/imap/connection.rs
498+++ b/melib/src/backends/imap/connection.rs
499 @@ -63,6 +63,7 @@ pub struct ImapExtensionUse {
500 pub idle: bool,
501 #[cfg(feature = "deflate_compression")]
502 pub deflate: bool,
503+ pub oauth2: bool,
504 }
505
506 impl Default for ImapExtensionUse {
507 @@ -72,6 +73,7 @@ impl Default for ImapExtensionUse {
508 idle: true,
509 #[cfg(feature = "deflate_compression")]
510 deflate: true,
511+ oauth2: false,
512 }
513 }
514 }
515 @@ -351,16 +353,39 @@ impl ImapStream {
516 .set_err_kind(crate::error::ErrorKind::Authentication));
517 }
518
519- let mut capabilities = None;
520- ret.send_command(
521- format!(
522- "LOGIN \"{}\" \"{}\"",
523- &server_conf.server_username, &server_conf.server_password
524- )
525- .as_bytes(),
526- )
527- .await?;
528+ match server_conf.protocol {
529+ ImapProtocol::IMAP {
530+ extension_use: ImapExtensionUse { oauth2, .. },
531+ } if oauth2 => {
532+ if !capabilities
533+ .iter()
534+ .any(|cap| cap.eq_ignore_ascii_case(b"AUTH=XOAUTH2"))
535+ {
536+ return Err(MeliError::new(format!(
537+ "Could not connect to {}: OAUTH2 is enabled but server did not return AUTH=XOAUTH2 capability. Returned capabilities were: {}",
538+ &server_conf.server_hostname,
539+ capabilities.iter().map(|capability|
540+ String::from_utf8_lossy(capability).to_string()).collect::<Vec<String>>().join(" ")
541+ )));
542+ }
543+ ret.send_command(
544+ format!("AUTHENTICATE XOAUTH2 {}", &server_conf.server_password).as_bytes(),
545+ )
546+ .await?;
547+ }
548+ _ => {
549+ ret.send_command(
550+ format!(
551+ "LOGIN \"{}\" \"{}\"",
552+ &server_conf.server_username, &server_conf.server_password
553+ )
554+ .as_bytes(),
555+ )
556+ .await?;
557+ }
558+ }
559 let tag_start = format!("M{} ", (ret.cmd_id - 1));
560+ let mut capabilities = None;
561
562 loop {
563 ret.read_lines(&mut res, &[], false).await?;
564 @@ -604,6 +629,7 @@ impl ImapConnection {
565 #[cfg(feature = "deflate_compression")]
566 deflate,
567 idle: _idle,
568+ oauth2: _,
569 },
570 } => {
571 if capabilities.contains(&b"CONDSTORE"[..]) && condstore {