googleapiclient 1.5 upgrade, rsa, pyasn1, pyasn1_moduels for native c…

…rypto

googleapiclient/__init__.py

@@ -12,4 +12,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "1.4.1"
+__version__ = "1.5.0"

googleapiclient/discovery.py

@@ -28,13 +28,17 @@
     'key2param',
     ]
 
-from six import StringIO
+from six import BytesIO
+from six.moves import http_client
 from six.moves.urllib.parse import urlencode, urlparse, urljoin, \
   urlunparse, parse_qsl
 
 # Standard library imports
 import copy
-from email.generator import Generator
+try:
+  from email.generator import BytesGenerator
+except ImportError:
+  from email.generator import Generator as BytesGenerator
 from email.mime.multipart import MIMEMultipart
 from email.mime.nonmultipart import MIMENonMultipart
 import json
@@ -101,6 +105,10 @@
 # Library-specific reserved words beyond Python keywords.
 RESERVED_WORDS = frozenset(['body'])
 
+# patch _write_lines to avoid munging '\r' into '\n'
+# ( https://bugs.python.org/issue18886 https://bugs.python.org/issue19003 )
+class _BytesGenerator(BytesGenerator):
+  _write_lines = BytesGenerator.write
 
 def fix_method_name(name):
   """Fix method names to avoid reserved word conflicts.
@@ -149,7 +157,9 @@ def build(serviceName,
           developerKey=None,
           model=None,
           requestBuilder=HttpRequest,
-          credentials=None):
+          credentials=None,
+          cache_discovery=True,
+          cache=None):
   """Construct a Resource for interacting with an API.
 
   Construct a Resource object for interacting with an API. The serviceName and
@@ -171,6 +181,9 @@ def build(serviceName,
       request.
     credentials: oauth2client.Credentials, credentials to be used for
       authentication.
+    cache_discovery: Boolean, whether or not to cache the discovery doc.
+    cache: googleapiclient.discovery_cache.base.CacheBase, an optional
+      cache object for the discovery documents.
 
   Returns:
     A Resource object with methods for interacting with the service.
@@ -185,22 +198,58 @@ def build(serviceName,
 
   requested_url = uritemplate.expand(discoveryServiceUrl, params)
 
+  try:
+    content = _retrieve_discovery_doc(requested_url, http, cache_discovery,
+                                      cache)
+  except HttpError as e:
+    if e.resp.status == http_client.NOT_FOUND:
+      raise UnknownApiNameOrVersion("name: %s  version: %s" % (serviceName,
+                                                               version))
+    else:
+      raise e
+
+  return build_from_document(content, base=discoveryServiceUrl, http=http,
+      developerKey=developerKey, model=model, requestBuilder=requestBuilder,
+      credentials=credentials)
+
+
+def _retrieve_discovery_doc(url, http, cache_discovery, cache=None):
+  """Retrieves the discovery_doc from cache or the internet.
+
+  Args:
+    url: string, the URL of the discovery document.
+    http: httplib2.Http, An instance of httplib2.Http or something that acts
+      like it through which HTTP requests will be made.
+    cache_discovery: Boolean, whether or not to cache the discovery doc.
+    cache: googleapiclient.discovery_cache.base.Cache, an optional cache
+      object for the discovery documents.
+
+  Returns:
+    A unicode string representation of the discovery document.
+  """
+  if cache_discovery:
+    from . import discovery_cache
+    from .discovery_cache import base
+    if cache is None:
+      cache = discovery_cache.autodetect()
+    if cache:
+      content = cache.get(url)
+      if content:
+        return content
+
+  actual_url = url
   # REMOTE_ADDR is defined by the CGI spec [RFC3875] as the environment
   # variable that contains the network address of the client sending the
   # request. If it exists then add that to the request for the discovery
   # document to avoid exceeding the quota on discovery requests.
   if 'REMOTE_ADDR' in os.environ:
-    requested_url = _add_query_parameter(requested_url, 'userIp',
-                                         os.environ['REMOTE_ADDR'])
-  logger.info('URL being requested: GET %s' % requested_url)
+    actual_url = _add_query_parameter(url, 'userIp', os.environ['REMOTE_ADDR'])
+  logger.info('URL being requested: GET %s', actual_url)
 
-  resp, content = http.request(requested_url)
+  resp, content = http.request(actual_url)
 
-  if resp.status == 404:
-    raise UnknownApiNameOrVersion("name: %s  version: %s" % (serviceName,
-                                                            version))
   if resp.status >= 400:
-    raise HttpError(resp, content, uri=requested_url)
+    raise HttpError(resp, content, uri=actual_url)
 
   try:
     content = content.decode('utf-8')
@@ -212,10 +261,9 @@ def build(serviceName,
   except ValueError as e:
     logger.error('Failed to parse as JSON: ' + content)
     raise InvalidJsonError()
-
-  return build_from_document(content, base=discoveryServiceUrl, http=http,
-      developerKey=developerKey, model=model, requestBuilder=requestBuilder,
-      credentials=credentials)
+  if cache_discovery and cache:
+    cache.set(url, content)
+  return content
 
 
 @positional(1)
@@ -254,6 +302,9 @@ def build_from_document(
     A Resource object with methods for interacting with the service.
   """
 
+  if http is None:
+    http = httplib2.Http()
+
   # future is no longer used.
   future = {}
 
@@ -753,8 +804,8 @@ def method(self, **kwargs):
           msgRoot.attach(msg)
           # encode the body: note that we can't use `as_string`, because
           # it plays games with `From ` lines.
-          fp = StringIO()
-          g = Generator(fp, mangle_from_=False)
+          fp = BytesIO()
+          g = _BytesGenerator(fp, mangle_from_=False)
           g.flatten(msgRoot, unixfrom=False)
           body = fp.getvalue()
 
@@ -854,7 +905,7 @@ def methodNext(self, previous_request, previous_response):
     # Retrieve nextPageToken from previous_response
     # Use as pageToken in previous_request to create new request.
 
-    if 'nextPageToken' not in previous_response:
+    if 'nextPageToken' not in previous_response or not previous_response['nextPageToken']:
       return None
 
     request = copy.copy(previous_request)

googleapiclient/discovery_cache/__init__.py

@@ -0,0 +1,42 @@
+# Copyright 2014 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Caching utility for the discovery document."""
+
+from __future__ import absolute_import
+
+import logging
+import datetime
+
+DISCOVERY_DOC_MAX_AGE = 60 * 60 * 24  # 1 day
+
+
+def autodetect():
+  """Detects an appropriate cache module and returns it.
+
+  Returns:
+    googleapiclient.discovery_cache.base.Cache, a cache object which
+    is auto detected, or None if no cache object is available.
+  """
+  try:
+    from google.appengine.api import memcache
+    from . import appengine_memcache
+    return appengine_memcache.cache
+  except Exception:
+    try:
+      from . import file_cache
+      return file_cache.cache
+    except Exception as e:
+      logging.warning(e, exc_info=True)
+      return None

googleapiclient/discovery_cache/appengine_memcache.py

@@ -0,0 +1,52 @@
+# Copyright 2014 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""App Engine memcache based cache for the discovery document."""
+
+import logging
+
+# This is only an optional dependency because we only import this
+# module when google.appengine.api.memcache is available.
+from google.appengine.api import memcache
+
+from . import base
+from ..discovery_cache import DISCOVERY_DOC_MAX_AGE
+
+NAMESPACE = 'google-api-client'
+
+
+class Cache(base.Cache):
+  """A cache with app engine memcache API."""
+
+  def __init__(self, max_age):
+      """Constructor.
+
+      Args:
+        max_age: Cache expiration in seconds.
+      """
+      self._max_age = max_age
+
+  def get(self, url):
+    try:
+      return memcache.get(url, namespace=NAMESPACE)
+    except Exception as e:
+      logging.warning(e, exc_info=True)
+
+  def set(self, url, content):
+    try:
+      memcache.set(url, content, time=int(self._max_age), namespace=NAMESPACE)
+    except Exception as e:
+      logging.warning(e, exc_info=True)
+
+cache = Cache(max_age=DISCOVERY_DOC_MAX_AGE)

googleapiclient/discovery_cache/base.py

@@ -0,0 +1,45 @@
+# Copyright 2014 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""An abstract class for caching the discovery document."""
+
+import abc
+
+
+class Cache(object):
+  """A base abstract cache class."""
+  __metaclass__ = abc.ABCMeta
+
+  @abc.abstractmethod
+  def get(self, url):
+    """Gets the content from the memcache with a given key.
+
+    Args:
+      url: string, the key for the cache.
+
+    Returns:
+      object, the value in the cache for the given key, or None if the key is
+      not in the cache.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def set(self, url, content):
+    """Sets the given key and content in the cache.
+
+    Args:
+      url: string, the key for the cache.
+      content: string, the discovery document.
+    """
+    raise NotImplementedError()