Package googleapiclient :: Module discovery |
[hide private]
[frames] | no frames]

# Source Code for Module googleapiclient.discovery

       1  # Copyright 2014 Google Inc. All Rights Reserved.
       2  #
       3  # Licensed under the Apache License, Version 2.0 (the "License");
       4  # you may not use this file except in compliance with the License.
       5  # You may obtain a copy of the License at
       6  #
       7  #      http://www.apache.org/licenses/LICENSE-2.0
       8  #
       9  # Unless required by applicable law or agreed to in writing, software
      10  # distributed under the License is distributed on an "AS IS" BASIS,
      11  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
      12  # See the License for the specific language governing permissions and
      13  # limitations under the License.
      14
      15  """Client for discovery based APIs.
      16
      17  A client library for Google's discovery based APIs.
      18  """
      19  from __future__ import absolute_import
      20  import six
      21  from six.moves import zip
      22
      23  __author__ = 'jcgregorio@google.com (Joe Gregorio)'
      24  __all__ = [
      25      'build',
      26      'build_from_document',
      27      'fix_method_name',
      28      'key2param',
      29      ]
      30
      31  from six import BytesIO
      32  from six.moves import http_client
      33  from six.moves.urllib.parse import urlencode, urlparse, urljoin, \
      34    urlunparse, parse_qsl
      35
      36  # Standard library imports
      37  import copy
      38  try:
      39    from email.generator import BytesGenerator
      40  except ImportError:
      41    from email.generator import Generator as BytesGenerator
      42  from email.mime.multipart import MIMEMultipart
      43  from email.mime.nonmultipart import MIMENonMultipart
      44  import json
      45  import keyword
      46  import logging
      47  import mimetypes
      48  import os
      49  import re
      50
      51  # Third-party imports
      52  import httplib2
      53  import uritemplate
      54
      55  # Local imports
      56  from googleapiclient import _auth
      57  from googleapiclient import mimeparse
      58  from googleapiclient.errors import HttpError
      59  from googleapiclient.errors import InvalidJsonError
      60  from googleapiclient.errors import MediaUploadSizeError
      61  from googleapiclient.errors import UnacceptableMimeTypeError
      62  from googleapiclient.errors import UnknownApiNameOrVersion
      63  from googleapiclient.errors import UnknownFileType
      64  from googleapiclient.http import BatchHttpRequest
      65  from googleapiclient.http import HttpMock
      66  from googleapiclient.http import HttpMockSequence
      67  from googleapiclient.http import HttpRequest
      68  from googleapiclient.http import MediaFileUpload
      69  from googleapiclient.http import MediaUpload
      70  from googleapiclient.model import JsonModel
      71  from googleapiclient.model import MediaModel
      72  from googleapiclient.model import RawModel
      73  from googleapiclient.schema import Schemas
      74  from oauth2client.client import GoogleCredentials
      75
      76  # Oauth2client < 3 has the positional helper in 'util', >= 3 has it
      77  # in '_helpers'.
      78  try:
      79    from oauth2client.util import _add_query_parameter
      80    from oauth2client.util import positional
      81  except ImportError:
      82    from oauth2client._helpers import _add_query_parameter
      83    from oauth2client._helpers import positional
      84
      85
      86  # The client library requires a version of httplib2 that supports RETRIES.
      87  httplib2.RETRIES = 1
      88
      89  logger = logging.getLogger(__name__)
      90
      91  URITEMPLATE = re.compile('{[^}]*}')
      92  VARNAME = re.compile('[a-zA-Z0-9_-]+')
      93  DISCOVERY_URI = ('https://www.googleapis.com/discovery/v1/apis/'
      94                   '{api}/{apiVersion}/rest')
      95  V1_DISCOVERY_URI = DISCOVERY_URI
      96  V2_DISCOVERY_URI = ('https://{api}.googleapis.com/$discovery/rest?'
      97                      'version={apiVersion}')
      98  DEFAULT_METHOD_DOC = 'A description of how to use this function'
      99  HTTP_PAYLOAD_METHODS = frozenset(['PUT', 'POST', 'PATCH'])
     100  _MEDIA_SIZE_BIT_SHIFTS = {'KB': 10, 'MB': 20, 'GB': 30, 'TB': 40}
     101  BODY_PARAMETER_DEFAULT_VALUE = {
     102      'description': 'The request body.',
     103      'type': 'object',
     104      'required': True,
     105  }
     106  MEDIA_BODY_PARAMETER_DEFAULT_VALUE = {
     107      'description': ('The filename of the media request body, or an instance '
     108                      'of a MediaUpload object.'),
     109      'type': 'string',
     110      'required': False,
     111  }
     112  MEDIA_MIME_TYPE_PARAMETER_DEFAULT_VALUE = {
     113      'description': ('The MIME type of the media request body, or an instance '
     114                      'of a MediaUpload object.'),
     115      'type': 'string',
     116      'required': False,
     117  }
     118
     119  # Parameters accepted by the stack, but not visible via discovery.
     120  # TODO(dhermes): Remove 'userip' in 'v2'.
     121  STACK_QUERY_PARAMETERS = frozenset(['trace', 'pp', 'userip', 'strict'])
     122  STACK_QUERY_PARAMETER_DEFAULT_VALUE = {'type': 'string', 'location': 'query'}
     123
     124  # Library-specific reserved words beyond Python keywords.
     125  RESERVED_WORDS = frozenset(['body'])
     126
     127  # patch _write_lines to avoid munging '\r' into '\n'
     128  # ( https://bugs.python.org/issue18886 https://bugs.python.org/issue19003 )
     129 -class _BytesGenerator(BytesGenerator):
     130    _write_lines = BytesGenerator.write
     131
     132 -def fix_method_name(name):
     133    """Fix method names to avoid reserved word conflicts.
     134
     135    Args:
     136      name: string, method name.
     137
     138    Returns:
     139      The name with a '_' prefixed if the name is a reserved word.
     140    """
     141    if keyword.iskeyword(name) or name in RESERVED_WORDS:
     142      return name + '_'
     143    else:
     144      return name
     145
     146
     147 -def key2param(key):
     148    """Converts key names into parameter names.
     149
     150    For example, converting "max-results" -> "max_results"
     151
     152    Args:
     153      key: string, the method key name.
     154
     155    Returns:
     156      A safe method name based on the key name.
     157    """
     158    result = []
     159    key = list(key)
     160    if not key[0].isalpha():
     161      result.append('x')
     162    for c in key:
     163      if c.isalnum():
     164        result.append(c)
     165      else:
     166        result.append('_')
     167
     168    return ''.join(result)
     169
     170
     171  @positional(2)
     172 -def build(serviceName,
     173            version,
     174            http=None,
     175            discoveryServiceUrl=DISCOVERY_URI,
     176            developerKey=None,
     177            model=None,
     178            requestBuilder=HttpRequest,
     179            credentials=None,
     180            cache_discovery=True,
     181            cache=None):
     182    """Construct a Resource for interacting with an API.
     183
     184    Construct a Resource object for interacting with an API. The serviceName and
     185    version are the names from the Discovery service.
     186
     187    Args:
     188      serviceName: string, name of the service.
     189      version: string, the version of the service.
     190      http: httplib2.Http, An instance of httplib2.Http or something that acts
     191        like it that HTTP requests will be made through.
     192      discoveryServiceUrl: string, a URI Template that points to the location of
     193        the discovery service. It should have two parameters {api} and
     194        {apiVersion} that when filled in produce an absolute URI to the discovery
     195        document for that service.
     196      developerKey: string, key obtained from
     197        https://code.google.com/apis/console.
     198      model: googleapiclient.Model, converts to and from the wire format.
     199      requestBuilder: googleapiclient.http.HttpRequest, encapsulator for an HTTP
     200        request.
     201      credentials: oauth2client.Credentials or
     202        google.auth.credentials.Credentials, credentials to be used for
     203        authentication.
     204      cache_discovery: Boolean, whether or not to cache the discovery doc.
     205      cache: googleapiclient.discovery_cache.base.CacheBase, an optional
     206        cache object for the discovery documents.
     207
     208    Returns:
     209      A Resource object with methods for interacting with the service.
     210    """
     211    params = {
     212        'api': serviceName,
     213        'apiVersion': version
     214        }
     215
     216    discovery_http = http if http is not None else httplib2.Http()
     217
     218    for discovery_url in (discoveryServiceUrl, V2_DISCOVERY_URI,):
     219      requested_url = uritemplate.expand(discovery_url, params)
     220
     221      try:
     222        content = _retrieve_discovery_doc(
     223          requested_url, discovery_http, cache_discovery, cache)
     224        return build_from_document(content, base=discovery_url, http=http,
     225            developerKey=developerKey, model=model, requestBuilder=requestBuilder,
     226            credentials=credentials)
     227      except HttpError as e:
     228        if e.resp.status == http_client.NOT_FOUND:
     229          continue
     230        else:
     231          raise e
     232
     233    raise UnknownApiNameOrVersion(
     234          "name: %s  version: %s" % (serviceName, version))
     235
     236
     237 -def _retrieve_discovery_doc(url, http, cache_discovery, cache=None):
     238    """Retrieves the discovery_doc from cache or the internet.
     239
     240    Args:
     241      url: string, the URL of the discovery document.
     242      http: httplib2.Http, An instance of httplib2.Http or something that acts
     243        like it through which HTTP requests will be made.
     244      cache_discovery: Boolean, whether or not to cache the discovery doc.
     245      cache: googleapiclient.discovery_cache.base.Cache, an optional cache
     246        object for the discovery documents.
     247
     248    Returns:
     249      A unicode string representation of the discovery document.
     250    """
     251    if cache_discovery:
     252      from . import discovery_cache
     253      from .discovery_cache import base
     254      if cache is None:
     255        cache = discovery_cache.autodetect()
     256      if cache:
     257        content = cache.get(url)
     258        if content:
     259          return content
     260
     261    actual_url = url
     262    # REMOTE_ADDR is defined by the CGI spec [RFC3875] as the environment
     263    # variable that contains the network address of the client sending the
     264    # request. If it exists then add that to the request for the discovery
     265    # document to avoid exceeding the quota on discovery requests.
     266    if 'REMOTE_ADDR' in os.environ:
     267      actual_url = _add_query_parameter(url, 'userIp', os.environ['REMOTE_ADDR'])
     268    logger.info('URL being requested: GET %s', actual_url)
     269
     270    resp, content = http.request(actual_url)
     271
     272    if resp.status >= 400:
     273      raise HttpError(resp, content, uri=actual_url)
     274
     275    try:
     276      content = content.decode('utf-8')
     277    except AttributeError:
     278      pass
     279
     280    try:
     281      service = json.loads(content)
     282    except ValueError as e:
     283      logger.error('Failed to parse as JSON: ' + content)
     284      raise InvalidJsonError()
     285    if cache_discovery and cache:
     286      cache.set(url, content)
     287    return content
     288
     289
     290  @positional(1)
     291 -def build_from_document(
     292      service,
     293      base=None,
     294      future=None,
     295      http=None,
     296      developerKey=None,
     297      model=None,
     298      requestBuilder=HttpRequest,
     299      credentials=None):
     300    """Create a Resource for interacting with an API.
     301
     302    Same as `build()`, but constructs the Resource object from a discovery
     303    document that is it given, as opposed to retrieving one over HTTP.
     304
     305    Args:
     306      service: string or object, the JSON discovery document describing the API.
     307        The value passed in may either be the JSON string or the deserialized
     308        JSON.
     309      base: string, base URI for all HTTP requests, usually the discovery URI.
     310        This parameter is no longer used as rootUrl and servicePath are included
     311        within the discovery document. (deprecated)
     312      future: string, discovery document with future capabilities (deprecated).
     313      http: httplib2.Http, An instance of httplib2.Http or something that acts
     314        like it that HTTP requests will be made through.
     315      developerKey: string, Key for controlling API usage, generated
     316        from the API Console.
     317      model: Model class instance that serializes and de-serializes requests and
     318        responses.
     319      requestBuilder: Takes an http request and packages it up to be executed.
     320      credentials: oauth2client.Credentials or
     321        google.auth.credentials.Credentials, credentials to be used for
     322        authentication.
     323
     324    Returns:
     325      A Resource object with methods for interacting with the service.
     326    """
     327
     328    if http is not None and credentials is not None:
     329      raise ValueError('Arguments http and credentials are mutually exclusive.')
     330
     331    if isinstance(service, six.string_types):
     332      service = json.loads(service)
     333
     334    if  'rootUrl' not in service and (isinstance(http, (HttpMock,
     335                                                        HttpMockSequence))):
     336        logger.error("You are using HttpMock or HttpMockSequence without" +
     337                     "having the service discovery doc in cache. Try calling " +
     338                     "build() without mocking once first to populate the " +
     339                     "cache.")
     340        raise InvalidJsonError()
     341
     342    base = urljoin(service['rootUrl'], service['servicePath'])
     343    schema = Schemas(service)
     344
     345    # If the http client is not specified, then we must construct an http client
     346    # to make requests. If the service has scopes, then we also need to setup
     347    # authentication.
     348    if http is None:
     349      # Does the service require scopes?
     350      scopes = list(
     351        service.get('auth', {}).get('oauth2', {}).get('scopes', {}).keys())
     352
     353      # If so, then the we need to setup authentication.
     354      if scopes:
     355        # If the user didn't pass in credentials, attempt to acquire application
     356        # default credentials.
     357        if credentials is None:
     358          credentials = _auth.default_credentials()
     359
     360        # The credentials need to be scoped.
     361        credentials = _auth.with_scopes(credentials, scopes)
     362
     363        # Create an authorized http instance
     364        http = _auth.authorized_http(credentials)
     365
     366      # If the service doesn't require scopes then there is no need for
     367      # authentication.
     368      else:
     369        http = httplib2.Http()
     370
     371    if model is None:
     372      features = service.get('features', [])
     373      model = JsonModel('dataWrapper' in features)
     374
     375    return Resource(http=http, baseUrl=base, model=model,
     376                    developerKey=developerKey, requestBuilder=requestBuilder,
     377                    resourceDesc=service, rootDesc=service, schema=schema)
     378
     379
     380 -def _cast(value, schema_type):
     381    """Convert value to a string based on JSON Schema type.
     382
     383    See http://tools.ietf.org/html/draft-zyp-json-schema-03 for more details on
     384    JSON Schema.
     385
     386    Args:
     387      value: any, the value to convert
     388      schema_type: string, the type that value should be interpreted as
     389
     390    Returns:
     391      A string representation of 'value' based on the schema_type.
     392    """
     393    if schema_type == 'string':
     394      if type(value) == type('') or type(value) == type(u''):
     395        return value
     396      else:
     397        return str(value)
     398    elif schema_type == 'integer':
     399      return str(int(value))
     400    elif schema_type == 'number':
     401      return str(float(value))
     402    elif schema_type == 'boolean':
     403      return str(bool(value)).lower()
     404    else:
     405      if type(value) == type('') or type(value) == type(u''):
     406        return value
     407      else:
     408        return str(value)
     409
     410
     411 -def _media_size_to_long(maxSize):
     412    """Convert a string media size, such as 10GB or 3TB into an integer.
     413
     414    Args:
     415      maxSize: string, size as a string, such as 2MB or 7GB.
     416
     417    Returns:
     418      The size as an integer value.
     419    """
     420    if len(maxSize) < 2:
     421      return 0
     422    units = maxSize[-2:].upper()
     423    bit_shift = _MEDIA_SIZE_BIT_SHIFTS.get(units)
     424    if bit_shift is not None:
     425      return int(maxSize[:-2]) << bit_shift
     426    else:
     427      return int(maxSize)
     428
     429
     430 -def _media_path_url_from_info(root_desc, path_url):
     431    """Creates an absolute media path URL.
     432
     433    Constructed using the API root URI and service path from the discovery
     434    document and the relative path for the API method.
     435
     436    Args:
     437      root_desc: Dictionary; the entire original deserialized discovery document.
     438      path_url: String; the relative URL for the API method. Relative to the API
     439          root, which is specified in the discovery document.
     440
     441    Returns:
     442      String; the absolute URI for media upload for the API method.
     443    """
     444    return '%(root)supload/%(service_path)s%(path)s' % {
     445        'root': root_desc['rootUrl'],
     446        'service_path': root_desc['servicePath'],
     447        'path': path_url,
     448    }
     449
     450
     451 -def _fix_up_parameters(method_desc, root_desc, http_method):
     452    """Updates parameters of an API method with values specific to this library.
     453
     454    Specifically, adds whatever global parameters are specified by the API to the
     455    parameters for the individual method. Also adds parameters which don't
     456    appear in the discovery document, but are available to all discovery based
     457    APIs (these are listed in STACK_QUERY_PARAMETERS).
     458
     459    SIDE EFFECTS: This updates the parameters dictionary object in the method
     460    description.
     461
     462    Args:
     463      method_desc: Dictionary with metadata describing an API method. Value comes
     464          from the dictionary of methods stored in the 'methods' key in the
     465          deserialized discovery document.
     466      root_desc: Dictionary; the entire original deserialized discovery document.
     467      http_method: String; the HTTP method used to call the API method described
     468          in method_desc.
     469
     470    Returns:
     471      The updated Dictionary stored in the 'parameters' key of the method
     472          description dictionary.
     473    """
     474    parameters = method_desc.setdefault('parameters', {})
     475
     476    # Add in the parameters common to all methods.
     477    for name, description in six.iteritems(root_desc.get('parameters', {})):
     478      parameters[name] = description
     479
     480    # Add in undocumented query parameters.
     481    for name in STACK_QUERY_PARAMETERS:
     482      parameters[name] = STACK_QUERY_PARAMETER_DEFAULT_VALUE.copy()
     483
     484    # Add 'body' (our own reserved word) to parameters if the method supports
     485    # a request payload.
     486    if http_method in HTTP_PAYLOAD_METHODS and 'request' in method_desc:
     487      body = BODY_PARAMETER_DEFAULT_VALUE.copy()
     488      body.update(method_desc['request'])
     489      parameters['body'] = body
     490
     491    return parameters
     492
     493
     494 -def _fix_up_media_upload(method_desc, root_desc, path_url, parameters):
     495    """Adds 'media_body' and 'media_mime_type' parameters if supported by method.
     496
     497    SIDE EFFECTS: If the method supports media upload and has a required body,
     498    sets body to be optional (required=False) instead. Also, if there is a
     499    'mediaUpload' in the method description, adds 'media_upload' key to
     500    parameters.
     501
     502    Args:
     503      method_desc: Dictionary with metadata describing an API method. Value comes
     504          from the dictionary of methods stored in the 'methods' key in the
     505          deserialized discovery document.
     506      root_desc: Dictionary; the entire original deserialized discovery document.
     507      path_url: String; the relative URL for the API method. Relative to the API
     508          root, which is specified in the discovery document.
     509      parameters: A dictionary describing method parameters for method described
     510          in method_desc.
     511
     512    Returns:
     513      Triple (accept, max_size, media_path_url) where:
     514        - accept is a list of strings representing what content types are
     515          accepted for media upload. Defaults to empty list if not in the
     516          discovery document.
     517        - max_size is a long representing the max size in bytes allowed for a
     518          media upload. Defaults to 0L if not in the discovery document.
     519        - media_path_url is a String; the absolute URI for media upload for the
     520          API method. Constructed using the API root URI and service path from
     521          the discovery document and the relative path for the API method. If
     522          media upload is not supported, this is None.
     523    """
     524    media_upload = method_desc.get('mediaUpload', {})
     525    accept = media_upload.get('accept', [])
     526    max_size = _media_size_to_long(media_upload.get('maxSize', ''))
     527    media_path_url = None
     528
     529    if media_upload:
     530      media_path_url = _media_path_url_from_info(root_desc, path_url)
     531      parameters['media_body'] = MEDIA_BODY_PARAMETER_DEFAULT_VALUE.copy()
     532      parameters['media_mime_type'] = MEDIA_MIME_TYPE_PARAMETER_DEFAULT_VALUE.copy()
     533      if 'body' in parameters:
     534        parameters['body']['required'] = False
     535
     536    return accept, max_size, media_path_url
     537
     538
     539 -def _fix_up_method_description(method_desc, root_desc):
     540    """Updates a method description in a discovery document.
     541
     542    SIDE EFFECTS: Changes the parameters dictionary in the method description with
     543    extra parameters which are used locally.
     544
     545    Args:
     546      method_desc: Dictionary with metadata describing an API method. Value comes
     547          from the dictionary of methods stored in the 'methods' key in the
     548          deserialized discovery document.
     549      root_desc: Dictionary; the entire original deserialized discovery document.
     550
     551    Returns:
     552      Tuple (path_url, http_method, method_id, accept, max_size, media_path_url)
     553      where:
     554        - path_url is a String; the relative URL for the API method. Relative to
     555          the API root, which is specified in the discovery document.
     556        - http_method is a String; the HTTP method used to call the API method
     557          described in the method description.
     558        - method_id is a String; the name of the RPC method associated with the
     559          API method, and is in the method description in the 'id' key.
     560        - accept is a list of strings representing what content types are
     561          accepted for media upload. Defaults to empty list if not in the
     562          discovery document.
     563        - max_size is a long representing the max size in bytes allowed for a
     564          media upload. Defaults to 0L if not in the discovery document.
     565        - media_path_url is a String; the absolute URI for media upload for the
     566          API method. Constructed using the API root URI and service path from
     567          the discovery document and the relative path for the API method. If
     568          media upload is not supported, this is None.
     569    """
     570    path_url = method_desc['path']
     571    http_method = method_desc['httpMethod']
     572    method_id = method_desc['id']
     573
     574    parameters = _fix_up_parameters(method_desc, root_desc, http_method)
     575    # Order is important. `_fix_up_media_upload` needs `method_desc` to have a
     576    # 'parameters' key and needs to know if there is a 'body' parameter because it
     577    # also sets a 'media_body' parameter.
     578    accept, max_size, media_path_url = _fix_up_media_upload(
     579        method_desc, root_desc, path_url, parameters)
     580
     581    return path_url, http_method, method_id, accept, max_size, media_path_url
     582
     583
     584 -def _urljoin(base, url):
     585    """Custom urljoin replacement supporting : before / in url."""
     586    # In general, it's unsafe to simply join base and url. However, for
     587    # the case of discovery documents, we know:
     588    #  * base will never contain params, query, or fragment
     589    #  * url will never contain a scheme or net_loc.
     590    # In general, this means we can safely join on /; we just need to
     591    # ensure we end up with precisely one / joining base and url. The
     592    # exception here is the case of media uploads, where url will be an
     593    # absolute url.
     594    if url.startswith('http://') or url.startswith('https://'):
     595      return urljoin(base, url)
     596    new_base = base if base.endswith('/') else base + '/'
     597    new_url = url[1:] if url.startswith('/') else url
     598    return new_base + new_url
     599
     600
     601  # TODO(dhermes): Convert this class to ResourceMethod and make it callable
     602 -class ResourceMethodParameters(object):
     603    """Represents the parameters associated with a method.
     604
     605    Attributes:
     606      argmap: Map from method parameter name (string) to query parameter name
     607          (string).
     608      required_params: List of required parameters (represented by parameter
     609          name as string).
     610      repeated_params: List of repeated parameters (represented by parameter
     611          name as string).
     612      pattern_params: Map from method parameter name (string) to regular
     613          expression (as a string). If the pattern is set for a parameter, the
     614          value for that parameter must match the regular expression.
     615      query_params: List of parameters (represented by parameter name as string)
     616          that will be used in the query string.
     617      path_params: Set of parameters (represented by parameter name as string)
     618          that will be used in the base URL path.
     619      param_types: Map from method parameter name (string) to parameter type. Type
     620          can be any valid JSON schema type; valid values are 'any', 'array',
     621          'boolean', 'integer', 'number', 'object', or 'string'. Reference:
     622          http://tools.ietf.org/html/draft-zyp-json-schema-03#section-5.1
     623      enum_params: Map from method parameter name (string) to list of strings,
     624         where each list of strings is the list of acceptable enum values.
     625    """
     626
     627 -  def __init__(self, method_desc):
     628      """Constructor for ResourceMethodParameters.
     629
     630      Sets default values and defers to set_parameters to populate.
     631
     632      Args:
     633        method_desc: Dictionary with metadata describing an API method. Value
     634            comes from the dictionary of methods stored in the 'methods' key in
     635            the deserialized discovery document.
     636      """
     637      self.argmap = {}
     638      self.required_params = []
     639      self.repeated_params = []
     640      self.pattern_params = {}
     641      self.query_params = []
     642      # TODO(dhermes): Change path_params to a list if the extra URITEMPLATE
     643      #                parsing is gotten rid of.
     644      self.path_params = set()
     645      self.param_types = {}
     646      self.enum_params = {}
     647
     648      self.set_parameters(method_desc)
     649
     650 -  def set_parameters(self, method_desc):
     651      """Populates maps and lists based on method description.
     652
     653      Iterates through each parameter for the method and parses the values from
     654      the parameter dictionary.
     655
     656      Args:
     657        method_desc: Dictionary with metadata describing an API method. Value
     658            comes from the dictionary of methods stored in the 'methods' key in
     659            the deserialized discovery document.
     660      """
     661      for arg, desc in six.iteritems(method_desc.get('parameters', {})):
     662        param = key2param(arg)
     663        self.argmap[param] = arg
     664
     665        if desc.get('pattern'):
     666          self.pattern_params[param] = desc['pattern']
     667        if desc.get('enum'):
     668          self.enum_params[param] = desc['enum']
     669        if desc.get('required'):
     670          self.required_params.append(param)
     671        if desc.get('repeated'):
     672          self.repeated_params.append(param)
     673        if desc.get('location') == 'query':
     674          self.query_params.append(param)
     675        if desc.get('location') == 'path':
     676          self.path_params.add(param)
     677        self.param_types[param] = desc.get('type', 'string')
     678
     679      # TODO(dhermes): Determine if this is still necessary. Discovery based APIs
     680      #                should have all path parameters already marked with
     681      #                'location: path'.
     682      for match in URITEMPLATE.finditer(method_desc['path']):
     683        for namematch in VARNAME.finditer(match.group(0)):
     684          name = key2param(namematch.group(0))
     685          self.path_params.add(name)
     686          if name in self.query_params:
     687            self.query_params.remove(name)
     688
     689
     690 -def createMethod(methodName, methodDesc, rootDesc, schema):
     691    """Creates a method for attaching to a Resource.
     692
     693    Args:
     694      methodName: string, name of the method to use.
     695      methodDesc: object, fragment of deserialized discovery document that
     696        describes the method.
     697      rootDesc: object, the entire deserialized discovery document.
     698      schema: object, mapping of schema names to schema descriptions.
     699    """
     700    methodName = fix_method_name(methodName)
     701    (pathUrl, httpMethod, methodId, accept,
     702     maxSize, mediaPathUrl) = _fix_up_method_description(methodDesc, rootDesc)
     703
     704    parameters = ResourceMethodParameters(methodDesc)
     705
     706    def method(self, **kwargs):
     707      # Don't bother with doc string, it will be over-written by createMethod.
     708
     709      for name in six.iterkeys(kwargs):
     710        if name not in parameters.argmap:
     711          raise TypeError('Got an unexpected keyword argument "%s"' % name)
     712
     713      # Remove args that have a value of None.
     714      keys = list(kwargs.keys())
     715      for name in keys:
     716        if kwargs[name] is None:
     717          del kwargs[name]
     718
     719      for name in parameters.required_params:
     720        if name not in kwargs:
     721          raise TypeError('Missing required parameter "%s"' % name)
     722
     723      for name, regex in six.iteritems(parameters.pattern_params):
     724        if name in kwargs:
     725          if isinstance(kwargs[name], six.string_types):
     726            pvalues = [kwargs[name]]
     727          else:
     728            pvalues = kwargs[name]
     729          for pvalue in pvalues:
     730            if re.match(regex, pvalue) is None:
     731              raise TypeError(
     732                  'Parameter "%s" value "%s" does not match the pattern "%s"' %
     733                  (name, pvalue, regex))
     734
     735      for name, enums in six.iteritems(parameters.enum_params):
     736        if name in kwargs:
     737          # We need to handle the case of a repeated enum
     738          # name differently, since we want to handle both
     739          # arg='value' and arg=['value1', 'value2']
     740          if (name in parameters.repeated_params and
     741              not isinstance(kwargs[name], six.string_types)):
     742            values = kwargs[name]
     743          else:
     744            values = [kwargs[name]]
     745          for value in values:
     746            if value not in enums:
     747              raise TypeError(
     748                  'Parameter "%s" value "%s" is not an allowed value in "%s"' %
     749                  (name, value, str(enums)))
     750
     751      actual_query_params = {}
     752      actual_path_params = {}
     753      for key, value in six.iteritems(kwargs):
     754        to_type = parameters.param_types.get(key, 'string')
     755        # For repeated parameters we cast each member of the list.
     756        if key in parameters.repeated_params and type(value) == type([]):
     757          cast_value = [_cast(x, to_type) for x in value]
     758        else:
     759          cast_value = _cast(value, to_type)
     760        if key in parameters.query_params:
     761          actual_query_params[parameters.argmap[key]] = cast_value
     762        if key in parameters.path_params:
     763          actual_path_params[parameters.argmap[key]] = cast_value
     764      body_value = kwargs.get('body', None)
     765      media_filename = kwargs.get('media_body', None)
     766      media_mime_type = kwargs.get('media_mime_type', None)
     767
     768      if self._developerKey:
     769        actual_query_params['key'] = self._developerKey
     770
     771      model = self._model
     772      if methodName.endswith('_media'):
     773        model = MediaModel()
     774      elif 'response' not in methodDesc:
     775        model = RawModel()
     776
     777      headers = {}
     778      headers, params, query, body = model.request(headers,
     779          actual_path_params, actual_query_params, body_value)
     780
     781      expanded_url = uritemplate.expand(pathUrl, params)
     782      url = _urljoin(self._baseUrl, expanded_url + query)
     783
     784      resumable = None
     785      multipart_boundary = ''
     786
     787      if media_filename:
     788        # Ensure we end up with a valid MediaUpload object.
     789        if isinstance(media_filename, six.string_types):
     790          if media_mime_type is None:
     791            logger.warning(
     792                'media_mime_type argument not specified: trying to auto-detect for %s',
     793                media_filename)
     794            media_mime_type, _ = mimetypes.guess_type(media_filename)
     795          if media_mime_type is None:
     796            raise UnknownFileType(media_filename)
     797          if not mimeparse.best_match([media_mime_type], ','.join(accept)):
     798            raise UnacceptableMimeTypeError(media_mime_type)
     799          media_upload = MediaFileUpload(media_filename,
     800                                         mimetype=media_mime_type)
     801        elif isinstance(media_filename, MediaUpload):
     802          media_upload = media_filename
     803        else:
     804          raise TypeError('media_filename must be str or MediaUpload.')
     805
     806        # Check the maxSize
     807        if media_upload.size() is not None and media_upload.size() > maxSize > 0:
     808          raise MediaUploadSizeError("Media larger than: %s" % maxSize)
     809
     810        # Use the media path uri for media uploads
     811        expanded_url = uritemplate.expand(mediaPathUrl, params)
     812        url = _urljoin(self._baseUrl, expanded_url + query)
     813        if media_upload.resumable():
     814          url = _add_query_parameter(url, 'uploadType', 'resumable')
     815
     816        if media_upload.resumable():
     817          # This is all we need to do for resumable, if the body exists it gets
     818          # sent in the first request, otherwise an empty body is sent.
     819          resumable = media_upload
     820        else:
     821          # A non-resumable upload
     822          if body is None:
     823            # This is a simple media upload
     824            headers['content-type'] = media_upload.mimetype()
     825            body = media_upload.getbytes(0, media_upload.size())
     826            url = _add_query_parameter(url, 'uploadType', 'media')
     827          else:
     828            # This is a multipart/related upload.
     829            msgRoot = MIMEMultipart('related')
     830            # msgRoot should not write out it's own headers
     831            setattr(msgRoot, '_write_headers', lambda self: None)
     832
     833            # attach the body as one part
     834            msg = MIMENonMultipart(*headers['content-type'].split('/'))
     835            msg.set_payload(body)
     836            msgRoot.attach(msg)
     837
     838            # attach the media as the second part
     839            msg = MIMENonMultipart(*media_upload.mimetype().split('/'))
     840            msg['Content-Transfer-Encoding'] = 'binary'
     841
     842            payload = media_upload.getbytes(0, media_upload.size())
     843            msg.set_payload(payload)
     844            msgRoot.attach(msg)
     845            # encode the body: note that we can't use `as_string`, because
     846            # it plays games with `From ` lines.
     847            fp = BytesIO()
     848            g = _BytesGenerator(fp, mangle_from_=False)
     849            g.flatten(msgRoot, unixfrom=False)
     850            body = fp.getvalue()
     851
     852            multipart_boundary = msgRoot.get_boundary()
     853            headers['content-type'] = ('multipart/related; '
     854                                       'boundary="%s"') % multipart_boundary
     855            url = _add_query_parameter(url, 'uploadType', 'multipart')
     856
     857      logger.info('URL being requested: %s %s' % (httpMethod,url))
     858      return self._requestBuilder(self._http,
     859                                  model.response,
     860                                  url,
     861                                  method=httpMethod,
     862                                  body=body,
     863                                  headers=headers,
     864                                  methodId=methodId,
     865                                  resumable=resumable)
     866
     867    docs = [methodDesc.get('description', DEFAULT_METHOD_DOC), '\n\n']
     868    if len(parameters.argmap) > 0:
     869      docs.append('Args:\n')
     870
     871    # Skip undocumented params and params common to all methods.
     872    skip_parameters = list(rootDesc.get('parameters', {}).keys())
     873    skip_parameters.extend(STACK_QUERY_PARAMETERS)
     874
     875    all_args = list(parameters.argmap.keys())
     876    args_ordered = [key2param(s) for s in methodDesc.get('parameterOrder', [])]
     877
     878    # Move body to the front of the line.
     879    if 'body' in all_args:
     880      args_ordered.append('body')
     881
     882    for name in all_args:
     883      if name not in args_ordered:
     884        args_ordered.append(name)
     885
     886    for arg in args_ordered:
     887      if arg in skip_parameters:
     888        continue
     889
     890      repeated = ''
     891      if arg in parameters.repeated_params:
     892        repeated = ' (repeated)'
     893      required = ''
     894      if arg in parameters.required_params:
     895        required = ' (required)'
     896      paramdesc = methodDesc['parameters'][parameters.argmap[arg]]
     897      paramdoc = paramdesc.get('description', 'A parameter')
     898      if '$ref' in paramdesc:
     899        docs.append(
     900            ('  %s: object, %s%s%s\n    The object takes the'
     901            ' form of:\n\n%s\n\n') % (arg, paramdoc, required, repeated,
     902              schema.prettyPrintByName(paramdesc['$ref'])))
     903      else:
     904        paramtype = paramdesc.get('type', 'string')
     905        docs.append('  %s: %s, %s%s%s\n' % (arg, paramtype, paramdoc, required,
     906                                            repeated))
     907      enum = paramdesc.get('enum', [])
     908      enumDesc = paramdesc.get('enumDescriptions', [])
     909      if enum and enumDesc:
     910        docs.append('    Allowed values\n')
     911        for (name, desc) in zip(enum, enumDesc):
     912          docs.append('      %s - %s\n' % (name, desc))
     913    if 'response' in methodDesc:
     914      if methodName.endswith('_media'):
     915        docs.append('\nReturns:\n  The media object as a string.\n\n    ')
     916      else:
     917        docs.append('\nReturns:\n  An object of the form:\n\n    ')
     918        docs.append(schema.prettyPrintSchema(methodDesc['response']))
     919
     920    setattr(method, '__doc__', ''.join(docs))
     921    return (methodName, method)
     922
     923
     924 -def createNextMethod(methodName):
     925    """Creates any _next methods for attaching to a Resource.
     926
     927    The _next methods allow for easy iteration through list() responses.
     928
     929    Args:
     930      methodName: string, name of the method to use.
     931    """
     932    methodName = fix_method_name(methodName)
     933
     934    def methodNext(self, previous_request, previous_response):
     935      """Retrieves the next page of results.
     936
     937  Args:
     938    previous_request: The request for the previous page. (required)
     939    previous_response: The response from the request for the previous page. (required)
     940
     941  Returns:
     942    A request object that you can call 'execute()' on to request the next
     943    page. Returns None if there are no more items in the collection.
     944      """
     945      # Retrieve nextPageToken from previous_response
     946      # Use as pageToken in previous_request to create new request.
     947
     948      if 'nextPageToken' not in previous_response or not previous_response['nextPageToken']:
     949        return None
     950
     951      request = copy.copy(previous_request)
     952
     953      pageToken = previous_response['nextPageToken']
     954      parsed = list(urlparse(request.uri))
     955      q = parse_qsl(parsed[4])
     956
     957      # Find and remove old 'pageToken' value from URI
     958      newq = [(key, value) for (key, value) in q if key != 'pageToken']
     959      newq.append(('pageToken', pageToken))
     960      parsed[4] = urlencode(newq)
     961      uri = urlunparse(parsed)
     962
     963      request.uri = uri
     964
     965      logger.info('URL being requested: %s %s' % (methodName,uri))
     966
     967      return request
     968
     969    return (methodName, methodNext)
     970
     971
     972 -class Resource(object):
     973    """A class for interacting with a resource."""
     974
     975 -  def __init__(self, http, baseUrl, model, requestBuilder, developerKey,
     976                 resourceDesc, rootDesc, schema):
     977      """Build a Resource from the API description.
     978
     979      Args:
     980        http: httplib2.Http, Object to make http requests with.
     981        baseUrl: string, base URL for the API. All requests are relative to this
     982            URI.
     983        model: googleapiclient.Model, converts to and from the wire format.
     984        requestBuilder: class or callable that instantiates an
     985            googleapiclient.HttpRequest object.
     986        developerKey: string, key obtained from
     987            https://code.google.com/apis/console
     988        resourceDesc: object, section of deserialized discovery document that
     989            describes a resource. Note that the top level discovery document
     990            is considered a resource.
     991        rootDesc: object, the entire deserialized discovery document.
     992        schema: object, mapping of schema names to schema descriptions.
     993      """
     994      self._dynamic_attrs = []
     995
     996      self._http = http
     997      self._baseUrl = baseUrl
     998      self._model = model
     999      self._developerKey = developerKey
    1000      self._requestBuilder = requestBuilder
    1001      self._resourceDesc = resourceDesc
    1002      self._rootDesc = rootDesc
    1003      self._schema = schema
    1004
    1005      self._set_service_methods()
    1006
    1007 -  def _set_dynamic_attr(self, attr_name, value):
    1008      """Sets an instance attribute and tracks it in a list of dynamic attributes.
    1009
    1010      Args:
    1011        attr_name: string; The name of the attribute to be set
    1012        value: The value being set on the object and tracked in the dynamic cache.
    1013      """
    1014      self._dynamic_attrs.append(attr_name)
    1015      self.__dict__[attr_name] = value
    1016
    1017 -  def __getstate__(self):
    1018      """Trim the state down to something that can be pickled.
    1019
    1020      Uses the fact that the instance variable _dynamic_attrs holds attrs that
    1021      will be wiped and restored on pickle serialization.
    1022      """
    1023      state_dict = copy.copy(self.__dict__)
    1024      for dynamic_attr in self._dynamic_attrs:
    1025        del state_dict[dynamic_attr]
    1026      del state_dict['_dynamic_attrs']
    1027      return state_dict
    1028
    1029 -  def __setstate__(self, state):
    1030      """Reconstitute the state of the object from being pickled.
    1031
    1032      Uses the fact that the instance variable _dynamic_attrs holds attrs that
    1033      will be wiped and restored on pickle serialization.
    1034      """
    1035      self.__dict__.update(state)
    1036      self._dynamic_attrs = []
    1037      self._set_service_methods()
    1038
    1039 -  def _set_service_methods(self):
    1040      self._add_basic_methods(self._resourceDesc, self._rootDesc, self._schema)
    1041      self._add_nested_resources(self._resourceDesc, self._rootDesc, self._schema)
    1042      self._add_next_methods(self._resourceDesc, self._schema)
    1043
    1044 -  def _add_basic_methods(self, resourceDesc, rootDesc, schema):
    1045      # If this is the root Resource, add a new_batch_http_request() method.
    1046      if resourceDesc == rootDesc:
    1047        batch_uri = '%s%s' % (
    1048          rootDesc['rootUrl'], rootDesc.get('batchPath', 'batch'))
    1049        def new_batch_http_request(callback=None):
    1050          """Create a BatchHttpRequest object based on the discovery document.
    1051
    1052          Args:
    1053            callback: callable, A callback to be called for each response, of the
    1054              form callback(id, response, exception). The first parameter is the
    1055              request id, and the second is the deserialized response object. The
    1056              third is an apiclient.errors.HttpError exception object if an HTTP
    1057              error occurred while processing the request, or None if no error
    1058              occurred.
    1059
    1060          Returns:
    1061            A BatchHttpRequest object based on the discovery document.
    1062          """
    1063          return BatchHttpRequest(callback=callback, batch_uri=batch_uri)
    1064        self._set_dynamic_attr('new_batch_http_request', new_batch_http_request)
    1065
    1066      # Add basic methods to Resource
    1067      if 'methods' in resourceDesc:
    1068        for methodName, methodDesc in six.iteritems(resourceDesc['methods']):
    1069          fixedMethodName, method = createMethod(
    1070              methodName, methodDesc, rootDesc, schema)
    1071          self._set_dynamic_attr(fixedMethodName,
    1072                                 method.__get__(self, self.__class__))
    1073          # Add in _media methods. The functionality of the attached method will
    1074          # change when it sees that the method name ends in _media.
    1075          if methodDesc.get('supportsMediaDownload', False):
    1076            fixedMethodName, method = createMethod(
    1077                methodName + '_media', methodDesc, rootDesc, schema)
    1078            self._set_dynamic_attr(fixedMethodName,
    1079                                   method.__get__(self, self.__class__))
    1080
    1081 -  def _add_nested_resources(self, resourceDesc, rootDesc, schema):
    1082      # Add in nested resources
    1083      if 'resources' in resourceDesc:
    1084
    1085        def createResourceMethod(methodName, methodDesc):
    1086          """Create a method on the Resource to access a nested Resource.
    1087
    1088          Args:
    1089            methodName: string, name of the method to use.
    1090            methodDesc: object, fragment of deserialized discovery document that
    1091              describes the method.
    1092          """
    1093          methodName = fix_method_name(methodName)
    1094
    1095          def methodResource(self):
    1096            return Resource(http=self._http, baseUrl=self._baseUrl,
    1097                            model=self._model, developerKey=self._developerKey,
    1098                            requestBuilder=self._requestBuilder,
    1099                            resourceDesc=methodDesc, rootDesc=rootDesc,
    1100                            schema=schema)
    1101
    1102          setattr(methodResource, '__doc__', 'A collection resource.')
    1103          setattr(methodResource, '__is_resource__', True)
    1104
    1105          return (methodName, methodResource)
    1106
    1107        for methodName, methodDesc in six.iteritems(resourceDesc['resources']):
    1108          fixedMethodName, method = createResourceMethod(methodName, methodDesc)
    1109          self._set_dynamic_attr(fixedMethodName,
    1110                                 method.__get__(self, self.__class__))
    1111
    1112 -  def _add_next_methods(self, resourceDesc, schema):
    1113      # Add _next() methods
    1114      # Look for response bodies in schema that contain nextPageToken, and methods
    1115      # that take a pageToken parameter.
    1116      if 'methods' in resourceDesc:
    1117        for methodName, methodDesc in six.iteritems(resourceDesc['methods']):
    1118          if 'response' in methodDesc:
    1119            responseSchema = methodDesc['response']
    1120            if '$ref' in responseSchema:
    1121              responseSchema = schema.get(responseSchema['$ref'])
    1122            hasNextPageToken = 'nextPageToken' in responseSchema.get('properties',
    1123                                                                     {})
    1124            hasPageToken = 'pageToken' in methodDesc.get('parameters', {})
    1125            if hasNextPageToken and hasPageToken:
    1126              fixedMethodName, method = createNextMethod(methodName + '_next')
    1127              self._set_dynamic_attr(fixedMethodName,
    1128                                     method.__get__(self, self.__class__))
    1129