1   
   2   
   3   
   4   
   5   
   6   
   7   
   8   
   9   
  10   
  11   
  12   
  13   
  14   
  15  """Classes to encapsulate a single HTTP request. 
  16   
  17  The classes implement a command pattern, with every 
  18  object supporting an execute() method that does the 
  19  actuall HTTP request. 
  20  """ 
  21  from __future__ import absolute_import 
  22  import six 
  23  from six.moves import http_client 
  24  from six.moves import range 
  25   
  26  __author__ = 'jcgregorio@google.com (Joe Gregorio)' 
  27   
  28  from six import BytesIO, StringIO 
  29  from six.moves.urllib.parse import urlparse, urlunparse, quote, unquote 
  30   
  31  import base64 
  32  import copy 
  33  import gzip 
  34  import httplib2 
  35  import json 
  36  import logging 
  37  import mimetypes 
  38  import os 
  39  import random 
  40  import socket 
  41  import sys 
  42  import time 
  43  import uuid 
  44   
  45   
  46  try: 
  47    import ssl 
  48  except ImportError: 
  49    _ssl_SSLError = object() 
  50  else: 
  51    _ssl_SSLError = ssl.SSLError 
  52   
  53  from email.generator import Generator 
  54  from email.mime.multipart import MIMEMultipart 
  55  from email.mime.nonmultipart import MIMENonMultipart 
  56  from email.parser import FeedParser 
  57   
  58   
  59   
  60  try: 
  61    from oauth2client import util 
  62  except ImportError: 
  63    from oauth2client import _helpers as util 
  64   
  65  from googleapiclient import mimeparse 
  66  from googleapiclient.errors import BatchError 
  67  from googleapiclient.errors import HttpError 
  68  from googleapiclient.errors import InvalidChunkSizeError 
  69  from googleapiclient.errors import ResumableUploadError 
  70  from googleapiclient.errors import UnexpectedBodyError 
  71  from googleapiclient.errors import UnexpectedMethodError 
  72  from googleapiclient.model import JsonModel 
  73   
  74   
  75  LOGGER = logging.getLogger(__name__) 
  76   
  77  DEFAULT_CHUNK_SIZE = 512*1024 
  78   
  79  MAX_URI_LENGTH = 2048 
  80   
  81  _TOO_MANY_REQUESTS = 429 
  85    """Determines whether a response should be retried. 
  86   
  87    Args: 
  88      resp_status: The response status received. 
  89      content: The response content body.  
  90   
  91    Returns: 
  92      True if the response should be retried, otherwise False. 
  93    """ 
  94     
  95    if resp_status >= 500: 
  96      return True 
  97   
  98     
  99    if resp_status == _TOO_MANY_REQUESTS: 
 100      return True 
 101   
 102     
 103     
 104    if resp_status == six.moves.http_client.FORBIDDEN: 
 105       
 106      if not content: 
 107        return False 
 108   
 109       
 110      try: 
 111        data = json.loads(content.decode('utf-8')) 
 112        reason = data['error']['errors'][0]['reason'] 
 113      except (UnicodeDecodeError, ValueError, KeyError): 
 114        LOGGER.warning('Invalid JSON content from response: %s', content) 
 115        return False 
 116   
 117      LOGGER.warning('Encountered 403 Forbidden with reason "%s"', reason) 
 118   
 119       
 120      if reason in ('userRateLimitExceeded', 'rateLimitExceeded', ): 
 121        return True 
 122   
 123     
 124    return False 
  125   
 126   
 127 -def _retry_request(http, num_retries, req_type, sleep, rand, uri, method, *args, 
 128                     **kwargs): 
  129    """Retries an HTTP request multiple times while handling errors. 
 130   
 131    If after all retries the request still fails, last error is either returned as 
 132    return value (for HTTP 5xx errors) or thrown (for ssl.SSLError). 
 133   
 134    Args: 
 135      http: Http object to be used to execute request. 
 136      num_retries: Maximum number of retries. 
 137      req_type: Type of the request (used for logging retries). 
 138      sleep, rand: Functions to sleep for random time between retries. 
 139      uri: URI to be requested. 
 140      method: HTTP method to be used. 
 141      args, kwargs: Additional arguments passed to http.request. 
 142   
 143    Returns: 
 144      resp, content - Response from the http request (may be HTTP 5xx). 
 145    """ 
 146    resp = None 
 147    content = None 
 148    for retry_num in range(num_retries + 1): 
 149      if retry_num > 0: 
 150         
 151        sleep_time = rand() * 2 ** retry_num 
 152        LOGGER.warning( 
 153            'Sleeping %.2f seconds before retry %d of %d for %s: %s %s, after %s', 
 154            sleep_time, retry_num, num_retries, req_type, method, uri, 
 155            resp.status if resp else exception) 
 156        sleep(sleep_time) 
 157   
 158      try: 
 159        exception = None 
 160        resp, content = http.request(uri, method, *args, **kwargs) 
 161       
 162      except _ssl_SSLError as ssl_error: 
 163        exception = ssl_error 
 164      except socket.error as socket_error: 
 165         
 166        if socket.errno.errorcode.get(socket_error.errno) not in ( 
 167            'WSAETIMEDOUT', 'ETIMEDOUT', 'EPIPE', 'ECONNABORTED', ): 
 168          raise 
 169        exception = socket_error 
 170   
 171      if exception: 
 172        if retry_num == num_retries: 
 173          raise exception 
 174        else: 
 175          continue 
 176   
 177      if not _should_retry_response(resp.status, content): 
 178        break 
 179   
 180    return resp, content 
  181   
 208   
 234   
 377   
 502   
 569   
 598   
 690   
 693    """Truncated stream. 
 694   
 695    Takes a stream and presents a stream that is a slice of the original stream. 
 696    This is used when uploading media in chunks. In later versions of Python a 
 697    stream can be passed to httplib in place of the string of data to send. The 
 698    problem is that httplib just blindly reads to the end of the stream. This 
 699    wrapper presents a virtual stream that only reads to the end of the chunk. 
 700    """ 
 701   
 702 -  def __init__(self, stream, begin, chunksize): 
  703      """Constructor. 
 704   
 705      Args: 
 706        stream: (io.Base, file object), the stream to wrap. 
 707        begin: int, the seek position the chunk begins at. 
 708        chunksize: int, the size of the chunk. 
 709      """ 
 710      self._stream = stream 
 711      self._begin = begin 
 712      self._chunksize = chunksize 
 713      self._stream.seek(begin) 
  714   
 715 -  def read(self, n=-1): 
  716      """Read n bytes. 
 717   
 718      Args: 
 719        n, int, the number of bytes to read. 
 720   
 721      Returns: 
 722        A string of length 'n', or less if EOF is reached. 
 723      """ 
 724       
 725      cur = self._stream.tell() 
 726      end = self._begin + self._chunksize 
 727      if n == -1 or cur + n > end: 
 728        n = end - cur 
 729      return self._stream.read(n) 
   730   
 733    """Encapsulates a single HTTP request.""" 
 734   
 735    @util.positional(4) 
 736 -  def __init__(self, http, postproc, uri, 
 737                 method='GET', 
 738                 body=None, 
 739                 headers=None, 
 740                 methodId=None, 
 741                 resumable=None): 
  742      """Constructor for an HttpRequest. 
 743   
 744      Args: 
 745        http: httplib2.Http, the transport object to use to make a request 
 746        postproc: callable, called on the HTTP response and content to transform 
 747                  it into a data object before returning, or raising an exception 
 748                  on an error. 
 749        uri: string, the absolute URI to send the request to 
 750        method: string, the HTTP method to use 
 751        body: string, the request body of the HTTP request, 
 752        headers: dict, the HTTP request headers 
 753        methodId: string, a unique identifier for the API method being called. 
 754        resumable: MediaUpload, None if this is not a resumbale request. 
 755      """ 
 756      self.uri = uri 
 757      self.method = method 
 758      self.body = body 
 759      self.headers = headers or {} 
 760      self.methodId = methodId 
 761      self.http = http 
 762      self.postproc = postproc 
 763      self.resumable = resumable 
 764      self.response_callbacks = [] 
 765      self._in_error_state = False 
 766   
 767       
 768      major, minor, params = mimeparse.parse_mime_type( 
 769          self.headers.get('content-type', 'application/json')) 
 770   
 771       
 772      self.body_size = len(self.body or '') 
 773   
 774       
 775      self.resumable_uri = None 
 776   
 777       
 778      self.resumable_progress = 0 
 779   
 780       
 781      self._rand = random.random 
 782      self._sleep = time.sleep 
  783   
 784    @util.positional(1) 
 785 -  def execute(self, http=None, num_retries=0): 
  786      """Execute the request. 
 787   
 788      Args: 
 789        http: httplib2.Http, an http object to be used in place of the 
 790              one the HttpRequest request object was constructed with. 
 791        num_retries: Integer, number of times to retry with randomized 
 792              exponential backoff. If all retries fail, the raised HttpError 
 793              represents the last request. If zero (default), we attempt the 
 794              request only once. 
 795   
 796      Returns: 
 797        A deserialized object model of the response body as determined 
 798        by the postproc. 
 799   
 800      Raises: 
 801        googleapiclient.errors.HttpError if the response was not a 2xx. 
 802        httplib2.HttpLib2Error if a transport error has occured. 
 803      """ 
 804      if http is None: 
 805        http = self.http 
 806   
 807      if self.resumable: 
 808        body = None 
 809        while body is None: 
 810          _, body = self.next_chunk(http=http, num_retries=num_retries) 
 811        return body 
 812   
 813       
 814   
 815      if 'content-length' not in self.headers: 
 816        self.headers['content-length'] = str(self.body_size) 
 817       
 818      if len(self.uri) > MAX_URI_LENGTH and self.method == 'GET': 
 819        self.method = 'POST' 
 820        self.headers['x-http-method-override'] = 'GET' 
 821        self.headers['content-type'] = 'application/x-www-form-urlencoded' 
 822        parsed = urlparse(self.uri) 
 823        self.uri = urlunparse( 
 824            (parsed.scheme, parsed.netloc, parsed.path, parsed.params, None, 
 825             None) 
 826            ) 
 827        self.body = parsed.query 
 828        self.headers['content-length'] = str(len(self.body)) 
 829   
 830       
 831      resp, content = _retry_request( 
 832            http, num_retries, 'request', self._sleep, self._rand, str(self.uri), 
 833            method=str(self.method), body=self.body, headers=self.headers) 
 834   
 835      for callback in self.response_callbacks: 
 836        callback(resp) 
 837      if resp.status >= 300: 
 838        raise HttpError(resp, content, uri=self.uri) 
 839      return self.postproc(resp, content) 
  840   
 841    @util.positional(2) 
 843      """add_response_headers_callback 
 844   
 845      Args: 
 846        cb: Callback to be called on receiving the response headers, of signature: 
 847   
 848        def cb(resp): 
 849          # Where resp is an instance of httplib2.Response 
 850      """ 
 851      self.response_callbacks.append(cb) 
  852   
 853    @util.positional(1) 
 855      """Execute the next step of a resumable upload. 
 856   
 857      Can only be used if the method being executed supports media uploads and 
 858      the MediaUpload object passed in was flagged as using resumable upload. 
 859   
 860      Example: 
 861   
 862        media = MediaFileUpload('cow.png', mimetype='image/png', 
 863                                chunksize=1000, resumable=True) 
 864        request = farm.animals().insert( 
 865            id='cow', 
 866            name='cow.png', 
 867            media_body=media) 
 868   
 869        response = None 
 870        while response is None: 
 871          status, response = request.next_chunk() 
 872          if status: 
 873            print "Upload %d%% complete." % int(status.progress() * 100) 
 874   
 875   
 876      Args: 
 877        http: httplib2.Http, an http object to be used in place of the 
 878              one the HttpRequest request object was constructed with. 
 879        num_retries: Integer, number of times to retry with randomized 
 880              exponential backoff. If all retries fail, the raised HttpError 
 881              represents the last request. If zero (default), we attempt the 
 882              request only once. 
 883   
 884      Returns: 
 885        (status, body): (ResumableMediaStatus, object) 
 886           The body will be None until the resumable media is fully uploaded. 
 887   
 888      Raises: 
 889        googleapiclient.errors.HttpError if the response was not a 2xx. 
 890        httplib2.HttpLib2Error if a transport error has occured. 
 891      """ 
 892      if http is None: 
 893        http = self.http 
 894   
 895      if self.resumable.size() is None: 
 896        size = '*' 
 897      else: 
 898        size = str(self.resumable.size()) 
 899   
 900      if self.resumable_uri is None: 
 901        start_headers = copy.copy(self.headers) 
 902        start_headers['X-Upload-Content-Type'] = self.resumable.mimetype() 
 903        if size != '*': 
 904          start_headers['X-Upload-Content-Length'] = size 
 905        start_headers['content-length'] = str(self.body_size) 
 906   
 907        resp, content = _retry_request( 
 908            http, num_retries, 'resumable URI request', self._sleep, self._rand, 
 909            self.uri, method=self.method, body=self.body, headers=start_headers) 
 910   
 911        if resp.status == 200 and 'location' in resp: 
 912          self.resumable_uri = resp['location'] 
 913        else: 
 914          raise ResumableUploadError(resp, content) 
 915      elif self._in_error_state: 
 916         
 917         
 918         
 919        headers = { 
 920            'Content-Range': 'bytes */%s' % size, 
 921            'content-length': '0' 
 922            } 
 923        resp, content = http.request(self.resumable_uri, 'PUT', 
 924                                     headers=headers) 
 925        status, body = self._process_response(resp, content) 
 926        if body: 
 927           
 928          return (status, body) 
 929   
 930      if self.resumable.has_stream(): 
 931        data = self.resumable.stream() 
 932        if self.resumable.chunksize() == -1: 
 933          data.seek(self.resumable_progress) 
 934          chunk_end = self.resumable.size() - self.resumable_progress - 1 
 935        else: 
 936           
 937          data = _StreamSlice(data, self.resumable_progress, 
 938                              self.resumable.chunksize()) 
 939          chunk_end = min( 
 940              self.resumable_progress + self.resumable.chunksize() - 1, 
 941              self.resumable.size() - 1) 
 942      else: 
 943        data = self.resumable.getbytes( 
 944            self.resumable_progress, self.resumable.chunksize()) 
 945   
 946         
 947        if len(data) < self.resumable.chunksize(): 
 948          size = str(self.resumable_progress + len(data)) 
 949   
 950        chunk_end = self.resumable_progress + len(data) - 1 
 951   
 952      headers = { 
 953          'Content-Range': 'bytes %d-%d/%s' % ( 
 954              self.resumable_progress, chunk_end, size), 
 955           
 956           
 957          'Content-Length': str(chunk_end - self.resumable_progress + 1) 
 958          } 
 959   
 960      for retry_num in range(num_retries + 1): 
 961        if retry_num > 0: 
 962          self._sleep(self._rand() * 2**retry_num) 
 963          LOGGER.warning( 
 964              'Retry #%d for media upload: %s %s, following status: %d' 
 965              % (retry_num, self.method, self.uri, resp.status)) 
 966   
 967        try: 
 968          resp, content = http.request(self.resumable_uri, method='PUT', 
 969                                       body=data, 
 970                                       headers=headers) 
 971        except: 
 972          self._in_error_state = True 
 973          raise 
 974        if not _should_retry_response(resp.status, content): 
 975          break 
 976   
 977      return self._process_response(resp, content) 
  978   
 980      """Process the response from a single chunk upload. 
 981   
 982      Args: 
 983        resp: httplib2.Response, the response object. 
 984        content: string, the content of the response. 
 985   
 986      Returns: 
 987        (status, body): (ResumableMediaStatus, object) 
 988           The body will be None until the resumable media is fully uploaded. 
 989   
 990      Raises: 
 991        googleapiclient.errors.HttpError if the response was not a 2xx or a 308. 
 992      """ 
 993      if resp.status in [200, 201]: 
 994        self._in_error_state = False 
 995        return None, self.postproc(resp, content) 
 996      elif resp.status == 308: 
 997        self._in_error_state = False 
 998         
 999        self.resumable_progress = int(resp['range'].split('-')[1]) + 1 
1000        if 'location' in resp: 
1001          self.resumable_uri = resp['location'] 
1002      else: 
1003        self._in_error_state = True 
1004        raise HttpError(resp, content, uri=self.uri) 
1005   
1006      return (MediaUploadProgress(self.resumable_progress, self.resumable.size()), 
1007              None) 
 1008   
1010      """Returns a JSON representation of the HttpRequest.""" 
1011      d = copy.copy(self.__dict__) 
1012      if d['resumable'] is not None: 
1013        d['resumable'] = self.resumable.to_json() 
1014      del d['http'] 
1015      del d['postproc'] 
1016      del d['_sleep'] 
1017      del d['_rand'] 
1018   
1019      return json.dumps(d) 
 1020   
1021    @staticmethod 
1023      """Returns an HttpRequest populated with info from a JSON object.""" 
1024      d = json.loads(s) 
1025      if d['resumable'] is not None: 
1026        d['resumable'] = MediaUpload.new_from_json(d['resumable']) 
1027      return HttpRequest( 
1028          http, 
1029          postproc, 
1030          uri=d['uri'], 
1031          method=d['method'], 
1032          body=d['body'], 
1033          headers=d['headers'], 
1034          methodId=d['methodId'], 
1035          resumable=d['resumable']) 
  1036   
1039    """Batches multiple HttpRequest objects into a single HTTP request. 
1040   
1041    Example: 
1042      from googleapiclient.http import BatchHttpRequest 
1043   
1044      def list_animals(request_id, response, exception): 
1045        \"\"\"Do something with the animals list response.\"\"\" 
1046        if exception is not None: 
1047          # Do something with the exception. 
1048          pass 
1049        else: 
1050          # Do something with the response. 
1051          pass 
1052   
1053      def list_farmers(request_id, response, exception): 
1054        \"\"\"Do something with the farmers list response.\"\"\" 
1055        if exception is not None: 
1056          # Do something with the exception. 
1057          pass 
1058        else: 
1059          # Do something with the response. 
1060          pass 
1061   
1062      service = build('farm', 'v2') 
1063   
1064      batch = BatchHttpRequest() 
1065   
1066      batch.add(service.animals().list(), list_animals) 
1067      batch.add(service.farmers().list(), list_farmers) 
1068      batch.execute(http=http) 
1069    """ 
1070   
1071    @util.positional(1) 
1072 -  def __init__(self, callback=None, batch_uri=None): 
 1073      """Constructor for a BatchHttpRequest. 
1074   
1075      Args: 
1076        callback: callable, A callback to be called for each response, of the 
1077          form callback(id, response, exception). The first parameter is the 
1078          request id, and the second is the deserialized response object. The 
1079          third is an googleapiclient.errors.HttpError exception object if an HTTP error 
1080          occurred while processing the request, or None if no error occurred. 
1081        batch_uri: string, URI to send batch requests to. 
1082      """ 
1083      if batch_uri is None: 
1084        batch_uri = 'https://www.googleapis.com/batch' 
1085      self._batch_uri = batch_uri 
1086   
1087       
1088      self._callback = callback 
1089   
1090       
1091      self._requests = {} 
1092   
1093       
1094      self._callbacks = {} 
1095   
1096       
1097      self._order = [] 
1098   
1099       
1100      self._last_auto_id = 0 
1101   
1102       
1103      self._base_id = None 
1104   
1105       
1106      self._responses = {} 
1107   
1108       
1109      self._refreshed_credentials = {} 
 1110   
1112      """Refresh the credentials and apply to the request. 
1113   
1114      Args: 
1115        request: HttpRequest, the request. 
1116        http: httplib2.Http, the global http object for the batch. 
1117      """ 
1118       
1119       
1120       
1121      creds = None 
1122      if request.http is not None and hasattr(request.http.request, 
1123          'credentials'): 
1124        creds = request.http.request.credentials 
1125      elif http is not None and hasattr(http.request, 'credentials'): 
1126        creds = http.request.credentials 
1127      if creds is not None: 
1128        if id(creds) not in self._refreshed_credentials: 
1129          creds.refresh(http) 
1130          self._refreshed_credentials[id(creds)] = 1 
1131   
1132       
1133       
1134      if request.http is None or not hasattr(request.http.request, 
1135          'credentials'): 
1136        creds.apply(request.headers) 
 1137   
1139      """Convert an id to a Content-ID header value. 
1140   
1141      Args: 
1142        id_: string, identifier of individual request. 
1143   
1144      Returns: 
1145        A Content-ID header with the id_ encoded into it. A UUID is prepended to 
1146        the value because Content-ID headers are supposed to be universally 
1147        unique. 
1148      """ 
1149      if self._base_id is None: 
1150        self._base_id = uuid.uuid4() 
1151   
1152      return '<%s+%s>' % (self._base_id, quote(id_)) 
 1153   
1155      """Convert a Content-ID header value to an id. 
1156   
1157      Presumes the Content-ID header conforms to the format that _id_to_header() 
1158      returns. 
1159   
1160      Args: 
1161        header: string, Content-ID header value. 
1162   
1163      Returns: 
1164        The extracted id value. 
1165   
1166      Raises: 
1167        BatchError if the header is not in the expected format. 
1168      """ 
1169      if header[0] != '<' or header[-1] != '>': 
1170        raise BatchError("Invalid value for Content-ID: %s" % header) 
1171      if '+' not in header: 
1172        raise BatchError("Invalid value for Content-ID: %s" % header) 
1173      base, id_ = header[1:-1].rsplit('+', 1) 
1174   
1175      return unquote(id_) 
 1176   
1178      """Convert an HttpRequest object into a string. 
1179   
1180      Args: 
1181        request: HttpRequest, the request to serialize. 
1182   
1183      Returns: 
1184        The request as a string in application/http format. 
1185      """ 
1186       
1187      parsed = urlparse(request.uri) 
1188      request_line = urlunparse( 
1189          ('', '', parsed.path, parsed.params, parsed.query, '') 
1190          ) 
1191      status_line = request.method + ' ' + request_line + ' HTTP/1.1\n' 
1192      major, minor = request.headers.get('content-type', 'application/json').split('/') 
1193      msg = MIMENonMultipart(major, minor) 
1194      headers = request.headers.copy() 
1195   
1196      if request.http is not None and hasattr(request.http.request, 
1197          'credentials'): 
1198        request.http.request.credentials.apply(headers) 
1199   
1200       
1201      if 'content-type' in headers: 
1202        del headers['content-type'] 
1203   
1204      for key, value in six.iteritems(headers): 
1205        msg[key] = value 
1206      msg['Host'] = parsed.netloc 
1207      msg.set_unixfrom(None) 
1208   
1209      if request.body is not None: 
1210        msg.set_payload(request.body) 
1211        msg['content-length'] = str(len(request.body)) 
1212   
1213       
1214      fp = StringIO() 
1215       
1216      g = Generator(fp, maxheaderlen=0) 
1217      g.flatten(msg, unixfrom=False) 
1218      body = fp.getvalue() 
1219   
1220      return status_line + body 
 1221   
1223      """Convert string into httplib2 response and content. 
1224   
1225      Args: 
1226        payload: string, headers and body as a string. 
1227   
1228      Returns: 
1229        A pair (resp, content), such as would be returned from httplib2.request. 
1230      """ 
1231       
1232      status_line, payload = payload.split('\n', 1) 
1233      protocol, status, reason = status_line.split(' ', 2) 
1234   
1235       
1236      parser = FeedParser() 
1237      parser.feed(payload) 
1238      msg = parser.close() 
1239      msg['status'] = status 
1240   
1241       
1242      resp = httplib2.Response(msg) 
1243      resp.reason = reason 
1244      resp.version = int(protocol.split('/', 1)[1].replace('.', '')) 
1245   
1246      content = payload.split('\r\n\r\n', 1)[1] 
1247   
1248      return resp, content 
 1249   
1251      """Create a new id. 
1252   
1253      Auto incrementing number that avoids conflicts with ids already used. 
1254   
1255      Returns: 
1256         string, a new unique id. 
1257      """ 
1258      self._last_auto_id += 1 
1259      while str(self._last_auto_id) in self._requests: 
1260        self._last_auto_id += 1 
1261      return str(self._last_auto_id) 
 1262   
1263    @util.positional(2) 
1264 -  def add(self, request, callback=None, request_id=None): 
 1265      """Add a new request. 
1266   
1267      Every callback added will be paired with a unique id, the request_id. That 
1268      unique id will be passed back to the callback when the response comes back 
1269      from the server. The default behavior is to have the library generate it's 
1270      own unique id. If the caller passes in a request_id then they must ensure 
1271      uniqueness for each request_id, and if they are not an exception is 
1272      raised. Callers should either supply all request_ids or nevery supply a 
1273      request id, to avoid such an error. 
1274   
1275      Args: 
1276        request: HttpRequest, Request to add to the batch. 
1277        callback: callable, A callback to be called for this response, of the 
1278          form callback(id, response, exception). The first parameter is the 
1279          request id, and the second is the deserialized response object. The 
1280          third is an googleapiclient.errors.HttpError exception object if an HTTP error 
1281          occurred while processing the request, or None if no errors occurred. 
1282        request_id: string, A unique id for the request. The id will be passed to 
1283          the callback with the response. 
1284   
1285      Returns: 
1286        None 
1287   
1288      Raises: 
1289        BatchError if a media request is added to a batch. 
1290        KeyError is the request_id is not unique. 
1291      """ 
1292      if request_id is None: 
1293        request_id = self._new_id() 
1294      if request.resumable is not None: 
1295        raise BatchError("Media requests cannot be used in a batch request.") 
1296      if request_id in self._requests: 
1297        raise KeyError("A request with this ID already exists: %s" % request_id) 
1298      self._requests[request_id] = request 
1299      self._callbacks[request_id] = callback 
1300      self._order.append(request_id) 
 1301   
1302 -  def _execute(self, http, order, requests): 
 1303      """Serialize batch request, send to server, process response. 
1304   
1305      Args: 
1306        http: httplib2.Http, an http object to be used to make the request with. 
1307        order: list, list of request ids in the order they were added to the 
1308          batch. 
1309        request: list, list of request objects to send. 
1310   
1311      Raises: 
1312        httplib2.HttpLib2Error if a transport error has occured. 
1313        googleapiclient.errors.BatchError if the response is the wrong format. 
1314      """ 
1315      message = MIMEMultipart('mixed') 
1316       
1317      setattr(message, '_write_headers', lambda self: None) 
1318   
1319       
1320      for request_id in order: 
1321        request = requests[request_id] 
1322   
1323        msg = MIMENonMultipart('application', 'http') 
1324        msg['Content-Transfer-Encoding'] = 'binary' 
1325        msg['Content-ID'] = self._id_to_header(request_id) 
1326   
1327        body = self._serialize_request(request) 
1328        msg.set_payload(body) 
1329        message.attach(msg) 
1330   
1331       
1332       
1333      fp = StringIO() 
1334      g = Generator(fp, mangle_from_=False) 
1335      g.flatten(message, unixfrom=False) 
1336      body = fp.getvalue() 
1337   
1338      headers = {} 
1339      headers['content-type'] = ('multipart/mixed; ' 
1340                                 'boundary="%s"') % message.get_boundary() 
1341   
1342      resp, content = http.request(self._batch_uri, method='POST', body=body, 
1343                                   headers=headers) 
1344   
1345      if resp.status >= 300: 
1346        raise HttpError(resp, content, uri=self._batch_uri) 
1347   
1348       
1349      header = 'content-type: %s\r\n\r\n' % resp['content-type'] 
1350       
1351       
1352      if six.PY3: 
1353        content = content.decode('utf-8') 
1354      for_parser = header + content 
1355   
1356      parser = FeedParser() 
1357      parser.feed(for_parser) 
1358      mime_response = parser.close() 
1359   
1360      if not mime_response.is_multipart(): 
1361        raise BatchError("Response not in multipart/mixed format.", resp=resp, 
1362                         content=content) 
1363   
1364      for part in mime_response.get_payload(): 
1365        request_id = self._header_to_id(part['Content-ID']) 
1366        response, content = self._deserialize_response(part.get_payload()) 
1367         
1368        if isinstance(content, six.text_type): 
1369          content = content.encode('utf-8') 
1370        self._responses[request_id] = (response, content) 
 1371   
1372    @util.positional(1) 
1374      """Execute all the requests as a single batched HTTP request. 
1375   
1376      Args: 
1377        http: httplib2.Http, an http object to be used in place of the one the 
1378          HttpRequest request object was constructed with. If one isn't supplied 
1379          then use a http object from the requests in this batch. 
1380   
1381      Returns: 
1382        None 
1383   
1384      Raises: 
1385        httplib2.HttpLib2Error if a transport error has occured. 
1386        googleapiclient.errors.BatchError if the response is the wrong format. 
1387      """ 
1388       
1389      if len(self._order) == 0: 
1390        return None 
1391   
1392       
1393      if http is None: 
1394        for request_id in self._order: 
1395          request = self._requests[request_id] 
1396          if request is not None: 
1397            http = request.http 
1398            break 
1399   
1400      if http is None: 
1401        raise ValueError("Missing a valid http object.") 
1402   
1403       
1404       
1405      if getattr(http.request, 'credentials', None) is not None: 
1406        creds = http.request.credentials 
1407        if not getattr(creds, 'access_token', None): 
1408          LOGGER.info('Attempting refresh to obtain initial access_token') 
1409          creds.refresh(http) 
1410   
1411      self._execute(http, self._order, self._requests) 
1412   
1413       
1414       
1415      redo_requests = {} 
1416      redo_order = [] 
1417   
1418      for request_id in self._order: 
1419        resp, content = self._responses[request_id] 
1420        if resp['status'] == '401': 
1421          redo_order.append(request_id) 
1422          request = self._requests[request_id] 
1423          self._refresh_and_apply_credentials(request, http) 
1424          redo_requests[request_id] = request 
1425   
1426      if redo_requests: 
1427        self._execute(http, redo_order, redo_requests) 
1428   
1429       
1430       
1431       
1432   
1433      for request_id in self._order: 
1434        resp, content = self._responses[request_id] 
1435   
1436        request = self._requests[request_id] 
1437        callback = self._callbacks[request_id] 
1438   
1439        response = None 
1440        exception = None 
1441        try: 
1442          if resp.status >= 300: 
1443            raise HttpError(resp, content, uri=request.uri) 
1444          response = request.postproc(resp, content) 
1445        except HttpError as e: 
1446          exception = e 
1447   
1448        if callback is not None: 
1449          callback(request_id, response, exception) 
1450        if self._callback is not None: 
1451          self._callback(request_id, response, exception) 
  1452   
1455    """Mock of HttpRequest. 
1456   
1457    Do not construct directly, instead use RequestMockBuilder. 
1458    """ 
1459   
1460 -  def __init__(self, resp, content, postproc): 
 1461      """Constructor for HttpRequestMock 
1462   
1463      Args: 
1464        resp: httplib2.Response, the response to emulate coming from the request 
1465        content: string, the response body 
1466        postproc: callable, the post processing function usually supplied by 
1467                  the model class. See model.JsonModel.response() as an example. 
1468      """ 
1469      self.resp = resp 
1470      self.content = content 
1471      self.postproc = postproc 
1472      if resp is None: 
1473        self.resp = httplib2.Response({'status': 200, 'reason': 'OK'}) 
1474      if 'reason' in self.resp: 
1475        self.resp.reason = self.resp['reason'] 
 1476   
1478      """Execute the request. 
1479   
1480      Same behavior as HttpRequest.execute(), but the response is 
1481      mocked and not really from an HTTP request/response. 
1482      """ 
1483      return self.postproc(self.resp, self.content) 
  1484   
1487    """A simple mock of HttpRequest 
1488   
1489      Pass in a dictionary to the constructor that maps request methodIds to 
1490      tuples of (httplib2.Response, content, opt_expected_body) that should be 
1491      returned when that method is called. None may also be passed in for the 
1492      httplib2.Response, in which case a 200 OK response will be generated. 
1493      If an opt_expected_body (str or dict) is provided, it will be compared to 
1494      the body and UnexpectedBodyError will be raised on inequality. 
1495   
1496      Example: 
1497        response = '{"data": {"id": "tag:google.c...' 
1498        requestBuilder = RequestMockBuilder( 
1499          { 
1500            'plus.activities.get': (None, response), 
1501          } 
1502        ) 
1503        googleapiclient.discovery.build("plus", "v1", requestBuilder=requestBuilder) 
1504   
1505      Methods that you do not supply a response for will return a 
1506      200 OK with an empty string as the response content or raise an excpetion 
1507      if check_unexpected is set to True. The methodId is taken from the rpcName 
1508      in the discovery document. 
1509   
1510      For more details see the project wiki. 
1511    """ 
1512   
1513 -  def __init__(self, responses, check_unexpected=False): 
 1514      """Constructor for RequestMockBuilder 
1515   
1516      The constructed object should be a callable object 
1517      that can replace the class HttpResponse. 
1518   
1519      responses - A dictionary that maps methodIds into tuples 
1520                  of (httplib2.Response, content). The methodId 
1521                  comes from the 'rpcName' field in the discovery 
1522                  document. 
1523      check_unexpected - A boolean setting whether or not UnexpectedMethodError 
1524                         should be raised on unsupplied method. 
1525      """ 
1526      self.responses = responses 
1527      self.check_unexpected = check_unexpected 
 1528   
1529 -  def __call__(self, http, postproc, uri, method='GET', body=None, 
1530                 headers=None, methodId=None, resumable=None): 
 1531      """Implements the callable interface that discovery.build() expects 
1532      of requestBuilder, which is to build an object compatible with 
1533      HttpRequest.execute(). See that method for the description of the 
1534      parameters and the expected response. 
1535      """ 
1536      if methodId in self.responses: 
1537        response = self.responses[methodId] 
1538        resp, content = response[:2] 
1539        if len(response) > 2: 
1540           
1541          expected_body = response[2] 
1542          if bool(expected_body) != bool(body): 
1543             
1544             
1545            raise UnexpectedBodyError(expected_body, body) 
1546          if isinstance(expected_body, str): 
1547            expected_body = json.loads(expected_body) 
1548          body = json.loads(body) 
1549          if body != expected_body: 
1550            raise UnexpectedBodyError(expected_body, body) 
1551        return HttpRequestMock(resp, content, postproc) 
1552      elif self.check_unexpected: 
1553        raise UnexpectedMethodError(methodId=methodId) 
1554      else: 
1555        model = JsonModel(False) 
1556        return HttpRequestMock(None, '{}', model.response) 
  1557   
1560    """Mock of httplib2.Http""" 
1561   
1562 -  def __init__(self, filename=None, headers=None): 
 1563      """ 
1564      Args: 
1565        filename: string, absolute filename to read response from 
1566        headers: dict, header to return with response 
1567      """ 
1568      if headers is None: 
1569        headers = {'status': '200'} 
1570      if filename: 
1571        f = open(filename, 'rb') 
1572        self.data = f.read() 
1573        f.close() 
1574      else: 
1575        self.data = None 
1576      self.response_headers = headers 
1577      self.headers = None 
1578      self.uri = None 
1579      self.method = None 
1580      self.body = None 
1581      self.headers = None 
 1582   
1583   
1584 -  def request(self, uri, 
1585                method='GET', 
1586                body=None, 
1587                headers=None, 
1588                redirections=1, 
1589                connection_type=None): 
 1590      self.uri = uri 
1591      self.method = method 
1592      self.body = body 
1593      self.headers = headers 
1594      return httplib2.Response(self.response_headers), self.data 
  1595   
1598    """Mock of httplib2.Http 
1599   
1600    Mocks a sequence of calls to request returning different responses for each 
1601    call. Create an instance initialized with the desired response headers 
1602    and content and then use as if an httplib2.Http instance. 
1603   
1604      http = HttpMockSequence([ 
1605        ({'status': '401'}, ''), 
1606        ({'status': '200'}, '{"access_token":"1/3w","expires_in":3600}'), 
1607        ({'status': '200'}, 'echo_request_headers'), 
1608        ]) 
1609      resp, content = http.request("http://examples.com") 
1610   
1611    There are special values you can pass in for content to trigger 
1612    behavours that are helpful in testing. 
1613   
1614    'echo_request_headers' means return the request headers in the response body 
1615    'echo_request_headers_as_json' means return the request headers in 
1616       the response body 
1617    'echo_request_body' means return the request body in the response body 
1618    'echo_request_uri' means return the request uri in the response body 
1619    """ 
1620   
1622      """ 
1623      Args: 
1624        iterable: iterable, a sequence of pairs of (headers, body) 
1625      """ 
1626      self._iterable = iterable 
1627      self.follow_redirects = True 
 1628   
1629 -  def request(self, uri, 
1630                method='GET', 
1631                body=None, 
1632                headers=None, 
1633                redirections=1, 
1634                connection_type=None): 
 1635      resp, content = self._iterable.pop(0) 
1636      if content == 'echo_request_headers': 
1637        content = headers 
1638      elif content == 'echo_request_headers_as_json': 
1639        content = json.dumps(headers) 
1640      elif content == 'echo_request_body': 
1641        if hasattr(body, 'read'): 
1642          content = body.read() 
1643        else: 
1644          content = body 
1645      elif content == 'echo_request_uri': 
1646        content = uri 
1647      if isinstance(content, six.text_type): 
1648        content = content.encode('utf-8') 
1649      return httplib2.Response(resp), content 
  1650   
1653    """Set the user-agent on every request. 
1654   
1655    Args: 
1656       http - An instance of httplib2.Http 
1657           or something that acts like it. 
1658       user_agent: string, the value for the user-agent header. 
1659   
1660    Returns: 
1661       A modified instance of http that was passed in. 
1662   
1663    Example: 
1664   
1665      h = httplib2.Http() 
1666      h = set_user_agent(h, "my-app-name/6.0") 
1667   
1668    Most of the time the user-agent will be set doing auth, this is for the rare 
1669    cases where you are accessing an unauthenticated endpoint. 
1670    """ 
1671    request_orig = http.request 
1672   
1673     
1674    def new_request(uri, method='GET', body=None, headers=None, 
1675                    redirections=httplib2.DEFAULT_MAX_REDIRECTS, 
1676                    connection_type=None): 
1677      """Modify the request headers to add the user-agent.""" 
1678      if headers is None: 
1679        headers = {} 
1680      if 'user-agent' in headers: 
1681        headers['user-agent'] = user_agent + ' ' + headers['user-agent'] 
1682      else: 
1683        headers['user-agent'] = user_agent 
1684      resp, content = request_orig(uri, method, body, headers, 
1685                          redirections, connection_type) 
1686      return resp, content 
 1687   
1688    http.request = new_request 
1689    return http 
1690   
1693    """Tunnel PATCH requests over POST. 
1694    Args: 
1695       http - An instance of httplib2.Http 
1696           or something that acts like it. 
1697   
1698    Returns: 
1699       A modified instance of http that was passed in. 
1700   
1701    Example: 
1702   
1703      h = httplib2.Http() 
1704      h = tunnel_patch(h, "my-app-name/6.0") 
1705   
1706    Useful if you are running on a platform that doesn't support PATCH. 
1707    Apply this last if you are using OAuth 1.0, as changing the method 
1708    will result in a different signature. 
1709    """ 
1710    request_orig = http.request 
1711   
1712     
1713    def new_request(uri, method='GET', body=None, headers=None, 
1714                    redirections=httplib2.DEFAULT_MAX_REDIRECTS, 
1715                    connection_type=None): 
1716      """Modify the request headers to add the user-agent.""" 
1717      if headers is None: 
1718        headers = {} 
1719      if method == 'PATCH': 
1720        if 'oauth_token' in headers.get('authorization', ''): 
1721          LOGGER.warning( 
1722              'OAuth 1.0 request made with Credentials after tunnel_patch.') 
1723        headers['x-http-method-override'] = "PATCH" 
1724        method = 'POST' 
1725      resp, content = request_orig(uri, method, body, headers, 
1726                          redirections, connection_type) 
1727      return resp, content 
 1728   
1729    http.request = new_request 
1730    return http 
1731